|
|
dd3773 |
diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst
|
|
|
dd3773 |
index d14793a..65aa3ad 100644
|
|
|
dd3773 |
--- a/Doc/using/cmdline.rst
|
|
|
dd3773 |
+++ b/Doc/using/cmdline.rst
|
|
|
dd3773 |
@@ -728,6 +728,45 @@ conflict.
|
|
|
dd3773 |
|
|
|
dd3773 |
.. versionadded:: 3.6
|
|
|
dd3773 |
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+.. envvar:: PYTHONCOERCECLOCALE
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ If set to the value ``0``, causes the main Python command line application
|
|
|
dd3773 |
+ to skip coercing the legacy ASCII-based C locale to a more capable UTF-8
|
|
|
dd3773 |
+ based alternative. Note that this setting is checked even when the
|
|
|
dd3773 |
+ :option:`-E` or :option:`-I` options are used, as it is handled prior to
|
|
|
dd3773 |
+ the processing of command line options.
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ If this variable is *not* set, or is set to a value other than ``0``, and
|
|
|
dd3773 |
+ the current locale reported for the ``LC_CTYPE`` category is the default
|
|
|
dd3773 |
+ ``C`` locale, then the Python CLI will attempt to configure one of the
|
|
|
dd3773 |
+ following locales for the given locale categories before loading the
|
|
|
dd3773 |
+ interpreter runtime:
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ * ``C.UTF-8`` (``LC_ALL``)
|
|
|
dd3773 |
+ * ``C.utf8`` (``LC_ALL``)
|
|
|
dd3773 |
+ * ``UTF-8`` (``LC_CTYPE``)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ If setting one of these locale categories succeeds, then the matching
|
|
|
dd3773 |
+ environment variables will be set (both ``LC_ALL`` and ``LANG`` for the
|
|
|
dd3773 |
+ ``LC_ALL`` category, and ``LC_CTYPE`` for the ``LC_CTYPE`` category) in
|
|
|
dd3773 |
+ the current process environment before the Python runtime is initialized.
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ Configuring one of these locales (either explicitly or via the above
|
|
|
dd3773 |
+ implicit locale coercion) will automatically set the error handler for
|
|
|
dd3773 |
+ :data:`sys.stdin` and :data:`sys.stdout` to ``surrogateescape``. This
|
|
|
dd3773 |
+ behavior can be overridden using :envvar:`PYTHONIOENCODING` as usual.
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ For debugging purposes, setting ``PYTHONCOERCECLOCALE=warn`` will cause
|
|
|
dd3773 |
+ Python to emit warning messages on ``stderr`` if either the locale coercion
|
|
|
dd3773 |
+ activates, or else if a locale that *would* have triggered coercion is
|
|
|
dd3773 |
+ still active when the Python runtime is initialized.
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ Availability: \*nix
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ .. versionadded:: 3.7
|
|
|
dd3773 |
+ See :pep:`538` for more details.
|
|
|
dd3773 |
+
|
|
|
dd3773 |
Debug-mode variables
|
|
|
dd3773 |
~~~~~~~~~~~~~~~~~~~~
|
|
|
dd3773 |
|
|
|
dd3773 |
diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py
|
|
|
dd3773 |
index 507dc48..c3cb720 100644
|
|
|
dd3773 |
--- a/Lib/test/support/script_helper.py
|
|
|
dd3773 |
+++ b/Lib/test/support/script_helper.py
|
|
|
dd3773 |
@@ -56,8 +56,35 @@ def interpreter_requires_environment():
|
|
|
dd3773 |
return __cached_interp_requires_environment
|
|
|
dd3773 |
|
|
|
dd3773 |
|
|
|
dd3773 |
-_PythonRunResult = collections.namedtuple("_PythonRunResult",
|
|
|
dd3773 |
- ("rc", "out", "err"))
|
|
|
dd3773 |
+class _PythonRunResult(collections.namedtuple("_PythonRunResult",
|
|
|
dd3773 |
+ ("rc", "out", "err"))):
|
|
|
dd3773 |
+ """Helper for reporting Python subprocess run results"""
|
|
|
dd3773 |
+ def fail(self, cmd_line):
|
|
|
dd3773 |
+ """Provide helpful details about failed subcommand runs"""
|
|
|
dd3773 |
+ # Limit to 80 lines to ASCII characters
|
|
|
dd3773 |
+ maxlen = 80 * 100
|
|
|
dd3773 |
+ out, err = self.out, self.err
|
|
|
dd3773 |
+ if len(out) > maxlen:
|
|
|
dd3773 |
+ out = b'(... truncated stdout ...)' + out[-maxlen:]
|
|
|
dd3773 |
+ if len(err) > maxlen:
|
|
|
dd3773 |
+ err = b'(... truncated stderr ...)' + err[-maxlen:]
|
|
|
dd3773 |
+ out = out.decode('ascii', 'replace').rstrip()
|
|
|
dd3773 |
+ err = err.decode('ascii', 'replace').rstrip()
|
|
|
dd3773 |
+ raise AssertionError("Process return code is %d\n"
|
|
|
dd3773 |
+ "command line: %r\n"
|
|
|
dd3773 |
+ "\n"
|
|
|
dd3773 |
+ "stdout:\n"
|
|
|
dd3773 |
+ "---\n"
|
|
|
dd3773 |
+ "%s\n"
|
|
|
dd3773 |
+ "---\n"
|
|
|
dd3773 |
+ "\n"
|
|
|
dd3773 |
+ "stderr:\n"
|
|
|
dd3773 |
+ "---\n"
|
|
|
dd3773 |
+ "%s\n"
|
|
|
dd3773 |
+ "---"
|
|
|
dd3773 |
+ % (self.rc, cmd_line,
|
|
|
dd3773 |
+ out,
|
|
|
dd3773 |
+ err))
|
|
|
dd3773 |
|
|
|
dd3773 |
|
|
|
dd3773 |
# Executing the interpreter in a subprocess
|
|
|
dd3773 |
@@ -115,30 +142,7 @@ def run_python_until_end(*args, **env_vars):
|
|
|
dd3773 |
def _assert_python(expected_success, *args, **env_vars):
|
|
|
dd3773 |
res, cmd_line = run_python_until_end(*args, **env_vars)
|
|
|
dd3773 |
if (res.rc and expected_success) or (not res.rc and not expected_success):
|
|
|
dd3773 |
- # Limit to 80 lines to ASCII characters
|
|
|
dd3773 |
- maxlen = 80 * 100
|
|
|
dd3773 |
- out, err = res.out, res.err
|
|
|
dd3773 |
- if len(out) > maxlen:
|
|
|
dd3773 |
- out = b'(... truncated stdout ...)' + out[-maxlen:]
|
|
|
dd3773 |
- if len(err) > maxlen:
|
|
|
dd3773 |
- err = b'(... truncated stderr ...)' + err[-maxlen:]
|
|
|
dd3773 |
- out = out.decode('ascii', 'replace').rstrip()
|
|
|
dd3773 |
- err = err.decode('ascii', 'replace').rstrip()
|
|
|
dd3773 |
- raise AssertionError("Process return code is %d\n"
|
|
|
dd3773 |
- "command line: %r\n"
|
|
|
dd3773 |
- "\n"
|
|
|
dd3773 |
- "stdout:\n"
|
|
|
dd3773 |
- "---\n"
|
|
|
dd3773 |
- "%s\n"
|
|
|
dd3773 |
- "---\n"
|
|
|
dd3773 |
- "\n"
|
|
|
dd3773 |
- "stderr:\n"
|
|
|
dd3773 |
- "---\n"
|
|
|
dd3773 |
- "%s\n"
|
|
|
dd3773 |
- "---"
|
|
|
dd3773 |
- % (res.rc, cmd_line,
|
|
|
dd3773 |
- out,
|
|
|
dd3773 |
- err))
|
|
|
dd3773 |
+ res.fail(cmd_line)
|
|
|
dd3773 |
return res
|
|
|
dd3773 |
|
|
|
dd3773 |
def assert_python_ok(*args, **env_vars):
|
|
|
dd3773 |
diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py
|
|
|
dd3773 |
new file mode 100644
|
|
|
dd3773 |
index 0000000..635c98f
|
|
|
dd3773 |
--- /dev/null
|
|
|
dd3773 |
+++ b/Lib/test/test_c_locale_coercion.py
|
|
|
dd3773 |
@@ -0,0 +1,371 @@
|
|
|
dd3773 |
+# Tests the attempted automatic coercion of the C locale to a UTF-8 locale
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+import unittest
|
|
|
dd3773 |
+import locale
|
|
|
dd3773 |
+import os
|
|
|
dd3773 |
+import sys
|
|
|
dd3773 |
+import sysconfig
|
|
|
dd3773 |
+import shutil
|
|
|
dd3773 |
+import subprocess
|
|
|
dd3773 |
+from collections import namedtuple
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+import test.support
|
|
|
dd3773 |
+from test.support.script_helper import (
|
|
|
dd3773 |
+ run_python_until_end,
|
|
|
dd3773 |
+ interpreter_requires_environment,
|
|
|
dd3773 |
+)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# Set our expectation for the default encoding used in the C locale
|
|
|
dd3773 |
+# for the filesystem encoding and the standard streams
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII
|
|
|
dd3773 |
+if sys.platform.startswith("aix"):
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING = "iso8859-1"
|
|
|
dd3773 |
+else:
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING = "ascii"
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# FS encoding is UTF-8 on macOS, other *nix platforms use the locale encoding
|
|
|
dd3773 |
+if sys.platform == "darwin":
|
|
|
dd3773 |
+ C_LOCALE_FS_ENCODING = "utf-8"
|
|
|
dd3773 |
+else:
|
|
|
dd3773 |
+ C_LOCALE_FS_ENCODING = C_LOCALE_STREAM_ENCODING
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# Note that the above is probably still wrong in some cases, such as:
|
|
|
dd3773 |
+# * Windows when PYTHONLEGACYWINDOWSFSENCODING is set
|
|
|
dd3773 |
+# * AIX and any other platforms that use latin-1 in the C locale
|
|
|
dd3773 |
+#
|
|
|
dd3773 |
+# Options for dealing with this:
|
|
|
dd3773 |
+# * Don't set PYTHON_COERCE_C_LOCALE on such platforms (e.g. Windows doesn't)
|
|
|
dd3773 |
+# * Fix the test expectations to match the actual platform behaviour
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# In order to get the warning messages to match up as expected, the candidate
|
|
|
dd3773 |
+# order here must much the target locale order in Python/pylifecycle.c
|
|
|
dd3773 |
+_C_UTF8_LOCALES = ("C.UTF-8", "C.utf8", "UTF-8")
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# There's no reliable cross-platform way of checking locale alias
|
|
|
dd3773 |
+# lists, so the only way of knowing which of these locales will work
|
|
|
dd3773 |
+# is to try them with locale.setlocale(). We do that in a subprocess
|
|
|
dd3773 |
+# to avoid altering the locale of the test runner.
|
|
|
dd3773 |
+#
|
|
|
dd3773 |
+# If the relevant locale module attributes exist, and we're not on a platform
|
|
|
dd3773 |
+# where we expect it to always succeed, we also check that
|
|
|
dd3773 |
+# `locale.nl_langinfo(locale.CODESET)` works, as if it fails, the interpreter
|
|
|
dd3773 |
+# will skip locale coercion for that particular target locale
|
|
|
dd3773 |
+_check_nl_langinfo_CODESET = bool(
|
|
|
dd3773 |
+ sys.platform not in ("darwin", "linux") and
|
|
|
dd3773 |
+ hasattr(locale, "nl_langinfo") and
|
|
|
dd3773 |
+ hasattr(locale, "CODESET")
|
|
|
dd3773 |
+)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+def _set_locale_in_subprocess(locale_name):
|
|
|
dd3773 |
+ cmd_fmt = "import locale; print(locale.setlocale(locale.LC_CTYPE, '{}'))"
|
|
|
dd3773 |
+ if _check_nl_langinfo_CODESET:
|
|
|
dd3773 |
+ # If there's no valid CODESET, we expect coercion to be skipped
|
|
|
dd3773 |
+ cmd_fmt += "; import sys; sys.exit(not locale.nl_langinfo(locale.CODESET))"
|
|
|
dd3773 |
+ cmd = cmd_fmt.format(locale_name)
|
|
|
dd3773 |
+ result, py_cmd = run_python_until_end("-c", cmd, __isolated=True)
|
|
|
dd3773 |
+ return result.rc == 0
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+_fields = "fsencoding stdin_info stdout_info stderr_info lang lc_ctype lc_all"
|
|
|
dd3773 |
+_EncodingDetails = namedtuple("EncodingDetails", _fields)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+class EncodingDetails(_EncodingDetails):
|
|
|
dd3773 |
+ # XXX (ncoghlan): Using JSON for child state reporting may be less fragile
|
|
|
dd3773 |
+ CHILD_PROCESS_SCRIPT = ";".join([
|
|
|
dd3773 |
+ "import sys, os",
|
|
|
dd3773 |
+ "print(sys.getfilesystemencoding())",
|
|
|
dd3773 |
+ "print(sys.stdin.encoding + ':' + sys.stdin.errors)",
|
|
|
dd3773 |
+ "print(sys.stdout.encoding + ':' + sys.stdout.errors)",
|
|
|
dd3773 |
+ "print(sys.stderr.encoding + ':' + sys.stderr.errors)",
|
|
|
dd3773 |
+ "print(os.environ.get('LANG', 'not set'))",
|
|
|
dd3773 |
+ "print(os.environ.get('LC_CTYPE', 'not set'))",
|
|
|
dd3773 |
+ "print(os.environ.get('LC_ALL', 'not set'))",
|
|
|
dd3773 |
+ ])
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ @classmethod
|
|
|
dd3773 |
+ def get_expected_details(cls, coercion_expected, fs_encoding, stream_encoding, env_vars):
|
|
|
dd3773 |
+ """Returns expected child process details for a given encoding"""
|
|
|
dd3773 |
+ _stream = stream_encoding + ":{}"
|
|
|
dd3773 |
+ # stdin and stdout should use surrogateescape either because the
|
|
|
dd3773 |
+ # coercion triggered, or because the C locale was detected
|
|
|
dd3773 |
+ stream_info = 2*[_stream.format("surrogateescape")]
|
|
|
dd3773 |
+ # stderr should always use backslashreplace
|
|
|
dd3773 |
+ stream_info.append(_stream.format("backslashreplace"))
|
|
|
dd3773 |
+ expected_lang = env_vars.get("LANG", "not set").lower()
|
|
|
dd3773 |
+ if coercion_expected:
|
|
|
dd3773 |
+ expected_lc_ctype = CLI_COERCION_TARGET.lower()
|
|
|
dd3773 |
+ else:
|
|
|
dd3773 |
+ expected_lc_ctype = env_vars.get("LC_CTYPE", "not set").lower()
|
|
|
dd3773 |
+ expected_lc_all = env_vars.get("LC_ALL", "not set").lower()
|
|
|
dd3773 |
+ env_info = expected_lang, expected_lc_ctype, expected_lc_all
|
|
|
dd3773 |
+ return dict(cls(fs_encoding, *stream_info, *env_info)._asdict())
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ @staticmethod
|
|
|
dd3773 |
+ def _handle_output_variations(data):
|
|
|
dd3773 |
+ """Adjust the output to handle platform specific idiosyncrasies
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ * Some platforms report ASCII as ANSI_X3.4-1968
|
|
|
dd3773 |
+ * Some platforms report ASCII as US-ASCII
|
|
|
dd3773 |
+ * Some platforms report UTF-8 instead of utf-8
|
|
|
dd3773 |
+ """
|
|
|
dd3773 |
+ data = data.replace(b"ANSI_X3.4-1968", b"ascii")
|
|
|
dd3773 |
+ data = data.replace(b"US-ASCII", b"ascii")
|
|
|
dd3773 |
+ data = data.lower()
|
|
|
dd3773 |
+ return data
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ @classmethod
|
|
|
dd3773 |
+ def get_child_details(cls, env_vars):
|
|
|
dd3773 |
+ """Retrieves fsencoding and standard stream details from a child process
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ Returns (encoding_details, stderr_lines):
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ - encoding_details: EncodingDetails for eager decoding
|
|
|
dd3773 |
+ - stderr_lines: result of calling splitlines() on the stderr output
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ The child is run in isolated mode if the current interpreter supports
|
|
|
dd3773 |
+ that.
|
|
|
dd3773 |
+ """
|
|
|
dd3773 |
+ result, py_cmd = run_python_until_end(
|
|
|
dd3773 |
+ "-c", cls.CHILD_PROCESS_SCRIPT,
|
|
|
dd3773 |
+ __isolated=True,
|
|
|
dd3773 |
+ **env_vars
|
|
|
dd3773 |
+ )
|
|
|
dd3773 |
+ if not result.rc == 0:
|
|
|
dd3773 |
+ result.fail(py_cmd)
|
|
|
dd3773 |
+ # All subprocess outputs in this test case should be pure ASCII
|
|
|
dd3773 |
+ adjusted_output = cls._handle_output_variations(result.out)
|
|
|
dd3773 |
+ stdout_lines = adjusted_output.decode("ascii").splitlines()
|
|
|
dd3773 |
+ child_encoding_details = dict(cls(*stdout_lines)._asdict())
|
|
|
dd3773 |
+ stderr_lines = result.err.decode("ascii").rstrip().splitlines()
|
|
|
dd3773 |
+ return child_encoding_details, stderr_lines
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# Details of the shared library warning emitted at runtime
|
|
|
dd3773 |
+LEGACY_LOCALE_WARNING = (
|
|
|
dd3773 |
+ "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
|
|
|
dd3773 |
+ "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
|
|
|
dd3773 |
+ "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
|
|
|
dd3773 |
+ "locales is recommended."
|
|
|
dd3773 |
+)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# Details of the CLI locale coercion warning emitted at runtime
|
|
|
dd3773 |
+CLI_COERCION_WARNING_FMT = (
|
|
|
dd3773 |
+ "Python detected LC_CTYPE=C: LC_CTYPE coerced to {} (set another locale "
|
|
|
dd3773 |
+ "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior)."
|
|
|
dd3773 |
+)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+AVAILABLE_TARGETS = None
|
|
|
dd3773 |
+CLI_COERCION_TARGET = None
|
|
|
dd3773 |
+CLI_COERCION_WARNING = None
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+def setUpModule():
|
|
|
dd3773 |
+ global AVAILABLE_TARGETS
|
|
|
dd3773 |
+ global CLI_COERCION_TARGET
|
|
|
dd3773 |
+ global CLI_COERCION_WARNING
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ if AVAILABLE_TARGETS is not None:
|
|
|
dd3773 |
+ # initialization already done
|
|
|
dd3773 |
+ return
|
|
|
dd3773 |
+ AVAILABLE_TARGETS = []
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ # Find the target locales available in the current system
|
|
|
dd3773 |
+ for target_locale in _C_UTF8_LOCALES:
|
|
|
dd3773 |
+ if _set_locale_in_subprocess(target_locale):
|
|
|
dd3773 |
+ AVAILABLE_TARGETS.append(target_locale)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ if AVAILABLE_TARGETS:
|
|
|
dd3773 |
+ # Coercion is expected to use the first available target locale
|
|
|
dd3773 |
+ CLI_COERCION_TARGET = AVAILABLE_TARGETS[0]
|
|
|
dd3773 |
+ CLI_COERCION_WARNING = CLI_COERCION_WARNING_FMT.format(CLI_COERCION_TARGET)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+class _LocaleHandlingTestCase(unittest.TestCase):
|
|
|
dd3773 |
+ # Base class to check expected locale handling behaviour
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def _check_child_encoding_details(self,
|
|
|
dd3773 |
+ env_vars,
|
|
|
dd3773 |
+ expected_fs_encoding,
|
|
|
dd3773 |
+ expected_stream_encoding,
|
|
|
dd3773 |
+ expected_warnings,
|
|
|
dd3773 |
+ coercion_expected):
|
|
|
dd3773 |
+ """Check the C locale handling for the given process environment
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ Parameters:
|
|
|
dd3773 |
+ expected_fs_encoding: expected sys.getfilesystemencoding() result
|
|
|
dd3773 |
+ expected_stream_encoding: expected encoding for standard streams
|
|
|
dd3773 |
+ expected_warning: stderr output to expect (if any)
|
|
|
dd3773 |
+ """
|
|
|
dd3773 |
+ result = EncodingDetails.get_child_details(env_vars)
|
|
|
dd3773 |
+ encoding_details, stderr_lines = result
|
|
|
dd3773 |
+ expected_details = EncodingDetails.get_expected_details(
|
|
|
dd3773 |
+ coercion_expected,
|
|
|
dd3773 |
+ expected_fs_encoding,
|
|
|
dd3773 |
+ expected_stream_encoding,
|
|
|
dd3773 |
+ env_vars
|
|
|
dd3773 |
+ )
|
|
|
dd3773 |
+ self.assertEqual(encoding_details, expected_details)
|
|
|
dd3773 |
+ if expected_warnings is None:
|
|
|
dd3773 |
+ expected_warnings = []
|
|
|
dd3773 |
+ self.assertEqual(stderr_lines, expected_warnings)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+class LocaleConfigurationTests(_LocaleHandlingTestCase):
|
|
|
dd3773 |
+ # Test explicit external configuration via the process environment
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def setUpClass():
|
|
|
dd3773 |
+ # This relies on setupModule() having been run, so it can't be
|
|
|
dd3773 |
+ # handled via the @unittest.skipUnless decorator
|
|
|
dd3773 |
+ if not AVAILABLE_TARGETS:
|
|
|
dd3773 |
+ raise unittest.SkipTest("No C-with-UTF-8 locale available")
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_external_target_locale_configuration(self):
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ # Explicitly setting a target locale should give the same behaviour as
|
|
|
dd3773 |
+ # is seen when implicitly coercing to that target locale
|
|
|
dd3773 |
+ self.maxDiff = None
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ expected_fs_encoding = "utf-8"
|
|
|
dd3773 |
+ expected_stream_encoding = "utf-8"
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ base_var_dict = {
|
|
|
dd3773 |
+ "LANG": "",
|
|
|
dd3773 |
+ "LC_CTYPE": "",
|
|
|
dd3773 |
+ "LC_ALL": "",
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ for env_var in ("LANG", "LC_CTYPE"):
|
|
|
dd3773 |
+ for locale_to_set in AVAILABLE_TARGETS:
|
|
|
dd3773 |
+ # XXX (ncoghlan): LANG=UTF-8 doesn't appear to work as
|
|
|
dd3773 |
+ # expected, so skip that combination for now
|
|
|
dd3773 |
+ # See https://bugs.python.org/issue30672 for discussion
|
|
|
dd3773 |
+ if env_var == "LANG" and locale_to_set == "UTF-8":
|
|
|
dd3773 |
+ continue
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ with self.subTest(env_var=env_var,
|
|
|
dd3773 |
+ configured_locale=locale_to_set):
|
|
|
dd3773 |
+ var_dict = base_var_dict.copy()
|
|
|
dd3773 |
+ var_dict[env_var] = locale_to_set
|
|
|
dd3773 |
+ self._check_child_encoding_details(var_dict,
|
|
|
dd3773 |
+ expected_fs_encoding,
|
|
|
dd3773 |
+ expected_stream_encoding,
|
|
|
dd3773 |
+ expected_warnings=None,
|
|
|
dd3773 |
+ coercion_expected=False)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+@test.support.cpython_only
|
|
|
dd3773 |
+@unittest.skipUnless(sysconfig.get_config_var("PY_COERCE_C_LOCALE"),
|
|
|
dd3773 |
+ "C locale coercion disabled at build time")
|
|
|
dd3773 |
+class LocaleCoercionTests(_LocaleHandlingTestCase):
|
|
|
dd3773 |
+ # Test implicit reconfiguration of the environment during CLI startup
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def _check_c_locale_coercion(self,
|
|
|
dd3773 |
+ fs_encoding, stream_encoding,
|
|
|
dd3773 |
+ coerce_c_locale,
|
|
|
dd3773 |
+ expected_warnings=None,
|
|
|
dd3773 |
+ coercion_expected=True,
|
|
|
dd3773 |
+ **extra_vars):
|
|
|
dd3773 |
+ """Check the C locale handling for various configurations
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ Parameters:
|
|
|
dd3773 |
+ fs_encoding: expected sys.getfilesystemencoding() result
|
|
|
dd3773 |
+ stream_encoding: expected encoding for standard streams
|
|
|
dd3773 |
+ coerce_c_locale: setting to use for PYTHONCOERCECLOCALE
|
|
|
dd3773 |
+ None: don't set the variable at all
|
|
|
dd3773 |
+ str: the value set in the child's environment
|
|
|
dd3773 |
+ expected_warnings: expected warning lines on stderr
|
|
|
dd3773 |
+ extra_vars: additional environment variables to set in subprocess
|
|
|
dd3773 |
+ """
|
|
|
dd3773 |
+ self.maxDiff = None
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ if not AVAILABLE_TARGETS:
|
|
|
dd3773 |
+ # Locale coercion is disabled when there aren't any target locales
|
|
|
dd3773 |
+ fs_encoding = C_LOCALE_FS_ENCODING
|
|
|
dd3773 |
+ stream_encoding = C_LOCALE_STREAM_ENCODING
|
|
|
dd3773 |
+ coercion_expected = False
|
|
|
dd3773 |
+ if expected_warnings:
|
|
|
dd3773 |
+ expected_warnings = [LEGACY_LOCALE_WARNING]
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ base_var_dict = {
|
|
|
dd3773 |
+ "LANG": "",
|
|
|
dd3773 |
+ "LC_CTYPE": "",
|
|
|
dd3773 |
+ "LC_ALL": "",
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ base_var_dict.update(extra_vars)
|
|
|
dd3773 |
+ for env_var in ("LANG", "LC_CTYPE"):
|
|
|
dd3773 |
+ for locale_to_set in ("", "C", "POSIX", "invalid.ascii"):
|
|
|
dd3773 |
+ # XXX (ncoghlan): *BSD platforms don't behave as expected in the
|
|
|
dd3773 |
+ # POSIX locale, so we skip that for now
|
|
|
dd3773 |
+ # See https://bugs.python.org/issue30672 for discussion
|
|
|
dd3773 |
+ if locale_to_set == "POSIX":
|
|
|
dd3773 |
+ continue
|
|
|
dd3773 |
+ with self.subTest(env_var=env_var,
|
|
|
dd3773 |
+ nominal_locale=locale_to_set,
|
|
|
dd3773 |
+ PYTHONCOERCECLOCALE=coerce_c_locale):
|
|
|
dd3773 |
+ var_dict = base_var_dict.copy()
|
|
|
dd3773 |
+ var_dict[env_var] = locale_to_set
|
|
|
dd3773 |
+ if coerce_c_locale is not None:
|
|
|
dd3773 |
+ var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale
|
|
|
dd3773 |
+ # Check behaviour on successful coercion
|
|
|
dd3773 |
+ self._check_child_encoding_details(var_dict,
|
|
|
dd3773 |
+ fs_encoding,
|
|
|
dd3773 |
+ stream_encoding,
|
|
|
dd3773 |
+ expected_warnings,
|
|
|
dd3773 |
+ coercion_expected)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_test_PYTHONCOERCECLOCALE_not_set(self):
|
|
|
dd3773 |
+ # This should coerce to the first available target locale by default
|
|
|
dd3773 |
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=None)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_PYTHONCOERCECLOCALE_not_zero(self):
|
|
|
dd3773 |
+ # *Any* string other than "0" is considered "set" for our purposes
|
|
|
dd3773 |
+ # and hence should result in the locale coercion being enabled
|
|
|
dd3773 |
+ for setting in ("", "1", "true", "false"):
|
|
|
dd3773 |
+ self._check_c_locale_coercion("utf-8", "utf-8", coerce_c_locale=setting)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_PYTHONCOERCECLOCALE_set_to_warn(self):
|
|
|
dd3773 |
+ # PYTHONCOERCECLOCALE=warn enables runtime warnings for legacy locales
|
|
|
dd3773 |
+ self._check_c_locale_coercion("utf-8", "utf-8",
|
|
|
dd3773 |
+ coerce_c_locale="warn",
|
|
|
dd3773 |
+ expected_warnings=[CLI_COERCION_WARNING])
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_PYTHONCOERCECLOCALE_set_to_zero(self):
|
|
|
dd3773 |
+ # The setting "0" should result in the locale coercion being disabled
|
|
|
dd3773 |
+ self._check_c_locale_coercion(C_LOCALE_FS_ENCODING,
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING,
|
|
|
dd3773 |
+ coerce_c_locale="0",
|
|
|
dd3773 |
+ coercion_expected=False)
|
|
|
dd3773 |
+ # Setting LC_ALL=C shouldn't make any difference to the behaviour
|
|
|
dd3773 |
+ self._check_c_locale_coercion(C_LOCALE_FS_ENCODING,
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING,
|
|
|
dd3773 |
+ coerce_c_locale="0",
|
|
|
dd3773 |
+ LC_ALL="C",
|
|
|
dd3773 |
+ coercion_expected=False)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ def test_LC_ALL_set_to_C(self):
|
|
|
dd3773 |
+ # Setting LC_ALL should render the locale coercion ineffective
|
|
|
dd3773 |
+ self._check_c_locale_coercion(C_LOCALE_FS_ENCODING,
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING,
|
|
|
dd3773 |
+ coerce_c_locale=None,
|
|
|
dd3773 |
+ LC_ALL="C",
|
|
|
dd3773 |
+ coercion_expected=False)
|
|
|
dd3773 |
+ # And result in a warning about a lack of locale compatibility
|
|
|
dd3773 |
+ self._check_c_locale_coercion(C_LOCALE_FS_ENCODING,
|
|
|
dd3773 |
+ C_LOCALE_STREAM_ENCODING,
|
|
|
dd3773 |
+ coerce_c_locale="warn",
|
|
|
dd3773 |
+ LC_ALL="C",
|
|
|
dd3773 |
+ expected_warnings=[LEGACY_LOCALE_WARNING],
|
|
|
dd3773 |
+ coercion_expected=False)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+def test_main():
|
|
|
dd3773 |
+ test.support.run_unittest(
|
|
|
dd3773 |
+ LocaleConfigurationTests,
|
|
|
dd3773 |
+ LocaleCoercionTests
|
|
|
dd3773 |
+ )
|
|
|
dd3773 |
+ test.support.reap_children()
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+if __name__ == "__main__":
|
|
|
dd3773 |
+ test_main()
|
|
|
dd3773 |
diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py
|
|
|
dd3773 |
index 38156b4..5922ed9 100644
|
|
|
dd3773 |
--- a/Lib/test/test_cmd_line.py
|
|
|
dd3773 |
+++ b/Lib/test/test_cmd_line.py
|
|
|
dd3773 |
@@ -153,6 +153,7 @@ class CmdLineTest(unittest.TestCase):
|
|
|
dd3773 |
env = os.environ.copy()
|
|
|
dd3773 |
# Use C locale to get ascii for the locale encoding
|
|
|
dd3773 |
env['LC_ALL'] = 'C'
|
|
|
dd3773 |
+ env['PYTHONCOERCECLOCALE'] = '0'
|
|
|
dd3773 |
code = (
|
|
|
dd3773 |
b'import locale; '
|
|
|
dd3773 |
b'print(ascii("' + undecodable + b'"), '
|
|
|
dd3773 |
diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py
|
|
|
dd3773 |
index 7866a5c..b41239a 100644
|
|
|
dd3773 |
--- a/Lib/test/test_sys.py
|
|
|
dd3773 |
+++ b/Lib/test/test_sys.py
|
|
|
dd3773 |
@@ -680,6 +680,7 @@ class SysModuleTest(unittest.TestCase):
|
|
|
dd3773 |
# Force the POSIX locale
|
|
|
dd3773 |
env = os.environ.copy()
|
|
|
dd3773 |
env["LC_ALL"] = "C"
|
|
|
dd3773 |
+ env["PYTHONCOERCECLOCALE"] = "0"
|
|
|
dd3773 |
code = '\n'.join((
|
|
|
dd3773 |
'import sys',
|
|
|
dd3773 |
'def dump(name):',
|
|
|
dd3773 |
diff --git a/Modules/main.c b/Modules/main.c
|
|
|
dd3773 |
index 585d696..96d8be4 100644
|
|
|
dd3773 |
--- a/Modules/main.c
|
|
|
dd3773 |
+++ b/Modules/main.c
|
|
|
dd3773 |
@@ -107,7 +107,11 @@ static const char usage_6[] =
|
|
|
dd3773 |
" predictable seed.\n"
|
|
|
dd3773 |
"PYTHONMALLOC: set the Python memory allocators and/or install debug hooks\n"
|
|
|
dd3773 |
" on Python memory allocators. Use PYTHONMALLOC=debug to install debug\n"
|
|
|
dd3773 |
-" hooks.\n";
|
|
|
dd3773 |
+" hooks.\n"
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+"PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n"
|
|
|
dd3773 |
+" coercion behavior. Use PYTHONCOERCECLOCALE=warn to request display of\n"
|
|
|
dd3773 |
+" locale coercion and locale compatibility warnings on stderr.\n";
|
|
|
dd3773 |
|
|
|
dd3773 |
static int
|
|
|
dd3773 |
usage(int exitcode, const wchar_t* program)
|
|
|
dd3773 |
diff --git a/Programs/_testembed.c b/Programs/_testembed.c
|
|
|
dd3773 |
index 813cf30..2a64092 100644
|
|
|
dd3773 |
--- a/Programs/_testembed.c
|
|
|
dd3773 |
+++ b/Programs/_testembed.c
|
|
|
dd3773 |
@@ -1,4 +1,5 @@
|
|
|
dd3773 |
#include <Python.h>
|
|
|
dd3773 |
+#include "pyconfig.h"
|
|
|
dd3773 |
#include "pythread.h"
|
|
|
dd3773 |
#include <stdio.h>
|
|
|
dd3773 |
|
|
|
dd3773 |
diff --git a/Programs/python.c b/Programs/python.c
|
|
|
dd3773 |
index a7afbc7..03f8295 100644
|
|
|
dd3773 |
--- a/Programs/python.c
|
|
|
dd3773 |
+++ b/Programs/python.c
|
|
|
dd3773 |
@@ -15,6 +15,21 @@ wmain(int argc, wchar_t **argv)
|
|
|
dd3773 |
}
|
|
|
dd3773 |
#else
|
|
|
dd3773 |
|
|
|
dd3773 |
+/* Access private pylifecycle helper API to better handle the legacy C locale
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * The legacy C locale assumes ASCII as the default text encoding, which
|
|
|
dd3773 |
+ * causes problems not only for the CPython runtime, but also other
|
|
|
dd3773 |
+ * components like GNU readline.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * Accordingly, when the CLI detects it, it attempts to coerce it to a
|
|
|
dd3773 |
+ * more capable UTF-8 based alternative.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * See the documentation of the PYTHONCOERCECLOCALE setting for more details.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+extern int _Py_LegacyLocaleDetected(void);
|
|
|
dd3773 |
+extern void _Py_CoerceLegacyLocale(void);
|
|
|
dd3773 |
+
|
|
|
dd3773 |
int
|
|
|
dd3773 |
main(int argc, char **argv)
|
|
|
dd3773 |
{
|
|
|
dd3773 |
@@ -25,7 +40,11 @@ main(int argc, char **argv)
|
|
|
dd3773 |
char *oldloc;
|
|
|
dd3773 |
|
|
|
dd3773 |
/* Force malloc() allocator to bootstrap Python */
|
|
|
dd3773 |
+#ifdef Py_DEBUG
|
|
|
dd3773 |
+ (void)_PyMem_SetupAllocators("malloc_debug");
|
|
|
dd3773 |
+# else
|
|
|
dd3773 |
(void)_PyMem_SetupAllocators("malloc");
|
|
|
dd3773 |
+# endif
|
|
|
dd3773 |
|
|
|
dd3773 |
argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
|
|
|
dd3773 |
argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1));
|
|
|
dd3773 |
@@ -49,7 +68,21 @@ main(int argc, char **argv)
|
|
|
dd3773 |
return 1;
|
|
|
dd3773 |
}
|
|
|
dd3773 |
|
|
|
dd3773 |
+#ifdef __ANDROID__
|
|
|
dd3773 |
+ /* Passing "" to setlocale() on Android requests the C locale rather
|
|
|
dd3773 |
+ * than checking environment variables, so request C.UTF-8 explicitly
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+ setlocale(LC_ALL, "C.UTF-8");
|
|
|
dd3773 |
+#else
|
|
|
dd3773 |
+ /* Reconfigure the locale to the default for this process */
|
|
|
dd3773 |
setlocale(LC_ALL, "");
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ if (_Py_LegacyLocaleDetected()) {
|
|
|
dd3773 |
+ _Py_CoerceLegacyLocale();
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ /* Convert from char to wchar_t based on the locale settings */
|
|
|
dd3773 |
for (i = 0; i < argc; i++) {
|
|
|
dd3773 |
argv_copy[i] = Py_DecodeLocale(argv[i], NULL);
|
|
|
dd3773 |
if (!argv_copy[i]) {
|
|
|
dd3773 |
@@ -70,7 +103,11 @@ main(int argc, char **argv)
|
|
|
dd3773 |
|
|
|
dd3773 |
/* Force again malloc() allocator to release memory blocks allocated
|
|
|
dd3773 |
before Py_Main() */
|
|
|
dd3773 |
+#ifdef Py_DEBUG
|
|
|
dd3773 |
+ (void)_PyMem_SetupAllocators("malloc_debug");
|
|
|
dd3773 |
+# else
|
|
|
dd3773 |
(void)_PyMem_SetupAllocators("malloc");
|
|
|
dd3773 |
+# endif
|
|
|
dd3773 |
|
|
|
dd3773 |
for (i = 0; i < argc; i++) {
|
|
|
dd3773 |
PyMem_RawFree(argv_copy2[i]);
|
|
|
dd3773 |
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
|
|
|
dd3773 |
index ecfdfee..4fee178 100644
|
|
|
dd3773 |
--- a/Python/pylifecycle.c
|
|
|
dd3773 |
+++ b/Python/pylifecycle.c
|
|
|
dd3773 |
@@ -167,6 +167,7 @@ Py_SetStandardStreamEncoding(const char *encoding, const char *errors)
|
|
|
dd3773 |
return 0;
|
|
|
dd3773 |
}
|
|
|
dd3773 |
|
|
|
dd3773 |
+
|
|
|
dd3773 |
/* Global initializations. Can be undone by Py_FinalizeEx(). Don't
|
|
|
dd3773 |
call this twice without an intervening Py_FinalizeEx() call. When
|
|
|
dd3773 |
initializations fail, a fatal error is issued and the function does
|
|
|
dd3773 |
@@ -301,6 +302,183 @@ import_init(PyInterpreterState *interp, PyObject *sysmod)
|
|
|
dd3773 |
}
|
|
|
dd3773 |
|
|
|
dd3773 |
|
|
|
dd3773 |
+/* Helper functions to better handle the legacy C locale
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * The legacy C locale assumes ASCII as the default text encoding, which
|
|
|
dd3773 |
+ * causes problems not only for the CPython runtime, but also other
|
|
|
dd3773 |
+ * components like GNU readline.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * Accordingly, when the CLI detects it, it attempts to coerce it to a
|
|
|
dd3773 |
+ * more capable UTF-8 based alternative as follows:
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * if (_Py_LegacyLocaleDetected()) {
|
|
|
dd3773 |
+ * _Py_CoerceLegacyLocale();
|
|
|
dd3773 |
+ * }
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * See the documentation of the PYTHONCOERCECLOCALE setting for more details.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * Locale coercion also impacts the default error handler for the standard
|
|
|
dd3773 |
+ * streams: while the usual default is "strict", the default for the legacy
|
|
|
dd3773 |
+ * C locale and for any of the coercion target locales is "surrogateescape".
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+int
|
|
|
dd3773 |
+_Py_LegacyLocaleDetected(void)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+#ifndef MS_WINDOWS
|
|
|
dd3773 |
+ /* On non-Windows systems, the C locale is considered a legacy locale */
|
|
|
dd3773 |
+ /* XXX (ncoghlan): some platforms (notably Mac OS X) don't appear to treat
|
|
|
dd3773 |
+ * the POSIX locale as a simple alias for the C locale, so
|
|
|
dd3773 |
+ * we may also want to check for that explicitly.
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+ const char *ctype_loc = setlocale(LC_CTYPE, NULL);
|
|
|
dd3773 |
+ return ctype_loc != NULL && strcmp(ctype_loc, "C") == 0;
|
|
|
dd3773 |
+#else
|
|
|
dd3773 |
+ /* Windows uses code pages instead of locales, so no locale is legacy */
|
|
|
dd3773 |
+ return 0;
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static const char *_C_LOCALE_WARNING =
|
|
|
dd3773 |
+ "Python runtime initialized with LC_CTYPE=C (a locale with default ASCII "
|
|
|
dd3773 |
+ "encoding), which may cause Unicode compatibility problems. Using C.UTF-8, "
|
|
|
dd3773 |
+ "C.utf8, or UTF-8 (if available) as alternative Unicode-compatible "
|
|
|
dd3773 |
+ "locales is recommended.\n";
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static int
|
|
|
dd3773 |
+_legacy_locale_warnings_enabled(void)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+ const char *coerce_c_locale = getenv("PYTHONCOERCECLOCALE");
|
|
|
dd3773 |
+ return (coerce_c_locale != NULL &&
|
|
|
dd3773 |
+ strncmp(coerce_c_locale, "warn", 5) == 0);
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static void
|
|
|
dd3773 |
+_emit_stderr_warning_for_legacy_locale(void)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+ if (_legacy_locale_warnings_enabled()) {
|
|
|
dd3773 |
+ if (_Py_LegacyLocaleDetected()) {
|
|
|
dd3773 |
+ fprintf(stderr, "%s", _C_LOCALE_WARNING);
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+typedef struct _CandidateLocale {
|
|
|
dd3773 |
+ const char *locale_name; /* The locale to try as a coercion target */
|
|
|
dd3773 |
+} _LocaleCoercionTarget;
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static _LocaleCoercionTarget _TARGET_LOCALES[] = {
|
|
|
dd3773 |
+ {"C.UTF-8"},
|
|
|
dd3773 |
+ {"C.utf8"},
|
|
|
dd3773 |
+ {"UTF-8"},
|
|
|
dd3773 |
+ {NULL}
|
|
|
dd3773 |
+};
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static char *
|
|
|
dd3773 |
+get_default_standard_stream_error_handler(void)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+ const char *ctype_loc = setlocale(LC_CTYPE, NULL);
|
|
|
dd3773 |
+ if (ctype_loc != NULL) {
|
|
|
dd3773 |
+ /* "surrogateescape" is the default in the legacy C locale */
|
|
|
dd3773 |
+ if (strcmp(ctype_loc, "C") == 0) {
|
|
|
dd3773 |
+ return "surrogateescape";
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+#ifdef PY_COERCE_C_LOCALE
|
|
|
dd3773 |
+ /* "surrogateescape" is the default in locale coercion target locales */
|
|
|
dd3773 |
+ const _LocaleCoercionTarget *target = NULL;
|
|
|
dd3773 |
+ for (target = _TARGET_LOCALES; target->locale_name; target++) {
|
|
|
dd3773 |
+ if (strcmp(ctype_loc, target->locale_name) == 0) {
|
|
|
dd3773 |
+ return "surrogateescape";
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ /* Otherwise return NULL to request the typical default error handler */
|
|
|
dd3773 |
+ return NULL;
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+#ifdef PY_COERCE_C_LOCALE
|
|
|
dd3773 |
+static const char *_C_LOCALE_COERCION_WARNING =
|
|
|
dd3773 |
+ "Python detected LC_CTYPE=C: LC_CTYPE coerced to %.20s (set another locale "
|
|
|
dd3773 |
+ "or PYTHONCOERCECLOCALE=0 to disable this locale coercion behavior).\n";
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+static void
|
|
|
dd3773 |
+_coerce_default_locale_settings(const _LocaleCoercionTarget *target)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ const char *newloc = target->locale_name;
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ /* Reset locale back to currently configured defaults */
|
|
|
dd3773 |
+ setlocale(LC_ALL, "");
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ /* Set the relevant locale environment variable */
|
|
|
dd3773 |
+ if (setenv("LC_CTYPE", newloc, 1)) {
|
|
|
dd3773 |
+ fprintf(stderr,
|
|
|
dd3773 |
+ "Error setting LC_CTYPE, skipping C locale coercion\n");
|
|
|
dd3773 |
+ return;
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ if (_legacy_locale_warnings_enabled()) {
|
|
|
dd3773 |
+ fprintf(stderr, _C_LOCALE_COERCION_WARNING, newloc);
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+ /* Reconfigure with the overridden environment variables */
|
|
|
dd3773 |
+ setlocale(LC_ALL, "");
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+void
|
|
|
dd3773 |
+_Py_CoerceLegacyLocale(void)
|
|
|
dd3773 |
+{
|
|
|
dd3773 |
+#ifdef PY_COERCE_C_LOCALE
|
|
|
dd3773 |
+ /* We ignore the Python -E and -I flags here, as the CLI needs to sort out
|
|
|
dd3773 |
+ * the locale settings *before* we try to do anything with the command
|
|
|
dd3773 |
+ * line arguments. For cross-platform debugging purposes, we also need
|
|
|
dd3773 |
+ * to give end users a way to force even scripts that are otherwise
|
|
|
dd3773 |
+ * isolated from their environment to use the legacy ASCII-centric C
|
|
|
dd3773 |
+ * locale.
|
|
|
dd3773 |
+ *
|
|
|
dd3773 |
+ * Ignoring -E and -I is safe from a security perspective, as we only use
|
|
|
dd3773 |
+ * the setting to turn *off* the implicit locale coercion, and anyone with
|
|
|
dd3773 |
+ * access to the process environment already has the ability to set
|
|
|
dd3773 |
+ * `LC_ALL=C` to override the C level locale settings anyway.
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+ const char *coerce_c_locale = getenv("PYTHONCOERCECLOCALE");
|
|
|
dd3773 |
+ if (coerce_c_locale == NULL || strncmp(coerce_c_locale, "0", 2) != 0) {
|
|
|
dd3773 |
+ /* PYTHONCOERCECLOCALE is not set, or is set to something other than "0" */
|
|
|
dd3773 |
+ const char *locale_override = getenv("LC_ALL");
|
|
|
dd3773 |
+ if (locale_override == NULL || *locale_override == '\0') {
|
|
|
dd3773 |
+ /* LC_ALL is also not set (or is set to an empty string) */
|
|
|
dd3773 |
+ const _LocaleCoercionTarget *target = NULL;
|
|
|
dd3773 |
+ for (target = _TARGET_LOCALES; target->locale_name; target++) {
|
|
|
dd3773 |
+ const char *new_locale = setlocale(LC_CTYPE,
|
|
|
dd3773 |
+ target->locale_name);
|
|
|
dd3773 |
+ if (new_locale != NULL) {
|
|
|
dd3773 |
+#if !defined(__APPLE__) && defined(HAVE_LANGINFO_H) && defined(CODESET)
|
|
|
dd3773 |
+ /* Also ensure that nl_langinfo works in this locale */
|
|
|
dd3773 |
+ char *codeset = nl_langinfo(CODESET);
|
|
|
dd3773 |
+ if (!codeset || *codeset == '\0') {
|
|
|
dd3773 |
+ /* CODESET is not set or empty, so skip coercion */
|
|
|
dd3773 |
+ new_locale = NULL;
|
|
|
dd3773 |
+ setlocale(LC_CTYPE, "");
|
|
|
dd3773 |
+ continue;
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+ /* Successfully configured locale, so make it the default */
|
|
|
dd3773 |
+ _coerce_default_locale_settings(target);
|
|
|
dd3773 |
+ return;
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ }
|
|
|
dd3773 |
+ /* No C locale warning here, as Py_Initialize will emit one later */
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
+}
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+
|
|
|
dd3773 |
void
|
|
|
dd3773 |
_Py_InitializeEx_Private(int install_sigs, int install_importlib)
|
|
|
dd3773 |
{
|
|
|
dd3773 |
@@ -315,11 +493,19 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
|
|
|
dd3773 |
initialized = 1;
|
|
|
dd3773 |
_Py_Finalizing = NULL;
|
|
|
dd3773 |
|
|
|
dd3773 |
-#ifdef HAVE_SETLOCALE
|
|
|
dd3773 |
+#ifdef __ANDROID__
|
|
|
dd3773 |
+ /* Passing "" to setlocale() on Android requests the C locale rather
|
|
|
dd3773 |
+ * than checking environment variables, so request C.UTF-8 explicitly
|
|
|
dd3773 |
+ */
|
|
|
dd3773 |
+ setlocale(LC_CTYPE, "C.UTF-8");
|
|
|
dd3773 |
+#else
|
|
|
dd3773 |
+#ifndef MS_WINDOWS
|
|
|
dd3773 |
/* Set up the LC_CTYPE locale, so we can obtain
|
|
|
dd3773 |
the locale's charset without having to switch
|
|
|
dd3773 |
locales. */
|
|
|
dd3773 |
setlocale(LC_CTYPE, "");
|
|
|
dd3773 |
+ _emit_stderr_warning_for_legacy_locale();
|
|
|
dd3773 |
+#endif
|
|
|
dd3773 |
#endif
|
|
|
dd3773 |
|
|
|
dd3773 |
if ((p = Py_GETENV("PYTHONDEBUG")) && *p != '\0')
|
|
|
dd3773 |
@@ -1247,12 +1433,8 @@ initstdio(void)
|
|
|
dd3773 |
}
|
|
|
dd3773 |
}
|
|
|
dd3773 |
if (!errors && !(pythonioencoding && *pythonioencoding)) {
|
|
|
dd3773 |
- /* When the LC_CTYPE locale is the POSIX locale ("C locale"),
|
|
|
dd3773 |
- stdin and stdout use the surrogateescape error handler by
|
|
|
dd3773 |
- default, instead of the strict error handler. */
|
|
|
dd3773 |
- char *loc = setlocale(LC_CTYPE, NULL);
|
|
|
dd3773 |
- if (loc != NULL && strcmp(loc, "C") == 0)
|
|
|
dd3773 |
- errors = "surrogateescape";
|
|
|
dd3773 |
+ /* Choose the default error handler based on the current locale */
|
|
|
dd3773 |
+ errors = get_default_standard_stream_error_handler();
|
|
|
dd3773 |
}
|
|
|
dd3773 |
}
|
|
|
dd3773 |
|
|
|
dd3773 |
diff --git a/configure.ac b/configure.ac
|
|
|
dd3773 |
index 3f2459a..7444486 100644
|
|
|
dd3773 |
--- a/configure.ac
|
|
|
dd3773 |
+++ b/configure.ac
|
|
|
dd3773 |
@@ -3360,6 +3360,40 @@ then
|
|
|
dd3773 |
fi
|
|
|
dd3773 |
AC_MSG_RESULT($with_pymalloc)
|
|
|
dd3773 |
|
|
|
dd3773 |
+# Check for --with-c-locale-coercion
|
|
|
dd3773 |
+AC_MSG_CHECKING(for --with-c-locale-coercion)
|
|
|
dd3773 |
+AC_ARG_WITH(c-locale-coercion,
|
|
|
dd3773 |
+ AS_HELP_STRING([--with(out)-c-locale-coercion],
|
|
|
dd3773 |
+ [disable/enable C locale coercion to a UTF-8 based locale]))
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+if test -z "$with_c_locale_coercion"
|
|
|
dd3773 |
+then
|
|
|
dd3773 |
+ with_c_locale_coercion="yes"
|
|
|
dd3773 |
+fi
|
|
|
dd3773 |
+if test "$with_c_locale_coercion" != "no"
|
|
|
dd3773 |
+then
|
|
|
dd3773 |
+ AC_DEFINE(PY_COERCE_C_LOCALE, 1,
|
|
|
dd3773 |
+ [Define if you want to coerce the C locale to a UTF-8 based locale])
|
|
|
dd3773 |
+fi
|
|
|
dd3773 |
+AC_MSG_RESULT($with_c_locale_coercion)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+# Check for --with-c-locale-warning
|
|
|
dd3773 |
+AC_MSG_CHECKING(for --with-c-locale-warning)
|
|
|
dd3773 |
+AC_ARG_WITH(c-locale-warning,
|
|
|
dd3773 |
+ AS_HELP_STRING([--with(out)-c-locale-warning],
|
|
|
dd3773 |
+ [disable/enable locale compatibility warning in the C locale]))
|
|
|
dd3773 |
+
|
|
|
dd3773 |
+if test -z "$with_c_locale_warning"
|
|
|
dd3773 |
+then
|
|
|
dd3773 |
+ with_c_locale_warning="yes"
|
|
|
dd3773 |
+fi
|
|
|
dd3773 |
+if test "$with_c_locale_warning" != "no"
|
|
|
dd3773 |
+then
|
|
|
dd3773 |
+ AC_DEFINE(PY_WARN_ON_C_LOCALE, 1,
|
|
|
dd3773 |
+ [Define to emit a locale compatibility warning in the C locale])
|
|
|
dd3773 |
+fi
|
|
|
dd3773 |
+AC_MSG_RESULT($with_c_locale_warning)
|
|
|
dd3773 |
+
|
|
|
dd3773 |
# Check for Valgrind support
|
|
|
dd3773 |
AC_MSG_CHECKING([for --with-valgrind])
|
|
|
dd3773 |
AC_ARG_WITH([valgrind],
|