Decode UTF-8 text from binary-mode streams in the Python loader

src-python/amazon/ion/simpleion.py

@@ -54,6 +54,7 @@
     ``simpleion.c_ext = False``
 
 """
+import codecs
 import io
 import warnings
 from contextlib import contextmanager
@@ -413,7 +414,9 @@ def load_python(fp, catalog=None, single_value=True, parse_eagerly=True):
         if maybe_ivm == _IVM:
             raw_reader = binary_reader()
         else:
-            raw_reader = text_reader()
+            # Text Ion is UTF-8 even when the caller opened the file in binary mode.
+            fp = codecs.getreader('utf-8')(fp)
+            raw_reader = text_reader(is_unicode=True)
     reader = blocking_reader(managed_reader(raw_reader, catalog), fp)
     if parse_eagerly:
         out = []  # top-level

tests/test_simpleion.py

@@ -1040,3 +1040,21 @@ def test_c_extension_recursion_is_recoverable():
     # Rejecting the input must leave the interpreter able to handle well-formed values.
     assert loads("[0]") == [0]
     assert dumps([0], binary=False) == '$ion_1_0 [0]'
+
+
+@mark.parametrize('ion_text', [
+    "'\U0001f3f30\U0001f308\ufe0f'",
+    '"caf\u00e9 \u4e2d \U0001f600"',
+    "'''\u4e2d\U0001f600'''",
+    "{'\u00e9': '\U0001f600'}",
+    "'\u4e2d'::\"\U0001f600\"",
+])
+@mark.parametrize('parse_eagerly', [True, False])
+def test_load_utf8_text_from_binary_stream(ion_text, parse_eagerly):
+    # A catalog forces the pure-Python path even when ion-c is installed.
+    catalog = SymbolTableCatalog()
+    expected = load(StringIO(ion_text), catalog=catalog)
+    stream = BytesIO(ion_text.encode('utf-8'))
+    actual = load(stream, catalog=catalog, parse_eagerly=parse_eagerly)
+    assert ion_equals(actual, expected)
+    assert not stream.closed