diff --git a/src-python/amazon/ion/simpleion.py b/src-python/amazon/ion/simpleion.py index 12336f18..5a001e2f 100644 --- a/src-python/amazon/ion/simpleion.py +++ b/src-python/amazon/ion/simpleion.py @@ -54,6 +54,7 @@ ``simpleion.c_ext = False`` """ +import codecs import io import warnings from contextlib import contextmanager @@ -413,7 +414,9 @@ def load_python(fp, catalog=None, single_value=True, parse_eagerly=True): if maybe_ivm == _IVM: raw_reader = binary_reader() else: - raw_reader = text_reader() + # Text Ion is UTF-8 even when the caller opened the file in binary mode. + fp = codecs.getreader('utf-8')(fp) + raw_reader = text_reader(is_unicode=True) reader = blocking_reader(managed_reader(raw_reader, catalog), fp) if parse_eagerly: out = [] # top-level diff --git a/tests/test_simpleion.py b/tests/test_simpleion.py index 3cf26414..9d57c7cf 100644 --- a/tests/test_simpleion.py +++ b/tests/test_simpleion.py @@ -1040,3 +1040,21 @@ def test_c_extension_recursion_is_recoverable(): # Rejecting the input must leave the interpreter able to handle well-formed values. assert loads("[0]") == [0] assert dumps([0], binary=False) == '$ion_1_0 [0]' + + +@mark.parametrize('ion_text', [ + "'\U0001f3f30\U0001f308\ufe0f'", + '"caf\u00e9 \u4e2d \U0001f600"', + "'''\u4e2d\U0001f600'''", + "{'\u00e9': '\U0001f600'}", + "'\u4e2d'::\"\U0001f600\"", +]) +@mark.parametrize('parse_eagerly', [True, False]) +def test_load_utf8_text_from_binary_stream(ion_text, parse_eagerly): + # A catalog forces the pure-Python path even when ion-c is installed. + catalog = SymbolTableCatalog() + expected = load(StringIO(ion_text), catalog=catalog) + stream = BytesIO(ion_text.encode('utf-8')) + actual = load(stream, catalog=catalog, parse_eagerly=parse_eagerly) + assert ion_equals(actual, expected) + assert not stream.closed