diff --git a/src/main/java/com/api/jsonata4java/JSONataUtils.java b/src/main/java/com/api/jsonata4java/JSONataUtils.java index ca4b096..25eef15 100644 --- a/src/main/java/com/api/jsonata4java/JSONataUtils.java +++ b/src/main/java/com/api/jsonata4java/JSONataUtils.java @@ -586,13 +586,19 @@ public static String encodeURIComponent(String s) throws URISyntaxException { } /** - * - * @param str - * @param start Location at which to begin extracting characters. If a negative - * number is given, it is treated as strLength - start where - * strLength is the length of the string. For example, - * str.substr(-3) is treated as str.substr(str.length - 3) - * @param length The number of characters to extract. If this argument is null, + * Returns a substring of {@code str} starting at code-point index {@code start} + * and spanning at most {@code length} code points. + * + *

Both {@code start} and {@code length} are counted in Unicode code points, + * not Java UTF-16 char units. This matches the JSONata specification, which + * treats every Unicode character (including supplementary characters such as + * emoji) as a single unit. + * + * @param str The source string. + * @param start Location at which to begin extracting characters (code-point index). + * If a negative number is given, it is treated as strLength + start + * where strLength is the code-point length of the string. + * @param length The number of code points to extract. If this argument is null, * all the characters from start to the end of the string are * extracted. * @return A new string containing the extracted section of the given string. If @@ -600,50 +606,41 @@ public static String encodeURIComponent(String s) throws URISyntaxException { */ static public String substr(String str, Integer start, Integer length) { - // below has to convert start and length for emojis and unicode - int origLen = str.length(); + String strData = Objects.requireNonNull(str); + int strCodePointCount = strData.codePointCount(0, strData.length()); - String strData = Objects.requireNonNull(str).intern(); - int strLen = strData.codePointCount(0, strData.length()); - if (start >= strLen) { + // If start is positive and is greater than or equal to the code-point + // length of the string, return an empty string. + if (start >= strCodePointCount) { return ""; } - // If start is negative, substr() uses it as a character index from the - // end of the string; the index of the last character is -1. - start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start)); - if (start < 0) { - start = 0; - } // If start is negative and abs(start) is larger than the length of the - // string, substr() uses 0 as the start index. - // If length is omitted, substr() extracts characters to the end of the - // string. - if (length == null) { - length = strData.length(); - } else if (length < 0) { - // If length is 0 or negative, substr() returns an empty string. - return ""; - } else if (length > strData.length()) { - length = strData.length(); - } - length = strData.offsetByCodePoints(0, length); + // If start is negative, treat it as an index from the end of the string. + // If the resulting index is still negative, clamp to 0. + int cpStart = start >= 0 ? start : Math.max(0, strCodePointCount + start); - if (start >= 0) { - // If start is positive and is greater than or equal to the length of - // the string, substr() returns an empty string. - if (start >= origLen) { - return ""; - } - } + // Convert the code-point start index to a UTF-16 offset. + int utf16Start = strData.offsetByCodePoints(0, cpStart); + + // How many code points remain after the start position. + int remaining = strCodePointCount - cpStart; - // collect length characters (unless it reaches the end of the string - // first, in which case it will return fewer) - int end = start + length; - if (end > origLen) { - end = origLen; + if (length == null) { + // If length is omitted, extract to the end of the string. + length = remaining; + } else if (length <= 0) { + // If length is 0 or negative, return an empty string. + return ""; + } else if (length > remaining) { + // Clamp length to the remaining code points. + length = remaining; } - return strData.substring(start, end); + // Convert the code-point length to a UTF-16 end offset, measured from + // the already-computed UTF-16 start offset. + int utf16End = strData.offsetByCodePoints(utf16Start, length); + + return strData.substring(utf16Start, utf16End); } } diff --git a/src/main/java/com/api/jsonata4java/expressions/functions/SubstringAfterFunction.java b/src/main/java/com/api/jsonata4java/expressions/functions/SubstringAfterFunction.java index de529f2..bbb34e8 100644 --- a/src/main/java/com/api/jsonata4java/expressions/functions/SubstringAfterFunction.java +++ b/src/main/java/com/api/jsonata4java/expressions/functions/SubstringAfterFunction.java @@ -22,7 +22,7 @@ package com.api.jsonata4java.expressions.functions; -import java.util.Objects; +import com.api.jsonata4java.JSONataUtils; import com.api.jsonata4java.expressions.EvaluateRuntimeException; import com.api.jsonata4java.expressions.ExpressionsVisitor; import com.api.jsonata4java.expressions.generated.MappingExpressionParser.ExprContext; @@ -142,8 +142,10 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex // Find chars in str final int index = str.indexOf(chars); if (index != -1) { - if ((index + chars.length()) < str.length()) { - result = new StringNode(substr(str, index + chars.length())); + int utf16After = index + chars.length(); + if (utf16After < str.length()) { + int cpStart = str.codePointCount(0, utf16After); + result = new StringNode(JSONataUtils.substr(str, cpStart, null)); } else { result = new StringNode(""); } @@ -159,58 +161,6 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex return result; } - /** - * - * @param str - * @param start Location at which to begin extracting characters. If a negative - * number is given, it is treated as strLength - start where - * strLength is the length of the string. For example, - * str.substr(-3) is treated as str.substr(str.length - 3) - * @return A new string containing the extracted section of the given string. If - * length is 0 or a negative number, an empty string is returned. - */ - private static String substr(String str, Integer start) { - - Integer length = null; - // below has to convert start and length for emojis and unicode - int origLen = str.length(); - - String strData = Objects.requireNonNull(str).intern(); - int strLen = strData.codePointCount(0, strData.length()); - // If start is negative, substr() uses it as a character index from the - // end of the string; the index of the last character is -1. - start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start)); - // If start is negative and abs(start) is larger than the length of the - // string, substr() uses 0 as the start index. - if (start < 0) { - start = 0; - } - // If length is omitted, substr() extracts characters to the end of the - // string. - if (length == null) { - length = strData.length(); - } - - length = strData.offsetByCodePoints(0, length); - - if (start >= 0) { - // If start is positive and is greater than or equal to the length of - // the string, substr() returns an empty string. - if (start >= origLen) { - return ""; - } - } - - // collect length characters (unless it reaches the end of the string - // first, in which case it will return fewer) - int end = start + length; - if (end > origLen) { - end = origLen; - } - - return strData.substring(start, end); - } - @Override public int getMaxArgs() { return 2; diff --git a/src/main/java/com/api/jsonata4java/expressions/functions/SubstringBeforeFunction.java b/src/main/java/com/api/jsonata4java/expressions/functions/SubstringBeforeFunction.java index 02c326a..21b41f2 100644 --- a/src/main/java/com/api/jsonata4java/expressions/functions/SubstringBeforeFunction.java +++ b/src/main/java/com/api/jsonata4java/expressions/functions/SubstringBeforeFunction.java @@ -22,7 +22,7 @@ package com.api.jsonata4java.expressions.functions; -import java.util.Objects; +import com.api.jsonata4java.JSONataUtils; import com.api.jsonata4java.expressions.EvaluateRuntimeException; import com.api.jsonata4java.expressions.ExpressionsVisitor; import com.api.jsonata4java.expressions.generated.MappingExpressionParser.ExprContext; @@ -142,7 +142,8 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex // Find chars in str final int index = str.indexOf(chars); if (index != -1) { - result = new StringNode(substr(str, 0, index)); + int cpLength = str.codePointCount(0, index); + result = new StringNode(JSONataUtils.substr(str, 0, cpLength)); } else { // argChars is not present... just return argString result = new StringNode(str); @@ -155,63 +156,6 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex return result; } - /** - * - * @param str - * @param start Location at which to begin extracting characters. If a negative - * number is given, it is treated as strLength - start where - * strLength is the length of the string. For example, - * str.substr(-3) is treated as str.substr(str.length - 3) - * @param length The number of characters to extract. If this argument is null, - * all the characters from start to the end of the string are - * extracted. - * @return A new string containing the extracted section of the given string. If - * length is 0 or a negative number, an empty string is returned. - */ - private static String substr(String str, Integer start, Integer length) { - - // below has to convert start and length for emojis and unicode - int origLen = str.length(); - - String strData = Objects.requireNonNull(str).intern(); - int strLen = strData.codePointCount(0, strData.length()); - // If start is negative, substr() uses it as a character index from the - // end of the string; the index of the last character is -1. - start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start)); - // If start is negative and abs(start) is larger than the length of the - // string, substr() uses 0 as the start index. - if (start < 0) { - start = 0; - } - // If length is omitted, substr() extracts characters to the end of the - // string. - if (length == null) { - length = strData.length(); - } else if (length < 0) { - // If length is 0 or negative, substr() returns an empty string. - return ""; - } - - length = strData.offsetByCodePoints(0, length); - - if (start >= 0) { - // If start is positive and is greater than or equal to the length of - // the string, substr() returns an empty string. - if (start >= origLen) { - return ""; - } - } - - // collect length characters (unless it reaches the end of the string - // first, in which case it will return fewer) - int end = start + length; - if (end > origLen) { - end = origLen; - } - - return strData.substring(start, end); - } - @Override public int getMaxArgs() { return 2; diff --git a/src/test/java/com/api/jsonata4java/expressions/SubstringAfterFunctionTests.java b/src/test/java/com/api/jsonata4java/expressions/SubstringAfterFunctionTests.java index 0ed7d70..8b66522 100644 --- a/src/test/java/com/api/jsonata4java/expressions/SubstringAfterFunctionTests.java +++ b/src/test/java/com/api/jsonata4java/expressions/SubstringAfterFunctionTests.java @@ -165,6 +165,20 @@ public static Collection data() { }, // { "$substringAfter('hello ', ' ')", "\"\"", null + }, + + // non-BMP (supplementary) character tests — issue #461 + { + "$substringAfter('😀-x', '-')", "\"x\"", null + }, // + { + "$substringAfter('a😀-x', '-')", "\"x\"", null + }, // + { + "$substringAfter('😀😀-x', '-')", "\"x\"", null + }, // + { + "$substringAfter('abc', '-')", "\"abc\"", null } }); } diff --git a/src/test/java/com/api/jsonata4java/expressions/SubstringBeforeFunctionTests.java b/src/test/java/com/api/jsonata4java/expressions/SubstringBeforeFunctionTests.java index a24a307..5a1994c 100644 --- a/src/test/java/com/api/jsonata4java/expressions/SubstringBeforeFunctionTests.java +++ b/src/test/java/com/api/jsonata4java/expressions/SubstringBeforeFunctionTests.java @@ -162,6 +162,20 @@ public static Collection data() { }, // { "$substringBefore(' hello world again', ' ')", "\"\"", null + }, + + // non-BMP (supplementary) character tests — issue #461 + { + "$substringBefore('x-😀', '-')", "\"x\"", null + }, // + { + "$substringBefore('😀-x', '-')", "\"😀\"", null + }, // + { + "$substringBefore('😀😀-x', '-')", "\"😀😀\"", null + }, // + { + "$substringBefore('abc', '-')", "\"abc\"", null } }); } diff --git a/src/test/java/com/api/jsonata4java/expressions/SubstringFunctionTests.java b/src/test/java/com/api/jsonata4java/expressions/SubstringFunctionTests.java index 3cea7d2..05b9e17 100644 --- a/src/test/java/com/api/jsonata4java/expressions/SubstringFunctionTests.java +++ b/src/test/java/com/api/jsonata4java/expressions/SubstringFunctionTests.java @@ -223,6 +223,50 @@ public static Collection data() { }, { "$substring(\"hello\", 1, noexisty)", "\"ello\"", null + }, + + // non-BMP (supplementary) character tests — issue #461 + { + "$substring(\"😀bc\", 1)", "\"bc\"", null + }, // + { + "$substring(\"𝒳bc\", 1)", "\"bc\"", null + }, // + { + "$substring(\"😀bc\", 0, 10)", "\"😀bc\"", null + }, // + { + "$substring(\"ab😀c\", 1, 2)", "\"b😀\"", null + }, // + { + "$substring(\"ab😀\", 0, 3)", "\"ab😀\"", null + }, // + { + "$substring(\"😀😀😀\", 1, 1)", "\"😀\"", null + }, // + { + "$substring(\"a😀c\", -2)", "\"😀c\"", null + }, // + { + "$substring(\"😀bc\", 0, 0)", "\"\"", null + }, // + { + "$substring(\"😀\", 5)", "\"\"", null + }, // + { + "$substring(\"😀bc\", 0)", "\"😀bc\"", null + }, // + { + "$substring(\"😀bc\", 0, 1)", "\"😀\"", null + }, // + { + "$substring(\"😀bc\", 0, -1)", "\"\"", null + }, // + { + "$substring('', 0)", "\"\"", null + }, // + { + "$substring('😀', 1)", "\"\"", null } }); // }