Skip to content

Fix substring handling for non-BMP Unicode characters #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sivaharish31 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Fix substring handling for non-BMP Unicode characters #462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 40 additions & 43 deletions src/main/java/com/api/jsonata4java/JSONataUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,64 +586,61 @@ public static String encodeURIComponent(String s) throws URISyntaxException {
}

/**
*
* @param str
* @param start Location at which to begin extracting characters. If a negative
* number is given, it is treated as strLength - start where
* strLength is the length of the string. For example,
* str.substr(-3) is treated as str.substr(str.length - 3)
* @param length The number of characters to extract. If this argument is null,
* Returns a substring of {@code str} starting at code-point index {@code start}
* and spanning at most {@code length} code points.
*
* <p>Both {@code start} and {@code length} are counted in Unicode code points,
* not Java UTF-16 char units. This matches the JSONata specification, which
* treats every Unicode character (including supplementary characters such as
* emoji) as a single unit.
*
* @param str The source string.
* @param start Location at which to begin extracting characters (code-point index).
* If a negative number is given, it is treated as strLength + start
* where strLength is the code-point length of the string.
* @param length The number of code points to extract. If this argument is null,
* all the characters from start to the end of the string are
* extracted.
* @return A new string containing the extracted section of the given string. If
* length is 0 or a negative number, an empty string is returned.
*/
static public String substr(String str, Integer start, Integer length) {

// below has to convert start and length for emojis and unicode
int origLen = str.length();
String strData = Objects.requireNonNull(str);
int strCodePointCount = strData.codePointCount(0, strData.length());

String strData = Objects.requireNonNull(str).intern();
int strLen = strData.codePointCount(0, strData.length());
if (start >= strLen) {
// If start is positive and is greater than or equal to the code-point
// length of the string, return an empty string.
if (start >= strCodePointCount) {
return "";
}
// If start is negative, substr() uses it as a character index from the
// end of the string; the index of the last character is -1.
start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start));
if (start < 0) {
start = 0;
} // If start is negative and abs(start) is larger than the length of the
// string, substr() uses 0 as the start index.
// If length is omitted, substr() extracts characters to the end of the
// string.
if (length == null) {
length = strData.length();
} else if (length < 0) {
// If length is 0 or negative, substr() returns an empty string.
return "";
} else if (length > strData.length()) {
length = strData.length();
}

length = strData.offsetByCodePoints(0, length);
// If start is negative, treat it as an index from the end of the string.
// If the resulting index is still negative, clamp to 0.
int cpStart = start >= 0 ? start : Math.max(0, strCodePointCount + start);

if (start >= 0) {
// If start is positive and is greater than or equal to the length of
// the string, substr() returns an empty string.
if (start >= origLen) {
return "";
}
}
// Convert the code-point start index to a UTF-16 offset.
int utf16Start = strData.offsetByCodePoints(0, cpStart);

// How many code points remain after the start position.
int remaining = strCodePointCount - cpStart;

// collect length characters (unless it reaches the end of the string
// first, in which case it will return fewer)
int end = start + length;
if (end > origLen) {
end = origLen;
if (length == null) {
// If length is omitted, extract to the end of the string.
length = remaining;
} else if (length <= 0) {
// If length is 0 or negative, return an empty string.
return "";
} else if (length > remaining) {
// Clamp length to the remaining code points.
length = remaining;
}

return strData.substring(start, end);
// Convert the code-point length to a UTF-16 end offset, measured from
// the already-computed UTF-16 start offset.
int utf16End = strData.offsetByCodePoints(utf16Start, length);

return strData.substring(utf16Start, utf16End);
}

}
60 changes: 5 additions & 55 deletions src/main/java/com/api/jsonata4java/expressions/functions/SubstringAfterFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

package com.api.jsonata4java.expressions.functions;

import java.util.Objects;
import com.api.jsonata4java.JSONataUtils;
import com.api.jsonata4java.expressions.EvaluateRuntimeException;
import com.api.jsonata4java.expressions.ExpressionsVisitor;
import com.api.jsonata4java.expressions.generated.MappingExpressionParser.ExprContext;
Expand Down Expand Up @@ -142,8 +142,10 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex
// Find chars in str
final int index = str.indexOf(chars);
if (index != -1) {
if ((index + chars.length()) < str.length()) {
result = new StringNode(substr(str, index + chars.length()));
int utf16After = index + chars.length();
if (utf16After < str.length()) {
int cpStart = str.codePointCount(0, utf16After);
result = new StringNode(JSONataUtils.substr(str, cpStart, null));
} else {
result = new StringNode("");
}
Expand All @@ -159,58 +161,6 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex
return result;
}

/**
*
* @param str
* @param start Location at which to begin extracting characters. If a negative
* number is given, it is treated as strLength - start where
* strLength is the length of the string. For example,
* str.substr(-3) is treated as str.substr(str.length - 3)
* @return A new string containing the extracted section of the given string. If
* length is 0 or a negative number, an empty string is returned.
*/
private static String substr(String str, Integer start) {

Integer length = null;
// below has to convert start and length for emojis and unicode
int origLen = str.length();

String strData = Objects.requireNonNull(str).intern();
int strLen = strData.codePointCount(0, strData.length());
// If start is negative, substr() uses it as a character index from the
// end of the string; the index of the last character is -1.
start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start));
// If start is negative and abs(start) is larger than the length of the
// string, substr() uses 0 as the start index.
if (start < 0) {
start = 0;
}
// If length is omitted, substr() extracts characters to the end of the
// string.
if (length == null) {
length = strData.length();
}

length = strData.offsetByCodePoints(0, length);

if (start >= 0) {
// If start is positive and is greater than or equal to the length of
// the string, substr() returns an empty string.
if (start >= origLen) {
return "";
}
}

// collect length characters (unless it reaches the end of the string
// first, in which case it will return fewer)
int end = start + length;
if (end > origLen) {
end = origLen;
}

return strData.substring(start, end);
}

@Override
public int getMaxArgs() {
return 2;
Expand Down
62 changes: 3 additions & 59 deletions src/main/java/com/api/jsonata4java/expressions/functions/SubstringBeforeFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@

package com.api.jsonata4java.expressions.functions;

import java.util.Objects;
import com.api.jsonata4java.JSONataUtils;
import com.api.jsonata4java.expressions.EvaluateRuntimeException;
import com.api.jsonata4java.expressions.ExpressionsVisitor;
import com.api.jsonata4java.expressions.generated.MappingExpressionParser.ExprContext;
Expand Down Expand Up @@ -142,7 +142,8 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex
// Find chars in str
final int index = str.indexOf(chars);
if (index != -1) {
result = new StringNode(substr(str, 0, index));
int cpLength = str.codePointCount(0, index);
result = new StringNode(JSONataUtils.substr(str, 0, cpLength));
} else {
// argChars is not present... just return argString
result = new StringNode(str);
Expand All @@ -155,63 +156,6 @@ public JsonNode invoke(ExpressionsVisitor expressionVisitor, Function_callContex
return result;
}

/**
*
* @param str
* @param start Location at which to begin extracting characters. If a negative
* number is given, it is treated as strLength - start where
* strLength is the length of the string. For example,
* str.substr(-3) is treated as str.substr(str.length - 3)
* @param length The number of characters to extract. If this argument is null,
* all the characters from start to the end of the string are
* extracted.
* @return A new string containing the extracted section of the given string. If
* length is 0 or a negative number, an empty string is returned.
*/
private static String substr(String str, Integer start, Integer length) {

// below has to convert start and length for emojis and unicode
int origLen = str.length();

String strData = Objects.requireNonNull(str).intern();
int strLen = strData.codePointCount(0, strData.length());
// If start is negative, substr() uses it as a character index from the
// end of the string; the index of the last character is -1.
start = strData.offsetByCodePoints(0, start >= 0 ? start : ((strLen + start) < 0 ? 0 : strLen + start));
// If start is negative and abs(start) is larger than the length of the
// string, substr() uses 0 as the start index.
if (start < 0) {
start = 0;
}
// If length is omitted, substr() extracts characters to the end of the
// string.
if (length == null) {
length = strData.length();
} else if (length < 0) {
// If length is 0 or negative, substr() returns an empty string.
return "";
}

length = strData.offsetByCodePoints(0, length);

if (start >= 0) {
// If start is positive and is greater than or equal to the length of
// the string, substr() returns an empty string.
if (start >= origLen) {
return "";
}
}

// collect length characters (unless it reaches the end of the string
// first, in which case it will return fewer)
int end = start + length;
if (end > origLen) {
end = origLen;
}

return strData.substring(start, end);
}

@Override
public int getMaxArgs() {
return 2;
Expand Down
14 changes: 14 additions & 0 deletions src/test/java/com/api/jsonata4java/expressions/SubstringAfterFunctionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,20 @@ public static Collection<Object[]> data() {
}, //
{
"$substringAfter('hello ', ' ')", "\"\"", null
},

// non-BMP (supplementary) character tests — issue #461
{
"$substringAfter('😀-x', '-')", "\"x\"", null
}, //
{
"$substringAfter('a😀-x', '-')", "\"x\"", null
}, //
{
"$substringAfter('😀😀-x', '-')", "\"x\"", null
}, //
{
"$substringAfter('abc', '-')", "\"abc\"", null
}
});
}
Expand Down
14 changes: 14 additions & 0 deletions src/test/java/com/api/jsonata4java/expressions/SubstringBeforeFunctionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,20 @@ public static Collection<Object[]> data() {
}, //
{
"$substringBefore(' hello world again', ' ')", "\"\"", null
},

// non-BMP (supplementary) character tests — issue #461
{
"$substringBefore('x-😀', '-')", "\"x\"", null
}, //
{
"$substringBefore('😀-x', '-')", "\"😀\"", null
}, //
{
"$substringBefore('😀😀-x', '-')", "\"😀😀\"", null
}, //
{
"$substringBefore('abc', '-')", "\"abc\"", null
}
});
}
Expand Down
44 changes: 44 additions & 0 deletions src/test/java/com/api/jsonata4java/expressions/SubstringFunctionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,50 @@ public static Collection<Object[]> data() {
},
{
"$substring(\"hello\", 1, noexisty)", "\"ello\"", null
},

// non-BMP (supplementary) character tests — issue #461
{
"$substring(\"😀bc\", 1)", "\"bc\"", null
}, //
{
"$substring(\"𝒳bc\", 1)", "\"bc\"", null
}, //
{
"$substring(\"😀bc\", 0, 10)", "\"😀bc\"", null
}, //
{
"$substring(\"ab😀c\", 1, 2)", "\"b😀\"", null
}, //
{
"$substring(\"ab😀\", 0, 3)", "\"ab😀\"", null
}, //
{
"$substring(\"😀😀😀\", 1, 1)", "\"😀\"", null
}, //
{
"$substring(\"a😀c\", -2)", "\"😀c\"", null
}, //
{
"$substring(\"😀bc\", 0, 0)", "\"\"", null
}, //
{
"$substring(\"😀\", 5)", "\"\"", null
}, //
{
"$substring(\"😀bc\", 0)", "\"😀bc\"", null
}, //
{
"$substring(\"😀bc\", 0, 1)", "\"😀\"", null
}, //
{
"$substring(\"😀bc\", 0, -1)", "\"\"", null
}, //
{
"$substring('', 0)", "\"\"", null
}, //
{
"$substring('😀', 1)", "\"\"", null
}
}); //
}
Expand Down