Unified: Handle multi-line string literals and other variations

unified/extractor/src/languages/swift/swift.rs

@@ -168,17 +168,23 @@ fn translation_rules() -> Vec<Rule<SwiftContext>> {
             tree!((builtin_expr #{node}))
         }),
         rule!((simpleStringLiteralExpr) @@node => (string_literal #{node})),
-        // String literal with a single constant segment (for some reason not typed as simpleStringLiteralExpr)
         rule!(
-            (stringLiteralExpr segments: (stringSegment) segments: _* @@rest) @@node
-            where rest.is_empty()
+            (stringLiteralExpr segments: _* @@segments) @@node
             =>
-            (string_literal #{node})
-        ),
-        rule!(
-            (stringLiteralExpr segments: _* @segments)
-            =>
-            (string_interpolation_expr element: {segments})
+            expr {
+                // Escapes and line breaks can split a constant literal into several segments.
+                if segments.iter().all(|segment| {
+                    ctx.ast()
+                        .get_node(*segment)
+                        .expect("matched string segment must exist")
+                        .kind_name() == "stringSegment"
+                }) {
+                    tree!((string_literal #{node}))
+                } else {
+                    let segments = ctx.translate(segments)?;
+                    tree!((string_interpolation_expr element: {segments}))
+                }
+            }
         ),
         rule!((stringSegment content: @@content) => (string_literal #{content})),
         rule!(

unified/extractor/tests/corpus/swift/literals/string-escapes.output

@@ -0,0 +1,148 @@
+let escaped = "a\nb\rc"
+let raw = #"\n"#
+let multiline = """
+    aa|
+    bb
+    """
+let interpolated = "a\(raw)b"
+
+---
+
+sourceFile
+  endOfFileToken: endOfFile
+  statements:
+    codeBlockItem
+      item:
+        variableDecl
+          attributes:
+          modifiers:
+          bindingSpecifier: let
+          bindings:
+            patternBinding
+              initializer:
+                initializerClause
+                  equal: =
+                  value:
+                    stringLiteralExpr
+                      closingQuote: "
+                      openingQuote: "
+                      segments:
+                        stringSegment
+                          content: stringSegment "a\\n"
+                        stringSegment
+                          content: stringSegment "b\\r"
+                        stringSegment
+                          content: stringSegment "c"
+              pattern:
+                identifierPattern
+                  identifier: identifier "escaped"
+    codeBlockItem
+      item:
+        variableDecl
+          attributes:
+          modifiers:
+          bindingSpecifier: let
+          bindings:
+            patternBinding
+              initializer:
+                initializerClause
+                  equal: =
+                  value:
+                    stringLiteralExpr
+                      closingPounds: rawStringPoundDelimiter "#"
+                      openingPounds: rawStringPoundDelimiter "#"
+                      closingQuote: "
+                      openingQuote: "
+                      segments:
+                        stringSegment
+                          content: stringSegment "\\n"
+              pattern:
+                identifierPattern
+                  identifier: identifier "raw"
+    codeBlockItem
+      item:
+        variableDecl
+          attributes:
+          modifiers:
+          bindingSpecifier: let
+          bindings:
+            patternBinding
+              initializer:
+                initializerClause
+                  equal: =
+                  value:
+                    stringLiteralExpr
+                      closingQuote: """
+                      openingQuote: """
+                      segments:
+                        stringSegment
+                          content: stringSegment "aa|\n"
+                        stringSegment
+                          content: stringSegment "bb"
+              pattern:
+                identifierPattern
+                  identifier: identifier "multiline"
+    codeBlockItem
+      item:
+        variableDecl
+          attributes:
+          modifiers:
+          bindingSpecifier: let
+          bindings:
+            patternBinding
+              initializer:
+                initializerClause
+                  equal: =
+                  value:
+                    stringLiteralExpr
+                      closingQuote: "
+                      openingQuote: "
+                      segments:
+                        stringSegment
+                          content: stringSegment "a"
+                        expressionSegment
+                          leftParen: (
+                          rightParen: )
+                          backslash: \
+                          expressions:
+                            labeledExpr
+                              expression:
+                                declReferenceExpr
+                                  baseName: identifier "raw"
+                        stringSegment
+                          content: stringSegment "b"
+              pattern:
+                identifierPattern
+                  identifier: identifier "interpolated"
+
+---
+
+top_level source="⟨body⟩"
+  body:
+    block source="⟨stmt⟩\n⟨stmt⟩\n⟨stmt⟩\n⟨stmt⟩"
+      stmt:
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "escaped" source="escaped"
+          value: string_literal "\"a\\nb\\rc\"" source="\"a\\nb\\rc\""
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "raw" source="raw"
+          value: string_literal "#\"\\n\"#" source="#\"\\n\"#"
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "multiline" source="multiline"
+          value: string_literal "\"\"\"\n    aa|\n    bb\n    \"\"\"" source="\"\"\"\n    aa|\n    bb\n    \"\"\""
+        variable_declaration source="⟨modifier⟩ ⟨pattern⟩ = ⟨value⟩"
+          modifier: modifier "let" source="let"
+          pattern: identifier "interpolated" source="interpolated"
+          value:
+            string_interpolation_expr source="\"⟨element⟩⟨element⟩⟨element⟩\""
+              element:
+                string_literal "a" source="a"
+                call_expr source="\\(⟨argument⟩)"
+                  callee: builtin_expr "interpolation" source=""
+                  argument:
+                    argument source="⟨value⟩"
+                      value: identifier "raw" source="raw"
+                string_literal "b" source="b"

unified/extractor/tests/corpus/swift/literals/string-escapes.swift

@@ -0,0 +1,7 @@
+let escaped = "a\nb\rc"
+let raw = #"\n"#
+let multiline = """
+    aa|
+    bb
+    """
+let interpolated = "a\(raw)b"

unified/ql/lib/codeql/unified/internal/FacadeAst.qll

@@ -8,6 +8,7 @@ private import unified as U
 private import AstPlugin
 private import codeql.files.FileSystem
 private import codeql.unified.internal.NameBinding as NameBinding
+private import codeql.unified.internal.SwiftStringLiterals as SwiftStringLiterals
 
 module Unified {
   private import Ast::Unified as G
@@ -109,15 +110,11 @@ module Unified {
   class Expr extends G::Expr {
     /** Gets the string value of this expression, if it is a known string constant. */
     string getStringValue() {
-      // TODO: we'll want to cook the string literals extractor-side, but for now
-      // just strip the quotes here and ignore escape sequences.
-      exists(string text | text = this.(StringLiteral).getValue() |
-        result = text.regexpCapture("\"(.*)\"", 1)
-        or
-        // Constant-segments of string interpolations are represented as string literals, but their raw text does not have quotes
-        not exists(text.regexpCapture("\"(.*)\"", 1)) and
-        result = text
-      )
+      result = SwiftStringLiterals::getStringValue(this.(StringLiteral))
+      or
+      // Interpolation segments have no delimiters; their raw text is not a complete literal.
+      this.getParent() instanceof StringInterpolationExpr and
+      result = this.(StringLiteral).getValue()
     }
 
     /** Gets the immediately-enclosing expression, skipping over intermediate sub-nodes like `Argument`, and without crossing a function boundary. */

unified/ql/lib/codeql/unified/internal/SwiftStringLiterals.qll

@@ -0,0 +1,90 @@
+/** Provides decoded values of Swift string literals. */
+overlay[local?]
+module;
+
+private import unified
+private import codeql.util.Numbers
+
+private predicate literalContents(StringLiteral literal, string text, string escapePrefix) {
+  exists(string raw, string hashes, string quote, string contents |
+    raw = literal.getValue().replaceAll("\r\n", "\n").replaceAll("\r", "\n") and
+    hashes = raw.regexpCapture("(?s)(#*)(\"\"\"|\")(.*)\\2\\1", 1) and
+    quote = raw.regexpCapture("(?s)(#*)(\"\"\"|\")(.*)\\2\\1", 2) and
+    contents = raw.regexpCapture("(?s)(#*)(\"\"\"|\")(.*)\\2\\1", 3) and
+    escapePrefix = "\\" + hashes
+  |
+    quote = "\"" and text = contents
+    or
+    quote = "\"\"\"" and
+    exists(string body, string indent |
+      body = contents.regexpCapture("(?s)\n(.*)\n([ \t]*)", 1) and
+      indent = contents.regexpCapture("(?s)\n(.*)\n([ \t]*)", 2) and
+      text = body.regexpReplaceAll("(?m)^" + indent, "")
+    )
+  )
+}
+
+private predicate decodedPart(StringLiteral literal, int start, int end, string value) {
+  exists(string text, string prefix |
+    literalContents(literal, text, prefix) and
+    start in [0 .. text.length() - 1]
+  |
+    not text.substring(start, start + prefix.length()) = prefix and
+    end = start + 1 and
+    value = text.charAt(start)
+    or
+    text.substring(start, start + prefix.length()) = prefix and
+    exists(int next | next = start + prefix.length() |
+      end = next + 1 and
+      (
+        text.charAt(next) = "0" and value = 0.toUnicode()
+        or
+        text.charAt(next) = "t" and value = "\t"
+        or
+        text.charAt(next) = "n" and value = "\n"
+        or
+        text.charAt(next) = "r" and value = "\r"
+        or
+        value = text.charAt(next) and value = ["\\", "\"", "'"]
+      )
+      or
+      exists(string hex |
+        hex =
+          text.substring(next, text.length()).regexpCapture("(?s)u\\{([0-9a-fA-F]{1,8})\\}.*", 1) and
+        end = next + hex.length() + 3 and
+        value = parseHexInt(hex).toUnicode()
+      )
+      or
+      exists(string continuation |
+        continuation = text.substring(next, text.length()).regexpCapture("(?s)([ \t]*\n).*", 1) and
+        end = next + continuation.length() and
+        value = ""
+      )
+    )
+  )
+}
+
+private predicate decodedOffset(StringLiteral literal, int offset) {
+  offset = 0
+  or
+  decodedPart(literal, _, offset, _) and
+  exists(int previous |
+    decodedOffset(literal, previous) and
+    decodedPart(literal, previous, offset, _)
+  )
+}
+
+/** Gets the decoded value of `literal`, if it is a complete, valid string literal. */
+string getStringValue(StringLiteral literal) {
+  exists(string text |
+    literalContents(literal, text, _) and
+    decodedOffset(literal, text.length())
+  |
+    result =
+      concat(int start, int end, string value |
+        decodedOffset(literal, start) and decodedPart(literal, start, end, value)
+      |
+        value order by start
+      )
+  )
+}

unified/ql/test/library-tests/string-literals/test.expected

@@ -0,0 +1,14 @@
+| test.swift:1:5:1:9 | empty |  |
+| test.swift:2:5:2:10 | simple | abc |
+| test.swift:3:5:3:11 | escaped | a\nb\rc\td\u0000e\\f"g'h |
+| test.swift:4:5:4:20 | escapedBackslash | \\n |
+| test.swift:5:5:5:11 | unicode | a\ud83d\ude00 |
+| test.swift:6:5:6:7 | raw | \\n\\"abc |
+| test.swift:7:5:7:14 | rawEscapes | \\n\na"\\ |
+| test.swift:8:5:8:16 | hashesInside | abc#def |
+| test.swift:9:5:9:13 | multiline | aa\|\nbb |
+| test.swift:13:5:13:19 | multilineIndent |   aa\nbb |
+| test.swift:17:5:17:22 | multilineEmptyLine | aa\n\nbb |
+| test.swift:22:5:22:25 | multilineContinuation | aabb |
+| test.swift:26:5:26:16 | rawMultiline | aa\\n\nbb |
+| test.swift:30:5:30:17 | interpolation |  |

unified/ql/test/library-tests/string-literals/test.ql

@@ -0,0 +1,5 @@
+import unified
+
+from VariableDeclaration decl
+where decl.getFile().fromSource()
+select decl.getPattern(), concat(decl.getValue().getStringValue())

unified/ql/test/library-tests/string-literals/test.swift

@@ -0,0 +1,30 @@
+let empty = ""
+let simple = "abc"
+let escaped = "a\nb\rc\td\0e\\f\"g\'h"
+let escapedBackslash = "\\n"
+let unicode = "\u{61}\u{1F600}"
+let raw = #"\n\"abc"#
+let rawEscapes = ##"\n\##n\##u{61}\##"\##\"##
+let hashesInside = #"abc#def"#
+let multiline = """
+    aa|
+    bb
+    """
+let multilineIndent = """
+      aa
+    bb
+    """
+let multilineEmptyLine = """
+    aa
+
+    bb
+    """
+let multilineContinuation = """
+    aa\
+    bb
+    """
+let rawMultiline = #"""
+    aa\n
+    bb
+    """#
+let interpolation = "a\(simple)b"