From 2ca3c25a95e65245937d6d4c8a934bbcdd6eabef Mon Sep 17 00:00:00 2001 From: Shahmir Varqha Date: Fri, 24 Jul 2026 22:24:26 +0800 Subject: [PATCH] feat(vnext): add dialect-aware statement index --- docs/adr/0001-language-service-and-session.md | 14 + docs/vnext/source-coordinates.md | 4 + docs/vnext/statement-index.md | 79 ++ src/vnext/__tests__/statement-index.test.ts | 745 ++++++++++++++ src/vnext/statement-index.ts | 925 ++++++++++++++++++ 5 files changed, 1767 insertions(+) create mode 100644 docs/vnext/statement-index.md create mode 100644 src/vnext/__tests__/statement-index.test.ts create mode 100644 src/vnext/statement-index.ts diff --git a/docs/adr/0001-language-service-and-session.md b/docs/adr/0001-language-service-and-session.md index fbc3d82..0920e47 100644 --- a/docs/adr/0001-language-service-and-session.md +++ b/docs/adr/0001-language-service-and-session.md @@ -202,6 +202,20 @@ document offsets and statement-relative offsets use distinct branded types. Statement lookup uses explicit cursor affinity at boundaries and EOF. It must not reintroduce the v0.x inclusive-end behavior. +The first statement-index implementation is an internal synchronous full-scan +oracle. Exact slot extents partition the analysis text, terminators associate +left, and trivia after a terminator associates with the following slot. Empty +documents and terminal delimiters retain explicit empty slots. Lookup requires +left or right affinity and uses binary search. + +The index does not contain statement text, line numbers, inferred kinds, parser +validity, or AST data. It materializes at most 10,000 slots and collapses an +unscanned remainder into an opaque slot on resource limits or unsupported +delimiter/procedural constructs. Opaque slots cannot be passed to later parsing +as exact source. Dialect lexical behavior uses internal configuration identity, +never a caller-controlled dialect ID. Incremental reuse and session attachment +remain a separate change tested against the full-scan oracle. + ## Request outcomes Every session feature request settles with the same top-level union: diff --git a/docs/vnext/source-coordinates.md b/docs/vnext/source-coordinates.md index 6db46d9..9a4932e 100644 --- a/docs/vnext/source-coordinates.md +++ b/docs/vnext/source-coordinates.md @@ -44,3 +44,7 @@ functions remain internal. This slice intentionally does not publish a source transformer or source-map SPI. Generated or reordered source requires a versioned segment-map design and evidence from real consumers before becoming public. + +The internal [statement index](./statement-index.md) scans `analysisText` and +uses its length-preserving offsets without publishing analysis-coordinate +ranges. diff --git a/docs/vnext/statement-index.md b/docs/vnext/statement-index.md new file mode 100644 index 0000000..92e4db8 --- /dev/null +++ b/docs/vnext/statement-index.md @@ -0,0 +1,79 @@ +# vNext Statement Index + +Status: internal full-scan correctness oracle + +The statement index is a synchronous, parser-free partition of +`analysisText`. It does not classify, parse, validate, or copy statements, and +it has no public `/vnext` export yet. + +Each exact slot contains: + +- An `extent` that participates in a contiguous partition of the complete text. +- A `source` range that includes leading and trailing trivia but excludes the + terminator. +- An optional one-code-unit `terminator`, owned by the slot on its left. +- `hasCode`, which distinguishes code from whitespace/comment-only slots. +- A lexical end state for incomplete quoted strings or block comments. + +An empty document has one empty slot. A document ending exactly in a semicolon +has an explicit trailing empty slot; trailing trivia instead forms the final +trivia-only slot. Consecutive semicolons retain their empty slots. + +The scanner returns an opaque suffix instead of guessed boundaries when it +encounters an unsupported custom delimiter, an unsupported BigQuery procedural +body, or a resource limit. Opaque slots omit `source`, `terminator`, and +`hasCode`, so later layers cannot accidentally parse them as exact statements. +At most 10,000 slots are materialized; a semicolon-dense remainder collapses +into one opaque slot. + +## Cursor affinity + +Point lookup always requires `left` or `right` affinity. At a shared extent +boundary, left selects the preceding slot and right selects the following slot. +At position zero both select the first slot. At EOF after a terminator, left +selects the terminated slot and right selects the explicit trailing empty slot. +At EOF in an unterminated final statement, both select that statement. + +The lookup uses binary search. It deliberately does not implement a hidden +"nearest code statement" fallback; completion, hover, gutter, and future +run-current-statement commands need different policies. + +## Dialect profiles + +Lexical behavior is carried by immutable internal profile identity. It is never +inferred from a caller-controlled dialect ID. This first oracle owns profiles +for: + +- PostgreSQL: doubled quotes, `E'...'`, dollar-quoted strings, and nested block + comments, following the + [PostgreSQL lexical contract](https://www.postgresql.org/docs/current/sql-syntax-lexical.html). + Dollar-tag and literal-prefix boundaries conservatively treat every + non-ASCII code point as identifier-like, covering the engines' permissive + Unicode behavior without exposing internal semicolons. SQL routine bodies + introduced by `BEGIN ATOMIC` are opaque in this slice. +- DuckDB: doubled quotes, escape strings, tagged dollar-quoted strings, and + nested comments. Dollar quoting follows + [DuckDB literal types](https://duckdb.org/docs/current/sql/data_types/literal_types). +- BigQuery: single, double, triple, raw, bytes, and raw-bytes strings; backtick + identifiers; `#` comments; and non-nesting block comments, following the + [GoogleSQL lexical contract](https://docs.cloud.google.com/bigquery/docs/reference/standard-sql/lexical). + Procedural bodies, including labeled loops, are opaque in this slice. +- Dremio: a compatibility profile limited to verified single-quoted strings, + double-quoted identifiers, and standard comments. It does not silently + inherit PostgreSQL extensions. + +Unterminated lexical constructs consume the remainder and report their opening +offset. Regular BigQuery strings also fail closed at a line break, because only +triple-quoted strings may span lines. + +## Complexity and sequencing + +A full build is linear in UTF-16 code units, retains only slot records and a +bounded dollar delimiter, and creates no statement substrings. Point lookup is +logarithmic in slot count. The scanner operates on `analysisText`; the current +length-preserving source transform makes its analysis ranges valid at the same +offsets in `originalText`. + +This implementation remains the correctness oracle. Incremental rescanning, +change mapping, cache reuse, and session attachment belong to a later slice and +must be tested against a fresh full build after arbitrary edits. diff --git a/src/vnext/__tests__/statement-index.test.ts b/src/vnext/__tests__/statement-index.test.ts new file mode 100644 index 0000000..0a17bd1 --- /dev/null +++ b/src/vnext/__tests__/statement-index.test.ts @@ -0,0 +1,745 @@ +import { describe, expect, it } from "vitest"; +import { + BIGQUERY_SQL_LEXICAL_PROFILE, + buildSqlStatementIndex, + DREMIO_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + findSqlStatementSlot, + MAX_SQL_STATEMENT_SLOTS, + POSTGRESQL_SQL_LEXICAL_PROFILE, + type SqlLexicalProfile, + type SqlStatementIndex, + type SqlStatementSlot, +} from "../statement-index.js"; +import { createMaskedSqlSource } from "../source.js"; + +function ranges(index: SqlStatementIndex) { + return index.slots.map((slot) => { + if (slot.boundaryQuality === "opaque") { + return { + extent: [slot.extent.from, slot.extent.to], + quality: slot.boundaryQuality, + reason: slot.endState.reason, + }; + } + return { + extent: [slot.extent.from, slot.extent.to], + hasCode: slot.hasCode, + quality: slot.boundaryQuality, + source: [slot.source.from, slot.source.to], + terminator: slot.terminator + ? [slot.terminator.from, slot.terminator.to] + : null, + }; + }); +} + +function exactSlot(slot: SqlStatementSlot) { + expect(slot.boundaryQuality).toBe("exact"); + if (slot.boundaryQuality === "opaque") { + throw new Error("Expected an exact SQL statement slot"); + } + return slot; +} + +function itemAt(items: readonly Value[], index: number): Value { + const item = items[index]; + if (item === undefined) { + throw new Error(`Expected test item at index ${index}`); + } + return item; +} + +function expectPartition(text: string, index: SqlStatementIndex): void { + expect(index.slots.length).toBeGreaterThan(0); + let cursor = 0; + for (const slot of index.slots) { + expect(slot.extent.from).toBe(cursor); + expect(slot.extent.to).toBeGreaterThanOrEqual(slot.extent.from); + expect(slot.extent.to).toBeLessThanOrEqual(text.length); + expect(Object.isFrozen(slot)).toBe(true); + expect(Object.isFrozen(slot.extent)).toBe(true); + if (slot.boundaryQuality === "exact") { + expect(slot.source.from).toBe(slot.extent.from); + expect(slot.source.to).toBeLessThanOrEqual(slot.extent.to); + if (slot.terminator) { + expect(slot.terminator.from).toBe(slot.source.to); + expect(slot.terminator.to).toBe(slot.extent.to); + expect(text.slice(slot.terminator.from, slot.terminator.to)).toBe(";"); + } else { + expect(slot.source.to).toBe(slot.extent.to); + } + } + cursor = slot.extent.to; + } + expect(cursor).toBe(text.length); + expect(Object.isFrozen(index)).toBe(true); + expect(Object.isFrozen(index.slots)).toBe(true); +} + +describe("statement partition", () => { + it.each([ + [ + "", + [ + { + extent: [0, 0], + hasCode: false, + quality: "exact", + source: [0, 0], + terminator: null, + }, + ], + ], + [ + "SELECT 1", + [ + { + extent: [0, 8], + hasCode: true, + quality: "exact", + source: [0, 8], + terminator: null, + }, + ], + ], + [ + "SELECT 1;", + [ + { + extent: [0, 9], + hasCode: true, + quality: "exact", + source: [0, 8], + terminator: [8, 9], + }, + { + extent: [9, 9], + hasCode: false, + quality: "exact", + source: [9, 9], + terminator: null, + }, + ], + ], + [ + "SELECT 1; \r\n -- next\n SELECT 2", + [ + { + extent: [0, 9], + hasCode: true, + quality: "exact", + source: [0, 8], + terminator: [8, 9], + }, + { + extent: [9, 30], + hasCode: true, + quality: "exact", + source: [9, 30], + terminator: null, + }, + ], + ], + [ + ";;", + [ + { + extent: [0, 1], + hasCode: false, + quality: "exact", + source: [0, 0], + terminator: [0, 1], + }, + { + extent: [1, 2], + hasCode: false, + quality: "exact", + source: [1, 1], + terminator: [1, 2], + }, + { + extent: [2, 2], + hasCode: false, + quality: "exact", + source: [2, 2], + terminator: null, + }, + ], + ], + ])("partitions %j without trimming or copying semantics", (text, expected) => { + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(ranges(index)).toEqual(expected); + expectPartition(text, index); + }); + + it("keeps trailing trivia in a following empty-code slot", () => { + const text = "SELECT 1; \n/* after */"; + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(ranges(index)).toEqual([ + { + extent: [0, 9], + hasCode: true, + quality: "exact", + source: [0, 8], + terminator: [8, 9], + }, + { + extent: [9, text.length], + hasCode: false, + quality: "exact", + source: [9, text.length], + terminator: null, + }, + ]); + }); + + it("preserves UTF-16 coordinates around astral and lone-surrogate text", () => { + const text = "SELECT '🦆;\ud800';\r\nSELECT 2"; + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(2); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe(13); + expect(exactSlot(itemAt(index.slots, 1)).source.from).toBe(14); + expectPartition(text, index); + }); +}); + +describe("cursor affinity", () => { + it("selects the requested side of a shared boundary", () => { + const text = "SELECT 1;SELECT 2"; + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + + expect(findSqlStatementSlot(index, 0, "left")).toBe(index.slots[0]); + expect(findSqlStatementSlot(index, 0, "right")).toBe(index.slots[0]); + expect(findSqlStatementSlot(index, 8, "left")).toBe(index.slots[0]); + expect(findSqlStatementSlot(index, 9, "left")).toBe(index.slots[0]); + expect(findSqlStatementSlot(index, 9, "right")).toBe(index.slots[1]); + expect(findSqlStatementSlot(index, text.length, "left")).toBe(index.slots[1]); + expect(findSqlStatementSlot(index, text.length, "right")).toBe(index.slots[1]); + }); + + it("distinguishes EOF after a terminator from EOF in an open statement", () => { + const terminated = buildSqlStatementIndex( + "SELECT 1;", + DUCKDB_SQL_LEXICAL_PROFILE, + ); + expect(findSqlStatementSlot(terminated, 9, "left")).toBe(terminated.slots[0]); + expect(findSqlStatementSlot(terminated, 9, "right")).toBe(terminated.slots[1]); + + const open = buildSqlStatementIndex("SELECT 1", DUCKDB_SQL_LEXICAL_PROFILE); + expect(findSqlStatementSlot(open, 8, "left")).toBe(open.slots[0]); + expect(findSqlStatementSlot(open, 8, "right")).toBe(open.slots[0]); + }); + + it("validates internal positions", () => { + const index = buildSqlStatementIndex("", DUCKDB_SQL_LEXICAL_PROFILE); + expect(() => findSqlStatementSlot(index, -1, "left")).toThrow(RangeError); + expect(() => findSqlStatementSlot(index, 1, "right")).toThrow(RangeError); + expect(() => findSqlStatementSlot(index, Number.NaN, "right")).toThrow( + RangeError, + ); + expect(() => + Reflect.apply(findSqlStatementSlot, undefined, [index, 0, "center"]), + ).toThrow(TypeError); + }); +}); + +describe("comments", () => { + it.each([ + "SELECT 1 -- ; hidden\r\n;SELECT 2", + "SELECT 1 /* ; hidden */;SELECT 2", + "/* comment only ; */;", + ])("ignores protected semicolons in %j", (text) => { + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(2); + expectPartition(text, index); + }); + + it("supports nested PostgreSQL and DuckDB block comments", () => { + const text = "SELECT 1 /* outer /* ; inner */ ; outer */; SELECT 2"; + for (const profile of [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + ]) { + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(2); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe(42); + } + }); + + it("supports BigQuery hash comments and its non-nesting block comments", () => { + const hash = buildSqlStatementIndex( + "SELECT 1 # ; hidden\n;SELECT 2", + BIGQUERY_SQL_LEXICAL_PROFILE, + ); + expect(hash.slots).toHaveLength(2); + + const nonNested = buildSqlStatementIndex( + "/* outer /* inner */;SELECT 2", + BIGQUERY_SQL_LEXICAL_PROFILE, + ); + expect(nonNested.slots).toHaveLength(2); + }); + + it("does not borrow BigQuery hash comments in other profiles", () => { + const index = buildSqlStatementIndex( + "SELECT 1 # ; SELECT 2", + DUCKDB_SQL_LEXICAL_PROFILE, + ); + expect(index.slots).toHaveLength(2); + }); +}); + +describe("PostgreSQL and DuckDB literals", () => { + it.each([ + "SELECT 'a;''b';SELECT 2", + 'SELECT "a;""b";SELECT 2', + "SELECT E'a\\';b';SELECT 2", + "SELECT $$a;b$$;SELECT 2", + "SELECT $tag$a;$$;b$tag$;SELECT 2", + ])("protects semicolons in %j", (text) => { + for (const profile of [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + ]) { + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(2); + expect(index.quality).toBe("exact"); + expectPartition(text, index); + } + }); + + it("requires a legal token boundary and case-matched dollar tag", () => { + const attached = buildSqlStatementIndex( + "SELECT value$tag$a;b$tag$;SELECT 2", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(attached.slots.length).toBeGreaterThan(2); + + const mismatched = buildSqlStatementIndex( + "SELECT $tag$a;b$TAG$;SELECT 2", + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(mismatched.slots).toHaveLength(1); + expect(mismatched.endState).toMatchObject({ + construct: "dollar-quoted-string", + kind: "unterminated", + }); + }); + + it("uses conservative Unicode rules for dollar tags and boundaries", () => { + for (const text of [ + "SELECT $é$a;b$é$;SELECT 2", + "SELECT $e\u0301$a;b$e\u0301$;SELECT 2", + "SELECT $𐐀$a;b$𐐀$;SELECT 2", + "SELECT $🦆$a;b$🦆$;SELECT 2", + "SELECT $—$a;b$—$;SELECT 2", + "SELECT $\u0301$a;b$\u0301$;SELECT 2", + ]) { + for (const profile of [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + ]) { + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(2); + expect(index.quality).toBe("exact"); + } + } + + for (const text of [ + "SELECT é$tag$a;b$tag$;SELECT 2", + "SELECT e\u0301$tag$a;b$tag$;SELECT 2", + "SELECT 𐐀$tag$a;b$tag$;SELECT 2", + "SELECT 🦆$tag$a;b$tag$;SELECT 2", + ]) { + for (const profile of [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + ]) { + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(3); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe( + text.indexOf(";"), + ); + } + } + }); + + it("does not apply E-string backslash escaping to an attached identifier", () => { + for (const text of [ + "SELECT nameE'a\\';SELECT 2", + "SELECT éE'a\\';SELECT 2", + "SELECT e\u0301E'a\\';SELECT 2", + "SELECT 𐐀E'a\\';SELECT 2", + "SELECT 🦆E'a\\';SELECT 2", + ]) { + for (const profile of [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + ]) { + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(2); + } + } + + const reviewedReproduction = "SELECT éE'a\\';b';SELECT 2"; + const index = buildSqlStatementIndex( + reviewedReproduction, + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe( + reviewedReproduction.indexOf(";"), + ); + }); +}); + +describe("BigQuery literals", () => { + it.each([ + "SELECT 'a\\';b';SELECT 2", + 'SELECT "a\\";b";SELECT 2', + "SELECT '''a;\n'b''';SELECT 2", + 'SELECT """a;\n"b""";SELECT 2', + "SELECT r'a\\';SELECT 2", + "SELECT br'''a\\;b''';SELECT 2", + "SELECT rb\"a\\;b\";SELECT 2", + "SELECT `project;a`;SELECT 2", + "SELECT `project\\`;a`;SELECT 2", + ])("protects semicolons in %j", (text) => { + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(2); + expect(index.quality).toBe("exact"); + expectPartition(text, index); + }); + + it("does not borrow BigQuery literal forms in Dremio", () => { + const index = buildSqlStatementIndex( + "SELECT `a;b`;SELECT 2", + DREMIO_SQL_LEXICAL_PROFILE, + ); + expect(index.slots).toHaveLength(3); + }); +}); + +describe("Dremio compatibility profile", () => { + it.each([ + "SELECT 'a;''b';SELECT 2", + 'SELECT "a;""b";SELECT 2', + "SELECT 1 -- ; hidden\n;SELECT 2", + "SELECT 1 /* ; hidden */;SELECT 2", + ])("protects verified lexical form %j", (text) => { + const index = buildSqlStatementIndex(text, DREMIO_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(2); + expect(index.quality).toBe("exact"); + }); + + it.each([ + ["SELECT 'a;b", "single-quoted-string"], + ['SELECT "a;b', "double-quoted-identifier"], + ["SELECT 1 /* a;b", "block-comment"], + ])("reports incomplete form %j", (text, construct) => { + const index = buildSqlStatementIndex(text, DREMIO_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(1); + expect(index.endState).toMatchObject({ construct, kind: "unterminated" }); + }); +}); + +describe("opaque boundaries", () => { + it.each([ + ["DELIMITER $$\nSELECT 1$$", "custom-delimiter"], + ["IF condition THEN\n SELECT 1;\nEND IF;", "procedural-block"], + ["LOOP\n SELECT 1;\nEND LOOP;", "procedural-block"], + ["WHILE condition DO\n SELECT 1;\nEND WHILE;", "procedural-block"], + ["REPEAT\n SELECT 1;\nUNTIL done\nEND REPEAT;", "procedural-block"], + ["FOR item IN (SELECT 1) DO\n SELECT item;\nEND FOR;", "procedural-block"], + ["BEGIN\n SELECT 1;\nEND;", "procedural-block"], + ["label: BEGIN\n SELECT 1;\nEND;", "procedural-block"], + ["label: FOR x IN (SELECT 1) DO\n SELECT x;\nEND FOR;", "procedural-block"], + ["é: FOR x IN (SELECT 1) DO\n SELECT x;\nEND FOR;", "procedural-block"], + ["𐐀: LOOP\n SELECT 1;\nEND LOOP;", "procedural-block"], + ["`label`: LOOP\n SELECT 1;\nEND LOOP;", "procedural-block"], + ["`label`: FOR x IN (SELECT 1) DO\n SELECT x;\nEND FOR;", "procedural-block"], + ["CREATE PROCEDURE p() BEGIN SELECT 1; END", "procedural-block"], + ["CREATE TEMP PROCEDURE p() BEGIN SELECT 1; END", "procedural-block"], + ["CREATE OR REPLACE TEMP PROCEDURE p() BEGIN SELECT 1; END", "procedural-block"], + ["BEGIN WORK;", "procedural-block"], + ])("fails closed for %j", (text, reason) => { + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.quality).toBe("opaque"); + expect(index.slots).toHaveLength(1); + expect(ranges(index)[0]).toMatchObject({ + extent: [0, text.length], + quality: "opaque", + reason, + }); + expectPartition(text, index); + }); + + it.each(["BEGIN;", "BEGIN TRANSACTION;"])( + "keeps documented transaction form %j exact", + (text) => { + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.quality).toBe("exact"); + expect(index.slots).toHaveLength(2); + }, + ); + + it("preserves exact preceding slots before an opaque suffix", () => { + const text = "SELECT 1; IF condition THEN SELECT 2; END IF;"; + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(2); + expect(itemAt(index.slots, 0).boundaryQuality).toBe("exact"); + expect(itemAt(index.slots, 1).boundaryQuality).toBe("opaque"); + expectPartition(text, index); + }); + + it.each([ + `CREATE FUNCTION f() RETURNS TABLE(x int) +LANGUAGE SQL +BEGIN /* body */ ATOMIC + SELECT 1; + SELECT 2; +END; +SELECT 3`, + `CREATE OR REPLACE PROCEDURE p() +LANGUAGE SQL +BEGIN ATOMIC + INSERT INTO t VALUES (1); +END; +SELECT 2`, + ])("fails closed for PostgreSQL BEGIN ATOMIC routine bodies", (text) => { + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(index.quality).toBe("opaque"); + expect(index.slots).toHaveLength(1); + expect(index.endState).toMatchObject({ + kind: "opaque", + reason: "procedural-block", + }); + }); + + it.each([ + `CREATE FUNCTION begin.atomic() +RETURNS int +LANGUAGE SQL +RETURN 1; +SELECT 2`, + `CREATE FUNCTION f(begin atomic) +RETURNS int +LANGUAGE SQL +RETURN 1; +SELECT 2`, + ])("does not confuse routine header identifiers with BEGIN ATOMIC", (text) => { + const index = buildSqlStatementIndex( + text, + POSTGRESQL_SQL_LEXICAL_PROFILE, + ); + expect(index.quality).toBe("exact"); + expect(index.slots).toHaveLength(2); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe( + text.indexOf(";"), + ); + }); + + it("caps materialized slots under a semicolon storm", () => { + const text = ";".repeat(MAX_SQL_STATEMENT_SLOTS + 100); + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(MAX_SQL_STATEMENT_SLOTS); + expect(index.quality).toBe("opaque"); + expect(index.slots.at(-1)).toMatchObject({ + boundaryQuality: "opaque", + endState: { kind: "opaque", reason: "resource-limit" }, + }); + expectPartition(text, index); + }); + + it("bounds dollar-quote delimiter retention", () => { + const text = `$${"a".repeat(300)}$payload`; + const index = buildSqlStatementIndex(text, POSTGRESQL_SQL_LEXICAL_PROFILE); + expect(index.quality).toBe("opaque"); + expect(index.endState).toMatchObject({ + kind: "opaque", + reason: "resource-limit", + }); + }); + + it("does not degrade a long dollar-prefixed identifier without a delimiter", () => { + const text = `$${"a".repeat(300)};SELECT 2`; + const index = buildSqlStatementIndex(text, POSTGRESQL_SQL_LEXICAL_PROFILE); + expect(index.quality).toBe("exact"); + expect(index.slots).toHaveLength(2); + }); +}); + +describe("incomplete input", () => { + it.each([ + ["SELECT 'a;b", "single-quoted-string"], + ['SELECT "a;b', "double-quoted-identifier"], + ["SELECT $$a;b", "dollar-quoted-string"], + ["SELECT 1 /* a;b", "block-comment"], + ])("reports %s without exposing internal semicolons", (text, construct) => { + const index = buildSqlStatementIndex(text, POSTGRESQL_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(1); + expect(index.quality).toBe("exact"); + expect(index.endState).toMatchObject({ + construct, + kind: "unterminated", + }); + expectPartition(text, index); + }); + + it.each([ + ["SELECT `a;b", "backtick-quoted-identifier"], + ["SELECT '''a;b", "triple-single-quoted-string"], + ['SELECT """a;b', "triple-double-quoted-string"], + ['SELECT "a;b', "double-quoted-string"], + ])("reports BigQuery %s", (text, construct) => { + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.endState).toMatchObject({ + construct, + kind: "unterminated", + }); + expect(index.slots).toHaveLength(1); + }); + + it("fails closed when a regular BigQuery string crosses a line", () => { + for (const text of [ + "SELECT 'a\n;b';SELECT 2", + "SELECT 'a\\\n;b';SELECT 2", + 'SELECT "a\\\r\n;b";SELECT 2', + ]) { + const index = buildSqlStatementIndex(text, BIGQUERY_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(1); + expect(index.endState).toMatchObject({ + kind: "unterminated", + }); + } + }); + + it("keeps a comment-only unterminated slot code-free", () => { + const index = buildSqlStatementIndex( + "/* comment ;", + DUCKDB_SQL_LEXICAL_PROFILE, + ); + expect(exactSlot(itemAt(index.slots, 0)).hasCode).toBe(false); + }); +}); + +describe("source masking", () => { + it("indexes analysis text while retaining original UTF-16 coordinates", () => { + const originalText = 'SELECT * FROM {fn("a;b")}; SELECT 2'; + const embeddedFrom = originalText.indexOf("{"); + const embeddedTo = originalText.indexOf("}") + 1; + const source = createMaskedSqlSource(originalText, [ + { from: embeddedFrom, language: "python", to: embeddedTo }, + ]); + const index = buildSqlStatementIndex( + source.analysisText, + DUCKDB_SQL_LEXICAL_PROFILE, + ); + + expect(index.slots).toHaveLength(2); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe(25); + expect(source.originalText.slice(0, 26)).toBe( + 'SELECT * FROM {fn("a;b")};', + ); + expectPartition(originalText, index); + }); +}); + +describe("deterministic properties", () => { + it("never promotes protected generated semicolons to terminators", () => { + const cases: readonly [ + SqlLexicalProfile, + (payload: string) => string, + ][] = [ + [POSTGRESQL_SQL_LEXICAL_PROFILE, (payload) => `'${payload}'`], + [POSTGRESQL_SQL_LEXICAL_PROFILE, (payload) => `"${payload}"`], + [POSTGRESQL_SQL_LEXICAL_PROFILE, (payload) => `$tag$${payload}$tag$`], + [DUCKDB_SQL_LEXICAL_PROFILE, (payload) => `E'${payload}'`], + [DUCKDB_SQL_LEXICAL_PROFILE, (payload) => `$d$${payload}$d$`], + [BIGQUERY_SQL_LEXICAL_PROFILE, (payload) => `'${payload}'`], + [BIGQUERY_SQL_LEXICAL_PROFILE, (payload) => `"""${payload}"""`], + [BIGQUERY_SQL_LEXICAL_PROFILE, (payload) => `r'${payload}'`], + [BIGQUERY_SQL_LEXICAL_PROFILE, (payload) => `\`${payload}\``], + [DREMIO_SQL_LEXICAL_PROFILE, (payload) => `'${payload}'`], + [DREMIO_SQL_LEXICAL_PROFILE, (payload) => `"${payload}"`], + ]; + + for (let length = 0; length < 40; length += 1) { + const payload = `${"a".repeat(length)};${"b".repeat(39 - length)}`; + for (const [profile, protect] of cases) { + const text = `SELECT ${protect(payload)};SELECT 2`; + const index = buildSqlStatementIndex(text, profile); + expect(index.slots).toHaveLength(2); + expect(exactSlot(itemAt(index.slots, 0)).terminator?.from).toBe( + text.lastIndexOf(";"), + ); + } + } + }); + + it("always returns a total frozen partition and total cursor lookup", () => { + let seed = 0x12_34_56_78; + const next = (limit: number) => { + seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0; + return seed % limit; + }; + const alphabet = [ + 0, + 9, + 10, + 13, + 32, + 34, + 35, + 36, + 39, + 42, + 45, + 47, + 59, + 65, + 92, + 96, + 0xd800, + 0xdc00, + ]; + const profiles: readonly SqlLexicalProfile[] = [ + POSTGRESQL_SQL_LEXICAL_PROFILE, + DUCKDB_SQL_LEXICAL_PROFILE, + BIGQUERY_SQL_LEXICAL_PROFILE, + DREMIO_SQL_LEXICAL_PROFILE, + ]; + + for (let iteration = 0; iteration < 250; iteration += 1) { + const codes = Array.from( + { length: next(80) }, + () => itemAt(alphabet, next(alphabet.length)), + ); + const text = String.fromCharCode(...codes); + const index = buildSqlStatementIndex( + text, + itemAt(profiles, next(profiles.length)), + ); + expectPartition(text, index); + for (let position = 0; position <= text.length; position += 1) { + expect(findSqlStatementSlot(index, position, "left")).toBeDefined(); + expect(findSqlStatementSlot(index, position, "right")).toBeDefined(); + } + expect( + ranges(buildSqlStatementIndex(text, itemAt(profiles, 0))), + ).toEqual( + ranges(buildSqlStatementIndex(text, itemAt(profiles, 0))), + ); + } + }); + + it("scans a full-size plain document without proportional temporary records", () => { + const text = "x".repeat(16 * 1024 * 1024); + const index = buildSqlStatementIndex(text, DUCKDB_SQL_LEXICAL_PROFILE); + expect(index.slots).toHaveLength(1); + expect(exactSlot(itemAt(index.slots, 0)).source.to).toBe(text.length); + }); +}); diff --git a/src/vnext/statement-index.ts b/src/vnext/statement-index.ts new file mode 100644 index 0000000..e328024 --- /dev/null +++ b/src/vnext/statement-index.ts @@ -0,0 +1,925 @@ +const analysisRangeBrand: unique symbol = Symbol("SqlAnalysisRange"); + +export const MAX_SQL_STATEMENT_SLOTS = 10_000; +const MAX_DOLLAR_QUOTE_DELIMITER_LENGTH = 256; +const MAX_PREFIX_TOKENS = 6; + +export interface SqlAnalysisRange { + readonly [analysisRangeBrand]: "SqlAnalysisRange"; + readonly from: number; + readonly to: number; +} + +export type SqlCursorAffinity = "left" | "right"; + +export type SqlUnterminatedConstruct = + | "backtick-quoted-identifier" + | "block-comment" + | "dollar-quoted-string" + | "double-quoted-identifier" + | "double-quoted-string" + | "single-quoted-string" + | "triple-double-quoted-string" + | "triple-single-quoted-string"; + +export type SqlOpaqueBoundaryReason = + | "custom-delimiter" + | "procedural-block" + | "resource-limit"; + +export type SqlLexicalEndState = + | { + readonly kind: "normal"; + } + | { + readonly construct: SqlUnterminatedConstruct; + readonly from: number; + readonly kind: "unterminated"; + } + | { + readonly from: number; + readonly kind: "opaque"; + readonly reason: SqlOpaqueBoundaryReason; + }; + +export interface ExactSqlStatementSlot { + readonly boundaryQuality: "exact"; + readonly endState: Exclude; + readonly extent: SqlAnalysisRange; + readonly hasCode: boolean; + readonly source: SqlAnalysisRange; + readonly terminator: SqlAnalysisRange | null; +} + +export interface OpaqueSqlStatementSlot { + readonly boundaryQuality: "opaque"; + readonly endState: Extract; + readonly extent: SqlAnalysisRange; +} + +export type SqlStatementSlot = + | ExactSqlStatementSlot + | OpaqueSqlStatementSlot; + +export interface SqlStatementIndex { + readonly endState: SqlLexicalEndState; + readonly quality: "exact" | "opaque"; + readonly slots: readonly SqlStatementSlot[]; +} + +type SqlSingleQuoteBackslash = "always" | "e-prefix" | "never"; +type SqlProceduralGuards = "bigquery" | "none" | "postgresql"; + +export interface SqlLexicalProfile { + readonly backtickQuotedIdentifiers: boolean; + readonly bigQueryStrings: boolean; + readonly dollarQuotedStrings: boolean; + readonly hashLineComments: boolean; + readonly nestedBlockComments: boolean; + readonly proceduralGuards: SqlProceduralGuards; + readonly singleQuoteBackslash: SqlSingleQuoteBackslash; +} + +export const POSTGRESQL_SQL_LEXICAL_PROFILE: SqlLexicalProfile = Object.freeze({ + backtickQuotedIdentifiers: false, + bigQueryStrings: false, + dollarQuotedStrings: true, + hashLineComments: false, + nestedBlockComments: true, + proceduralGuards: "postgresql", + singleQuoteBackslash: "e-prefix", +}); + +export const DUCKDB_SQL_LEXICAL_PROFILE: SqlLexicalProfile = Object.freeze({ + backtickQuotedIdentifiers: false, + bigQueryStrings: false, + dollarQuotedStrings: true, + hashLineComments: false, + nestedBlockComments: true, + proceduralGuards: "none", + singleQuoteBackslash: "e-prefix", +}); + +export const BIGQUERY_SQL_LEXICAL_PROFILE: SqlLexicalProfile = Object.freeze({ + backtickQuotedIdentifiers: true, + bigQueryStrings: true, + dollarQuotedStrings: false, + hashLineComments: true, + nestedBlockComments: false, + proceduralGuards: "bigquery", + singleQuoteBackslash: "always", +}); + +export const DREMIO_SQL_LEXICAL_PROFILE: SqlLexicalProfile = Object.freeze({ + backtickQuotedIdentifiers: false, + bigQueryStrings: false, + dollarQuotedStrings: false, + hashLineComments: false, + nestedBlockComments: false, + proceduralGuards: "none", + singleQuoteBackslash: "never", +}); + +const NORMAL_END_STATE: Extract< + SqlLexicalEndState, + { readonly kind: "normal" } +> = Object.freeze({ kind: "normal" }); + +function createAnalysisRange(from: number, to: number): SqlAnalysisRange { + const range: SqlAnalysisRange = { + [analysisRangeBrand]: "SqlAnalysisRange", + from, + to, + }; + return Object.freeze(range); +} + +function createUnterminatedEndState( + construct: SqlUnterminatedConstruct, + from: number, +): Extract { + return Object.freeze({ construct, from, kind: "unterminated" }); +} + +function createOpaqueEndState( + reason: SqlOpaqueBoundaryReason, + from: number, +): Extract { + return Object.freeze({ from, kind: "opaque", reason }); +} + +function createExactSlot( + from: number, + sourceTo: number, + extentTo: number, + hasCode: boolean, + endState: ExactSqlStatementSlot["endState"], +): ExactSqlStatementSlot { + return Object.freeze({ + boundaryQuality: "exact", + endState, + extent: createAnalysisRange(from, extentTo), + hasCode, + source: createAnalysisRange(from, sourceTo), + terminator: + sourceTo === extentTo ? null : createAnalysisRange(sourceTo, extentTo), + }); +} + +function createOpaqueSlot( + from: number, + to: number, + reason: SqlOpaqueBoundaryReason, + detectedAt: number, +): OpaqueSqlStatementSlot { + return Object.freeze({ + boundaryQuality: "opaque", + endState: createOpaqueEndState(reason, detectedAt), + extent: createAnalysisRange(from, to), + }); +} + +function isAsciiIdentifierStart(code: number): boolean { + return ( + code === 95 || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) + ); +} + +function isAsciiIdentifierContinue(code: number): boolean { + return isAsciiIdentifierStart(code) || (code >= 48 && code <= 57); +} + +function codePointLengthAt(text: string, index: number): 0 | 1 | 2 { + const codePoint = text.codePointAt(index); + if (codePoint === undefined) { + return 0; + } + return codePoint > 0xffff ? 2 : 1; +} + +function sqlIdentifierStartLengthAt(text: string, index: number): 0 | 1 | 2 { + const code = text.charCodeAt(index); + if (code <= 0x7f) { + return isAsciiIdentifierStart(code) ? 1 : 0; + } + return codePointLengthAt(text, index); +} + +function sqlIdentifierContinueLengthAt( + text: string, + index: number, +): 0 | 1 | 2 { + const code = text.charCodeAt(index); + if (code <= 0x7f) { + return isAsciiIdentifierContinue(code) ? 1 : 0; + } + return codePointLengthAt(text, index); +} + +function previousCodePointIndex(text: string, index: number): number { + if (index <= 0) { + return -1; + } + const last = text.charCodeAt(index - 1); + if ( + last >= 0xdc00 && + last <= 0xdfff && + index >= 2 + ) { + const first = text.charCodeAt(index - 2); + if (first >= 0xd800 && first <= 0xdbff) { + return index - 2; + } + } + return index - 1; +} + +function hasSqlIdentifierBefore(text: string, index: number): boolean { + const previous = previousCodePointIndex(text, index); + if (previous < 0) { + return false; + } + return ( + text.charCodeAt(previous) === 36 || + sqlIdentifierContinueLengthAt(text, previous) > 0 + ); +} + +function isSqlWhitespace(code: number): boolean { + return ( + code === 9 || + code === 10 || + code === 11 || + code === 12 || + code === 13 || + code === 32 + ); +} + +function hasPrefixAtTokenBoundary( + text: string, + quoteAt: number, + prefix: string, +): boolean { + const prefixFrom = quoteAt - prefix.length; + if (prefixFrom < 0) { + return false; + } + for (let index = 0; index < prefix.length; index += 1) { + const actual = text.charCodeAt(prefixFrom + index) | 32; + if (actual !== prefix.charCodeAt(index)) { + return false; + } + } + return ( + prefixFrom === 0 || + !hasSqlIdentifierBefore(text, prefixFrom) + ); +} + +function hasEscapeStringPrefix(text: string, quoteAt: number): boolean { + return hasPrefixAtTokenBoundary(text, quoteAt, "e"); +} + +function isBigQueryRawString(text: string, quoteAt: number): boolean { + return ( + hasPrefixAtTokenBoundary(text, quoteAt, "r") || + hasPrefixAtTokenBoundary(text, quoteAt, "br") || + hasPrefixAtTokenBoundary(text, quoteAt, "rb") + ); +} + +interface QuoteScanResult { + readonly closed: boolean; + readonly to: number; +} + +function scanQuoted( + text: string, + from: number, + quote: number, + quoteLength: 1 | 3, + backslashEscapes: boolean, + doubledQuoteEscapes: boolean, + stopAtLineBreak: boolean, +): QuoteScanResult { + let cursor = from + quoteLength; + while (cursor < text.length) { + const code = text.charCodeAt(cursor); + if (stopAtLineBreak && (code === 10 || code === 13)) { + return { closed: false, to: text.length }; + } + if (backslashEscapes && code === 92) { + const escapedCode = text.charCodeAt(cursor + 1); + if ( + stopAtLineBreak && + (escapedCode === 10 || escapedCode === 13) + ) { + return { closed: false, to: text.length }; + } + cursor += Math.min(2, text.length - cursor); + continue; + } + if (code !== quote) { + cursor += 1; + continue; + } + if (quoteLength === 3) { + if ( + text.charCodeAt(cursor + 1) === quote && + text.charCodeAt(cursor + 2) === quote + ) { + return { closed: true, to: cursor + 3 }; + } + cursor += 1; + continue; + } + if (doubledQuoteEscapes && text.charCodeAt(cursor + 1) === quote) { + cursor += 2; + continue; + } + return { closed: true, to: cursor + 1 }; + } + return { closed: false, to: text.length }; +} + +interface BlockCommentScanResult { + readonly closed: boolean; + readonly to: number; +} + +function scanBlockComment( + text: string, + from: number, + nested: boolean, +): BlockCommentScanResult { + let cursor = from + 2; + let depth = 1; + while (cursor < text.length) { + const code = text.charCodeAt(cursor); + const next = text.charCodeAt(cursor + 1); + if (nested && code === 47 && next === 42) { + depth += 1; + cursor += 2; + continue; + } + if (code === 42 && next === 47) { + depth -= 1; + cursor += 2; + if (depth === 0) { + return { closed: true, to: cursor }; + } + continue; + } + cursor += 1; + } + return { closed: false, to: text.length }; +} + +interface DollarQuoteScanResult { + readonly detectedAt: number; + readonly endState: ExactSqlStatementSlot["endState"] | null; + readonly opaqueReason: SqlOpaqueBoundaryReason | null; + readonly to: number; +} + +function scanDollarQuote( + text: string, + from: number, +): DollarQuoteScanResult | null { + if (hasSqlIdentifierBefore(text, from)) { + return null; + } + const first = text.charCodeAt(from + 1); + let cursor = from + 1; + let delimiterTooLong = false; + if (first !== 36) { + const firstLength = sqlIdentifierStartLengthAt(text, cursor); + if (firstLength === 0) { + return null; + } + cursor += firstLength; + while (cursor < text.length) { + const continueLength = sqlIdentifierContinueLengthAt(text, cursor); + if (continueLength === 0) { + break; + } + if (cursor - from + 1 > MAX_DOLLAR_QUOTE_DELIMITER_LENGTH) { + delimiterTooLong = true; + } + cursor += continueLength; + } + if (text.charCodeAt(cursor) !== 36) { + return null; + } + if (delimiterTooLong) { + return { + detectedAt: from, + endState: null, + opaqueReason: "resource-limit", + to: text.length, + }; + } + } + const delimiterTo = cursor + 1; + const delimiter = text.slice(from, delimiterTo); + const closeAt = text.indexOf(delimiter, delimiterTo); + if (closeAt < 0) { + return { + detectedAt: from, + endState: createUnterminatedEndState("dollar-quoted-string", from), + opaqueReason: null, + to: text.length, + }; + } + return { + detectedAt: from, + endState: NORMAL_END_STATE, + opaqueReason: null, + to: closeAt + delimiter.length, + }; +} + +class SqlPrefixGuard { + readonly #mode: SqlProceduralGuards; + readonly #tokens: string[] = []; + #labelColonAt: number | null = null; + #parenthesisDepth = 0; + #postgresRoutine = false; + #previousRoutineWord: string | null = null; + #previousRoutineWordAt = 0; + #reason: SqlOpaqueBoundaryReason | null = null; + #reasonAt = 0; + + constructor(mode: SqlProceduralGuards) { + this.#mode = mode; + } + + reset(): void { + this.#tokens.length = 0; + this.#labelColonAt = null; + this.#parenthesisDepth = 0; + this.#postgresRoutine = false; + this.#previousRoutineWord = null; + this.#previousRoutineWordAt = 0; + this.#reason = null; + this.#reasonAt = 0; + } + + recordWord(text: string, from: number, to: number): void { + if (this.#reason !== null) { + return; + } + if (this.#postgresRoutine) { + const token = + to - from <= 32 ? text.slice(from, to).toUpperCase() : ""; + if (this.#previousRoutineWord === "BEGIN" && token === "ATOMIC") { + if (this.#parenthesisDepth === 0) { + this.#setReason("procedural-block", this.#previousRoutineWordAt); + return; + } + } + this.#previousRoutineWord = token; + this.#previousRoutineWordAt = from; + return; + } + if (this.#tokens.length >= MAX_PREFIX_TOKENS) { + return; + } + const token = + to - from <= 32 ? text.slice(from, to).toUpperCase() : ""; + this.#tokens.push(token); + + if (this.#tokens.length === 1 && token === "DELIMITER") { + this.#setReason("custom-delimiter", from); + return; + } + if ( + this.#mode === "postgresql" && + isCreatePostgresqlRoutinePrefix(this.#tokens) + ) { + this.#postgresRoutine = true; + this.#previousRoutineWord = token; + this.#previousRoutineWordAt = from; + return; + } + if (this.#mode !== "bigquery") { + return; + } + if ( + this.#tokens.length === 1 && + (token === "FOR" || + token === "IF" || + token === "LOOP" || + token === "REPEAT" || + token === "WHILE") + ) { + this.#setReason("procedural-block", from); + return; + } + if ( + this.#tokens.length === 2 && + this.#tokens[0] === "BEGIN" && + token !== "TRANSACTION" + ) { + this.#setReason("procedural-block", from); + return; + } + if ( + this.#labelColonAt !== null && + this.#tokens.length === 2 && + (token === "BEGIN" || + token === "FOR" || + token === "LOOP" || + token === "REPEAT" || + token === "WHILE") + ) { + this.#setReason("procedural-block", this.#labelColonAt); + return; + } + if (isCreateProcedurePrefix(this.#tokens)) { + this.#setReason("procedural-block", from); + } + } + + recordQuotedIdentifier(): void { + if ( + this.#reason === null && + this.#mode === "bigquery" && + this.#tokens.length < MAX_PREFIX_TOKENS + ) { + this.#tokens.push(""); + } + } + + recordNonWord(code: number): void { + if (this.#mode !== "postgresql" || !this.#postgresRoutine) { + return; + } + if (code === 40) { + this.#parenthesisDepth += 1; + } else if (code === 41 && this.#parenthesisDepth > 0) { + this.#parenthesisDepth -= 1; + } + this.#previousRoutineWord = null; + } + + recordColon(at: number): void { + if ( + this.#mode === "bigquery" && + this.#reason === null && + this.#tokens.length === 1 + ) { + this.#labelColonAt = at; + } + } + + get reason(): SqlOpaqueBoundaryReason | null { + return this.#reason; + } + + get reasonAt(): number { + return this.#reasonAt; + } + + #setReason(reason: SqlOpaqueBoundaryReason, at: number): void { + this.#reason = reason; + this.#reasonAt = at; + } +} + +function isCreatePostgresqlRoutinePrefix(tokens: readonly string[]): boolean { + if (tokens[0] !== "CREATE") { + return false; + } + if (tokens[1] === "FUNCTION" || tokens[1] === "PROCEDURE") { + return true; + } + return ( + tokens[1] === "OR" && + tokens[2] === "REPLACE" && + (tokens[3] === "FUNCTION" || tokens[3] === "PROCEDURE") + ); +} + +function isCreateProcedurePrefix(tokens: readonly string[]): boolean { + if (tokens[0] !== "CREATE") { + return false; + } + if (tokens[1] === "PROCEDURE") { + return true; + } + if ( + (tokens[1] === "TEMP" || tokens[1] === "TEMPORARY") && + tokens[2] === "PROCEDURE" + ) { + return true; + } + if (tokens[1] !== "OR" || tokens[2] !== "REPLACE") { + return false; + } + return ( + tokens[3] === "PROCEDURE" || + ((tokens[3] === "TEMP" || tokens[3] === "TEMPORARY") && + tokens[4] === "PROCEDURE") + ); +} + +function quoteConstruct( + quote: number, + quoteLength: 1 | 3, + bigQueryStrings: boolean, +): SqlUnterminatedConstruct { + if (quoteLength === 3) { + return quote === 39 + ? "triple-single-quoted-string" + : "triple-double-quoted-string"; + } + if (quote === 39) { + return "single-quoted-string"; + } + return bigQueryStrings + ? "double-quoted-string" + : "double-quoted-identifier"; +} + +/** Builds the bounded, parser-free statement partition for one analysis text. */ +export function buildSqlStatementIndex( + analysisText: string, + profile: SqlLexicalProfile, +): SqlStatementIndex { + const slots: SqlStatementSlot[] = []; + const prefixGuard = new SqlPrefixGuard(profile.proceduralGuards); + let slotFrom = 0; + let hasCode = false; + let cursor = 0; + let finalEndState: ExactSqlStatementSlot["endState"] = NORMAL_END_STATE; + + const finishOpaque = ( + reason: SqlOpaqueBoundaryReason, + detectedAt: number, + ): SqlStatementIndex => { + const slot = createOpaqueSlot( + slotFrom, + analysisText.length, + reason, + detectedAt, + ); + slots.push(slot); + return Object.freeze({ + endState: slot.endState, + quality: "opaque", + slots: Object.freeze(slots), + }); + }; + + while (cursor < analysisText.length) { + const code = analysisText.charCodeAt(cursor); + const next = analysisText.charCodeAt(cursor + 1); + + if (isSqlWhitespace(code)) { + cursor += 1; + continue; + } + if (code === 45 && next === 45) { + cursor += 2; + while ( + cursor < analysisText.length && + analysisText.charCodeAt(cursor) !== 10 && + analysisText.charCodeAt(cursor) !== 13 + ) { + cursor += 1; + } + continue; + } + if (profile.hashLineComments && code === 35) { + cursor += 1; + while ( + cursor < analysisText.length && + analysisText.charCodeAt(cursor) !== 10 && + analysisText.charCodeAt(cursor) !== 13 + ) { + cursor += 1; + } + continue; + } + if (code === 47 && next === 42) { + const comment = scanBlockComment( + analysisText, + cursor, + profile.nestedBlockComments, + ); + if (!comment.closed) { + finalEndState = createUnterminatedEndState("block-comment", cursor); + } + cursor = comment.to; + continue; + } + + if ( + profile.dollarQuotedStrings && + code === 36 + ) { + const dollarQuote = scanDollarQuote(analysisText, cursor); + if (dollarQuote) { + hasCode = true; + prefixGuard.recordNonWord(code); + if (dollarQuote.opaqueReason !== null) { + return finishOpaque( + dollarQuote.opaqueReason, + dollarQuote.detectedAt, + ); + } + if (dollarQuote.endState?.kind === "unterminated") { + finalEndState = dollarQuote.endState; + } + cursor = dollarQuote.to; + continue; + } + } + + if ( + code === 39 || + code === 34 || + (profile.backtickQuotedIdentifiers && code === 96) + ) { + hasCode = true; + if (code === 96) { + prefixGuard.recordQuotedIdentifier(); + prefixGuard.recordNonWord(code); + const quote = scanQuoted( + analysisText, + cursor, + code, + 1, + true, + false, + false, + ); + if (!quote.closed) { + finalEndState = createUnterminatedEndState( + "backtick-quoted-identifier", + cursor, + ); + } + cursor = quote.to; + continue; + } + + const isTriple = + profile.bigQueryStrings && + analysisText.charCodeAt(cursor + 1) === code && + analysisText.charCodeAt(cursor + 2) === code; + const quoteLength = isTriple ? 3 : 1; + const rawBigQueryString = + profile.bigQueryStrings && isBigQueryRawString(analysisText, cursor); + const backslashEscapes = + !rawBigQueryString && + (profile.bigQueryStrings || + (code === 39 && + (profile.singleQuoteBackslash === "always" || + (profile.singleQuoteBackslash === "e-prefix" && + hasEscapeStringPrefix(analysisText, cursor))))); + const doubledQuoteEscapes = !profile.bigQueryStrings; + if (code === 34 && !profile.bigQueryStrings) { + prefixGuard.recordQuotedIdentifier(); + } + prefixGuard.recordNonWord(code); + const quote = scanQuoted( + analysisText, + cursor, + code, + quoteLength, + backslashEscapes, + doubledQuoteEscapes, + profile.bigQueryStrings && quoteLength === 1, + ); + if (!quote.closed) { + finalEndState = createUnterminatedEndState( + quoteConstruct(code, quoteLength, profile.bigQueryStrings), + cursor, + ); + } + cursor = quote.to; + continue; + } + + const wordStartLength = sqlIdentifierStartLengthAt(analysisText, cursor); + if (wordStartLength > 0) { + const wordFrom = cursor; + cursor += wordStartLength; + while (cursor < analysisText.length) { + const continueLength = sqlIdentifierContinueLengthAt( + analysisText, + cursor, + ); + if (continueLength === 0) { + break; + } + cursor += continueLength; + } + prefixGuard.recordWord(analysisText, wordFrom, cursor); + hasCode = true; + if (prefixGuard.reason !== null) { + return finishOpaque(prefixGuard.reason, prefixGuard.reasonAt); + } + continue; + } + + if (code === 58) { + prefixGuard.recordColon(cursor); + } + prefixGuard.recordNonWord(code); + if (code === 59) { + if (slots.length >= MAX_SQL_STATEMENT_SLOTS - 1) { + return finishOpaque("resource-limit", cursor); + } + slots.push( + createExactSlot( + slotFrom, + cursor, + cursor + 1, + hasCode, + NORMAL_END_STATE, + ), + ); + slotFrom = cursor + 1; + hasCode = false; + finalEndState = NORMAL_END_STATE; + prefixGuard.reset(); + cursor += 1; + continue; + } + + hasCode = true; + cursor += 1; + } + + slots.push( + createExactSlot( + slotFrom, + analysisText.length, + analysisText.length, + hasCode, + finalEndState, + ), + ); + return Object.freeze({ + endState: finalEndState, + quality: "exact", + slots: Object.freeze(slots), + }); +} + +/** Finds one slot with explicit behavior at shared extent boundaries. */ +export function findSqlStatementSlot( + index: SqlStatementIndex, + position: number, + affinity: SqlCursorAffinity, +): SqlStatementSlot { + const slots = index.slots; + const documentLength = getStatementSlot(slots, slots.length - 1).extent.to; + if ( + !Number.isSafeInteger(position) || + position < 0 || + position > documentLength + ) { + throw new RangeError("SQL statement position is outside the document"); + } + if (affinity !== "left" && affinity !== "right") { + throw new TypeError("SQL cursor affinity must be left or right"); + } + + let low = 0; + let high = slots.length; + while (low < high) { + const middle = low + Math.floor((high - low) / 2); + if (getStatementSlot(slots, middle).extent.from <= position) { + low = middle + 1; + } else { + high = middle; + } + } + let slotIndex = Math.max(0, low - 1); + if ( + affinity === "left" && + position > 0 && + getStatementSlot(slots, slotIndex).extent.from === position + ) { + slotIndex -= 1; + } + return getStatementSlot(slots, Math.max(0, slotIndex)); +} + +function getStatementSlot( + slots: readonly SqlStatementSlot[], + index: number, +): SqlStatementSlot { + const slot = slots[index]; + if (!slot) { + throw new Error("SQL statement index must contain at least one slot"); + } + return slot; +}