Skip to content

Commit df2362a

Browse files
committed
Preserve consumed delimiters during statement recovery
The lexer could return an error after consuming a bracket opener or entering a malformed HEX literal. Keep that delimiter until recovery finishes the construct, so following statements remain visible. Cover bracket tails, EXPLAIN, ParseNext recovery and Reader buffer boundaries. Treat an unterminated HEX literal like other unclosed quotes.
1 parent 26f51ed commit df2362a

2 files changed

Lines changed: 114 additions & 8 deletions

File tree

‎parse_test.go‎

Lines changed: 91 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -842,6 +842,94 @@ func TestDDLTailUnterminated(t *testing.T) {
842842
}
843843
}
844844

845+
func TestDDLRecoveryOpeners(t *testing.T) {
846+
for _, first := range []string{
847+
"CREATE TABLE t ([a;b] INTEGER)",
848+
"CREATE TABLE t ([a'b] INTEGER)",
849+
"CREATE TABLE t ([a--b] INTEGER)",
850+
"CREATE TABLE t(a TEXT DEFAULT X'g;h')",
851+
// These errors are already past a closing delimiter, or inside a
852+
// fully consumed executable comment, and must not reopen it.
853+
"CREATE TABLE t '['",
854+
"CREATE TABLE t(a TEXT DEFAULT X'a')",
855+
"CREATE TABLE t (`` INT)",
856+
"CREATE TABLE t (/*! [ */ a INT)",
857+
} {
858+
t.Run(first, func(t *testing.T) {
859+
testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false)
860+
})
861+
}
862+
}
863+
864+
func TestExplainTailSemicolons(t *testing.T) {
865+
first := "EXPLAIN SELECT 'a;b'"
866+
input := first + "; SELECT 1"
867+
if _, err := Parse(input); err != nil {
868+
t.Fatal(err)
869+
}
870+
queries, statements, err := ParseMultiple(NewStringTokenizer(input))
871+
if err != nil || len(queries) != 2 || len(statements) != 2 || queries[0] != first || queries[1] != "SELECT 1" {
872+
t.Fatalf("ParseMultiple(%q): %v, %v, %v", input, queries, statements, err)
873+
}
874+
if _, ok := statements[0].(*Explain); !ok || String(statements[1]) != "select 1" {
875+
t.Fatalf("unexpected statements: %T, %v", statements[0], statements[1])
876+
}
877+
for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} {
878+
if stmt, err := ParseNext(tokenizer); err != nil {
879+
t.Fatal(err)
880+
} else if _, ok := stmt.(*Explain); !ok || tokenizer.Position != len(first)+1 {
881+
t.Fatalf("EXPLAIN boundary: %T at %d", stmt, tokenizer.Position)
882+
}
883+
if stmt, err := ParseNext(tokenizer); err != nil || String(stmt) != "select 1" {
884+
t.Fatalf("second statement: %v, %v", stmt, err)
885+
}
886+
if _, err := ParseNext(tokenizer); err != io.EOF {
887+
t.Fatalf("after last statement: %v", err)
888+
}
889+
}
890+
}
891+
892+
func TestParseNextRecoverySemicolons(t *testing.T) {
893+
for _, first := range []string{"SELECT ) 'a;b'", "SELECT X'g;h'", "SELECT [a'b]"} {
894+
input := first + "; SELECT 1"
895+
for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} {
896+
if _, err := ParseNext(tokenizer); err == nil {
897+
t.Fatalf("ParseNext(%q): want first-statement error", input)
898+
}
899+
if tokenizer.Position != len(first)+1 {
900+
t.Errorf("ParseNext(%q): recovered at %d, want %d", input, tokenizer.Position, len(first)+1)
901+
}
902+
if stmt, err := ParseNext(tokenizer); err != nil || String(stmt) != "select 1" {
903+
t.Errorf("recovered statement from %q: %v, %v", input, stmt, err)
904+
}
905+
if _, err := ParseNext(tokenizer); err != io.EOF {
906+
t.Errorf("after last statement: %v", err)
907+
}
908+
}
909+
}
910+
}
911+
912+
func TestParseNextRecoveryUnterminatedHex(t *testing.T) {
913+
input := "select x'78 from t; select 1 from t"
914+
for _, tokenizer := range []*Tokenizer{NewStringTokenizer(input), NewTokenizer(strings.NewReader(input))} {
915+
if _, err := ParseNext(tokenizer); err == nil {
916+
t.Fatal("want an unterminated hex literal error")
917+
}
918+
if _, err := ParseNext(tokenizer); err != io.EOF {
919+
t.Fatalf("unterminated hex literal: got %v, want EOF", err)
920+
}
921+
}
922+
}
923+
924+
func TestDDLTailReaderBoundary(t *testing.T) {
925+
prefix := "create index i on t(a) "
926+
for _, tail := range []string{"'a;b'", "/* a;b */"} {
927+
// The construct opens in the first Reader buffer and ends in the next.
928+
first := prefix + strings.Repeat(" ", defaultBufSize-2-len(prefix)) + tail
929+
testDDLTail(t, first+"; select 1", []string{first, "select 1"}, false)
930+
}
931+
}
932+
845933
func testDDLTail(t *testing.T, input string, want []string, backslash bool) {
846934
t.Helper()
847935
if !backslash {
@@ -1355,8 +1443,9 @@ var (
13551443
input: "select 0xH from t",
13561444
output: "syntax error at position 10 near '0x'",
13571445
}, {
1358-
input: "select x'78 from t",
1359-
output: "syntax error at position 12 near '78'",
1446+
input: "select x'78 from t",
1447+
output: "syntax error at position 12 near '78'",
1448+
excludeMulti: true,
13601449
}, {
13611450
input: "select x'777' from t",
13621451
output: "syntax error at position 14 near '777'",

‎token.go‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ type Tokenizer struct {
4242
lastChar uint16
4343
Position int
4444
lastToken []byte
45+
errorDelimiter uint16 // Unclosed delimiter at a lexical error.
4546
LastError error
4647
posVarIndex int
4748
ParseTree Statement
@@ -354,6 +355,12 @@ func (tkn *Tokenizer) Error(err string) {
354355
}
355356
tkn.LastError = errors.New(buf.String())
356357

358+
// Finish a construct whose opener the failing token already consumed.
359+
if tkn.errorDelimiter != 0 {
360+
tkn.skipToDelimiter(tkn.errorDelimiter)
361+
tkn.errorDelimiter = 0
362+
}
363+
357364
// Try and re-sync to the next statement
358365
if tkn.lastChar != ';' {
359366
tkn.skipStatement()
@@ -363,6 +370,7 @@ func (tkn *Tokenizer) Error(err string) {
363370
// Scan scans the tokenizer for the next token and returns
364371
// the token type and an optional value.
365372
func (tkn *Tokenizer) Scan() (int, []byte) {
373+
tkn.errorDelimiter = 0
366374
scanAgain:
367375
if tkn.specialComment != nil {
368376
// Enter specialComment scan mode.
@@ -502,6 +510,9 @@ scanAgain:
502510
case '`':
503511
return tkn.scanLiteralIdentifier()
504512
default:
513+
if ch == '[' {
514+
tkn.errorDelimiter = ']'
515+
}
505516
return LEX_ERROR, []byte{byte(ch)}
506517
}
507518
}
@@ -519,12 +530,7 @@ func (tkn *Tokenizer) skipStatement() {
519530
case '`':
520531
tkn.scanLiteralIdentifier()
521532
case '[':
522-
for tkn.lastChar != ']' && tkn.lastChar != eofChar {
523-
tkn.next()
524-
}
525-
if tkn.lastChar == ']' {
526-
tkn.next()
527-
}
533+
tkn.skipToDelimiter(']')
528534
case '#':
529535
tkn.scanCommentType1("#")
530536
case '-':
@@ -545,6 +551,16 @@ func (tkn *Tokenizer) skipStatement() {
545551
}
546552
}
547553

554+
// skipToDelimiter resumes after an already consumed opening delimiter.
555+
func (tkn *Tokenizer) skipToDelimiter(delim uint16) {
556+
for tkn.lastChar != delim && tkn.lastChar != eofChar {
557+
tkn.next()
558+
}
559+
if tkn.lastChar == delim {
560+
tkn.next()
561+
}
562+
}
563+
548564
func (tkn *Tokenizer) skipBlank() {
549565
ch := tkn.lastChar
550566
for ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t' {
@@ -576,6 +592,7 @@ func (tkn *Tokenizer) scanHex() (int, []byte) {
576592
buffer := &bytes2.Buffer{}
577593
tkn.scanMantissa(16, buffer)
578594
if tkn.lastChar != '\'' {
595+
tkn.errorDelimiter = '\''
579596
return LEX_ERROR, buffer.Bytes()
580597
}
581598
tkn.next()

0 commit comments

Comments
 (0)