Improve lexer performance

Author: antonmedv

parser/lexer/lexer.go

@@ -13,6 +13,11 @@ func Lex(source *file.Source) ([]Token, error) {
 		input:  source.Content(),
 		tokens: make([]Token, 0),
 	}
+
+	l.loc = file.Location{1, 0}
+	l.prev = l.loc
+	l.startLoc = l.loc
+
 	for state := root; state != nil; {
 		state = state(l)
 	}
@@ -28,8 +33,10 @@ type lexer struct {
 	input      string
 	state      stateFn
 	tokens     []Token
-	start, end int // current position in input
-	width      int // last rune with
+	start, end int           // current position in input
+	width      int           // last rune with
+	startLoc   file.Location // start location
+	prev, loc  file.Location // prev location of end location, end location
 	err        *file.Error
 }
 
@@ -43,6 +50,15 @@ func (l *lexer) next() rune {
 	r, w := utf8.DecodeRuneInString(l.input[l.end:])
 	l.width = w
 	l.end += w
+
+	l.prev = l.loc
+	if r == '\n' {
+		l.loc.Line++
+		l.loc.Column = 0
+	} else {
+		l.loc.Column++
+	}
+
 	return r
 }
 
@@ -54,6 +70,7 @@ func (l *lexer) peek() rune {
 
 func (l *lexer) backup() {
 	l.end -= l.width
+	l.loc = l.prev
 }
 
 func (l *lexer) emit(t Kind) {
@@ -62,19 +79,21 @@ func (l *lexer) emit(t Kind) {
 
 func (l *lexer) emitValue(t Kind, value string) {
 	l.tokens = append(l.tokens, Token{
-		Location: l.loc(l.start),
+		Location: l.startLoc,
 		Kind:     t,
 		Value:    value,
 	})
 	l.start = l.end
+	l.startLoc = l.loc
 }
 
 func (l *lexer) emitEOF() {
 	l.tokens = append(l.tokens, Token{
-		Location: l.loc(l.start - 1), // Point to previous position for better error messages.
+		Location: l.prev, // Point to previous position for better error messages.
 		Kind:     EOF,
 	})
 	l.start = l.end
+	l.startLoc = l.loc
 }
 
 func (l *lexer) word() string {
@@ -83,6 +102,7 @@ func (l *lexer) word() string {
 
 func (l *lexer) ignore() {
 	l.start = l.end
+	l.startLoc = l.loc
 }
 
 func (l *lexer) accept(valid string) bool {
@@ -101,9 +121,13 @@ func (l *lexer) acceptRun(valid string) {
 
 func (l *lexer) acceptWord(word string) bool {
 	pos := l.end
+	loc := l.loc
+	prev := l.prev
 	for _, ch := range word {
 		if l.next() != ch {
 			l.end = pos
+			l.loc = loc
+			l.prev = prev
 			return false
 		}
 	}
@@ -113,32 +137,13 @@ func (l *lexer) acceptWord(word string) bool {
 func (l *lexer) error(format string, args ...interface{}) stateFn {
 	if l.err == nil { // show first error
 		l.err = &file.Error{
-			Location: l.loc(l.end - 1),
+			Location: l.loc,
 			Message:  fmt.Sprintf(format, args...),
 		}
 	}
 	return nil
 }
 
-func (l *lexer) loc(pos int) file.Location {
-	line, column := 1, 0
-	for i, ch := range []rune(l.input) {
-		if i == pos {
-			break
-		}
-		if ch == '\n' {
-			line++
-			column = 0
-		} else {
-			column++
-		}
-	}
-	return file.Location{
-		Line:   line,
-		Column: column,
-	}
-}
-
 func digitVal(ch rune) int {
 	switch {
 	case '0' <= ch && ch <= '9':

parser/lexer/lexer_test.go

@@ -1,6 +1,8 @@
 package lexer_test
 
 import (
+	"fmt"
+	"github.com/stretchr/testify/assert"
 	"strings"
 	"testing"
 
@@ -91,19 +93,6 @@ var lexTests = []lexTest{
 	},
 }
 
-var lexErrorTests = []lexErrorTest{
-	{
-		`
-		"\xQA"
-		`,
-		`invalid char escape (2:6)`,
-	},
-	{
-		`id "hello`,
-		`literal not terminated (1:9)`,
-	},
-}
-
 func compareTokens(i1, i2 []Token) bool {
 	if len(i1) != len(i2) {
 		return false
@@ -132,15 +121,34 @@ func TestLex(t *testing.T) {
 	}
 }
 
+const errorTests = `
+"\xQA"
+invalid char escape (1:5)
+ | "\xQA"
+ | ....^
+
+id "hello
+literal not terminated (1:10)
+ | id "hello
+ | .........^
+`
+
 func TestLex_error(t *testing.T) {
-	for _, test := range lexErrorTests {
-		out, err := Lex(file.NewSource(test.input))
-		if err == nil {
-			t.Errorf("%s:\nexpected error\n%v", test.input, out)
-			continue
+	tests := strings.Split(strings.Trim(errorTests, "\n"), "\n\n")
+
+	for _, test := range tests {
+
+		input := strings.SplitN(test, "\n", 2)
+		if len(input) != 2 {
+			t.Errorf("syntax error in test: %q", test)
+			break
 		}
-		if !strings.HasPrefix(err.Error(), test.err) || test.err == "" {
-			t.Errorf("%s:\ngot\n\t%+v\nexpected\n\t%v", test.input, err.Error(), test.err)
+
+		_, err := Lex(file.NewSource(input[0]))
+		if err == nil {
+			err = fmt.Errorf("<nil>")
 		}
+
+		assert.Equal(t, input[1], err.Error(), input[0])
 	}
 }