Add support to lexer to support "not in" operator

Author: antonmedv

parser2/lexer/lexer.go

@@ -28,8 +28,8 @@ type lexer struct {
 	input      string
 	state      stateFn
 	tokens     []Token
-	start, end int // Current position in input.
-	width      int // Last rune with.
+	start, end int // current position in input
+	width      int // last rune with
 	err        *file.Error
 }
 
@@ -91,6 +91,17 @@ func (l *lexer) acceptRun(valid string) {
 	l.backup()
 }
 
+func (l *lexer) acceptWord(word string) bool {
+	pos := l.end
+	for _, ch := range word {
+		if l.next() != ch {
+			l.end = pos
+			return false
+		}
+	}
+	return true
+}
+
 func (l *lexer) error(format string, args ...interface{}) stateFn {
 	if l.err == nil { // show first error
 		l.err = &file.Error{

parser2/lexer/lexer_test.go

@@ -20,13 +20,15 @@ type lexErrorTest struct {
 
 var lexTests = []lexTest{
 	{
-		"1 02 1e3 1.2e-4 1_000_000",
+		".5 1 02 1e3 1.2e-4 1_000_000 _42",
 		[]Token{
+			{Kind: Number, Value: ".5"},
 			{Kind: Number, Value: "1"},
 			{Kind: Number, Value: "02"},
 			{Kind: Number, Value: "1e3"},
 			{Kind: Number, Value: "1.2e-4"},
 			{Kind: Number, Value: "1_000_000"},
+			{Kind: Identifier, Value: "_42"},
 			{Kind: EOF},
 		},
 	},
@@ -44,18 +46,32 @@ var lexTests = []lexTest{
 		},
 	},
 	{
-		"a and orb",
+		"a and orb().val",
 		[]Token{
 			{Kind: Identifier, Value: "a"},
 			{Kind: Operator, Value: "and"},
 			{Kind: Identifier, Value: "orb"},
+			{Kind: Bracket, Value: "("},
+			{Kind: Bracket, Value: ")"},
+			{Kind: Operator, Value: "."},
+			{Kind: Identifier, Value: "val"},
 			{Kind: EOF},
 		},
 	},
 	{
-		`not in`,
+		`not in not abc not i not(false) not  in`,
 		[]Token{
 			{Kind: Operator, Value: "not in"},
+			{Kind: Operator, Value: "not"},
+			{Kind: Identifier, Value: "abc"},
+			{Kind: Operator, Value: "not"},
+			{Kind: Identifier, Value: "i"},
+			{Kind: Operator, Value: "not"},
+			{Kind: Bracket, Value: "("},
+			{Kind: Identifier, Value: "false"},
+			{Kind: Bracket, Value: ")"},
+			{Kind: Operator, Value: "not"},
+			{Kind: Operator, Value: "in"},
 			{Kind: EOF},
 		},
 	},

parser2/lexer/state.go

@@ -23,48 +23,52 @@ func root(l *lexer) stateFn {
 		l.emitValue(String, str)
 	case '0' <= r && r <= '9':
 		l.backup()
-		return lexNumber
+		return number
 	case strings.ContainsRune("([{", r):
 		l.emit(Bracket)
 	case strings.ContainsRune(")]}", r):
 		l.emit(Bracket)
-	case strings.ContainsRune(".,?:", r):
-		l.emit(Punctuation)
-	case strings.ContainsRune("!%&*+-/<=>^|~", r):
+	case strings.ContainsRune(",?!:%#&*+-/<=>^|", r):
 		l.backup()
-		return lexOperator
+		return operator
+	case r == '.':
+		l.backup()
+		return dot
 	case isAlphaNumeric(r):
 		l.backup()
-		return lexName
+		return identifier
 	default:
 		return l.error("unrecognized character: %#U", r)
 	}
 	return root
 }
 
-func lexNumber(l *lexer) stateFn {
+func number(l *lexer) stateFn {
 	if !l.scanNumber() {
 		return l.error("bad number syntax: %q", l.word())
 	}
 	l.emit(Number)
 	return root
 }
 
+const digits = "0123456789"
+
 func (l *lexer) scanNumber() bool {
 	// Is it hex?
-	digits := "0123456789_"
-	l.acceptRun(digits)
+	l.accept(digits)
+	l.acceptRun(digits + "_")
 	if l.accept(".") {
 		// Lookup for .. operator: if after dot there is another dot (1..2), it maybe a range operator.
 		if l.peek() == '.' {
 			l.backup()
 			return true
 		}
-		l.acceptRun(digits)
+		l.accept(digits)
+		l.acceptRun(digits + "_")
 	}
 	if l.accept("eE") {
 		l.accept("+-")
-		l.acceptRun("0123456789_")
+		l.acceptRun(digits)
 	}
 	// Next thing mustn't be alphanumeric.
 	if isAlphaNumeric(l.peek()) {
@@ -74,37 +78,51 @@ func (l *lexer) scanNumber() bool {
 	return true
 }
 
-func lexOperator(l *lexer) stateFn {
+func operator(l *lexer) stateFn {
 	l.next()
-	l.accept("|&=*")
+	l.accept(".|&=*")
 	l.emit(Operator)
 	return root
 }
 
-func lexName(l *lexer) stateFn {
-Loop:
+func dot(l *lexer) stateFn {
+	l.next()
+	if l.accept(digits) {
+		l.backup()
+		return number
+	}
+	l.accept(".")
+	l.emit(Operator)
+	return root
+}
+
+func identifier(l *lexer) stateFn {
+loop:
 	for {
 		switch r := l.next(); {
 		case isAlphaNumeric(r):
-			// absorb.
+			// absorb
 		default:
 			l.backup()
 			switch l.word() {
 			case "not":
-				l.emit(Operator)
-			case "in":
-				l.emit(Operator)
-			case "or":
-				l.emit(Operator)
-			case "and":
-				l.emit(Operator)
-			case "matches":
+				return not
+			case "in", "or", "and", "matches", "contains", "startsWith", "endsWith":
 				l.emit(Operator)
 			default:
 				l.emit(Identifier)
 			}
-			break Loop
+			break loop
 		}
 	}
 	return root
 }
+
+func not(l *lexer) stateFn {
+	if l.acceptWord(" in") {
+		l.emit(Operator)
+	} else {
+		l.emit(Operator)
+	}
+	return root
+}

parser2/lexer/token.go

@@ -8,13 +8,12 @@ import (
 type Kind string
 
 const (
-	Identifier  Kind = "Identifier"
-	Number           = "Number"
-	String           = "String"
-	Operator         = "Operator"
-	Bracket          = "Bracket"
-	Punctuation      = "Punctuation"
-	EOF              = "EOF"
+	Identifier Kind = "Identifier"
+	Number          = "Number"
+	String          = "String"
+	Operator        = "Operator"
+	Bracket         = "Bracket"
+	EOF             = "EOF"
 )
 
 type Token struct {