z_mirror/luapls

git clone https://git.raiguard.me/z_mirror/luapls

Caleb HeuerRefactor lexer to support unicode and clean things up91fac8d

main
branchesmainall refs →
3.9 KiB129 linesraw
1package lexer
2
3import (
4	"testing"
5
6	"github.com/raiguard/luapls/lua/token"
7
8	"github.com/stretchr/testify/assert"
9	"github.com/stretchr/testify/require"
10)
11
12func TestOperators(t *testing.T) {
13	input := "+-++%^=/==>>=<=<~="
14	tokens := []token.Token{
15		{Type: token.PLUS, Literal: "+", Pos: 0},
16		{Type: token.MINUS, Literal: "-", Pos: 1},
17		{Type: token.PLUS, Literal: "+", Pos: 2},
18		{Type: token.PLUS, Literal: "+", Pos: 3},
19		{Type: token.MOD, Literal: "%", Pos: 4},
20		{Type: token.POW, Literal: "^", Pos: 5},
21		{Type: token.ASSIGN, Literal: "=", Pos: 6},
22		{Type: token.SLASH, Literal: "/", Pos: 7},
23		{Type: token.EQUAL, Literal: "==", Pos: 8},
24		{Type: token.GT, Literal: ">", Pos: 10},
25		{Type: token.GEQ, Literal: ">=", Pos: 11},
26		{Type: token.LEQ, Literal: "<=", Pos: 13},
27		{Type: token.LT, Literal: "<", Pos: 15},
28		{Type: token.NEQ, Literal: "~=", Pos: 16},
29		{Type: token.EOF, Literal: "", Pos: 18},
30	}
31	testLexer(t, input, tokens)
32}
33
34func TestKeywords(t *testing.T) {
35	input := "local while for"
36	tokens := []token.Token{
37		{Type: token.LOCAL, Literal: "local", Pos: 0},
38		{Type: token.WHILE, Literal: "while", Pos: 6},
39		{Type: token.FOR, Literal: "for", Pos: 12},
40		{Type: token.EOF, Literal: "", Pos: 15},
41	}
42	testLexer(t, input, tokens)
43}
44
45func TestAssignment(t *testing.T) {
46	input := `
47local foo = 123
482 + 2 == 4`
49	tokens := []token.Token{
50		{Type: token.LOCAL, Literal: "local", Pos: 1},
51		{Type: token.IDENT, Literal: "foo", Pos: 7},
52		{Type: token.ASSIGN, Literal: "=", Pos: 11},
53		{Type: token.NUMBER, Literal: "123", Pos: 13},
54		{Type: token.NUMBER, Literal: "2", Pos: 17},
55		{Type: token.PLUS, Literal: "+", Pos: 19},
56		{Type: token.NUMBER, Literal: "2", Pos: 21},
57		{Type: token.EQUAL, Literal: "==", Pos: 23},
58		{Type: token.NUMBER, Literal: "4", Pos: 26},
59		{Type: token.EOF, Literal: "", Pos: 27},
60	}
61	testLexer(t, input, tokens)
62}
63
64func TestNumbers(t *testing.T) {
65	input := "3 3.0 3.1416 314.16e-2 0.31416E1 0xff 0x0.1E 0xA23p-4 0X1.921FB54442D18P+1"
66	tokens := []token.Token{
67		{Type: token.NUMBER, Literal: "3", Pos: 0},
68		{Type: token.NUMBER, Literal: "3.0", Pos: 2},
69		{Type: token.NUMBER, Literal: "3.1416", Pos: 6},
70		{Type: token.NUMBER, Literal: "314.16e-2", Pos: 13},
71		{Type: token.NUMBER, Literal: "0.31416E1", Pos: 23},
72		{Type: token.NUMBER, Literal: "0xff", Pos: 33},
73		{Type: token.NUMBER, Literal: "0x0.1E", Pos: 38},
74		{Type: token.NUMBER, Literal: "0xA23p-4", Pos: 45},
75		{Type: token.NUMBER, Literal: "0X1.921FB54442D18P+1", Pos: 54},
76		{Type: token.EOF, Literal: "", Pos: 74},
77	}
78	testLexer(t, input, tokens)
79}
80
81func TestStrings(t *testing.T) {
82	input := "'314.16e-2' \"foo bar\""
83	tokens := []token.Token{
84		{Type: token.STRING, Literal: "'314.16e-2'", Pos: 0},
85		{Type: token.STRING, Literal: "\"foo bar\"", Pos: 12},
86		{Type: token.EOF, Literal: "", Pos: 21},
87	}
88	testLexer(t, input, tokens)
89}
90
91func TestRawStrings(t *testing.T) {
92	input := `[====[asdkflasdkfjs]===]
93  19"38'44'"al]====]
94[===[[[==[=]]==]====][=[]=]===]
95`
96	tokens := []token.Token{
97		{Type: token.RAWSTRING, Literal: "[====[asdkflasdkfjs]===]\n  19\"38'44'\"al]====]", Pos: 0},
98		{Type: token.RAWSTRING, Literal: "[===[[[==[=]]==]====][=[]=]===]", Pos: 46},
99		{Type: token.EOF, Literal: "", Pos: 78},
100	}
101	testLexer(t, input, tokens)
102
103	l := New(input)
104	for l.Next().Type != token.EOF {
105	}
106	require.ElementsMatch(t, []int{24, 45, 77}, l.GetLineBreaks())
107}
108
109func TestLabel(t *testing.T) {
110	input := "::foo_BAR_314::"
111	tokens := []token.Token{
112		{Type: token.LABEL, Literal: "::", Pos: 0},
113		{Type: token.IDENT, Literal: "foo_BAR_314", Pos: 2},
114		{Type: token.LABEL, Literal: "::", Pos: 13},
115		{Type: token.EOF, Literal: "", Pos: 15},
116	}
117	testLexer(t, input, tokens)
118}
119
120func testLexer(t *testing.T, input string, tokens []token.Token) {
121	l := New(input)
122	for _, expected := range tokens {
123		actual := l.Next()
124		// Compare strings for better test output
125		assert.Equal(t, expected.Type.String(), actual.Type.String())
126		assert.Equal(t, expected.Literal, actual.Literal)
127		assert.Equal(t, expected.Pos, actual.Pos)
128	}
129}