Initial commit

lexer.lua

@@ -0,0 +1,117 @@
+local Lexer = {}
+Lexer.__index = Lexer
+
+function Lexer:eat(byte_count)
+	for c in self.source:sub(self.i, self.i + byte_count - 1):gmatch"." do
+		if c == "\n" then
+			self.current_row = self.current_row + 1
+			self.current_col = 1
+		else
+			self.current_col = self.current_col + 1
+		end
+	end
+	
+	self.i = self.i + byte_count
+end
+
+function Lexer:match(pattern)
+	local m = self.source:sub(self.i):match("^" .. pattern)
+	if m then
+		self:eat(#m)
+		self.last_match = m
+		return m
+	end
+end
+
+function Lexer:add(token_type, token_data)
+	table.insert(self.result, {type = token_type, data = token_data, x = self.current_col, y = self.current_row})
+end
+
+function Lexer:get()
+	if self:match"[0-9]+r[0-9a-zA-Z]+" or self:match"[0-9]+" then
+		self:add("num", self.last_match)
+	elseif self:match"if" then
+		self:add("if", nil)
+	elseif self:match"string" then
+		self:add("stringkw", nil)
+	elseif self:match"loop" then
+		self:add("loop", nil)
+	elseif self:match"return" then
+		self:add("return", nil)
+	elseif self:match"export" then
+		self:add("export", nil)
+	elseif self:match"[a-zA-Z_][a-zA-Z0-9_]*" then
+		self:add("ident", self.last_match)
+	elseif self:match"@[a-zA-Z_][a-zA-Z0-9_]*" then
+		self:add("mod", self.last_match)
+	elseif self:match"'" then
+		local j = self.i
+		local value = ""
+		while j <= #self.source do
+			local b = self.source:sub(j, j)
+			
+			if b == "'" then
+				break
+			elseif b == "\\" then
+				j = j + 1
+				b = self.source:sub(j, j)
+				if b == "n" then
+					b = "\n"
+				elseif b == "b" then
+					b = "\b"
+				elseif b == "e" then
+					b = "\x1B"
+				elseif b == "r" then
+					b = "\r"
+				elseif b == "t" then
+					b = "\t"
+				elseif b == "x" then
+					b = string.char(tonumber(self.source:sub(j + 1, j + 2), 16))
+					j = j + 2
+				else
+					error("Unknown escape sequence")
+				end
+			end
+			
+			value = value .. b
+			j = j + 1
+		end
+		self:eat(j - self.i + 1)
+		self:add("string", value)
+	elseif self:match"==" then
+		self:add("==", nil)
+	elseif self:match"!=" then
+		self:add("!=", nil)
+	elseif self:match">=" then
+		self:add(">=", nil)
+	elseif self:match"<=" then
+		self:add("<=", nil)
+	elseif self:match":" then
+		self:add(":", nil)
+	elseif self:match"%(" then
+		self:add("(", nil)
+	elseif self:match"%)" then
+		self:add(")", nil)
+	elseif self:match"->" then
+		self:add("->", nil)
+	elseif self:match"{" then
+		self:add("{", nil)
+	elseif self:match"}" then
+		self:add("}", nil)
+	elseif self:match"%s" then
+		--self:add("ws", nil)
+	else
+		self:add(self.source:sub(self.i, self.i), nil)
+		self:eat(1)
+	end
+end
+
+function Lexer:go()
+	while self.i <= #self.source do
+		self:get()
+	end
+end
+
+return function(source_code)
+	return setmetatable({i = 1, source = source_code, result = {}, current_row = 1, current_col = 1}, Lexer)
+end