local Lexer = {} Lexer.__index = Lexer function Lexer:eat(byte_count) for c in self.source:sub(self.i, self.i + byte_count - 1):gmatch"." do if c == "\n" then self.current_row = self.current_row + 1 self.current_col = 1 else self.current_col = self.current_col + 1 end end self.i = self.i + byte_count end function Lexer:match(pattern) local m = self.source:sub(self.i):match("^" .. pattern) if m then self:eat(#m) self.last_match = m return m end end function Lexer:add(token_type, token_data) table.insert(self.result, {type = token_type, data = token_data, x = self.current_col, y = self.current_row}) end function Lexer:get() if self:match"[0-9]+r[0-9a-zA-Z]+" or self:match"[0-9]+" then self:add("num", self.last_match) elseif self:match"if" then self:add("if", nil) elseif self:match"string" then self:add("stringkw", nil) elseif self:match"loop" then self:add("loop", nil) elseif self:match"return" then self:add("return", nil) elseif self:match"export" then self:add("export", nil) elseif self:match"[a-zA-Z_][a-zA-Z0-9_]*" then self:add("ident", self.last_match) elseif self:match"@[a-zA-Z_][a-zA-Z0-9_]*" then self:add("mod", self.last_match) elseif self:match"'" then local j = self.i local value = "" while j <= #self.source do local b = self.source:sub(j, j) if b == "'" then break elseif b == "\\" then j = j + 1 b = self.source:sub(j, j) if b == "n" then b = "\n" elseif b == "b" then b = "\b" elseif b == "e" then b = "\x1B" elseif b == "r" then b = "\r" elseif b == "t" then b = "\t" elseif b == "x" then b = string.char(tonumber(self.source:sub(j + 1, j + 2), 16)) j = j + 2 else error("Unknown escape sequence") end end value = value .. b j = j + 1 end self:eat(j - self.i + 1) self:add("string", value) elseif self:match"==" then self:add("==", nil) elseif self:match"!=" then self:add("!=", nil) elseif self:match">=" then self:add(">=", nil) elseif self:match"<=" then self:add("<=", nil) elseif self:match":" then self:add(":", nil) elseif self:match"%(" then self:add("(", nil) elseif self:match"%)" then self:add(")", nil) elseif self:match"->" then self:add("->", nil) elseif self:match"{" then self:add("{", nil) elseif self:match"}" then self:add("}", nil) elseif self:match"%s" then --self:add("ws", nil) else self:add(self.source:sub(self.i, self.i), nil) self:eat(1) end end function Lexer:go() while self.i <= #self.source do self:get() end end return function(source_code) return setmetatable({i = 1, source = source_code, result = {}, current_row = 1, current_col = 1}, Lexer) end