lex.lua 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215
  1. --[[
  2. Titty has two main design goals:
  3. 1. Should look natural from the outside, Lua code's PoV, and
  4. 2. Should compile to Lua code that also appears natural from within
  5. The second is less important than the first.
  6. ]]
  7. local inspect = require"inspect".inspect
  8. local function codepoints(str)
  9. local it, state, v1, v2 = utf8.codes(str)
  10. local idx = 0
  11. return function()
  12. if not it then return nil end
  13. v1, v2 = it(state, v1)
  14. if v1 then
  15. local r = idx
  16. idx = idx + 1
  17. return r, v2
  18. else
  19. it = nil
  20. end
  21. end
  22. end
  23. local function is_digit(cp)
  24. return cp and (cp >= 48 and cp <= 57)
  25. end
  26. local function is_ident_start(cp)
  27. return cp and ((cp >= 65 and cp <= 90) or (cp >= 97 and cp <= 122) or cp == 95)
  28. end
  29. local function is_ident_nonstart(cp)
  30. return cp and (is_ident_start(cp) or is_digit(cp))
  31. end
  32. local function forall(tbl, f, startI)
  33. startI = startI or 1
  34. for k, v in pairs(tbl) do
  35. if not f(v) then
  36. return false
  37. end
  38. end
  39. return true
  40. end
  41. local function is_whitespace(cp)
  42. return cp and (cp == 32 or cp == 10 or cp == 9)
  43. end
  44. local function clear(tbl)
  45. for k in pairs(tbl) do
  46. tbl[k] = nil
  47. end
  48. end
  49. local function lex(cpgetraw)
  50. local row, column, idx = 0, 0, 0
  51. local pull
  52. local function cpget()
  53. if pull then
  54. local ret = pull
  55. pull = nil
  56. return ret
  57. else
  58. local cp
  59. idx, cp = cpgetraw()
  60. column = column + 1
  61. if cp == 10 then
  62. column = 0
  63. row = row + 1
  64. end
  65. return cp
  66. end
  67. end
  68. local buf = {}
  69. return function()
  70. local cp
  71. while true do
  72. cp = cpget()
  73. if not is_whitespace(cp) then
  74. break
  75. end
  76. end
  77. if not cp then
  78. return nil
  79. end
  80. local rowStart, columnStart = row, column
  81. if cp == 40 then
  82. return "(", "(", rowStart, columnStart
  83. elseif cp == 41 then
  84. return ")", ")", rowStart, columnStart
  85. elseif cp == 123 then
  86. return "{", "{", rowStart, columnStart
  87. elseif cp == 125 then
  88. return "}", "}", rowStart, columnStart
  89. elseif cp == 44 then
  90. return ",", ",", rowStart, columnStart
  91. elseif cp == 46 then
  92. return ".", ".", rowStart, columnStart
  93. elseif cp == 58 then
  94. return ":", ":", rowStart, columnStart
  95. elseif cp == 61 then
  96. local after = cpget()
  97. if after == 61 then
  98. return "==", "==", rowStart, columnStart
  99. else
  100. pull = after
  101. end
  102. return "=", "=", rowStart, columnStart
  103. elseif cp == 43 then
  104. return "+", "+", rowStart, columnStart
  105. elseif cp == 45 then
  106. return "-", "-", rowStart, columnStart
  107. elseif cp == 42 then
  108. local after = cpget()
  109. if after == 42 then
  110. return "**", "**", rowStart, columnStart
  111. else
  112. pull = after
  113. end
  114. return "*", "*", rowStart, columnStart
  115. elseif cp == 47 then
  116. return "/", "/", rowStart, columnStart
  117. elseif cp == 37 then
  118. return "%", "%", rowStart, columnStart
  119. elseif cp == 35 then
  120. return "#", "#", rowStart, columnStart
  121. elseif cp == 91 then
  122. return "[", "[", rowStart, columnStart
  123. elseif cp == 93 then
  124. return "]", "]", rowStart, columnStart
  125. elseif cp == 39 then
  126. while true do
  127. cp = cpget()
  128. if cp ~= 39 then
  129. buf[#buf + 1] = cp
  130. else
  131. break
  132. end
  133. end
  134. local ret = utf8.char(table.unpack(buf))
  135. clear(buf)
  136. return "string", ret, rowStart, columnStart
  137. elseif is_ident_start(cp) then
  138. buf[1] = cp
  139. while true do
  140. cp = cpget()
  141. if is_ident_nonstart(cp) then
  142. buf[#buf + 1] = cp
  143. else
  144. pull = cp
  145. break
  146. end
  147. end
  148. local ret = utf8.char(table.unpack(buf))
  149. clear(buf)
  150. if ret == "func" or ret == "end" or ret == "if"
  151. or ret == "while" or ret == "do" or ret == "then"
  152. or ret == "elseif" or ret == "type" or ret == "interf"
  153. or ret == "let" or ret == "return" or ret == "else"
  154. or ret == "for" or ret == "import" or ret == "constr"
  155. or ret == "loop" or ret == "break" or ret == "nil" then
  156. return ret, ret, rowStart, columnStart
  157. end
  158. return "id", ret, rowStart, columnStart
  159. elseif is_digit(cp) then
  160. buf[1] = cp
  161. local dotFound = false
  162. while true do
  163. cp = cpget()
  164. if is_digit(cp) then
  165. buf[#buf + 1] = cp
  166. elseif cp == 46 and not dotFound then
  167. dotFound = true
  168. buf[#buf + 1] = cp
  169. else
  170. pull = cp
  171. break
  172. end
  173. end
  174. local ret = utf8.char(table.unpack(buf))
  175. clear(buf)
  176. return "num", ret, rowStart, columnStart
  177. else
  178. error(string.format("%i:%i unknown character %q (code point %i)", row, column, utf8.char(cp), cp))
  179. end
  180. end
  181. end
  182. return {lex = lex, codepoints = codepoints}