fhdskjgfdsjglfdskjgfd lexer
This commit is contained in:
parent
63fe8290a5
commit
696d4910b1
2 changed files with 177 additions and 91 deletions
235
lex.ha
235
lex.ha
|
|
@ -1,10 +1,12 @@
|
|||
use ascii; // TODO: maybe use unicode?
|
||||
use ascii;
|
||||
use bufio;
|
||||
use encoding::utf8;
|
||||
use fmt;
|
||||
use io;
|
||||
use memio;
|
||||
use os;
|
||||
use unicode;
|
||||
|
||||
// Testing dependency
|
||||
use fmt;
|
||||
use strings;
|
||||
|
||||
// my cod prob sux :(
|
||||
|
|
@ -33,7 +35,7 @@ export fn close(lex: *lexer) void = {
|
|||
io::close(&lex.strbuf)!;
|
||||
};
|
||||
|
||||
export fn next(lex: *lexer) (token | io::EOF | error) = {
|
||||
export fn lex(lex: *lexer) (token | io::EOF | error) = {
|
||||
const rn = match (nextrunews(lex)?) {
|
||||
case io::EOF =>
|
||||
return io::EOF;
|
||||
|
|
@ -43,28 +45,35 @@ export fn next(lex: *lexer) (token | io::EOF | error) = {
|
|||
|
||||
switch (rn) {
|
||||
case '(' =>
|
||||
return punctuation::LEFT_PAREN: token;
|
||||
return comment{ v = scancomment(lex)? };
|
||||
case ')' =>
|
||||
return punctuation::RIGHT_PAREN: token;
|
||||
return lex.loc: invalid;
|
||||
case '[' =>
|
||||
return punctuation::LEFT_SQUARE_BRACKET: token;
|
||||
return quotstart;
|
||||
case ']' =>
|
||||
return punctuation::RIGHT_SQUARE_BRACKET: token;
|
||||
return quotend;
|
||||
case '{' =>
|
||||
return punctuation::LEFT_CURLY_BRACKET: token;
|
||||
return mapstart;
|
||||
case '}' =>
|
||||
return punctuation::RIGHT_CURLY_BRACKET: token;
|
||||
return mapend;
|
||||
case '\\' =>
|
||||
return punctuation::BACKSLASH: token;
|
||||
case ':' =>
|
||||
return punctuation::COLON: token;
|
||||
case '"' =>
|
||||
match (scanstr(lex)?) {
|
||||
case let s: str =>
|
||||
return s;
|
||||
case io::EOF =>
|
||||
return io::EOF;
|
||||
let v = scanword(lex)?;
|
||||
if (len(v) == 0) {
|
||||
return lex.loc: invalid;
|
||||
} else {
|
||||
return symbol{ v = v, kw = false };
|
||||
};
|
||||
case ':' =>
|
||||
let v = scanword(lex)?;
|
||||
if (len(v) == 0) {
|
||||
return lex.loc: invalid;
|
||||
} else {
|
||||
return symbol{ v = v, kw = true };
|
||||
};
|
||||
case '\'' =>
|
||||
return scanchar(lex)?;
|
||||
case '"' =>
|
||||
return scanstr(lex)?;
|
||||
case =>
|
||||
yield;
|
||||
};
|
||||
|
|
@ -110,7 +119,7 @@ fn nextrunews(lex: *lexer) (rune | io::EOF | error) = {
|
|||
for (true) {
|
||||
match (nextrune(lex)?) {
|
||||
case let rn: rune =>
|
||||
if (ascii::isspace(rn)) {
|
||||
if (isspace(rn)) {
|
||||
continue;
|
||||
};
|
||||
return rn;
|
||||
|
|
@ -135,7 +144,7 @@ fn scanword(lex: *lexer) (str | error) = {
|
|||
case io::EOF =>
|
||||
break;
|
||||
};
|
||||
if (ascii::isspace(rn)) {
|
||||
if (isspace(rn) || isdelimiter(rn)) {
|
||||
unget(lex, rn);
|
||||
break;
|
||||
};
|
||||
|
|
@ -144,14 +153,37 @@ fn scanword(lex: *lexer) (str | error) = {
|
|||
return memio::string(&lex.strbuf)!;
|
||||
};
|
||||
|
||||
fn scanstr(lex: *lexer) (str | io::EOF | error) = {
|
||||
fn scancomment(lex: *lexer) (str | error) = {
|
||||
memio::reset(&lex.strbuf);
|
||||
for (true) {
|
||||
const rn = match (nextrune(lex)?) {
|
||||
case let rn: rune =>
|
||||
yield rn;
|
||||
case io::EOF =>
|
||||
return ("comment", lex.loc.0, lex.loc.1): unterminated;
|
||||
};
|
||||
|
||||
switch (rn) {
|
||||
case '(' =>
|
||||
return lex.loc: invalid;
|
||||
case ')' =>
|
||||
break;
|
||||
case =>
|
||||
memio::appendrune(&lex.strbuf, rn)!;
|
||||
};
|
||||
};
|
||||
|
||||
return memio::string(&lex.strbuf)!;
|
||||
};
|
||||
|
||||
fn scanstr(lex: *lexer) (str | error) = {
|
||||
memio::reset(&lex.strbuf);
|
||||
for (true) {
|
||||
const rn = match (nextrune(lex)?) {
|
||||
case let rn: rune =>
|
||||
yield rn;
|
||||
case io::EOF =>
|
||||
return ("string literal", lex.loc.0, lex.loc.1): unterminated;
|
||||
};
|
||||
|
||||
switch (rn) {
|
||||
|
|
@ -165,6 +197,22 @@ fn scanstr(lex: *lexer) (str | io::EOF | error) = {
|
|||
return memio::string(&lex.strbuf)!;
|
||||
};
|
||||
|
||||
fn scanchar(lex: *lexer) (rune | error) = {
|
||||
const rn = match (nextrune(lex)?) {
|
||||
case let rn: rune =>
|
||||
yield rn;
|
||||
case io::EOF =>
|
||||
return ("character literal", lex.loc.0, lex.loc.1): unterminated;
|
||||
};
|
||||
|
||||
switch (rn) {
|
||||
case '\\' =>
|
||||
return scanescape(lex)?;
|
||||
case =>
|
||||
return rn;
|
||||
};
|
||||
};
|
||||
|
||||
fn scanescape(lex: *lexer) (rune | error) = {
|
||||
const rn = match (nextrune(lex)?) {
|
||||
case let rn: rune =>
|
||||
|
|
@ -178,79 +226,110 @@ fn scanescape(lex: *lexer) (rune | error) = {
|
|||
return '"';
|
||||
case '\\' =>
|
||||
return '\\';
|
||||
case '\n' =>
|
||||
case 'n' =>
|
||||
return '\n';
|
||||
case 't' =>
|
||||
return '\t';
|
||||
case 's' =>
|
||||
return ' ';
|
||||
case =>
|
||||
return lex.loc: invalid;
|
||||
};
|
||||
};
|
||||
|
||||
// Tests! :)
|
||||
|
||||
fn tnext(lex: *lexer) token = {
|
||||
match (next(lex)!) {
|
||||
case let t: token =>
|
||||
return t;
|
||||
fn isspace(rn: rune) bool = {
|
||||
if (ascii::isspace(rn)) {
|
||||
return true;
|
||||
} else {
|
||||
switch (unicode::rune_gc(rn)) {
|
||||
case unicode::gc::Zs =>
|
||||
return true;
|
||||
case =>
|
||||
assert(false);
|
||||
return word { v = "" };
|
||||
return false;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@test fn test_next() void = {
|
||||
let lex = newlexer(&memio::fixed(
|
||||
strings::toutf8("\"hello\" \\greeting def")),
|
||||
"<string>");
|
||||
defer close(&lex);
|
||||
|
||||
let tk = tnext(&lex);
|
||||
assert(tk is str && tk: str == "hello");
|
||||
let tk = tnext(&lex);
|
||||
assert(tk is punctuation && tk: punctuation == punctuation::BACKSLASH);
|
||||
let tk = tnext(&lex);
|
||||
assert(tk is word && (tk: word).v == "greeting");
|
||||
let tk = tnext(&lex);
|
||||
assert(tk is word && (tk: word).v == "def");
|
||||
def delimiters = `()[]{}\:'`;
|
||||
fn isdelimiter(rn: rune) bool = {
|
||||
match (strings::index(delimiters, rn)) {
|
||||
case size =>
|
||||
return true;
|
||||
case =>
|
||||
return false;
|
||||
};
|
||||
};
|
||||
|
||||
@test fn test_nextrune() void = {
|
||||
let lex = newlexer(&memio::fixed(strings::toutf8("a\nb")),
|
||||
"<string>");
|
||||
defer close(&lex);
|
||||
@test fn lex() void = {
|
||||
const cases: [_](str, []token) = [
|
||||
(
|
||||
`"hello" \greeting def`,
|
||||
[
|
||||
"hello",
|
||||
mksym("greeting"),
|
||||
mkword("def"),
|
||||
]
|
||||
),
|
||||
(
|
||||
`[dup *] (a -- a) \square def`,
|
||||
[
|
||||
quotstart,
|
||||
mkword("dup"),
|
||||
mkword("*"),
|
||||
quotend,
|
||||
mkcomment("a -- a"),
|
||||
mksym("square"),
|
||||
mkword("def"),
|
||||
]
|
||||
),
|
||||
(`'\s`, [' '])
|
||||
];
|
||||
|
||||
assert(nextrune(&lex)! == 'a');
|
||||
assert(nextrune(&lex)! == '\n');
|
||||
assert(nextrune(&lex)! == 'b');
|
||||
assert(lex.loc.0 == 2u && lex.loc.1 == 1u);
|
||||
for (let i = 0z; i < len(cases); i += 1) {
|
||||
const src = strings::toutf8(cases[i].0);
|
||||
const src = memio::fixed(src);
|
||||
const lexer = newlexer(&src, "<string>");
|
||||
defer close(&lexer);
|
||||
|
||||
for (let j = 0z; j < len(cases[i].1); j += 1) {
|
||||
const want = cases[i].1[j];
|
||||
const have = lex(&lexer)! as token;
|
||||
assert(tokeq(want, have));
|
||||
};
|
||||
|
||||
@test fn test_nextrunews() void = {
|
||||
let lex = newlexer(&memio::fixed(strings::toutf8("\n a")),
|
||||
"<string>");
|
||||
defer close(&lex);
|
||||
|
||||
assert(nextrunews(&lex)! == 'a');
|
||||
assert(lex.loc.0 == 2u && lex.loc.1 == 2u);
|
||||
assert(lex(&lexer) is io::EOF);
|
||||
};
|
||||
};
|
||||
|
||||
@test fn test_scanword() void = {
|
||||
let lex = newlexer(&memio::fixed(strings::toutf8("string->number .")),
|
||||
"<string>");
|
||||
defer close(&lex);
|
||||
|
||||
assert(scanword(&lex)! == "string->number");
|
||||
};
|
||||
|
||||
@test fn test_scanstr() void = {
|
||||
let lex = newlexer(&memio::fixed(strings::toutf8("\"\\\\hello\\\"world!\\\n\"")),
|
||||
"<string>");
|
||||
defer close(&lex);
|
||||
|
||||
assert(nextrune(&lex)! == '"');
|
||||
match (scanstr(&lex)!) {
|
||||
case io::EOF =>
|
||||
assert(false);
|
||||
fn tokeq(have: token, want: token) bool = {
|
||||
match (want) {
|
||||
case quotstart =>
|
||||
return have is quotstart;
|
||||
case quotend =>
|
||||
return have is quotend;
|
||||
case mapstart =>
|
||||
return have is mapstart;
|
||||
case mapend =>
|
||||
return have is mapend;
|
||||
case let w: word =>
|
||||
return (have as word).v == w.v;
|
||||
case let s: str =>
|
||||
assert(s == "\\hello\"world!\n");
|
||||
return have as str == s;
|
||||
case let s: symbol =>
|
||||
return (have as symbol).v == s.v;
|
||||
case let c: comment =>
|
||||
return (have as comment).v == c.v;
|
||||
case let r: rune =>
|
||||
return have as rune == r;
|
||||
};
|
||||
};
|
||||
|
||||
fn mkword(v: const str) word =
|
||||
word{ v = v };
|
||||
|
||||
fn mkcomment(v: const str) comment =
|
||||
comment{ v = v };
|
||||
|
||||
fn mksym(v: const str, kw: bool = false) symbol =
|
||||
symbol{ v = v, kw = kw };
|
||||
|
||||
|
|
|
|||
23
types.ha
23
types.ha
|
|
@ -2,16 +2,20 @@ use io;
|
|||
use fmt;
|
||||
|
||||
export type invalid = !(uint, uint);
|
||||
export type error = !(invalid | io::error);
|
||||
export type unterminated = !(const str, uint, uint);
|
||||
export type error = !(invalid | unterminated | io::error);
|
||||
|
||||
export type punctuation = enum uint {
|
||||
LEFT_PAREN, RIGHT_PAREN,
|
||||
LEFT_SQUARE_BRACKET, RIGHT_SQUARE_BRACKET,
|
||||
LEFT_CURLY_BRACKET, RIGHT_CURLY_BRACKET,
|
||||
BACKSLASH, COLON,
|
||||
};
|
||||
export type quotstart = void;
|
||||
export type quotend = void;
|
||||
export type mapstart = void;
|
||||
export type mapend = void;
|
||||
|
||||
export type comment = struct { v: str };
|
||||
export type word = struct { v: str };
|
||||
export type token = (punctuation | word | str);
|
||||
export type symbol = struct { v: str, kw: bool };
|
||||
|
||||
export type token = (quotstart | quotend | mapstart | mapend |
|
||||
word | symbol | comment | str | rune);
|
||||
|
||||
export fn strerror(err: error) const str = {
|
||||
static let buf: [64]u8 = [0...];
|
||||
|
|
@ -19,6 +23,9 @@ export fn strerror(err: error) const str = {
|
|||
case let err: invalid =>
|
||||
return fmt::bsprintf(buf,
|
||||
"{}:{}: Invalid token found", err.0, err.1);
|
||||
case let err: unterminated =>
|
||||
return fmt::bsprintf(buf,
|
||||
"{}:{}: Unterminated {} found", err.1, err.2, err.0);
|
||||
case let err: io::error =>
|
||||
return io::strerror(err);
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue