diff --git a/kojote/lex/+test.ha b/kojote/lex/+test.ha index 5ece579..b534f71 100644 --- a/kojote/lex/+test.ha +++ b/kojote/lex/+test.ha @@ -68,9 +68,9 @@ type dummytoken = (ty, value); if (!tokeq(have, want)) { fmt::printf("Expected:\n\t")!; - fmt::println(tokpp(want.0, want.1))!; + fmt::println(tokstr((want.0, want.1, location{ ... })))!; fmt::printf("Got:\n\t")!; - fmt::println(tokpp(have.0, have.1))!; + fmt::println(tokstr(have))!; assert(false); }; }; @@ -91,41 +91,3 @@ fn tokeq(have: token, want: dummytoken) bool = yield want.1 is bool && (want.1 as bool) == b; }; -fn tokpp(ty: ty, value: value) const str = { - static let buf: [128]u8 = [0...]; - - switch (ty) { - case ty::QUOT_START => - return "["; - case ty::QUOT_END => - return "]"; - case ty::MAP_START => - return "{"; - case ty::MAP_END => - return "}"; - case ty::COMMENT => - return fmt::bsprintf(buf, "({})", value as str); - case ty::WORD => - return value as str; - case ty::SYMBOL => - return fmt::bsprintf(buf, "\\{}", value as str); - case ty::KEYWORD => - return fmt::bsprintf(buf, ":{}", value as str); - case ty::STRING => - return fmt::bsprintf(buf, "\"{}\"", value as str); - case ty::CHAR => - let rn = value as rune; - for (let i = 0z; i < len(longcharnames); i += 1) { - if (longcharnames[i].1 == rn) { - return fmt::bsprintf(buf, "#\\{}", - longcharnames[i].0); - }; - }; - return fmt::bsprintf(buf, "#\\{}", rn); - case ty::NUMBER => - return value as str; - case ty::BOOLEAN => - return fmt::bsprintf(buf, "#{}", - if (value as bool) 't' else 'f'); - }; -}; diff --git a/kojote/lex/lexer.ha b/kojote/lex/lexer.ha index 93cc318..150c7be 100644 --- a/kojote/lex/lexer.ha +++ b/kojote/lex/lexer.ha @@ -34,7 +34,8 @@ export type lexer = struct { unread: (rune | void), }; -// Creates a new [[lexer]] for the given [[io::handle]]. The path is borrowed +// Creates a new [[lexer]] for the given [[io::handle]]. +// The path is borrowed. export fn newlexer(in: io::handle, path: str) lexer = { return lexer { in = in, diff --git a/kojote/lex/token.ha b/kojote/lex/token.ha new file mode 100644 index 0000000..8297fdb --- /dev/null +++ b/kojote/lex/token.ha @@ -0,0 +1,51 @@ +use ascii; +use fmt; + +// Returns a string representation of the token. +// +// This string representation may not correspond one to one with the source, +// but it is guaranteed to be a syntactically-valid construct, such that +// parsing it results in the same token. +export fn tokstr(tok: token) str = { + static let buf: [128]u8 = [0...]; + + switch (tok.0) { + case ty::QUOT_START => + return "["; + case ty::QUOT_END => + return "]"; + case ty::MAP_START => + return "{"; + case ty::MAP_END => + return "}"; + case ty::COMMENT => + return fmt::bsprintf(buf, "({})", tok.1 as str); + case ty::WORD => + return tok.1 as str; + case ty::SYMBOL => + return fmt::bsprintf(buf, "\\{}", tok.1 as str); + case ty::KEYWORD => + return fmt::bsprintf(buf, ":{}", tok.1 as str); + case ty::STRING => + // TODO: escape string before printing + return fmt::bsprintf(buf, "\"{}\"", tok.1 as str); + case ty::CHAR => + let rn = tok.1 as rune; + for (let i = 0z; i < len(longcharnames); i += 1) { + if (longcharnames[i].1 == rn) { + return fmt::bsprintf(buf, "#\\{}", + longcharnames[i].0); + }; + }; + if (ascii::isgraph(rn)) { + return fmt::bsprintf(buf, "#\\{}", rn); + } else { + return fmt::bsprintf(buf, "#\\x{x};", rn: u32); + }; + case ty::NUMBER => + return tok.1 as str; + case ty::BOOLEAN => + return fmt::bsprintf(buf, "#{}", + if (tok.1 as bool) 't' else 'f'); + }; +};