lex: token representation and quoting stuff

* `escape` => `quote` and `quotestr`
  `quote` writes the result to a I/O handle, `quotestr` returns it as a
  heap-allocated string.
* `tokstr` => `tokrepr` and `tokstr`
  same deal

Makefile

@@ -7,7 +7,8 @@ HARESRCDIR=${SRCDIR}/hare
 HARE=hare
 HAREPATH=vendor/hare-unicode:${HARESRCDIR}/stdlib:${HARESRCDIR}/third-party
 
-.PHONY: check build clean
+.PHONY: check build clean print-harepath
+
 
 check:
 	@env HAREPATH=${HAREPATH} hare test -T +test
@@ -17,3 +18,6 @@ build:
 
 clean:
 	rm -f kj
+
+print-harepath:
+	@echo ${HAREPATH}

kojote/lex/+test.ha

@@ -2,11 +2,10 @@ use memio;
 use fmt;
 use strings;
 use io;
-
-type dummytoken = (ty, value);
+use os;
 
 @test fn next() void = {
-	const cases: [_](str, []dummytoken) = [
+	const cases: [_](str, [](ty, value)) = [
 		(`"hello" \greeting def`,
 			[
 				(ty::STRING, "hello"),
@@ -28,11 +27,12 @@ type dummytoken = (ty, value);
 				(ty::BOOLEAN, true),
 				(ty::BOOLEAN, false),
 			]),
-		(`#\a #\space #\nul`,
+		(`#\a #\space #\nul #\嶲`,
 			[
 				(ty::CHAR, 'a'),
 				(ty::CHAR, ' '),
 				(ty::CHAR, '\0'),
+				(ty::CHAR, '嶲'),
 			]),
 		(`"\x0a;\x2014;\x2f9f4;"`,
 			[
@@ -47,16 +47,17 @@ type dummytoken = (ty, value);
 			]),
 	];
 
-	for (let i = 0z; i < len(cases); i += 1) {
-		const src = strings::toutf8(cases[i].0);
+	for (let tcase .. cases) {
+		const src = strings::toutf8(tcase.0);
 		const src = memio::fixed(src);
 		const lexer = newlexer(&src, "<string>");
 		defer close(&lexer);
 
-		for (let j = 0z; j < len(cases[i].1); j += 1) {
-			const want = cases[i].1[j];
+		for (let want .. tcase.1) {
 			const have = match (next(&lexer)) {
 			case let tok: token =>
+				tokrepr(os::stdout, tok)!;
+				fmt::print(' ')!;
 				yield tok;
 			case io::EOF =>
 				assert(false, "reached EOF");
@@ -67,27 +68,28 @@ type dummytoken = (ty, value);
 			};
 
 			if (!tokeq(have, want)) {
-				fmt::printf("Expected:\n\t")!;
-				fmt::println(tokstr((want.0, want.1, location{ ... })))!;
-				fmt::printf("Got:\n\t")!;
-				fmt::println(tokstr(have))!;
-				assert(false);
+				fmt::errorf("Expected:\n\t")!;
+				tokrepr(os::stderr,
+					(want.0, want.1, location{ ... }))!;
+				fmt::errorf("\nGot:\n\t")!;
+				tokrepr(os::stderr, have)!;
+				assert(false, "test case doesn't match expectation");
 			};
 		};
 
-		assert(next(&lexer) is io::EOF);
+		assert(next(&lexer) is io::EOF, "didn't reach EOF at the end of test case");
 	};
 };
 
-fn tokeq(have: token, want: dummytoken) bool =
+fn tokeq(have: token, want: (ty, value)) bool =
 	have.0 == want.0 && match (have.1) {
-	case void =>
-		yield true;
-	case let s: str =>
-		yield want.1 is str && (want.1 as str) == s;
-	case let r: rune =>
-		yield want.1 is rune && (want.1 as rune) == r;
-	case let b: bool =>
-		yield want.1 is bool && (want.1 as bool) == b;
-	};
+		case void =>
+			yield true;
+		case let s: str =>
+			yield want.1 is str && (want.1 as str) == s;
+		case let r: rune =>
+			yield want.1 is rune && (want.1 as rune) == r;
+		case let b: bool =>
+			yield want.1 is bool && (want.1 as bool) == b;
+		};
 

kojote/lex/lexer.ha

@@ -1,29 +1,31 @@
 use ascii;
+use strings;
 use bufio;
 use encoding::utf8;
 use io;
 use memio;
 use unicode;
 use strconv;
-
-use fmt;
-use strings;
+use sort;
 
 def longcharnames: [_](str, rune) = [
-	("nul", '\u0000'),
-	("alarm", '\u0007'),
-	("backspace", '\u0008'),
-	("tab", '\u0009'),
-	("newline", '\u000a'),
-	("linefeed", '\u000a'),
-	("vtab", '\u000b'),
-	("page", '\u000c'),
-	("return", '\u000d'),
-	("esc", '\u001b'),
-	("space", '\u0020'),
-	("delete", '\u007f'),
+	("alarm", '\x07'),
+	("backspace", '\x08'),
+	("delete", '\x7f'),
+	("esc", '\x1b'),
+	("linefeed", '\x0a'),
+	("newline", '\x0a'),
+	("nul", '\0'),
+	("page", '\x0c'),
+	("return", '\x0d'),
+	("space", ' '),
+	("tab", '\x09'),
+	("vtab", '\x0b'),
 ];
 
+fn longcharnames_namecmp(a: const *opaque, b: const *opaque) int =
+	strings::compare((a: *(str, rune)).0, (b: *(str, rune)).0);
+
 export type lexer = struct {
 	in: io::handle,
 	strbuf: memio::stream,
@@ -264,13 +266,17 @@ fn scanchar(lex: *lexer) (token | error) = {
 			memio::appendrune(&namebuf, rn)!;
 			memio::concat(&namebuf, scanword(lex)?)!;
 			const name = memio::string(&namebuf)!;
-			for (let i = 0z; i < len(longcharnames); i += 1) {
-				if (name == longcharnames[i].0) {
-					return mktoken(lex, ty::CHAR,
-						longcharnames[i].1);
-				};
+
+			match (sort::search(longcharnames: []opaque,
+					size((str, rune)),
+					&name: const *opaque,
+					&longcharnames_namecmp)) {
+			case let ix: size =>
+				return mktoken(lex, ty::CHAR,
+					longcharnames[ix].1);
+			case void =>
+				return mkerror(lex, "invalid named character literal");
 			};
-			return mkerror(lex, "invalid named character literal");
 		};
 	};
 };

kojote/lex/quote.ha

@@ -0,0 +1,71 @@
+use ascii;
+use encoding::utf8;
+use fmt;
+use io;
+use memio;
+use sort;
+use strings;
+
+use unicode;
+
+// Sorted to use with [[sort::search]].
+def escapetable: [_](rune, rune) = [
+	('\0',   '0'),
+	('\a',   'a'),
+	('\b',   'b'),
+	('\t',   't'),
+	('\n',   'n'),
+	('\v',   'v'),
+	('\f',   'f'),
+	('\r',   'r'),
+	('\x1b', 'e'),
+	('"',    '"'),
+	('\\',   '\\'),
+];
+
+fn escapetable_cmpfunc(a: const *opaque, b: const *opaque) int =
+	(a: *(rune, rune)).0: int - (b: *(rune, rune)).0: int;
+
+// Quotes a Kojote string and writes it to the provided I/O handle.
+export fn quote(sink: io::handle, s: str) (size | io::error) = {
+	if (len(s) == 0) {
+		return io::writeall(sink, ['\"', '\"'])?;
+	};
+
+	let z = io::writeall(sink, ['\"'])?;
+
+	const iter = strings::iter(s);
+	for (let rn => strings::next(&iter)) {
+		if (isgraph(rn)) {
+			z += io::writeall(sink, utf8::encoderune(rn))?;
+			continue;
+		};
+
+		match (sort::search(
+				escapetable: []opaque,
+				size((rune, rune)),
+				&rn: const *opaque,
+				&escapetable_cmpfunc)) {
+		case let ix: size =>
+			z += io::writeall(sink, ['\\'])?;
+			z += io::writeall(sink,
+				utf8::encoderune(escapetable[ix].1))?;
+		case void =>
+			let buf: [16]u8 = [0...];
+			const esc = fmt::bsprintf(buf, `\x{:x};`, rn: u32);
+			z += io::writeall(sink, strings::toutf8(esc))?;
+		};
+	};
+
+	z += io::writeall(sink, ['\"'])?;
+	return z;
+};
+
+// Quotes a Kojote string and returns a new string. The caller must free the
+// return value.
+export fn quotestr(s: str) str = {
+	const sink = memio::dynamic();
+	quote(&sink, s)!;
+	return memio::string(&sink)!;
+};
+

kojote/lex/token.ha

@@ -1,51 +1,76 @@
 use ascii;
 use fmt;
+use strings;
+use encoding::utf8;
+use memio;
+use io;
+use sort;
 
-// Returns a string representation of the token.
-//
-// This string representation may not correspond one to one with the source,
-// but it is guaranteed to be a syntactically-valid construct, such that
-// parsing it results in the same token.
-export fn tokstr(tok: token) str = {
-	static let buf: [128]u8 = [0...];
+def punctrepr: [_]u8 = ['[', ']', '{', '}'];
+
+// Writes the human representation of a token to the provided I/O handle.
+export fn tokrepr(sink: io::handle, tok: token) (size | io::error) = {
+	let z = 0z;
+
+	if (tok.0: size < len(punctrepr)) {
+		return io::writeall(sink, [punctrepr[tok.0]])?;
+	};
 
 	switch (tok.0) {
-	case ty::QUOT_START =>
-		return "[";
-	case ty::QUOT_END =>
-		return "]";
-	case ty::MAP_START =>
-		return "{";
-	case ty::MAP_END =>
-		return "}";
 	case ty::COMMENT =>
-		return fmt::bsprintf(buf, "({})", tok.1 as str);
+		z += io::writeall(sink, ['('])?;
+		z += io::writeall(sink, strings::toutf8(tok.1 as str))?;
+		z += io::writeall(sink, [')'])?;
 	case ty::WORD =>
-		return tok.1 as str;
+		z += io::writeall(sink, strings::toutf8(tok.1 as str))?;
 	case ty::SYMBOL =>
-		return fmt::bsprintf(buf, "\\{}", tok.1 as str);
+		z += io::writeall(sink, ['\\'])?;
+		z += io::writeall(sink, strings::toutf8(tok.1 as str))?;
 	case ty::KEYWORD =>
-		return fmt::bsprintf(buf, ":{}", tok.1 as str);
+		z += io::writeall(sink, [':'])?;
+		z += io::writeall(sink, strings::toutf8(tok.1 as str))?;
 	case ty::STRING =>
-		// TODO: escape string before printing
-		return fmt::bsprintf(buf, "\"{}\"", tok.1 as str);
+		z += quote(sink, tok.1 as str)?;
 	case ty::CHAR =>
 		let rn = tok.1 as rune;
-		for (let i = 0z; i < len(longcharnames); i += 1) {
-			if (longcharnames[i].1 == rn) {
-				return fmt::bsprintf(buf, "#\\{}",
-					longcharnames[i].0);
+		let named = false;
+
+		z += io::writeall(sink, ['#', '\\'])?;
+
+		for (let ch .. longcharnames) {
+			if (ch.1 == rn) {
+				z += io::writeall(sink, strings::toutf8(ch.0))?;
+				named = true;
+				break;
 			};
 		};
-		if (ascii::isgraph(rn)) {
-			return fmt::bsprintf(buf, "#\\{}", rn);
-		} else {
-			return fmt::bsprintf(buf, "#\\x{x};", rn: u32);
+
+		if (!named) {
+			if (isgraph(rn)) {
+				z += io::writeall(sink, utf8::encoderune(rn))?;
+			} else {
+				z += fmt::fprintf(sink, "x{:x};", rn: u32)?;
+			};
 		};
 	case ty::NUMBER =>
-		return tok.1 as str;
+		z += io::writeall(sink, strings::toutf8(tok.1 as str))?;
 	case ty::BOOLEAN =>
-		return fmt::bsprintf(buf, "#{}",
-			if (tok.1 as bool) 't' else 'f');
+		z += io::writeall(sink, ['#'])?;
+		z += io::writeall(sink,
+			if (tok.1 as bool) ['t'] else ['f'])?;
+	case =>
+		// unreachable
+		abort();
 	};
+
+	return z;
 };
+
+// Returns the human representation of a token as a new string. The caller must
+// free the return value.
+export fn tokstr(tok: token) str = {
+	const sink = memio::dynamic();
+	tokrepr(&sink, tok)!;
+	return memio::string(&sink)!;
+};
+

kojote/lex/util.ha

@@ -0,0 +1,14 @@
+use unicode;
+
+def graphtable: [unicode::gc::Zs]bool = [
+	false, false, false, false, false,   // C
+	true, true, true, true, true,        // L
+	true, true, true,                    // M
+	true, true, true,                    // N
+	true, true, true, true, true, true,  // P
+	true, true, true, true,              // S
+	false, false, false,                 // Z
+];
+
+fn isgraph(r: rune) bool =
+	if (r == ' ') true else graphtable[unicode::rune_gc(r)];