lex: move token pretty-printing to another file

kojote/lex/+test.ha

@@ -68,9 +68,9 @@ type dummytoken = (ty, value);
 
 			if (!tokeq(have, want)) {
 				fmt::printf("Expected:\n\t")!;
-				fmt::println(tokpp(want.0, want.1))!;
+				fmt::println(tokstr((want.0, want.1, location{ ... })))!;
 				fmt::printf("Got:\n\t")!;
-				fmt::println(tokpp(have.0, have.1))!;
+				fmt::println(tokstr(have))!;
 				assert(false);
 			};
 		};
@@ -91,41 +91,3 @@ fn tokeq(have: token, want: dummytoken) bool =
 		yield want.1 is bool && (want.1 as bool) == b;
 	};
 
-fn tokpp(ty: ty, value: value) const str = {
-	static let buf: [128]u8 = [0...];
-
-	switch (ty) {
-	case ty::QUOT_START =>
-		return "[";
-	case ty::QUOT_END =>
-		return "]";
-	case ty::MAP_START =>
-		return "{";
-	case ty::MAP_END =>
-		return "}";
-	case ty::COMMENT =>
-		return fmt::bsprintf(buf, "({})", value as str);
-	case ty::WORD =>
-		return value as str;
-	case ty::SYMBOL =>
-		return fmt::bsprintf(buf, "\\{}", value as str);
-	case ty::KEYWORD =>
-		return fmt::bsprintf(buf, ":{}", value as str);
-	case ty::STRING =>
-		return fmt::bsprintf(buf, "\"{}\"", value as str);
-	case ty::CHAR =>
-		let rn = value as rune;
-		for (let i = 0z; i < len(longcharnames); i += 1) {
-			if (longcharnames[i].1 == rn) {
-				return fmt::bsprintf(buf, "#\\{}",
-					longcharnames[i].0);
-			};
-		};
-		return fmt::bsprintf(buf, "#\\{}", rn);
-	case ty::NUMBER =>
-		return value as str;
-	case ty::BOOLEAN =>
-		return fmt::bsprintf(buf, "#{}",
-			if (value as bool) 't' else 'f');
-	};
-};

kojote/lex/lexer.ha

@@ -34,7 +34,8 @@ export type lexer = struct {
 	unread: (rune | void),
 };
 
-// Creates a new [[lexer]] for the given [[io::handle]]. The path is borrowed
+// Creates a new [[lexer]] for the given [[io::handle]].
+// The path is borrowed.
 export fn newlexer(in: io::handle, path: str) lexer = {
 	return lexer {
 		in = in,

kojote/lex/token.ha

@@ -0,0 +1,51 @@
+use ascii;
+use fmt;
+
+// Returns a string representation of the token.
+//
+// This string representation may not correspond one to one with the source,
+// but it is guaranteed to be a syntactically-valid construct, such that
+// parsing it results in the same token.
+export fn tokstr(tok: token) str = {
+	static let buf: [128]u8 = [0...];
+
+	switch (tok.0) {
+	case ty::QUOT_START =>
+		return "[";
+	case ty::QUOT_END =>
+		return "]";
+	case ty::MAP_START =>
+		return "{";
+	case ty::MAP_END =>
+		return "}";
+	case ty::COMMENT =>
+		return fmt::bsprintf(buf, "({})", tok.1 as str);
+	case ty::WORD =>
+		return tok.1 as str;
+	case ty::SYMBOL =>
+		return fmt::bsprintf(buf, "\\{}", tok.1 as str);
+	case ty::KEYWORD =>
+		return fmt::bsprintf(buf, ":{}", tok.1 as str);
+	case ty::STRING =>
+		// TODO: escape string before printing
+		return fmt::bsprintf(buf, "\"{}\"", tok.1 as str);
+	case ty::CHAR =>
+		let rn = tok.1 as rune;
+		for (let i = 0z; i < len(longcharnames); i += 1) {
+			if (longcharnames[i].1 == rn) {
+				return fmt::bsprintf(buf, "#\\{}",
+					longcharnames[i].0);
+			};
+		};
+		if (ascii::isgraph(rn)) {
+			return fmt::bsprintf(buf, "#\\{}", rn);
+		} else {
+			return fmt::bsprintf(buf, "#\\x{x};", rn: u32);
+		};
+	case ty::NUMBER =>
+		return tok.1 as str;
+	case ty::BOOLEAN =>
+		return fmt::bsprintf(buf, "#{}",
+			if (tok.1 as bool) 't' else 'f');
+	};
+};