Home /Modules/test-suites/reg_lexer.cpp

reg_lexer.cpp

test-suites/regression/cpp/reg_lexer.cpp · 479 line(s) · c

// reg_lexer.cpp -- regression suite for impl/frontend/lexer.cpp
//
// Contract (SLEELA.syntax lexical grammar, lexer.h): the tokenizer turns source
// text into a vector of Tokens that always ends with Eof, tracks 1-based
// line/column, and reports every lexical error by throwing std::runtime_error.
//
// Pinned grammar facts (each one is a documented rule or a defect that was
// fixed once and must not return):
//   * comments are  #...  //...  < ... > (one line, at line start)  and
//     << ... >> (block, at line start).  C-style /* */ is NOT a comment.
//   * '-' continues an identifier only when an identifier character follows
//     (jdk-25, version-mual, non-sealed); `j--`, `j-=1` and `x - 1` split.
//   * numeric forms: 0x 0b 0o prefixes, '_' separators, L/l, f/F/d/D, exponents.
//   * `gc`, `mem`, `ran` are contextual builtins, never reserved words.
#include "../../../impl/frontend/lexer.h"
#include "../c/regression.h"

#include <cstring>
#include <stdexcept>
#include <string>
#include <vector>

using sleela::Lexer;
using sleela::Tok;
using sleela::Token;

namespace {

std::vector<Token> lex(const std::string &src) { return Lexer(src).tokenize(); }

// Returns the error message, or "" when lexing succeeded.
std::string lex_error(const std::string &src) {
    try { (void)lex(src); } catch (const std::runtime_error &e) { return e.what(); }
    return "";
}

bool contains(const std::string &hay, const char *needle) {
    return hay.find(needle) != std::string::npos;
}

// "kind:text" rendering of the stream without the trailing Eof.
std::string kinds(const std::string &src) {
    std::string out;
    for (const Token &t : lex(src)) {
        if (t.kind == Tok::Eof) break;
        if (!out.empty()) out += ' ';
        out += sleela::tokName(t.kind);
    }
    return out;
}

std::string texts(const std::string &src) {
    std::string out;
    for (const Token &t : lex(src)) {
        if (t.kind == Tok::Eof) break;
        if (!out.empty()) out += '|';
        out += t.text;
    }
    return out;
}

struct Vocab { const char *src; Tok kind; const char *text; };

// Every keyword the lexer recognises, independent of lexer.cpp's table.
const Vocab kKeywords[] = {
    {"class", Tok::KwClass, "class"}, {"interface", Tok::KwInterface, "interface"},
    {"enum", Tok::KwEnum, "enum"}, {"record", Tok::KwRecord, "record"}, {"struct", Tok::KwStruct, "struct"},
    {"static", Tok::KwStatic, "static"}, {"protected", Tok::KwProtected, "protected"},
    {"public", Tok::KwPublic, "public"}, {"private", Tok::KwPrivate, "private"},
    {"final", Tok::KwFinal, "final"}, {"abstract", Tok::KwAbstract, "abstract"}, {"native", Tok::KwNative, "native"},
    {"synchronized", Tok::KwSynchronized, "synchronized"}, {"volatile", Tok::KwVolatile, "volatile"},
    {"transient", Tok::KwTransient, "transient"}, {"strictfp", Tok::KwStrictfp, "strictfp"},
    {"sealed", Tok::KwSealed, "sealed"}, {"non-sealed", Tok::KwNonSealed, "non-sealed"},
    {"const", Tok::KwConst, "const"}, {"usualmem", Tok::KwUsualMem, "usualmem"}, {"usualgc", Tok::KwUsualGc, "usualgc"},
    {"package", Tok::KwPackage, "package"}, {"require", Tok::KwRequire, "require"},
    {"import", Tok::KwImport, "import"}, {"yield", Tok::KwYield, "yield"},
    {"if", Tok::KwIf, "if"}, {"else", Tok::KwElse, "else"}, {"switch", Tok::KwSwitch, "switch"},
    {"case", Tok::KwCase, "case"}, {"default", Tok::KwDefault, "default"},
    {"while", Tok::KwWhile, "while"}, {"for", Tok::KwFor, "for"}, {"do", Tok::KwDo, "do"},
    {"break", Tok::KwBreak, "break"}, {"continue", Tok::KwContinue, "continue"},
    {"return", Tok::KwReturn, "return"}, {"throw", Tok::KwThrow, "throw"}, {"try", Tok::KwTry, "try"},
    {"catch", Tok::KwCatch, "catch"}, {"finally", Tok::KwFinally, "finally"},
    {"extends", Tok::KwExtends, "extends"}, {"implements", Tok::KwImplements, "implements"},
    {"throws", Tok::KwThrows, "throws"}, {"super", Tok::KwSuper, "super"}, {"this", Tok::KwThis, "this"},
    {"instanceof", Tok::KwInstanceof, "instanceof"}, {"new", Tok::KwNew, "new"}, {"null", Tok::KwNull, "null"},
    {"true", Tok::KwTrue, "true"}, {"false", Tok::KwFalse, "false"}, {"print", Tok::KwPrint, "print"},
    {"void", Tok::KwVoid, "void"}, {"int", Tok::KwIntT, "int"}, {"long", Tok::KwLongT, "long"},
    {"int64", Tok::KwInt64T, "int64"}, {"int128", Tok::KwInt128T, "int128"}, {"double", Tok::KwDoubleT, "double"},
    {"boolean", Tok::KwBoolT, "boolean"}, {"String", Tok::KwStringT, "String"},
};

// Operators and punctuation; the token text must equal the spelling.
struct OpVocab { const char *src; Tok kind; bool text_checked; };
const OpVocab kOps[] = {
    {"@", Tok::At, true}, {"(", Tok::LParen, true}, {")", Tok::RParen, true}, {"{", Tok::LBrace, true},
    {"}", Tok::RBrace, true}, {"[", Tok::LBracket, true}, {"]", Tok::RBracket, true}, {";", Tok::Semicolon, true},
    {",", Tok::Comma, true}, {".", Tok::Dot, true}, {"=", Tok::Assign, true}, {"+", Tok::Plus, true},
    {"-", Tok::Minus, true}, {"*", Tok::Star, true}, {"/", Tok::Slash, true}, {"%", Tok::Percent, true},
    {"+=", Tok::PlusAssign, true}, {"-=", Tok::MinusAssign, true}, {"*=", Tok::StarAssign, true},
    {"/=", Tok::SlashAssign, true}, {"%=", Tok::PercentAssign, true}, {"&=", Tok::AndAssign, true},
    {"|=", Tok::OrAssign, true}, {"^=", Tok::XorAssign, true}, {"<<=", Tok::ShlAssign, true},
    {">>=", Tok::ShrAssign, true}, {">>>=", Tok::UshrAssign, true}, {"++", Tok::Increment, true},
    {"--", Tok::Decrement, true}, {"<<", Tok::ShiftLeft, true}, {">>", Tok::ShiftRight, true},
    {">>>", Tok::UnsignedShiftRight, true}, {"==", Tok::EqEq, true}, {"!=", Tok::NotEq, true},
    {"<", Tok::Lt, true}, {"<=", Tok::Le, true}, {">", Tok::Gt, true}, {">=", Tok::Ge, true},
    {"&&", Tok::AndAnd, true}, {"||", Tok::OrOr, true}, {"&", Tok::BitAnd, true}, {"|", Tok::BitOr, true},
    {"^", Tok::BitXor, true}, {"!", Tok::Not, true}, {"~", Tok::BitNot, true}, {"?", Tok::Question, true},
    {":", Tok::Colon, true}, {"::", Tok::DoubleColon, true}, {":::", Tok::TripleColon, true}, {"->", Tok::Arrow, true},
};

// ----------------------------------------------------------------------- cases

void t_keywords() {
    for (const Vocab &v : kKeywords) {
        const std::vector<Token> t = lex(v.src);
        REG_REQUIRE(t.size() == 2);
        reg_note_check();
        if (t[0].kind != v.kind || t[0].text != v.text)
            reg_fail(__FILE__, __LINE__, "keyword \"%s\" lexed as %s", v.src, sleela::tokName(t[0].kind));
        REG_CHECK(std::string(sleela::tokName(v.kind)) == v.text);   // name <-> spelling agree
        REG_CHECK(t[1].kind == Tok::Eof);
        // a longer identifier that merely starts with a keyword is an identifier
        const std::string longer = std::string(v.src) + "x";
        const std::vector<Token> u = lex(longer);
        REG_CHECK(u.size() == 2 && u[0].kind == Tok::Ident && u[0].text == longer);
    }
    REG_CHECK(lex("Class")[0].kind == Tok::Ident);                    // case-sensitive
    REG_CHECK(lex("STRING")[0].kind == Tok::Ident);
    REG_CHECK(lex("string")[0].kind == Tok::Ident);                   // only `String` is the type
}

void t_operators_longest_match() {
    for (const OpVocab &v : kOps) {
        const std::string src = std::string("a ") + v.src + " b";
        const std::vector<Token> t = lex(src);
        REG_REQUIRE(t.size() == 4);
        reg_note_check();
        if (t[1].kind != v.kind)
            reg_fail(__FILE__, __LINE__, "operator \"%s\" lexed as %s", v.src, sleela::tokName(t[1].kind));
        if (v.text_checked) REG_CHECK(t[1].text == v.src);
        REG_CHECK(std::string(sleela::tokName(v.kind)) == v.src);
    }
    // maximal munch on adjacent operators
    REG_CHECK(kinds("a>>>=b") == "identifier >>>= identifier");
    REG_CHECK(kinds("a>>>b") == "identifier >>> identifier");
    REG_CHECK(kinds("a>>=b") == "identifier >>= identifier");
    REG_CHECK(kinds("a>>b") == "identifier >> identifier");
    REG_CHECK(kinds("a<<=b") == "identifier <<= identifier");
    REG_CHECK(kinds("a+++b") == "identifier ++ + identifier");
    REG_CHECK(kinds("a---b") == "identifier -- - identifier");
    REG_CHECK(kinds("a&&&b") == "identifier && & identifier");
    REG_CHECK(kinds("a::::b") == "identifier ::: : identifier");
    REG_CHECK(kinds("a::::::b") == "identifier ::: ::: identifier");
    REG_CHECK(kinds("a->b") == "identifier -> identifier");
    REG_CHECK(kinds("a=>b") == "identifier = > identifier");
    REG_CHECK(kinds("a===b") == "identifier == = identifier");
    REG_CHECK(kinds("!!a") == "! ! identifier");
}

// Regression (was LEX-CMP-TEXT): the token text of !=, <= and >= is the full
// operator, so diagnostics that print Token::text show what was written.
void t_comparison_token_text_is_the_full_operator() {
    REG_CHECK(lex("a != b")[1].text == "!=");
    REG_CHECK(lex("a <= b")[1].text == "<=");
    REG_CHECK(lex("a >= b")[1].text == ">=");
    REG_CHECK(lex("a ! b")[1].text == "!");
    REG_CHECK(lex("a < b")[1].text == "<");
    REG_CHECK(lex("a > b")[1].text == ">");
    REG_CHECK(texts("a!=b<=c>=d") == "a|!=|b|<=|c|>=|d");
}

void t_identifiers_and_hyphen_rule() {
    // SLEELA.syntax 43.6 rule 12: '-' joins identifier characters only when an
    // identifier character follows it.
    REG_CHECK(kinds("jdk-25") == "identifier");
    REG_CHECK(texts("jdk-25 version-mual") == "jdk-25|version-mual");
    REG_CHECK(texts("a-b") == "a-b");                                  // pinned: one identifier
    REG_CHECK(texts("i-1") == "i-1");                                  // pinned: one identifier
    REG_CHECK(kinds("j--") == "identifier --");
    REG_CHECK(kinds("j-=1") == "identifier -= int-literal");
    REG_CHECK(kinds("x - 1") == "identifier - int-literal");
    REG_CHECK(kinds("x -1") == "identifier - int-literal");
    REG_CHECK(kinds("x- 1") == "identifier - int-literal");            // trailing '-' ends the name
    REG_CHECK(kinds("a->b") == "identifier -> identifier");            // '>' is not an identifier char
    REG_CHECK(texts("_under_score9") == "_under_score9");
    REG_CHECK(kinds("9lives") == "int-literal identifier");            // digits do not start identifiers
    // contextual builtins are ordinary identifiers to the lexer
    REG_CHECK(kinds("gc mem ran") == "identifier identifier identifier");
    REG_CHECK(sleela::isContextualBuiltin("gc") && sleela::isContextualBuiltin("mem") &&
              sleela::isContextualBuiltin("ran") && sleela::isContextualBuiltin("Munction") &&
              sleela::isContextualBuiltin("synchroOpen") && sleela::isContextualBuiltin("bestOfNew") &&
              sleela::isContextualBuiltin("ideaCanvasCreate"));
    REG_CHECK(!sleela::isContextualBuiltin("") && !sleela::isContextualBuiltin("foo") &&
              !sleela::isContextualBuiltin("GC") && !sleela::isContextualBuiltin("class"));
    REG_CHECK(sleela::isGcHintWord("gc") && sleela::isGcHintWord("mem"));
    REG_CHECK(!sleela::isGcHintWord("ran") && !sleela::isGcHintWord("") && !sleela::isGcHintWord("Mem"));
}

void t_numeric_literals() {
    struct N { const char *src; Tok kind; const char *text; };
    const N ok[] = {
        {"0", Tok::Int, "0"}, {"42", Tok::Int, "42"}, {"1_000_000", Tok::Int, "1000000"},
        {"10L", Tok::Int, "10"}, {"10l", Tok::Int, "10"}, {"0x1F", Tok::Int, "0x1F"}, {"0XFF", Tok::Int, "0XFF"},
        {"0xFF_FF", Tok::Int, "0xFFFF"}, {"0x1FL", Tok::Int, "0x1F"}, {"0b101_1", Tok::Int, "0b1011"},
        {"0B11", Tok::Int, "0B11"}, {"0o17", Tok::Int, "017"}, {"0O7", Tok::Int, "07"},
        {"3.14", Tok::Double, "3.14"}, {"1e5", Tok::Double, "1e5"}, {"1E5", Tok::Double, "1E5"},
        {"1.5e-3", Tok::Double, "1.5e-3"}, {"2e+10", Tok::Double, "2e+10"}, {"1_0.2_5", Tok::Double, "10.25"},
        {"2f", Tok::Double, "2.0"}, {"2F", Tok::Double, "2.0"}, {"2d", Tok::Double, "2.0"}, {"2.5d", Tok::Double, "2.5"},
        {"2.5f", Tok::Double, "2.5"},
    };
    for (const N &n : ok) {
        const std::vector<Token> t = lex(n.src);
        reg_note_check();
        if (t.size() != 2 || t[0].kind != n.kind || t[0].text != n.text)
            reg_fail(__FILE__, __LINE__, "literal %s: got %s{%s}", n.src,
                     sleela::tokName(t[0].kind), t[0].text.c_str());
    }
    // a trailing '.' is not part of the number; 1.e5 is not a float
    REG_CHECK(kinds("7.") == "int-literal .");
    REG_CHECK(kinds("1.e5") == "int-literal . identifier");
    REG_CHECK(kinds("1..2") == "int-literal . . int-literal");
    REG_CHECK(kinds("1_") == "int-literal identifier");                // '_' needs a digit after it
    REG_CHECK(kinds("10Lx") == "int-literal identifier");              // suffix must end the token
    // malformed radix literals
    REG_CHECK(contains(lex_error("0x"), "hexadecimal literal has no digits"));
    REG_CHECK(contains(lex_error("0xG"), "hexadecimal literal has no digits"));
    REG_CHECK(contains(lex_error("0b"), "binary literal has no digits"));
    REG_CHECK(contains(lex_error("0b2"), "binary literal has no digits"));
    REG_CHECK(contains(lex_error("0o"), "octal literal has no digits"));
    REG_CHECK(contains(lex_error("0o8"), "octal literal has no digits"));
}

void t_string_literals() {
    REG_CHECK(texts("\"hello\"") == "hello");
    REG_CHECK(texts("\"\"") == "");
    REG_CHECK(texts("\"a\\nb\\tc\\r\\b\\f\\\\\\\"\"") == std::string("a\nb\tc\r\b\f\\\""));
    REG_CHECK(texts("\"\\u00e9\"") == "\xC3\xA9");                     // U+00E9 -> UTF-8
    REG_CHECK(texts("\"\\u20AC\"") == "\xE2\x82\xAC");
    REG_CHECK(texts("\"\\u0041\"") == "A");
    REG_CHECK(texts("\"\\uE000\"") == "\xEE\x80\x80");
    REG_CHECK(texts("\"caf\xC3\xA9\"") == "caf\xC3\xA9");             // raw UTF-8 passes through
    {   // \0 yields an embedded NUL, not a terminator
        const std::vector<Token> t = lex("\"a\\0b\"");
        REG_CHECK(t[0].text.size() == 3 && t[0].text[1] == '\0');
    }
    REG_CHECK(texts("\"\\q\"") == "q");                                // CHARACTERIZATION: unknown escape keeps the char
    REG_CHECK(texts("\"line\nbreak\"") == "line\nbreak");               // CHARACTERIZATION: literal may span lines
    REG_CHECK(texts("\"// not a comment\"") == "// not a comment");
    REG_CHECK(texts("\"# not a comment\"") == "# not a comment");
    REG_CHECK(texts("\"/* x */\"") == "/* x */");
    // errors
    REG_CHECK(contains(lex_error("\"abc"), "unterminated string literal"));
    REG_CHECK(contains(lex_error("\"abc\\"), "unterminated escape sequence"));
    REG_CHECK(contains(lex_error("\"\\u12\""), "invalid unicode escape"));
    REG_CHECK(contains(lex_error("\"\\u12"), "unterminated unicode escape"));
    REG_CHECK(contains(lex_error("\"\\uZZZZ\""), "invalid unicode escape"));
    REG_CHECK(contains(lex_error("\"\\uD83D\""), "surrogate"));          // lone high
    REG_CHECK(contains(lex_error("\"\\uDE00\""), "surrogate"));          // lone low
    REG_CHECK(contains(lex_error("\"\\uD83D\\uDE00\""), "surrogate"));   // pairs are not recombined
    REG_CHECK(lex_error("\"\\uD7FF\"").empty() && lex_error("\"\\uE000\"").empty());
}

void t_char_literals() {
    struct C { const char *src; const char *code; };
    const C ok[] = {
        {"'a'", "97"}, {"'A'", "65"}, {"' '", "32"}, {"'\\n'", "10"}, {"'\\t'", "9"}, {"'\\r'", "13"},
        {"'\\b'", "8"}, {"'\\f'", "12"}, {"'\\s'", "32"}, {"'\\\\'", "92"}, {"'\\''", "39"}, {"'\\\"'", "34"},
        {"'\\0'", "0"}, {"'\\7'", "7"}, {"'\\101'", "65"}, {"'\\377'", "255"}, {"'\\u0041'", "65"},
        {"'\\uuu0041'", "65"}, {"'\\uFFFF'", "65535"}, {"'\xC3\xA9'", "233"}, {"'\xE2\x82\xAC'", "8364"},
        {"'\"'", "34"},
    };
    for (const C &c : ok) {
        const std::vector<Token> t = lex(c.src);
        reg_note_check();
        if (t.size() != 2 || t[0].kind != Tok::Char || t[0].text != c.code)
            reg_fail(__FILE__, __LINE__, "char literal %s: got %s", c.src, t.empty() ? "?" : t[0].text.c_str());
    }
    REG_CHECK(contains(lex_error("''"), "empty character literal"));
    REG_CHECK(contains(lex_error("'ab'"), "exactly one character"));
    REG_CHECK(contains(lex_error("'a"), "exactly one character"));
    REG_CHECK(contains(lex_error("'"), "unterminated character literal"));
    REG_CHECK(contains(lex_error("'\n'"), "unterminated character literal"));
    REG_CHECK(contains(lex_error("'\\q'"), "invalid escape sequence"));
    REG_CHECK(contains(lex_error("'\\u00G0'"), "invalid unicode escape"));
    REG_CHECK(contains(lex_error("'\\u12'"), "invalid unicode escape"));
    REG_CHECK(contains(lex_error("'\xF0\x9F\x98\x80'"), "basic multilingual plane"));
    REG_CHECK(contains(lex_error("'\x80'"), "invalid UTF-8"));
    REG_CHECK(contains(lex_error("'\\400'"), "exactly one character"));  // octal escapes stop at 255
}

void t_comments() {
    REG_CHECK(kinds("a # to end of line\nb") == "identifier identifier");
    REG_CHECK(kinds("a // to end of line\nb") == "identifier identifier");
    REG_CHECK(kinds("a // c") == "identifier");
    REG_CHECK(kinds("#sleela 3.1\nclass") == "class");
    REG_CHECK(kinds("a\n< single line comment >\nb") == "identifier identifier");
    REG_CHECK(kinds("a\n    < indented comment >\nb") == "identifier identifier");
    REG_CHECK(kinds("a\n\t< tab indented >\nb") == "identifier identifier");
    REG_CHECK(kinds("a\n<< block >>\nb") == "identifier identifier");
    REG_CHECK(kinds("a\n<< multi\nline\nblock >>\nb") == "identifier identifier");
    REG_CHECK(kinds("a\n  << x >> b") == "identifier identifier");     // code may follow a block on its line
    REG_CHECK(kinds("<< head >>\nclass") == "class");
    // a block comment accepts ANY characters, including the ones that are
    // lexical errors in code (apostrophe, backtick, dollar)
    REG_CHECK(kinds("a\n<< don't `use` $this 'q' \"unterminated >>\nb") == "identifier identifier");
    // ... and angle brackets that are not a close
    REG_CHECK(kinds("a\n<< a > b >> c") == "identifier identifier");
    REG_CHECK(contains(lex_error("a\n<< never closed"), "unterminated angle-bracket block comment"));
    REG_CHECK(contains(lex_error("<< x > y"), "unterminated angle-bracket block comment"));
    // C-style comments are NOT comments: the body is tokenised as code
    REG_CHECK(kinds("/* c */") == "/ * identifier * /");
    REG_CHECK(kinds("a /* c */ b") == "identifier / * identifier * / identifier");
    REG_CHECK(!lex_error("/* don't */").empty());                      // the apostrophe is a lexical error
    REG_CHECK(!lex_error("/* `x` */").empty());
    // `<` mid-line is an operator, never a comment opener
    REG_CHECK(kinds("a < b > c") == "identifier < identifier > identifier");
    REG_CHECK(kinds("x; < c >") == "identifier ; < identifier >");     // inline trailing "comment" is code
    REG_CHECK(kinds("a << b") == "identifier << identifier");
    // a '<' line with no closing '>' on that line is not a comment
    REG_CHECK(kinds("a\n< no close\nb") == "identifier < identifier identifier identifier");
    REG_CHECK(!lex_error("a\n< it's open\nb").empty());                // so its apostrophe is an error
    // '#' and '//' win even inside what would be a block opener's line
    REG_CHECK(kinds("a // << not a block\nb") == "identifier identifier");
}

void t_positions() {
    const std::vector<Token> t = lex("ab cd\n  ef\r\n\tgh");
    REG_REQUIRE(t.size() == 5);
    REG_CHECK(t[0].line == 1 && t[0].col == 1);
    REG_CHECK(t[1].line == 1 && t[1].col == 4);
    REG_CHECK(t[2].line == 2 && t[2].col == 3);
    REG_CHECK(t[3].line == 3 && t[3].col == 2);                        // a tab is one column
    REG_CHECK(t[4].kind == Tok::Eof && t[4].line == 3 && t[4].col == 4);
    const std::vector<Token> e = lex("");
    REG_CHECK(e.size() == 1 && e[0].kind == Tok::Eof && e[0].line == 1 && e[0].col == 1);
    const std::vector<Token> w = lex("  \n\n  ");
    REG_CHECK(w.size() == 1 && w[0].kind == Tok::Eof && w[0].line == 3 && w[0].col == 3);
    // multi-line string literal advances the line counter
    const std::vector<Token> s = lex("\"a\nb\" x");
    REG_REQUIRE(s.size() == 3);
    REG_CHECK(s[0].line == 1 && s[1].line == 2 && s[1].col == 4);
    // positions inside error messages
    REG_CHECK(contains(lex_error("a\n  $"), "line 2, col 4"));
    REG_CHECK(contains(lex_error("a $"), "unexpected character '$'"));
    REG_CHECK(contains(lex_error("`"), "unexpected character '`'"));
    REG_CHECK(contains(lex_error("\\"), "unexpected character"));
}

void t_non_ascii_outside_literals_is_rejected() {
    REG_CHECK(!lex_error("caf\xC3\xA9").empty());                      // identifiers are ASCII only
    REG_CHECK(!lex_error("\xE2\x82\xAC").empty());
    REG_CHECK(!lex_error(std::string("a\0b", 3)).empty());              // NUL byte in source
    REG_CHECK(lex_error("# caf\xC3\xA9 comment\nx").empty());           // fine inside comments
    REG_CHECK(lex_error("// \xFF\xFE\nx").empty());
}

// ----------------------------------------------------------- generated sources

struct Piece { std::string src; Tok kind; std::string text; bool text_checked; };

std::vector<Piece> build_vocabulary() {
    std::vector<Piece> v;
    for (const Vocab &k : kKeywords) v.push_back({k.src, k.kind, k.text, true});
    for (const OpVocab &o : kOps) v.push_back({o.src, o.kind, o.src, o.text_checked});
    const char *ids[] = {"x", "foo", "Foo9", "_t", "a_b", "jdk-25", "version-mual", "gc", "mem", "ran", "Munction"};
    for (const char *i : ids) v.push_back({i, Tok::Ident, i, true});
    v.push_back({"0", Tok::Int, "0", true}); v.push_back({"123", Tok::Int, "123", true});
    v.push_back({"0xBEEF", Tok::Int, "0xBEEF", true}); v.push_back({"1_000", Tok::Int, "1000", true});
    v.push_back({"2.5", Tok::Double, "2.5", true}); v.push_back({"1e9", Tok::Double, "1e9", true});
    v.push_back({"\"s\\n\"", Tok::Str, "s\n", true}); v.push_back({"\"\"", Tok::Str, "", true});
    v.push_back({"'z'", Tok::Char, "122", true}); v.push_back({"'\\n'", Tok::Char, "10", true});
    return v;
}

// Generate random token streams with random whitespace and check kinds, text,
// line and column of every token against positions tracked by the generator.
void t_generated_streams() {
    const std::vector<Piece> vocab = build_vocabulary();
    reg_seed(0x1E8E4ull);
    for (int it = 0; it < 3000; ++it) {
        std::string src; int line = 1, col = 1; bool after_newline = true;   // a '<' at a line start would open a comment
        struct Want { Tok kind; std::string text; bool text_checked; int line, col; };
        std::vector<Want> want;
        const unsigned count = 1 + reg_below(25);
        for (unsigned i = 0; i < count; ++i) {
            // the first token on every line is a plain identifier, so '<' / '<<' is never at a line start
            const Piece use = after_newline ? Piece{"q", Tok::Ident, "q", true}
                                            : vocab[reg_below((uint32_t)vocab.size())];
            want.push_back({use.kind, use.text, use.text_checked, line, col});
            src += use.src;
            // advance expected position over the token's source text (strings/chars are single-line here)
            col += (int)use.src.size();
            after_newline = false;
            switch (reg_below(5)) {
                case 0: src += ' '; col += 1; break;
                case 1: src += "  "; col += 2; break;
                case 2: src += '\t'; col += 1; break;
                case 3: src += '\n'; line += 1; col = 1; after_newline = true; break;
                default: src += "\r\n"; line += 1; col = 1; after_newline = true; break;
            }
        }
        std::vector<Token> got;
        try { got = lex(src); }
        catch (const std::exception &e) { REG_CHECK(!"generated stream failed to lex"); reg_fail(__FILE__, __LINE__, "%s", e.what()); return; }
        REG_REQUIRE(got.size() == want.size() + 1);
        for (size_t k = 0; k < want.size(); ++k) {
            reg_note_check();
            if (got[k].kind != want[k].kind || (want[k].text_checked && got[k].text != want[k].text) ||
                got[k].line != want[k].line || got[k].col != want[k].col) {
                reg_fail(__FILE__, __LINE__, "stream mismatch at token %zu in iteration %d: kind %s/%s line %d/%d col %d/%d",
                         k, it, sleela::tokName(got[k].kind), sleela::tokName(want[k].kind),
                         got[k].line, want[k].line, got[k].col, want[k].col);
                return;
            }
        }
        REG_CHECK(got.back().kind == Tok::Eof);
    }
}

// Robustness: arbitrary bytes either lex to a well-formed stream or throw
// std::runtime_error; never crash, hang, or leave positions non-monotonic.
void t_fuzz_never_crashes() {
    reg_seed(0xF022ull);
    const char *snippets[] = {"<<", ">>", "<", ">", "'", "\"", "\\", "#", "//", "/*", "0x", "0b", "0o", "1e", ".", "::", "-", "\n", "\r", "\t"};
    int accepted = 0, rejected = 0;
    for (int it = 0; it < 40000; ++it) {
        std::string src; const unsigned n = reg_below(40);
        for (unsigned i = 0; i < n; ++i) {
            if (reg_below(3) == 0) src += snippets[reg_below(sizeof snippets / sizeof snippets[0])];
            else src += (char)reg_below(256);
        }
        try {
            const std::vector<Token> t = lex(src);
            ++accepted;
            if (t.empty() || t.back().kind != Tok::Eof) { REG_CHECK(!"stream does not end with Eof"); return; }
            int pl = 1;
            for (const Token &k : t) {
                if (k.line < pl || k.line < 1 || k.col < 1) { REG_CHECK(!"line/col not monotonic"); return; }
                pl = k.line;
            }
            for (size_t i = 0; i + 1 < t.size(); ++i)
                if (t[i].kind == Tok::Eof) { REG_CHECK(!"Eof in the middle of the stream"); return; }
        } catch (const std::runtime_error &) {
            ++rejected;
        }
    }
    REG_CHECK(accepted > 0 && rejected > 0);                          // the generator reaches both outcomes
}

void t_large_input_is_linear() {
    std::string src;
    for (int i = 0; i < 200000; ++i) src += "foo = bar + 12345; ";
    const std::vector<Token> t = lex(src);
    REG_CHECK(t.size() == 200000u * 6u + 1u);
    std::string comment = "<<";
    comment.append(4u << 20, 'x');
    comment += ">>\nend";
    REG_CHECK(kinds(comment) == "identifier");
}

}  // namespace

int main() {
    REG_RUN(t_keywords);
    REG_RUN(t_operators_longest_match);
    REG_RUN(t_comparison_token_text_is_the_full_operator);
    REG_RUN(t_identifiers_and_hyphen_rule);
    REG_RUN(t_numeric_literals);
    REG_RUN(t_string_literals);
    REG_RUN(t_char_literals);
    REG_RUN(t_comments);
    REG_RUN(t_positions);
    REG_RUN(t_non_ascii_outside_literals_is_rejected);
    REG_RUN(t_generated_streams);
    REG_RUN(t_fuzz_never_crashes);
    REG_RUN(t_large_input_is_linear);
    return reg_finish("lexer");
}