Home /Modules/impl/xlt_lexer.cpp

xlt_lexer.cpp

impl/xlt/xlt_lexer.cpp · 132 line(s) · c

// ===========================================================================
// xlt_lexer.cpp  --  Implementation of the XLT (.xlt) tokenizer.
// ===========================================================================
#include "xlt_lexer.h"

#include <cctype>

namespace xlt {

const char* tokName(Tok t) {
    switch (t) {
        case Tok::Pragma:    return "pragma";
        case Tok::Ident:     return "identifier";
        case Tok::String:    return "string";
        case Tok::Integer:   return "integer";
        case Tok::Version:   return "version";
        case Tok::LBrace:    return "{";
        case Tok::RBrace:    return "}";
        case Tok::Semicolon: return ";";
        case Tok::Eof:       return "<eof>";
    }
    return "?";
}

namespace {
bool isIdentStart(char c) { return std::isalpha((unsigned char)c) || c == '_'; }
bool isIdentChar(char c)  { return std::isalnum((unsigned char)c) || c == '_'; }
}

std::vector<Token> lex(const std::string& src, const std::string& file,
                       DiagnosticBag& diags) {
    std::vector<Token> toks;
    size_t i = 0;
    int line = 1, col = 1;
    auto advance = [&](int n = 1) { while (n-- > 0) { if (src[i] == '\n') { line++; col = 1; } else { col++; } i++; } };

    while (i < src.size()) {
        char c = src[i];
        // whitespace
        if (c == ' ' || c == '\t' || c == '\r' || c == '\n') { advance(); continue; }
        // line comment
        if (c == '/' && i + 1 < src.size() && src[i + 1] == '/') {
            while (i < src.size() && src[i] != '\n') advance();
            continue;
        }
        int startLine = line, startCol = col;
        // pragma: '#' + identifier
        if (c == '#') {
            advance();
            std::string name;
            while (i < src.size() && isIdentChar(src[i])) { name += src[i]; advance(); }
            toks.push_back(Token{ Tok::Pragma, name, startLine, startCol });
            continue;
        }
        // punctuation
        if (c == '{') { advance(); toks.push_back(Token{ Tok::LBrace, "{", startLine, startCol }); continue; }
        if (c == '}') { advance(); toks.push_back(Token{ Tok::RBrace, "}", startLine, startCol }); continue; }
        if (c == ';') { advance(); toks.push_back(Token{ Tok::Semicolon, ";", startLine, startCol }); continue; }
        // string
        if (c == '"') {
            advance();
            std::string out;
            while (i < src.size() && src[i] != '"') {
                if (src[i] == '\\' && i + 1 < src.size()) { advance(); out += src[i]; advance(); }
                else { out += src[i]; advance(); }
            }
            if (i >= src.size()) {
                diags.error("XLT-E-0001", file, startLine, "unterminated string literal", "XLT-LEX-01");
            } else { advance(); }
            toks.push_back(Token{ Tok::String, out, startLine, startCol });
            continue;
        }
        // number / version
        if (std::isdigit((unsigned char)c)) {
            // Radix-prefixed integer literals, matching the SLeeLa frontend
            // (SLEELA.syntax 1.4): 0x/0X hex, 0b/0B binary, 0o/0O octal, with
            // optional '_' digit separators. The prefix is kept in the token
            // text; the parser decodes the value. A non-prefixed run stays a
            // decimal integer, or a dotted Version (e.g. the #xlt 1.0 pragma).
            auto isHex = [](char d) {
                return (d >= '0' && d <= '9') || (d >= 'a' && d <= 'f') || (d >= 'A' && d <= 'F');
            };
            if (c == '0' && i + 1 < src.size() &&
                (src[i + 1] == 'x' || src[i + 1] == 'X' ||
                 src[i + 1] == 'b' || src[i + 1] == 'B' ||
                 src[i + 1] == 'o' || src[i + 1] == 'O')) {
                char radix = src[i + 1];
                std::string num; num += src[i]; num += radix; advance(2);
                bool any = false;
                while (i < src.size()) {
                    char d = src[i];
                    if (d == '_') { advance(); continue; }
                    bool ok = (radix == 'x' || radix == 'X') ? isHex(d)
                            : (radix == 'b' || radix == 'B') ? (d == '0' || d == '1')
                            : (d >= '0' && d <= '7');  // octal
                    if (!ok) break;
                    num += d; any = true; advance();
                }
                if (!any) {
                    diags.error("XLT-E-0003", file, startLine,
                                std::string("radix literal '") + num + "' has no digits", "XLT-LEX-03");
                }
                toks.push_back(Token{ Tok::Integer, num, startLine, startCol });
                continue;
            }
            std::string num; bool dotted = false;
            while (i < src.size() && (std::isdigit((unsigned char)src[i]) || src[i] == '.' || src[i] == '_')) {
                if (src[i] == '.') dotted = true;
                if (src[i] != '_') num += src[i];
                advance();
            }
            toks.push_back(Token{ dotted ? Tok::Version : Tok::Integer, num, startLine, startCol });
            continue;
        }
        // identifier / keyword
        if (isIdentStart(c)) {
            std::string id;
            while (i < src.size() && isIdentChar(src[i])) { id += src[i]; advance(); }
            toks.push_back(Token{ Tok::Ident, id, startLine, startCol });
            continue;
        }
        // unknown character
        diags.error("XLT-E-0002", file, startLine,
                    std::string("unexpected character '") + c + "'", "XLT-LEX-02");
        advance();
    }
    toks.push_back(Token{ Tok::Eof, "", line, col });
    return toks;
}

} // namespace xlt