Home /Modules/impl/sst_lexer.cpp

sst_lexer.cpp

impl/nordshrift/sst_lexer.cpp · 377 line(s) · c

// ===========================================================================
// sst_lexer.cpp  --  Spec-conformant .sst tokenizer (NS-SST-0001 Part I).
//
// Strategy: process the source line by line. Blank and comment-only lines are
// skipped for structure (IND-04). For each significant line we compute its
// indentation, compare against an indent stack to emit INDENT / DEDENT, then
// tokenize the line's content and emit a terminating NEWLINE.
//
// The indent unit (2 or 4 spaces) is fixed by the first indented line (IND-01);
// every indentation must be a whole multiple of it, else NSS-E-0002.
// ===========================================================================
#include "sst_lexer.h"

#include <cctype>
#include <cstdint>

namespace nordshrift {

const char* tokName(Tok t) {
    switch (t) {
        case Tok::Newline:  return "<newline>";
        case Tok::Indent:   return "<indent>";
        case Tok::Dedent:   return "<dedent>";
        case Tok::Pragma:   return "pragma";
        case Tok::Ident:    return "identifier";
        case Tok::String:   return "string";
        case Tok::Integer:  return "integer";
        case Tok::Version:  return "version";
        case Tok::Colon:    return "':'";
        case Tok::Comma:    return "','";
        case Tok::LBracket: return "'['";
        case Tok::RBracket: return "']'";
        case Tok::Eof:      return "<eof>";
    }
    return "?";
}

namespace {

// Validate that a byte string is well-formed UTF-8 (NSS-E-0001).
bool validUtf8(const std::string& s) {
    size_t i = 0, n = s.size();
    while (i < n) {
        unsigned char c = (unsigned char)s[i];
        int extra;
        if (c < 0x80) extra = 0;
        else if ((c & 0xE0) == 0xC0) extra = 1;
        else if ((c & 0xF0) == 0xE0) extra = 2;
        else if ((c & 0xF8) == 0xF0) extra = 3;
        else return false;
        if (i + (size_t)extra >= n) return false;
        for (int k = 1; k <= extra; k++)
            if (((unsigned char)s[i + k] & 0xC0) != 0x80) return false;
        i += (size_t)extra + 1;
    }
    return true;
}

bool identStart(char c) { return std::isalpha((unsigned char)c) || c == '_'; }
bool identChar(char c)  { return std::isalnum((unsigned char)c) || c == '_' || c == '-'; }

// Net bracket depth of a line: '[' minus ']', skipping brackets that appear
// inside string literals or after a // line comment.
int bracketDelta(const std::string& s) {
    int depth = 0; bool inStr = false;
    for (size_t i = 0; i < s.size(); i++) {
        char c = s[i];
        if (inStr) {
            if (c == '\\') { i++; continue; }
            if (c == '"') inStr = false;
            continue;
        }
        if (c == '"') { inStr = true; continue; }
        if (c == '/' && i + 1 < s.size() && s[i+1] == '/') break;
        if (c == '[') depth++;
        else if (c == ']') depth--;
    }
    return depth;
}

struct Lexer {
    const std::string& src;
    const std::string& file;
    DiagnosticBag& diags;
    std::vector<Token> out;

    int indentUnit = 0;                 // 0 = not yet determined
    std::vector<int> indentStack{0};    // column widths of open blocks
    std::string pendingDoc;             // last /// doc comment seen
    bool docPending = false;

    Lexer(const std::string& s, const std::string& f, DiagnosticBag& d)
        : src(s), file(f), diags(d) {}

    // Split into logical lines with LF/CRLF normalized (§1.1).
    std::vector<std::string> splitLines() {
        std::vector<std::string> lines;
        std::string cur;
        for (size_t i = 0; i < src.size(); i++) {
            char c = src[i];
            if (c == '\r') { if (i + 1 < src.size() && src[i+1] == '\n') i++; lines.push_back(cur); cur.clear(); }
            else if (c == '\n') { lines.push_back(cur); cur.clear(); }
            else cur += c;
        }
        lines.push_back(cur);
        return lines;
    }

    void emit(Tok k, const std::string& text, int line, int col) {
        Token t{ k, text, line, col, "" };
        if (docPending) { t.doc = pendingDoc; docPending = false; pendingDoc.clear(); }
        out.push_back(t);
    }

    std::vector<Token> run() {
        if (!validUtf8(src)) {
            diags.error("NSS-E-0001", file, 0, "file is not valid UTF-8", "§1.1");
            out.push_back(Token{ Tok::Eof, "", 0, 0, "" });
            return out;
        }
        auto lines = splitLines();
        bool inBlockComment = false;

        for (size_t li = 0; li < lines.size(); li++) {
            std::string raw = lines[li];
            int lineNo = (int)li + 1;

            // Multiline string stitching (§1.5): if this line opens a """ that
            // does not close on the same line, absorb following lines up to the
            // closing """ into one logical line, joining with '\n'. Leading
            // whitespace up to the closing-""" indent is stripped per line.
            {
                size_t open = raw.find("\"\"\"");
                if (open != std::string::npos &&
                    raw.find("\"\"\"", open + 3) == std::string::npos) {
                    std::string joined = raw;
                    size_t j = li + 1;
                    for (; j < lines.size(); j++) {
                        const std::string& nx = lines[j];
                        size_t close = nx.find("\"\"\"");
                        int strip = 0; while (strip < (int)nx.size() && nx[strip] == ' ') strip++;
                        joined += "\n" + nx.substr((size_t)strip);
                        if (close != std::string::npos) break;
                    }
                    li = j;
                    raw = joined;
                }
            }

            // List continuation stitching (§1.5): a bracketed list may wrap over
            // several physical lines; the continuation lines' indentation is not
            // block structure. If more '[' than ']' remain open on this line,
            // absorb following lines (space-joined) until the brackets balance.
            {
                int depth = bracketDelta(raw);
                if (depth > 0) {
                    std::string joined = raw;
                    size_t j = li + 1;
                    for (; j < lines.size() && depth > 0; j++) {
                        const std::string& nx = lines[j];
                        int strip = 0; while (strip < (int)nx.size() && nx[strip] == ' ') strip++;
                        std::string content = nx.substr((size_t)strip);
                        joined += " " + content;
                        depth += bracketDelta(content);
                    }
                    li = j - 1;
                    raw = joined;
                }
            }

            // measure leading spaces (tabs are not an indent unit here)
            int indent = 0;
            while (indent < (int)raw.size() && raw[indent] == ' ') indent++;
            std::string body = raw.substr((size_t)indent);
            // strip trailing whitespace (IND-05)
            while (!body.empty() && (body.back() == ' ' || body.back() == '\t')) body.pop_back();

            // handle an ongoing /* ... */ block comment
            if (inBlockComment) {
                size_t end = body.find("*/");
                if (end == std::string::npos) continue;
                body = body.substr(end + 2);
                // strip leading spaces of remainder
                size_t p = 0; while (p < body.size() && body[p] == ' ') p++;
                body = body.substr(p);
                inBlockComment = false;
                if (body.empty()) continue;
            }

            if (body.empty()) continue;                 // blank line (IND-04)

            // whole-line comments
            if (body.rfind("///", 0) == 0) {             // doc comment (§1.3)
                std::string text = body.substr(3);
                if (!text.empty() && text[0] == ' ') text.erase(0, 1);
                if (docPending) pendingDoc += "\n" + text; else pendingDoc = text;
                docPending = true;
                continue;
            }
            if (body.rfind("//", 0) == 0) continue;      // line comment
            if (body.rfind("/*", 0) == 0) {              // block comment start
                size_t end = body.find("*/", 2);
                if (end == std::string::npos) { inBlockComment = true; continue; }
                // inline block comment consuming the whole line
                std::string rest = body.substr(end + 2);
                size_t p = 0; while (p < rest.size() && rest[p] == ' ') p++;
                if (p >= rest.size()) continue;
                body = rest.substr(p);                   // (rare) code after */
            }

            // ---- indentation bookkeeping ----
            if (indent > indentStack.back()) {
                if (indentUnit == 0) {
                    if (indent != 2 && indent != 4) {
                        diags.error("NSS-E-0002", file, lineNo,
                            "first indent must be 2 or 4 spaces (found " + std::to_string(indent) + ")",
                            "IND-01");
                    }
                    indentUnit = (indent == 4) ? 4 : 2;
                }
                if (indentUnit && (indent - indentStack.back()) != indentUnit) {
                    diags.error("NSS-E-0002", file, lineNo,
                        "inconsistent indentation: expected " +
                        std::to_string(indentStack.back() + indentUnit) + " spaces, found " +
                        std::to_string(indent), "IND-01");
                }
                indentStack.push_back(indent);
                emit(Tok::Indent, "", lineNo, 1);
            } else {
                while (indent < indentStack.back()) {
                    indentStack.pop_back();
                    emit(Tok::Dedent, "", lineNo, 1);
                }
                if (indent != indentStack.back()) {
                    diags.error("NSS-E-0002", file, lineNo,
                        "dedent does not match any enclosing block indentation",
                        "IND-03");
                    indentStack.push_back(indent);   // recover
                }
            }

            // ---- tokenize the line body ----
            lexLineBody(body, lineNo);
            emit(Tok::Newline, "", lineNo, (int)raw.size() + 1);
        }

        // close any open blocks at EOF
        while (indentStack.size() > 1) { indentStack.pop_back(); emit(Tok::Dedent, "", (int)lines.size(), 1); }
        if (docPending)
            diags.warning("NSS-W-0001", file, (int)lines.size(),
                "documentation comment not followed by a directive or block", "§1.3");
        out.push_back(Token{ Tok::Eof, "", (int)lines.size() + 1, 1, "" });
        return out;
    }

    // Tokenize one line's content (after indentation). `base` is the line's
    // indent column, used to strip a multiline-string's leading whitespace.
    void lexLineBody(const std::string& s, int lineNo) {
        size_t i = 0, n = s.size();

        // pragma: line begins with '#'
        if (n > 0 && s[0] == '#') {
            size_t p = 1;
            std::string name;
            while (p < n && identChar(s[p])) name += s[p++];
            emit(Tok::Pragma, name, lineNo, 1);
            while (p < n && s[p] == ' ') p++;
            // the rest of the pragma line is its operand (version or ident)
            lexRest(s, p, lineNo);
            return;
        }
        lexRest(s, i, lineNo);
    }

    // Tokenize the remainder of a line starting at offset i.
    void lexRest(const std::string& s, size_t i, int lineNo) {
        size_t n = s.size();
        while (i < n) {
            char c = s[i];
            if (c == ' ' || c == '\t') { i++; continue; }
            if (c == '/' && i + 1 < n && s[i+1] == '/') break;             // trailing // comment
            if (c == '/' && i + 1 < n && s[i+1] == '*') {                  // inline /* */
                size_t end = s.find("*/", i + 2);
                if (end == std::string::npos) break;                       // rest is comment
                i = end + 2; continue;
            }
            int col = (int)i + 1;
            if (c == ':') { emit(Tok::Colon, ":", lineNo, col); i++; continue; }
            if (c == ',') { emit(Tok::Comma, ",", lineNo, col); i++; continue; }
            if (c == '[') { emit(Tok::LBracket, "[", lineNo, col); i++; continue; }
            if (c == ']') { emit(Tok::RBracket, "]", lineNo, col); i++; continue; }
            if (c == '"') { lexString(s, i, lineNo); continue; }
            if (std::isdigit((unsigned char)c)) { lexNumberOrVersion(s, i, lineNo); continue; }
            if (identStart(c)) {
                std::string id; int start = col;
                while (i < n && identChar(s[i])) id += s[i++];
                emit(Tok::Ident, id, lineNo, start);
                continue;
            }
            // unknown char: skip (a later parse error will report context)
            i++;
        }
    }

    void lexString(const std::string& s, size_t& i, int lineNo) {
        int col = (int)i + 1;
        // triple-quoted?
        if (s.compare(i, 3, "\"\"\"") == 0) {
            // single-line triple quote; multiline handled by caller merging is
            // out of scope here -- most sheets keep """ content on the line or
            // the parser stitches description text. Read until closing """.
            size_t end = s.find("\"\"\"", i + 3);
            std::string val = (end == std::string::npos) ? s.substr(i + 3)
                                                         : s.substr(i + 3, end - (i + 3));
            i = (end == std::string::npos) ? s.size() : end + 3;
            emit(Tok::String, val, lineNo, col);
            return;
        }
        i++; // opening quote
        std::string val;
        while (i < s.size() && s[i] != '"') {
            char c = s[i++];
            if (c == '\\' && i < s.size()) {
                char e = s[i++];
                switch (e) {
                    case 'n': val += '\n'; break;
                    case 't': val += '\t'; break;
                    case 'r': val += '\r'; break;
                    case '\\': val += '\\'; break;
                    case '"': val += '"'; break;
                    case 'u': {
                        // \uXXXX -> keep as-is decoded to a codepoint's bytes is
                        // beyond scope; store the escape literally for fidelity.
                        val += "\\u";
                        for (int k = 0; k < 4 && i < s.size(); k++) val += s[i++];
                        break;
                    }
                    default: val += e; break;
                }
            } else val += c;
        }
        if (i < s.size()) i++; // closing quote
        emit(Tok::String, val, lineNo, col);
    }

    void lexNumberOrVersion(const std::string& s, size_t& i, int lineNo) {
        int col = (int)i + 1;
        std::string raw;
        int dots = 0;
        while (i < s.size() && (std::isdigit((unsigned char)s[i]) || s[i] == '_' || s[i] == '.')) {
            if (s[i] == '.') {
                // only consume '.' if followed by a digit (version segment)
                if (i + 1 < s.size() && std::isdigit((unsigned char)s[i+1])) { dots++; raw += s[i++]; }
                else break;
            } else {
                raw += s[i++];
            }
        }
        if (dots >= 1) {
            emit(Tok::Version, raw, lineNo, col);
        } else {
            std::string digits;
            for (char ch : raw) if (ch != '_') digits += ch;
            emit(Tok::Integer, digits, lineNo, col);
        }
    }
};

} // namespace

std::vector<Token> lex(const std::string& src, const std::string& file, DiagnosticBag& diags) {
    Lexer lx(src, file, diags);
    return lx.run();
}

} // namespace nordshrift