Home /Modules/sleela-terminal/lexer.cpp

lexer.cpp

sleela-terminal/lexer.cpp · 316 line(s) · c

// SleelaTerminal(TM) shell -- L2 lexer implementation.
// Original SLeeLa work (see NOTICE).

#include "lexer.hpp"

#include <cctype>

namespace sleela::sh {

namespace {

// Is this token position the start of a command word (so a `name=value` word
// should be an assignment, and if/while/... are keywords)? True at the start
// and immediately after a command separator or opening keyword.
bool atCommandStart(const std::vector<Token>& out) {
    if (out.empty()) return true;
    switch (out.back().kind) {
        case Tok::Pipe: case Tok::Semi: case Tok::Newline:
        case Tok::AndIf: case Tok::OrIf: case Tok::Amp: case Tok::Bang:
        case Tok::If: case Tok::Then: case Tok::Elif: case Tok::Else:
        case Tok::While: case Tok::Until: case Tok::Do:
        case Tok::In: case Tok::LBrace: case Tok::RParen:
            return true;
        default:
            return false;
    }
}

// A well-formed assignment word is `name=...` where name is [A-Za-z_][A-Za-z0-9_]*.
bool looksLikeAssignment(const std::string& w) {
    const auto eq = w.find('=');
    if (eq == std::string::npos || eq == 0) return false;
    if (!(std::isalpha(static_cast<unsigned char>(w[0])) || w[0] == '_')) return false;
    for (std::size_t i = 1; i < eq; ++i) {
        if (!(std::isalnum(static_cast<unsigned char>(w[i])) || w[i] == '_'))
            return false;
    }
    return true;
}

// Compound-command openers: recognised only at command-start position.
Tok openerKind(const std::string& w) {
    if (w == "if")    return Tok::If;
    if (w == "while") return Tok::While;
    if (w == "until") return Tok::Until;
    if (w == "for")   return Tok::For;
    if (w == "case")  return Tok::Case;
    return Tok::Word;
}

// Structural keywords: recognised whenever an unquoted bareword matches, since
// they only make sense inside a compound command (then/do/in/...). This lets
// e.g. `for i in ...` see `in` even though it follows the word `i`.
Tok structuralKind(const std::string& w) {
    if (w == "then")  return Tok::Then;
    if (w == "elif")  return Tok::Elif;
    if (w == "else")  return Tok::Else;
    if (w == "fi")    return Tok::Fi;
    if (w == "do")    return Tok::Do;
    if (w == "done")  return Tok::Done;
    if (w == "in")    return Tok::In;
    if (w == "esac")  return Tok::Esac;
    return Tok::Word;
}

} // namespace

bool lex(const std::string& src, std::vector<Token>& out, LexError& err) {
    out.clear();
    std::size_t i = 0;
    const std::size_t n = src.size();
    int line = 1, col = 1;

    auto adv = [&](std::size_t k = 1) {
        for (std::size_t t = 0; t < k && i < n; ++t) {
            if (src[i] == '\n') { ++line; col = 1; } else { ++col; }
            ++i;
        }
    };
    auto push = [&](Tok kind, std::string text, int ln, int cl) {
        Token tk; tk.kind = kind; tk.text = std::move(text); tk.line = ln; tk.col = cl;
        out.push_back(std::move(tk));
    };

    while (i < n) {
        const char c = src[i];

        // whitespace (not newline)
        if (c == ' ' || c == '\t' || c == '\r') { adv(); continue; }

        // comment
        if (c == '#') { while (i < n && src[i] != '\n') adv(); continue; }

        // newline
        if (c == '\n') { push(Tok::Newline, "\n", line, col); adv(); continue; }

        // operators (maximal munch)
        const char nx = (i + 1 < n) ? src[i + 1] : '\0';
        if (c == '|') {
            if (nx == '|') { push(Tok::OrIf, "||", line, col); adv(2); }
            else { push(Tok::Pipe, "|", line, col); adv(); }
            continue;
        }
        if (c == '&') {
            if (nx == '&') { push(Tok::AndIf, "&&", line, col); adv(2); continue; }
            push(Tok::Amp, "&", line, col); adv(); continue;   // background
        }
        if (c == ';') { push(Tok::Semi, ";", line, col); adv(); continue; }
        if (c == '<') {
            if (nx == '<') {
                // here-document: << or <<-
                const bool dash = (i + 2 < n && src[i + 2] == '-');
                const int hline = line, hcol = col;
                adv(dash ? 3 : 2);
                push(dash ? Tok::DLessDash : Tok::DLess, dash ? "<<-" : "<<", hline, hcol);
                // read the delimiter word (skip spaces)
                while (i < n && (src[i] == ' ' || src[i] == '\t')) adv();
                std::string delim;
                bool quotedDelim = false;
                while (i < n && src[i] != '\n' && src[i] != ' ' && src[i] != '\t' &&
                       src[i] != ';' && src[i] != '|' && src[i] != '&') {
                    if (src[i] == '\'' || src[i] == '"') { quotedDelim = true; adv(); continue; }
                    delim.push_back(src[i]); adv();
                }
                if (delim.empty()) { err = LexError{"expected here-document delimiter after <<", hline, hcol}; return false; }
                push(Tok::Word, delim, hline, hcol);
                // consume the rest of the current line up to and including \n
                while (i < n && src[i] != '\n') adv();
                if (i < n) adv();  // the newline
                // collect body lines until a line equal to the delimiter
                std::string bodyText;
                bool closed = false;
                while (i < n) {
                    // read one physical line
                    std::string ln;
                    while (i < n && src[i] != '\n') { ln.push_back(src[i]); adv(); }
                    if (i < n) adv();  // consume newline
                    // for <<- strip leading tabs from the line (and the delim compare)
                    std::string cmp = ln;
                    if (dash) {
                        std::size_t t = 0; while (t < cmp.size() && cmp[t] == '\t') ++t;
                        cmp = cmp.substr(t);
                        std::size_t t2 = 0; while (t2 < ln.size() && ln[t2] == '\t') ++t2;
                        ln = ln.substr(t2);
                    }
                    if (cmp == delim) { closed = true; break; }
                    bodyText += ln;
                    bodyText.push_back('\n');
                }
                if (!closed) { err = LexError{"unterminated here-document (missing " + delim + ")", hline, hcol}; return false; }
                // A here-doc body is expandable unless the delimiter was quoted.
                // Encode expandability as a leading flag char the parser strips:
                //   'E' = expand $ in the body, 'L' = literal.
                push(Tok::HeredocBody, (quotedDelim ? "L" : "E") + bodyText, hline, hcol);
                continue;
            }
            push(Tok::Less, "<", line, col); adv(); continue;
        }
        if (c == '>') {
            if (nx == '>') { push(Tok::DGreat, ">>", line, col); adv(2); }
            else { push(Tok::Great, ">", line, col); adv(); }
            continue;
        }
        // Grouping/definition punctuation. A leading '$' before '(' or '{' is
        // handled inside the word loop as a substitution span, so a bare '(' or
        // '{' here is structural.
        if (c == '(') { push(Tok::LParen, "(", line, col); adv(); continue; }
        if (c == ')') { push(Tok::RParen, ")", line, col); adv(); continue; }
        // '{' / '}' are structural (brace group `{ cmd; }` / function body) only
        // when they stand alone -- i.e. followed by whitespace, end-of-input, or
        // an operator. When immediately followed by word content (e.g. `{a,b}`,
        // `{m..n}`), the brace begins an ordinary word so that brace expansion
        // in the L4 layer can act on it. This mirrors the test used inside the
        // word loop below for a word-initial brace.
        if (c == '{' || c == '}') {
            const char after = (i + 1 < n) ? src[i + 1] : '\0';
            const bool standalone = (after == '\0' || after == ' ' || after == '\t' ||
                                     after == '\n' || after == '\r' || after == ';' ||
                                     after == '|' || after == '&');
            if (standalone) {
                push(c == '{' ? Tok::LBrace : Tok::RBrace,
                     std::string(1, c), line, col);
                adv();
                continue;
            }
            // not standalone -> fall through to the word scanner below
        }

        // a word (possibly with quoted spans), read until a delimiter
        const int wline = line, wcol = col;
        std::string word;
        bool sawQuote = false;
        bool sawQuoteBeforeEq = false;
        while (i < n) {
            const char d = src[i];
            if (d == ' ' || d == '\t' || d == '\r' || d == '\n' ||
                d == '|' || d == '&' || d == ';' || d == '<' || d == '>' ||
                d == '#' || d == '(' || d == ')') {
                break;
            }
            // '{' and '}' are only structural (function body / brace group) when
            // they stand alone; when adjacent to word characters they belong to
            // the word (e.g. brace expansion item{1,2,3}, or ${...}). Break the
            // word only for a standalone brace at the very start of a word.
            if ((d == '{' || d == '}') && word.empty()) {
                // standalone if followed by whitespace/eol/operator
                const char after = (i + 1 < n) ? src[i + 1] : '\0';
                if (after == '\0' || after == ' ' || after == '\t' ||
                    after == '\n' || after == '\r' || after == ';' ||
                    after == '|' || after == '&') {
                    break;  // let the operator scanner emit LBrace/RBrace
                }
            }
            if (d == '\'') {
                // single quote: literal until the next single quote
                sawQuote = true;
                if (word.find('=') == std::string::npos) sawQuoteBeforeEq = true;
                adv();  // consume opening '
                while (i < n && src[i] != '\'') { word.push_back(src[i]); adv(); }
                if (i >= n) { err = LexError{"unterminated single quote", wline, wcol}; return false; }
                adv();  // consume closing '
                continue;
            }
            if (d == '"') {
                // double quote: keep contents; expansion handled later. We keep
                // the raw characters (including $), and support \" and \\ escapes.
                sawQuote = true;
                if (word.find('=') == std::string::npos) sawQuoteBeforeEq = true;
                adv();  // consume opening "
                while (i < n && src[i] != '"') {
                    if (src[i] == '\\' && i + 1 < n &&
                        (src[i + 1] == '"' || src[i + 1] == '\\' || src[i + 1] == '$')) {
                        word.push_back(src[i + 1]); adv(2);
                    } else {
                        word.push_back(src[i]); adv();
                    }
                }
                if (i >= n) { err = LexError{"unterminated double quote", wline, wcol}; return false; }
                adv();  // consume closing "
                continue;
            }
            if (d == '\\') {
                // backslash escape of the next character
                if (i + 1 < n) { word.push_back(src[i + 1]); adv(2); }
                else { adv(); }
                continue;
            }
            // A $(( ... )) arithmetic span, or a ${ ... } parameter span, is a
            // single word span even when it contains spaces or operators.
            if (d == '$' && i + 2 < n && src[i + 1] == '(' && src[i + 2] == '(') {
                word.append("$((");
                adv(3);
                int depth = 1;
                while (i < n && depth > 0) {
                    if (src[i] == '(') { ++depth; word.push_back(src[i]); adv(); }
                    else if (src[i] == ')') {
                        // "))" closes the arithmetic span when depth reaches 0.
                        if (i + 1 < n && src[i + 1] == ')' && depth == 1) {
                            word.append("))"); adv(2); depth = 0;
                        } else { --depth; word.push_back(src[i]); adv(); }
                    } else { word.push_back(src[i]); adv(); }
                }
                if (depth != 0) { err = LexError{"unterminated $(( ))", wline, wcol}; return false; }
                continue;
            }
            // A $( ... ) command-substitution span is a single word span with
            // balanced parentheses (checked after $(( above).
            if (d == '$' && i + 1 < n && src[i + 1] == '(') {
                word.append("$(");
                adv(2);
                int depth = 1;
                while (i < n && depth > 0) {
                    if (src[i] == '(') ++depth;
                    else if (src[i] == ')') --depth;
                    if (depth > 0) { word.push_back(src[i]); adv(); }
                    else { word.push_back(')'); adv(); }  // closing ')'
                }
                if (depth != 0) { err = LexError{"unterminated $( )", wline, wcol}; return false; }
                continue;
            }
            if (d == '$' && i + 1 < n && src[i + 1] == '{') {
                word.append("${");
                adv(2);
                while (i < n && src[i] != '}') { word.push_back(src[i]); adv(); }
                if (i >= n) { err = LexError{"unterminated ${ }", wline, wcol}; return false; }
                word.push_back('}'); adv();
                continue;
            }
            word.push_back(d);
            adv();
        }

        // classify the word
        const bool cmdStart = atCommandStart(out);
        Tok structural = sawQuote ? Tok::Word : structuralKind(word);
        if (!sawQuoteBeforeEq && cmdStart && looksLikeAssignment(word)) {
            push(Tok::Assignment, word, wline, wcol);
        } else if (structural != Tok::Word) {
            // then/do/done/in/esac/... are keywords wherever they appear bare.
            push(structural, word, wline, wcol);
        } else if (!sawQuote && cmdStart && word == "!") {
            // '!' at command-start negates the pipeline that follows.
            push(Tok::Bang, "!", wline, wcol);
        } else if (!sawQuote && cmdStart) {
            push(openerKind(word), word, wline, wcol);
        } else {
            push(Tok::Word, word, wline, wcol);
        }
    }

    push(Tok::Eof, "", line, col);
    return true;
}

} // namespace sleela::sh