Home /Modules/impl/sst_lexer.h

sst_lexer.h

impl/nordshrift/sst_lexer.h · 64 line(s) · c

// ===========================================================================
// sst_lexer.h  --  Spec-conformant tokenizer for Nordshrift .sst sheets.
//
// Implements NS-SST-0001 Part I (Lexical Conventions). The .sst format is
// indentation-significant: the lexer produces synthetic INDENT / DEDENT
// tokens (Part XIV, §14.1) so the parser can work over block structure without
// re-deriving indentation. One NEWLINE token terminates each logical line.
//
// This replaces the earlier brace-delimited component-language lexer; the .sst
// file is now a build-control sheet, not a program.
// ===========================================================================
#ifndef NORDSHRIFT_SST_LEXER_H
#define NORDSHRIFT_SST_LEXER_H

#include <string>
#include <vector>

#include "diagnostics.h"

namespace nordshrift {

enum class Tok {
    // structure
    Newline,      // end of a logical line
    Indent,       // block open  (one indent unit deeper)
    Dedent,       // block close
    // atoms
    Pragma,       // '#' + identifier   (text = pragma name, e.g. "nordshrift")
    Ident,        // identifier / bare keyword (keywords are contextual)
    String,       // "..." or """...""" (text = decoded contents)
    Integer,      // 1_000            (text = digits, separators removed)
    Version,      // 1.0.0 or 1.0     (text = raw)
    // punctuation
    Colon,        // ':'
    Comma,        // ','
    LBracket,     // '['
    RBracket,     // ']'
    // sentinel
    Eof
};

struct Token {
    Tok         kind;
    std::string text;      // lexeme / decoded value
    int         line;
    int         col;
    // For a directive/block line, the documentation comment (///) that
    // immediately preceded it, if any (Part I §1.3). Empty otherwise.
    std::string doc;
};

// Tokenize a .sst source. Lexical diagnostics (NSS-E-0001, NSS-E-0002,
// NSS-W-0001) are appended to `diags`. Returns the token stream (always
// terminated by Eof). On a fatal lexical error the returned stream may be
// partial; callers should check diags.hasErrors().
std::vector<Token> lex(const std::string& src, const std::string& file,
                       DiagnosticBag& diags);

const char* tokName(Tok t);

} // namespace nordshrift

#endif // NORDSHRIFT_SST_LEXER_H