lexer.cpp
impl/frontend/lexer.cpp · 219 line(s) · c
// ===============================================================================
// lexer.cpp -- Implementation of the Java-compatible Sleela tokenizer.
// ===============================================================================
#include "lexer.h"
#include <cctype>
#include <cstdint>
#include <stdexcept>
#include <unordered_map>
namespace sleela {
namespace {
std::string utf8FromCodePoint(uint32_t cp) {
std::string out;
if (cp <= 0x7F) {
out.push_back(static_cast<char>(cp));
} else if (cp <= 0x7FF) {
out.push_back(static_cast<char>(0xC0 | ((cp >> 6) & 0x1F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else if (cp <= 0xFFFF) {
if (cp >= 0xD800 && cp <= 0xDFFF) {
throw std::runtime_error("invalid Unicode surrogate half in string literal");
}
out.push_back(static_cast<char>(0xE0 | ((cp >> 12) & 0x0F)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else if (cp <= 0x10FFFF) {
out.push_back(static_cast<char>(0xF0 | ((cp >> 18) & 0x07)));
out.push_back(static_cast<char>(0x80 | ((cp >> 12) & 0x3F)));
out.push_back(static_cast<char>(0x80 | ((cp >> 6) & 0x3F)));
out.push_back(static_cast<char>(0x80 | (cp & 0x3F)));
} else {
throw std::runtime_error("invalid Unicode code point in string literal");
}
return out;
}
}
const char* tokName(Tok t) {
switch(t) {
case Tok::Int:return "int-literal"; case Tok::Double:return "double-literal"; case Tok::Str:return "string-literal"; case Tok::Char:return "char-literal";
case Tok::Ident:return "identifier"; case Tok::At:return "@";
case Tok::KwClass:return "class"; case Tok::KwInterface:return "interface"; case Tok::KwEnum:return "enum"; case Tok::KwRecord:return "record"; case Tok::KwStruct:return "struct";
case Tok::KwStatic:return "static"; case Tok::KwProtected:return "protected"; case Tok::KwPublic:return "public"; case Tok::KwPrivate:return "private";
case Tok::KwFinal:return "final"; case Tok::KwAbstract:return "abstract"; case Tok::KwNative:return "native";
case Tok::KwSynchronized:return "synchronized"; case Tok::KwVolatile:return "volatile"; case Tok::KwTransient:return "transient";
case Tok::KwStrictfp:return "strictfp"; case Tok::KwSealed:return "sealed"; case Tok::KwNonSealed:return "non-sealed"; case Tok::KwConst:return "const"; case Tok::KwUsualMem:return "usualmem"; case Tok::KwUsualGc:return "usualgc"; case Tok::KwAccurateMem:return "accuratemem";
case Tok::KwPackage:return "package"; case Tok::KwRequire:return "require"; case Tok::KwImport:return "import"; case Tok::KwYield:return "yield";
case Tok::KwIf:return "if"; case Tok::KwElse:return "else"; case Tok::KwSwitch:return "switch"; case Tok::KwCase:return "case"; case Tok::KwDefault:return "default";
case Tok::KwWhile:return "while"; case Tok::KwFor:return "for"; case Tok::KwDo:return "do"; case Tok::KwBreak:return "break"; case Tok::KwContinue:return "continue";
case Tok::KwReturn:return "return"; case Tok::KwThrow:return "throw"; case Tok::KwTry:return "try"; case Tok::KwCatch:return "catch"; case Tok::KwFinally:return "finally"; case Tok::KwAssert:return "assert";
case Tok::KwExtends:return "extends"; case Tok::KwImplements:return "implements"; case Tok::KwThrows:return "throws"; case Tok::KwSuper:return "super"; case Tok::KwThis:return "this"; case Tok::KwInstanceof:return "instanceof";
case Tok::KwNew:return "new"; case Tok::KwNull:return "null"; case Tok::KwTrue:return "true"; case Tok::KwFalse:return "false"; case Tok::KwPrint:return "print";
case Tok::KwVoid:return "void"; case Tok::KwIntT:return "int"; case Tok::KwLongT:return "long"; case Tok::KwInt64T:return "int64"; case Tok::KwInt128T:return "int128"; case Tok::KwDoubleT:return "double"; case Tok::KwBoolT:return "boolean"; case Tok::KwStringT:return "String";
case Tok::LParen:return "("; case Tok::RParen:return ")"; case Tok::LBrace:return "{"; case Tok::RBrace:return "}";
case Tok::LBracket:return "["; case Tok::RBracket:return "]"; case Tok::Semicolon:return ";"; case Tok::Comma:return ","; case Tok::Dot:return ".";
case Tok::Assign:return "="; case Tok::Plus:return "+"; case Tok::Minus:return "-"; case Tok::Star:return "*"; case Tok::Slash:return "/";
case Tok::Percent:return "%"; case Tok::PlusAssign:return "+="; case Tok::MinusAssign:return "-="; case Tok::StarAssign:return "*="; case Tok::SlashAssign:return "/="; case Tok::PercentAssign:return "%="; case Tok::AndAssign:return "&="; case Tok::OrAssign:return "|="; case Tok::XorAssign:return "^="; case Tok::ShlAssign:return "<<="; case Tok::ShrAssign:return ">>="; case Tok::UshrAssign:return ">>>=";
case Tok::Increment:return "++"; case Tok::Decrement:return "--"; case Tok::ShiftLeft:return "<<"; case Tok::ShiftRight:return ">>"; case Tok::UnsignedShiftRight:return ">>>";
case Tok::EqEq:return "=="; case Tok::NotEq:return "!="; case Tok::Lt:return "<"; case Tok::Le:return "<=";
case Tok::Gt:return ">"; case Tok::Ge:return ">="; case Tok::AndAnd:return "&&"; case Tok::OrOr:return "||"; case Tok::BitAnd:return "&"; case Tok::BitOr:return "|"; case Tok::BitXor:return "^"; case Tok::BitNot:return "~";
case Tok::Not:return "!"; case Tok::Question:return "?"; case Tok::Colon:return ":"; case Tok::DoubleColon:return "::"; case Tok::TripleColon:return ":::"; case Tok::Arrow:return "->";
case Tok::Eof:return "<eof>";
}
return "?";
}
char Lexer::peek(int off) const { size_t p=pos_+(size_t)off; return p<src_.size()?src_[p]: '\0'; }
char Lexer::advance(){char c=src_[pos_++];if(c=='\n'){line_++;col_=1;}else col_++;return c;}
bool Lexer::match(char c){if(peek()==c){advance();return true;}return false;}
void Lexer::error(const std::string& msg) const {throw std::runtime_error("Lexical error (line "+std::to_string(line_)+", col "+std::to_string(col_)+"): "+msg);}
void Lexer::skipTrivia(){
auto atLinePrefix=[&](){size_t p=pos_;while(p>0&&src_[p-1]!='\n'){char b=src_[p-1];if(b!=' '&&b!='\t'&&b!='\r')return false;--p;}return true;};
for(;;){char c=peek();
if(c==' '||c=='\t'||c=='\r'||c=='\n')advance();
else if(c=='#')while(!atEnd()&&peek()!='\n')advance();
else if(c=='/'&&peek(1)=='/')while(!atEnd()&&peek()!='\n')advance();
else if(atLinePrefix()&&c=='<'&&peek(1)=='<'){advance();advance();bool closed=false;while(!atEnd()){if(peek()=='>'&&peek(1)=='>'){advance();advance();closed=true;break;}advance();}if(!closed)error("unterminated angle-bracket block comment");}
else if(atLinePrefix()&&c=='<'&&peek(1)!='<'){size_t e=pos_+1;while(e<src_.size()&&src_[e]!='\n'&&src_[e]!='>')++e;if(e<src_.size()&&src_[e]=='>')while(!atEnd()&&peek()!='\n')advance();else break;}
else break;
}
}
static bool identChar(char c){return std::isalnum((unsigned char)c)||c=='_';}
Token Lexer::makeNumber(){
int L=line_,C=col_;std::string s;
// Radix-prefixed integer literals: 0x/0X hex, 0b/0B binary, 0o/0O octal.
// The prefix is kept in the token text; the parser decodes the value.
if(peek()=='0'){
char p1=peek(1);
auto isHex=[](char c){return (c>='0'&&c<='9')||(c>='a'&&c<='f')||(c>='A'&&c<='F');};
if(p1=='x'||p1=='X'){
s+=advance();s+=advance(); // consume "0x"
if(!isHex(peek()))error("hexadecimal literal has no digits after '0x'");
while(isHex(peek())||peek()=='_'){char c=advance();if(c!='_')s+=c;}
if((peek()=='L'||peek()=='l')&&!identChar(peek(1)))advance();
return Token{Tok::Int,s,L,C};
}
if(p1=='b'||p1=='B'){
s+=advance();s+=advance(); // consume "0b"
if(peek()!='0'&&peek()!='1')error("binary literal has no digits after '0b'");
while(peek()=='0'||peek()=='1'||peek()=='_'){char c=advance();if(c!='_')s+=c;}
if((peek()=='L'||peek()=='l')&&!identChar(peek(1)))advance();
return Token{Tok::Int,s,L,C};
}
if(p1=='o'||p1=='O'){
advance();advance(); // consume "0o" (not kept)
s+='0'; // keep a leading 0 so stoll base 0 reads octal
if(peek()<'0'||peek()>'7')error("octal literal has no digits after '0o'");
while((peek()>='0'&&peek()<='7')||peek()=='_'){char c=advance();if(c!='_')s+=c;}
if((peek()=='L'||peek()=='l')&&!identChar(peek(1)))advance();
return Token{Tok::Int,s,L,C};
}
}
// Java numeric literal forms (SLEELA.syntax 43.6): `_` between digits
// (1_000_000), the long suffix L/l, and the floating suffixes d/D/f/F.
auto digitsWithUnderscores=[&]{ while(std::isdigit((unsigned char)peek())||(peek()=='_'&&std::isdigit((unsigned char)peek(1)))){char c=advance();if(c!='_')s+=c;} };
bool d=false;digitsWithUnderscores();if(peek()=='.'&&std::isdigit((unsigned char)peek(1))){d=true;s+=advance();digitsWithUnderscores();}if(peek()=='e'||peek()=='E'){d=true;s+=advance();if(peek()=='+'||peek()=='-')s+=advance();while(std::isdigit((unsigned char)peek()))s+=advance();}
if(!d&&(peek()=='L'||peek()=='l')&&!identChar(peek(1))){advance();return Token{Tok::Int,s,L,C};}
if((peek()=='d'||peek()=='D'||peek()=='f'||peek()=='F')&&!identChar(peek(1))){advance();if(!d)s+=".0";return Token{Tok::Double,s,L,C};}
return Token{d?Tok::Double:Tok::Int,s,L,C};}
// Syntax 3.1: character literal 'x'. It denotes the UTF-16 code unit of the
// character as an integer constant (SLeeLa has no separate char value kind).
Token Lexer::makeChar(){
int L=line_,C=col_;advance();
if(atEnd()||peek()=='\n')error("unterminated character literal");
if(peek()=='\'')error("empty character literal");
uint32_t cp=0;char c=advance();
if(c=='\\'){
if(atEnd())error("unterminated escape sequence");
char e=advance();
switch(e){
case'n':cp='\n';break;case't':cp='\t';break;case'r':cp='\r';break;case'b':cp='\b';break;case'f':cp='\f';break;
case's':cp=' ';break;case'\\':cp='\\';break;case'\'':cp='\'';break;case'"':cp='"';break;
case'0':case'1':case'2':case'3':case'4':case'5':case'6':case'7':{cp=(uint32_t)(e-'0');int n=1;while(n<3&&peek()>='0'&&peek()<='7'&&cp*8+(uint32_t)(peek()-'0')<=255){cp=cp*8+(uint32_t)(advance()-'0');n++;}break;}
case'u':{while(peek()=='u')advance();for(int i=0;i<4;i++){if(atEnd())error("unterminated unicode escape");char hex=advance();int v=0;if(hex>='0'&&hex<='9')v=hex-'0';else if(hex>='a'&&hex<='f')v=10+(hex-'a');else if(hex>='A'&&hex<='F')v=10+(hex-'A');else error("invalid unicode escape");cp=(cp<<4)|(uint32_t)v;}break;}
default:error(std::string("invalid escape sequence '\\")+e+"' in character literal");
}
}else if((unsigned char)c<0x80){cp=(unsigned char)c;}
else{
int extra=((unsigned char)c>=0xF0)?3:((unsigned char)c>=0xE0)?2:((unsigned char)c>=0xC0)?1:-1;
if(extra<0)error("invalid UTF-8 in character literal");
cp=(uint32_t)((unsigned char)c&(0x3F>>extra));
for(int i=0;i<extra;i++){if(atEnd())error("invalid UTF-8 in character literal");cp=(cp<<6)|((uint32_t)(unsigned char)advance()&0x3F);}
}
if(cp>0xFFFF)error("character literal outside the basic multilingual plane");
if(atEnd()||peek()!='\'')error("a character literal holds exactly one character");
advance();
return Token{Tok::Char,std::to_string(cp),L,C};
}
Token Lexer::makeString(){int L=line_,C=col_;advance();std::string out;while(!atEnd()&&peek()!='"'){char c=advance();if(c=='\\'){if(atEnd())error("unterminated escape sequence");char e=advance();switch(e){case'n':out+='\n';break;case't':out+='\t';break;case'r':out+='\r';break;case'b':out+='\b';break;case'f':out+='\f';break;case'0':out+='\0';break;case'\\':out+='\\';break;case'"':out+='"';break;case'u':{uint32_t cp=0;for(int i=0;i<4;i++){if(atEnd())error("unterminated unicode escape");char hex=advance();int v=0;if(hex>='0'&&hex<='9')v=hex-'0';else if(hex>='a'&&hex<='f')v=10+(hex-'a');else if(hex>='A'&&hex<='F')v=10+(hex-'A');else error("invalid unicode escape");cp=(cp<<4)|v;}out+=utf8FromCodePoint(cp);break;}default:out+=e;break;}}else out+=c;}if(atEnd())error("unterminated string literal");advance();return Token{Tok::Str,out,L,C};}
static const std::unordered_map<std::string,int> kContextualBuiltins13={
{"Munction",1},{"start",1},{"connect",1},{"enable",1},{"send",1},{"thatch",1},{"consume",1},{"latch",1},{"closeWithReceipt",1},{"reception",1},
{"synchroOpen",1},{"synchroDispatch",1},{"synchroReport",1},{"synchroClose",1},{"synchroSent",1},{"synchroReceived",1},{"synchroMean",1},{"synchroMin",1},{"synchroMax",1},{"synchroP95",1},{"synchroLoss",1},
{"bestOfNew",1},{"bestOfWeight",1},{"bestOfMinVersion",1},{"bestOfCostBudget",1},{"bestOfCandidate",1},{"bestOfRecord",1},{"bestOfScore",1},{"bestOfBest",1},{"bestOfChoice",1},{"bestOfReport",1},{"bestOfClose",1},
{"bestOfMean",1},{"bestOfLoss",1},{"bestOfJitter",1},{"bestOfCertainty",1},{"bestOfCandidateArch",1},{"bestOfArchRealized",1},{"bestOfArch",1},{"bestOfArchParam",1},{"bestOfArchState",1},
/* Idea System(TM) realtime canvas (idea/realtime) native-bridge built-ins. */
{"ideaCanvasCreate",1},{"ideaCanvasDestroy",1},{"ideaCanvasCols",1},{"ideaCanvasRows",1},{"ideaCanvasCount",1},
{"ideaCanvasSetParams",1},{"ideaCanvasExcite",1},{"ideaCanvasStep",1},{"ideaCanvasAdvance",1},{"ideaCanvasTimestep",1},
{"ideaCanvasIntensityAt",1},{"ideaCanvasHasRenderer",1},{"ideaCanvasOpenWindow",1},{"ideaCanvasShouldClose",1},{"ideaCanvasPresent",1},{"ideaCanvasRun",1},
{"ideaCanvasSetBackendPolicy",1},{"ideaCanvasPolicy",1},{"ideaCanvasActiveBackend",1},{"ideaCanvasGpuAvailable",1},{"ideaCanvasMeasuredHz",1},{"ideaCanvasMigrate",1},
/* GC-hint words (syntax 1.9). `gc` and `mem` are CONTEXTUAL, not reserved:
* they tokenize as ordinary identifiers and are only special as the whole
* right-hand side of an assignment (`x = gc N;` / `x = mem N;`), so they
* remain usable as identifiers everywhere else. The parser recognises the
* hint positionally; this registry entry documents them as contextual.
* `ran` is the System / already-Ran-in-RAM namespace qualifier: `ran::mem`,
* `ran::gc` name the system meaning explicitly so any word (mem, gc, ...)
* stays usable as a class/variable name. `ran` is contextual too -- it is
* special only immediately before `::`. */
{"gc",1},{"mem",1},{"ran",1}
};
bool isContextualBuiltin(const std::string& name){return kContextualBuiltins13.find(name)!=kContextualBuiltins13.end();}
// True for the two GC-hint spellings recognised as the RHS of `x = gc/mem N;`.
bool isGcHintWord(const std::string& name){return name=="gc"||name=="mem";}
Token Lexer::makeIdentOrKeyword(){
static const std::unordered_map<std::string,Tok> kw={
{"class",Tok::KwClass},{"interface",Tok::KwInterface},{"enum",Tok::KwEnum},{"record",Tok::KwRecord},{"struct",Tok::KwStruct},
{"static",Tok::KwStatic},{"protected",Tok::KwProtected},{"public",Tok::KwPublic},{"private",Tok::KwPrivate},
{"final",Tok::KwFinal},{"abstract",Tok::KwAbstract},{"native",Tok::KwNative},{"synchronized",Tok::KwSynchronized},
{"volatile",Tok::KwVolatile},{"transient",Tok::KwTransient},{"strictfp",Tok::KwStrictfp},{"sealed",Tok::KwSealed},{"non-sealed",Tok::KwNonSealed},{"const",Tok::KwConst},{"usualmem",Tok::KwUsualMem},{"usualgc",Tok::KwUsualGc},
{"package",Tok::KwPackage},{"require",Tok::KwRequire},{"import",Tok::KwImport},{"yield",Tok::KwYield},
{"if",Tok::KwIf},{"else",Tok::KwElse},{"switch",Tok::KwSwitch},{"case",Tok::KwCase},{"default",Tok::KwDefault},
{"while",Tok::KwWhile},{"for",Tok::KwFor},{"do",Tok::KwDo},{"break",Tok::KwBreak},{"continue",Tok::KwContinue},
{"return",Tok::KwReturn},{"throw",Tok::KwThrow},{"try",Tok::KwTry},{"catch",Tok::KwCatch},{"finally",Tok::KwFinally},
{"default",Tok::KwDefault},{"extends",Tok::KwExtends},{"implements",Tok::KwImplements},{"throws",Tok::KwThrows},{"super",Tok::KwSuper},{"this",Tok::KwThis},{"instanceof",Tok::KwInstanceof},
{"new",Tok::KwNew},{"null",Tok::KwNull},{"true",Tok::KwTrue},{"false",Tok::KwFalse},{"print",Tok::KwPrint},
{"void",Tok::KwVoid},{"int",Tok::KwIntT},{"long",Tok::KwLongT},{"int64",Tok::KwInt64T},{"int128",Tok::KwInt128T},{"double",Tok::KwDoubleT},{"boolean",Tok::KwBoolT},{"String",Tok::KwStringT}
};
// A '-' continues an identifier (jdk-25, version-mual) only when an identifier
// character follows it, so Java `j--`, `j-=1` and a trailing '-' end the name
// (SLEELA.syntax 43.6 rule 12).
int L=line_,C=col_;std::string s;while(std::isalnum((unsigned char)peek())||peek()=='_'||(peek()=='-'&&identChar(peek(1))))s+=advance();
auto it=kw.find(s);return Token{it!=kw.end()?it->second:Tok::Ident,s,L,C};
}
std::vector<Token> Lexer::tokenize(){
std::vector<Token> toks;
for(;;){skipTrivia();if(atEnd()){toks.push_back(Token{Tok::Eof,"",line_,col_});break;}int L=line_,C=col_;char c=peek();
if(std::isdigit((unsigned char)c)){toks.push_back(makeNumber());continue;} if(c=='"'){toks.push_back(makeString());continue;} if(c=='\''){toks.push_back(makeChar());continue;}
if(std::isalpha((unsigned char)c)||c=='_'){toks.push_back(makeIdentOrKeyword());continue;}
advance();auto emit=[&](Tok k,const char*txt){toks.push_back(Token{k,txt,L,C});};
switch(c){
case'@':emit(Tok::At,"@");break;case'(':emit(Tok::LParen,"(");break;case')':emit(Tok::RParen,")");break;
case'{':emit(Tok::LBrace,"{");break;case'}':emit(Tok::RBrace,"}");break;case'[':emit(Tok::LBracket,"[");break;case']':emit(Tok::RBracket,"]");break;
case';':emit(Tok::Semicolon,";");break;case',':emit(Tok::Comma,",");break;case'.':emit(Tok::Dot,".");break;
case'=':if(match('='))emit(Tok::EqEq,"==");else emit(Tok::Assign,"=");break;
case'!':if(match('='))emit(Tok::NotEq,"!=");else emit(Tok::Not,"!");break;
case'+':if(match('+'))emit(Tok::Increment,"++");else if(match('='))emit(Tok::PlusAssign,"+=");else emit(Tok::Plus,"+");break;
case'-':if(match('-'))emit(Tok::Decrement,"--");else if(match('='))emit(Tok::MinusAssign,"-=");else if(match('>'))emit(Tok::Arrow,"->");else emit(Tok::Minus,"-");break;
case'*':if(match('='))emit(Tok::StarAssign,"*=");else emit(Tok::Star,"*");break;case'/':if(match('='))emit(Tok::SlashAssign,"/=");else emit(Tok::Slash,"/");break;
case'%':if(match('='))emit(Tok::PercentAssign,"%=");else emit(Tok::Percent,"%");break;
case'<':if(match('<')){if(match('='))emit(Tok::ShlAssign,"<<=");else emit(Tok::ShiftLeft,"<<");}else if(match('='))emit(Tok::Le,"<=");else emit(Tok::Lt,"<");break;
case'>':if(match('>')){if(match('>')){if(match('='))emit(Tok::UshrAssign,">>>=");else emit(Tok::UnsignedShiftRight,">>>");}else if(match('='))emit(Tok::ShrAssign,">>=");else emit(Tok::ShiftRight,">>");}else if(match('='))emit(Tok::Ge,">=");else emit(Tok::Gt,">");break;
case'&':if(match('&'))emit(Tok::AndAnd,"&&");else if(match('='))emit(Tok::AndAssign,"&=");else emit(Tok::BitAnd,"&");break;
case'|':if(match('|'))emit(Tok::OrOr,"||");else if(match('='))emit(Tok::OrAssign,"|=");else emit(Tok::BitOr,"|");break;
case'^':if(match('='))emit(Tok::XorAssign,"^=");else emit(Tok::BitXor,"^");break;case'~':emit(Tok::BitNot,"~");break;
case'?':emit(Tok::Question,"?");break;case':':if(match(':')){if(match(':'))emit(Tok::TripleColon,":::");else emit(Tok::DoubleColon,"::");}else emit(Tok::Colon,":");break;
default:error(std::string("unexpected character '")+c+"'");
}
}
return toks;
}
} // namespace sleela