// docs.cpp -- `sleela docs` subcommand implementation. See docs.h. // =========================================================================== #include "docs.h" #include "library_index.h" #include #include #include #include #include #include #include #include #include #include #include #include #include namespace fs = std::filesystem; namespace sleela { namespace { // ---- small text helpers --------------------------------------------------- std::string lower(std::string s){ for(char& c:s) c=(char)std::tolower((unsigned char)c); return s; } std::string trim(const std::string& s){ size_t a=s.find_first_not_of(" \t\r\n"); if(a==std::string::npos) return ""; size_t b=s.find_last_not_of(" \t\r\n"); return s.substr(a,b-a+1); } std::string squeeze(const std::string& s){ return std::regex_replace(trim(s), std::regex("\\s+"), " "); } // A levenshtein-free "nearness": a topic is near a candidate when it is a // case-insensitive substring of the candidate, or the candidate of it, or they // share a long common prefix. Returns a score 0..100 (higher = closer); 0 = no. int nearness(const std::string& topicL, const std::string& candL){ if(topicL==candL) return 100; if(candL.find(topicL)!=std::string::npos) return 85; // topic inside candidate if(topicL.find(candL)!=std::string::npos) return 75; // candidate inside topic size_t n=std::min(topicL.size(),candL.size()), p=0; while(p=3) return (int)(40 + p); // shared prefix return 0; } // ---- the parsed, rich doc model ------------------------------------------- struct Tag { std::string name, value; }; struct Member { std::string kind, name, sig, desc; std::vector tags; // @param/@returns/... harvested above it }; struct ClassDoc { std::string kind, name, extends, summary; std::vector fields, methods; std::vector tags; // class-level @intent/@thesis/@axiom/... bool hasMetadata=false; }; struct UnitDoc { std::string package, name, path, definition, syntax; std::vector classes; }; bool isMetaName(const std::string& n){ static const std::regex r("^(metaVersion|metaId|metaAuthor|metaDate|metaCopyright|metaTrademark|" "metaCorporation|metaOrigin|metaBirthDate|metaScienceRating|metaNationalSchedule)$"); static const std::regex a("^(get|set)(MetaVersion|MetaId|Author|Date|Copyright|Trademark|" "Corporation|Origin|BirthDate|ScienceRating|NationalSchedule)$"); return std::regex_match(n,r) || std::regex_match(n,a); } // Pull the plain text out of a single comment line (`<`, `<<`/`>>`, `//`), or // empty string when the line is a decorative rule, or a sentinel-only line. std::string commentText(const std::string& raw){ std::string s=trim(raw); if(s.rfind("<<",0)==0){ s=s.substr(2); if(s.size()>=2 && s.substr(s.size()-2)==">>") s=s.substr(0,s.size()-2); } else if(!s.empty() && s[0]=='<' && s.rfind("<=",0)!=0){ s=s.substr(1); if(!s.empty() && s.back()=='>') s.pop_back(); } else if(s.rfind("//",0)==0){ s=s.substr(2); } else if(s.rfind("#",0)==0){ return ""; } else { return ""; } s=trim(s); for(const char* sent : {"@doc","@page","@intent","@title"}) if(s.rfind(sent,0)==0) s=trim(s.substr(std::string(sent).size())); bool rule=!s.empty(); for(char c:s) if(c!='-'&&c!=' '){ rule=false; break; } if(rule) return ""; return s; } // Collect the raw comment lines (delimiters stripped) of the contiguous run // ending at `end`, top-to-bottom. Used by both the plain-prose and the tagged // gatherers so they agree on run boundaries. std::vector gatherRaw(const std::vector& lines, int end){ std::vector pieces; int i=end; while(i>=0){ std::string s=trim(lines[i]); if(s.empty()) break; bool isComment = (!s.empty() && (s[0]=='<' || s.rfind("//",0)==0 || s.rfind("<<",0)==0)); if(!isComment){ if(!pieces.empty() && s.find(';')==std::string::npos && s.find('{')==std::string::npos && s.find('(')==std::string::npos){ pieces.push_back(s); --i; continue; } break; } // keep the raw (post-delimiter) text, preserving a leading @tag token std::string t=trim(lines[i]); if(t.rfind("<<",0)==0){ t=t.substr(2); if(t.size()>=2 && t.substr(t.size()-2)==">>") t=t.substr(0,t.size()-2); } else if(!t.empty() && t[0]=='<'){ t=t.substr(1); if(!t.empty() && t.back()=='>') t.pop_back(); } else if(t.rfind("//",0)==0){ t=t.substr(2); } pieces.push_back(trim(t)); --i; } std::reverse(pieces.begin(),pieces.end()); return pieces; } // Split a comment run into (free prose, tags). A line whose first token is an // @word starts a tag; following non-@ lines continue that tag's value; lines // before any @tag are summary prose. Recognised SleelaDoc tags only. void splitDoc(const std::vector& raw, std::string& proseOut, std::vector& tagsOut){ static const std::regex tagHead("^@([A-Za-z]+)\\b\\s*(.*)$"); std::vector prose; Tag* curTag=nullptr; for(auto line : raw){ std::string s=trim(line); bool rule=!s.empty(); for(char c:s) if(c!='-'&&c!=' '){ rule=false; break; } if(rule||s.empty()) continue; std::smatch m; if(std::regex_match(s,m,tagHead)){ tagsOut.push_back(Tag{lower(m[1].str()), trim(m[2].str())}); curTag=&tagsOut.back(); } else if(curTag){ if(!curTag->value.empty()) curTag->value+=" "; curTag->value+=s; } else { prose.push_back(s); } } std::string p; for(auto& x:prose){ if(x.empty()) continue; if(!p.empty()) p+=" "; p+=x; } proseOut=squeeze(p); } // Gather the contiguous comment run that ends at line index `end` (inclusive), // walking upward, and join it as one prose string. std::string gatherAbove(const std::vector& lines, int end){ std::vector pieces; int i=end; while(i>=0){ std::string s=trim(lines[i]); if(s.empty()) break; std::string t=commentText(lines[i]); bool isComment = (!s.empty() && (s[0]=='<' || s.rfind("//",0)==0 || s.rfind("<<",0)==0)); if(!isComment){ // allow interior lines of a << >> block (no delimiter) only mid-run if(!pieces.empty() && s.find(';')==std::string::npos && s.find('{')==std::string::npos && s.find('(')==std::string::npos){ pieces.push_back(s); --i; continue; } break; } pieces.push_back(t); --i; } std::reverse(pieces.begin(),pieces.end()); std::string out; for(const auto& p:pieces){ if(p.empty()) continue; if(!out.empty()) out+=" "; out+=p; } return squeeze(out); } // Parse one .sleela unit into a rich doc model (mirrors the Python generator's // grammar closely enough for the terminal rendering). Symbol paths are stored // RELATIVE to the index root, so resolve against it before opening. UnitDoc parseUnit(const library::Symbol& sym, const std::string& libRoot){ UnitDoc u; u.package=sym.package; u.name=sym.name; fs::path full = fs::path(sym.path).is_absolute() ? fs::path(sym.path) : (fs::path(libRoot)/sym.path); // Display a stable, repo-relative path (lib/<...>), not an absolute build // path, so a page is identical on every machine and clean inside the JAR. fs::path libp(libRoot); std::string libName = libp.filename().string().empty()? "lib" : libp.filename().string(); u.path = (fs::path(libName)/sym.path).generic_string(); std::ifstream in(full); if(!in) return u; std::stringstream ss; ss<(text.size(),1600)); if(std::regex_search(head,m,std::regex("Definition:\\s*([^>]+)"))) u.definition=squeeze(m[1].str()); if(std::regex_search(text,m,std::regex("#sleela\\s+([0-9.]+)"))) u.syntax=m[1].str(); std::vector lines; { std::stringstream ls(text); std::string ln; while(std::getline(ls,ln)) lines.push_back(ln); } static const std::regex classRe("^(class|struct|interface)\\s+([A-Za-z_][A-Za-z0-9_]*)(?:\\s+extends\\s+([A-Za-z_][A-Za-z0-9_]*))?"); static const std::regex fieldRe("^(?:static\\s+)?(?:final\\s+)?([A-Za-z_][A-Za-z0-9_]*(?:\\s*\\[\\s*\\])?)\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*(?:=.*)?;\\s*$"); static const std::regex methRe("^(?:static\\s+)?(?:final\\s+)?([A-Za-z_][A-Za-z0-9_]*(?:\\s*\\[\\s*\\])?)\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\("); ClassDoc* cur=nullptr; bool inBlock=false; int depth=0; for(int idx=0;idx<(int)lines.size();++idx){ std::string s=trim(lines[idx]); if(inBlock){ if(s.find(">>")!=std::string::npos) inBlock=false; continue; } if(s.rfind("<<",0)==0 && s.find(">>")==std::string::npos){ inBlock=true; continue; } if(!s.empty() && (s[0]=='<' || s.rfind("//",0)==0)) continue; if(s.rfind("#",0)==0) continue; int opens=(int)std::count(s.begin(),s.end(),'{'); int closes=(int)std::count(s.begin(),s.end(),'}'); std::smatch cm; if(std::regex_search(s,cm,classRe)){ u.classes.push_back(ClassDoc{cm[1].str(),cm[2].str(),cm[3].matched?cm[3].str():"", "", {}, {}, {}, false}); cur=&u.classes.back(); { std::string prose; std::vector tg; splitDoc(gatherRaw(lines,idx-1),prose,tg); cur->summary=prose; cur->tags=tg; } depth += opens-closes; // usually enters the class body (depth 1) continue; } if(!cur){ depth += opens-closes; continue; } // Members live at the class-body level (depth==1). Deeper lines are // method-body locals (int i = 0; return this; ...) and must be skipped. bool atClassBody = (depth==1); std::smatch mm; if(atClassBody && std::regex_search(s,mm,fieldRe)){ std::string nm=mm[2].str(); if(isMetaName(nm)){ cur->hasMetadata=true; } else { std::string prose; std::vector tg; splitDoc(gatherRaw(lines,idx-1),prose,tg); cur->fields.push_back(Member{"field",nm,squeeze(s.substr(0,s.size()-1)),prose,tg}); } } else if(atClassBody && std::regex_search(s,mm,methRe)){ std::string nm=mm[2].str(); if(isMetaName(nm)){ cur->hasMetadata=true; } else { std::string sig=s; size_t br=sig.find('{'); if(br!=std::string::npos) sig=trim(sig.substr(0,br)); std::string prose; std::vector tg; splitDoc(gatherRaw(lines,idx-1),prose,tg); cur->methods.push_back(Member{"method",nm,squeeze(sig),prose,tg}); } } depth += opens-closes; if(depth<0) depth=0; } return u; } // ---- rendering ------------------------------------------------------------- void rule(std::ostream& o,char c='='){ o< names)->std::string{ for(const char* nm:names) for(const auto& t:c.tags) if(t.name==nm) return t.value; return std::string(); }; auto allTags=[&](const char* name)->std::vector{ std::vector v; for(const auto& t:c.tags) if(t.name==name) v.push_back(t.value); return v; }; if(!u.definition.empty()){ o<<"\nDEFINITION\n "<(cue.size(),180)))<<"\n"; o<<" * Audience: INT (the attentive reader) studies the RELATIONSHIPS;\n" " Swarmy (the habitual reader) should slow at the Careful cue.\n"; // MASTERY LEVELS: a graduated study ladder over this class. Author-supplied // @mastery lines (if any) are listed first as the class's own guidance; the // standard five-rung ladder always follows so every page teaches the method. o<<"\nMASTERY LEVELS\n"; { auto ms=allTags("mastery"); for(size_t k=0;k units; for(const auto& s:ix.symbols()) if(s.package==pkg) units.push_back(&s); std::sort(units.begin(),units.end(),[](auto*a,auto*b){return a->namename;}); o<<"\n"<96) one=one.substr(0,93)+"..."; o<<" "<name; if(!one.empty()) o<<"\n "<name=="SLPackage"); if(facadeOnly){ o<<"\nThis is a FACADE package: its /lib entry is only the package marker.\n" <<"Find and document its real files on disk with the live OS search:\n" <<" sleela docs --facade "<)\n\n"; } else { o<<"\nRead one in full with: sleela docs "<\n" <<" or: sleela docs \n\n"; } } void listAll(std::ostream& o, const library::Index& ix){ std::map pkgCount; for(const auto& s:ix.symbols()) ++pkgCount[s.package]; rule(o); o<<" SleelaDoc -- all documentable topics\n"; rule(o); o<<"\n"< e.g. sleela docs crypto\n"; o<<" sleela docs e.g. sleela docs stringbuilder\n"; o<<" sleela docs the exact unit\n\n"; o<<"PACKAGES\n"; for(const auto& kv:pkgCount){ if(kv.first=="__root__") continue; o<<" "< hits; std::map pkgSeen; for(const auto& s:ix.symbols()){ int sc=nearness(tL,lower(s.name)); if(sc>0) hits.push_back({sc,s.package,s.name}); int pc=nearness(tL,lower(s.package)); if(pc>0 && !pkgSeen[s.package]++) hits.push_back({pc-1,s.package,"(whole package)"}); } std::sort(hits.begin(),hits.end(),[](const Hit&a,const Hit&b){return a.score>b.score;}); o<<"No exact topic '"<=12) break; o<<" "</TOPICS.index one "packageUnit" line per unit // //.txt the rendered class page // //_package.txt the rendered package overview // The corpus is always byte-identical to live output because both go through // the same renderClass/renderPackage writers. // Where the corpus lives. Overridable with SLEELA_DOCS_DIR; else /api/ // sleela/docs under the discovered /lib root's parent (the repo/install root). std::string corpusDir(const library::Index& ix){ if(const char* e=std::getenv("SLEELA_DOCS_DIR"); e&&*e) return e; fs::path lib(ix.root()); fs::path base = lib.has_parent_path()? lib.parent_path() : lib; // repo/install root return (base/"api"/"sleela"/"docs").generic_string(); } // Emit one cached page if it exists; returns true when served from cache. bool serveCached(std::ostream& o, const std::string& dir, const std::string& pkg, const std::string& unit){ fs::path p = fs::path(dir)/pkg/(unit+".txt"); std::ifstream in(p); if(!in) return false; o<> byPkg; for(const auto& s:ix.symbols()){ if(s.path.find("early_access")!=std::string::npos) continue; if(!corpusIncludes(s.package)) continue; byPkg[s.package].push_back(&s); } size_t units=0, pages=0; for(auto& kv:byPkg){ const std::string& pkg=kv.first; if(pkg=="__root__") continue; fs::create_directories(fs::path(outDir)/pkg,ec); // package overview { std::ostringstream po; renderPackage(po,ix,pkg); std::ofstream pf((fs::path(outDir)/pkg/"_package.txt").string()); pf<namename;}); for(auto* s:us){ ++units; UnitDoc u=parseUnit(*s,ix.root()); if(u.classes.empty()) continue; std::ostringstream cp; for(const auto& c:u.classes) renderClass(cp,u,c); std::ofstream cf((fs::path(outDir)/pkg/(s->name+".txt")).string()); cf<name<<"\n"; ++pages; } } std::cout<<"SleelaDoc corpus written to "<` searches // the filesystem AT RUNTIME for files belonging to the topic and renders a // facade report. The search is bounded by three user-overridable caps and // always prints an honest, partial report naming whichever cap stopped it: // --facade-max-mb N working-memory ceiling (default 6 MB) // --facade-max-files N files examined ceiling (default 6000) // --facade-max-seconds N wall-clock ceiling (default 360 = 6 min) // --facade-root DIR where to search (default: the SLeeLa root; may // be widened up to "/" at the user's choice) struct FacadeCaps { size_t maxBytes=6*1024*1024; size_t maxFiles=6000; long maxSeconds=360; }; struct FacadeHit { std::string path; std::string kind; std::string peek; uintmax_t size=0; }; std::string classifyKind(const fs::path& p){ std::string e=lower(p.extension().string()); if(e==".sleela") return "source (.sleela)"; if(e==".md"||e==".txt"||e==".rst"||e==".adoc") return "document"; if(e==".c"||e==".h"||e==".cpp"||e==".hpp"||e==".cc"||e==".mm") return "native source"; if(e==".py"||e==".sh"||e==".ps1"||e==".bat") return "script"; if(e==".json"||e==".xml"||e==".csv"||e==".yaml"||e==".yml"||e==".sst"||e==".sheet") return "data"; if(e==".key"||e==".pem"||e==".bin") return "binary/key"; return "other"; } bool looksText(const fs::path& p){ std::string e=lower(p.extension().string()); static const std::vector bin={".key",".pem",".bin",".png",".jpg",".jpeg",".gif",".zip",".jar",".gz",".o",".a",".so",".class"}; for(const auto& b:bin) if(e==b) return false; return true; } // First non-empty, non-trivial line of a text file, trimmed to 120 chars. std::string firstLinePeek(const fs::path& p){ std::ifstream in(p); if(!in) return ""; std::string ln; int guard=0; while(std::getline(in,ln) && guard++<40){ std::string s=trim(ln); if(s.empty()) continue; // skip pure comment/marker lines for a more useful peek if(s.size()>120) s=s.substr(0,117)+"..."; return s; } return ""; } // Should this directory be skipped during the walk? bool skipDir(const std::string& name){ return name==".git" || name=="node_modules" || name=="build" || name==".cache" || name=="__pycache__" || name==".svn"; } int renderFacade(std::ostream& o, const library::Index& ix, const std::string& topic, const std::string& rootArg, const FacadeCaps& caps){ // Resolve the search root: explicit arg, else env, else the SLeeLa root // (parent of /lib) so sibling top-level subject dirs are found by default. std::string root=rootArg; if(root.empty()) if(const char* e=std::getenv("SLEELA_FACADE_PATH"); e&&*e) root=e; if(root.empty()){ fs::path lib(ix.root()); root=(lib.has_parent_path()?lib.parent_path():lib).generic_string(); } fs::path rootP(root); std::error_code ec; if(!fs::is_directory(rootP,ec)){ o<<"sleela docs --facade: search root is not a directory: "< hits; size_t filesSeen=0, bytesHeld=0, dirsSeen=0; auto t0=std::chrono::steady_clock::now(); const char* stopReason=nullptr; bool topicDirMatch=false; auto elapsedS=[&]{ return (long)std::chrono::duration_cast(std::chrono::steady_clock::now()-t0).count(); }; // PHASE 1 -- locate directories named (case-insensitive) anywhere // under the root. A facade topic's real material lives in a directory of // that name; finding those first lets the file caps apply to the RELEVANT // subtree(s) rather than being spent walking unrelated trees. The dir scan // is itself bounded by the file/time caps (counting directories visited). std::vector topicDirs; // If the root directory itself is named the topic, it is a match -- a user // who points --facade-root straight at the topic dir expects its contents. { fs::path canon=fs::weakly_canonical(rootP,ec); if(ec){ ec.clear(); canon=rootP; } if(lower(canon.filename().string())==tL) topicDirs.push_back(rootP); } { fs::recursive_directory_iterator dit(rootP, fs::directory_options::skip_permission_denied, ec), dend; if(ec){ o<<"(cannot read search root: "<=caps.maxSeconds){ stopReason="time"; break; } std::error_code dec; if(!dit->is_directory(dec)){ dec.clear(); continue; } const fs::path& d=dit->path(); if(skipDir(d.filename().string())){ dit.disable_recursion_pending(); continue; } if(++dirsSeen>=caps.maxFiles){ stopReason="files"; break; } // reuse the files cap for the dir scan if(lower(d.filename().string())==tL) topicDirs.push_back(d); } } // PHASE 2 -- within each matched topic directory, collect its FOLDERS and // its regular files (recursively), honoring the three caps. The folders are // the shape of the subject; the files follow as a matter of natural motion. // Every directory and file under a topic-named directory belongs to the // topic. The matched topic directories themselves are folders of record. std::vector folders; auto addFolder=[&](const fs::path& d){ FacadeHit f; f.path=d.generic_string(); f.kind="folder"; f.size=0; size_t cost=f.path.size()+64; if(bytesHeld+cost>caps.maxBytes){ stopReason="memory"; return false; } bytesHeld+=cost; folders.push_back(std::move(f)); return true; }; if(!stopReason) for(const auto& base:topicDirs){ topicDirMatch=true; if(!addFolder(base)) break; // the topic folder itself fs::recursive_directory_iterator it(base, fs::directory_options::skip_permission_denied, ec), end; if(ec){ ec.clear(); continue; } for(; it!=end; it.increment(ec)){ if(ec){ ec.clear(); continue; } if(elapsedS()>=caps.maxSeconds){ stopReason="time"; break; } std::error_code fec; if(it->is_directory(fec)){ if(skipDir(it->path().filename().string())){ it.disable_recursion_pending(); continue; } if(++dirsSeen>=caps.maxFiles){ stopReason="files"; break; } if(!addFolder(it->path())){ break; } // folders shown implicitly continue; } if(fec){ fec.clear(); continue; } if(!it->is_regular_file(fec)){ fec.clear(); continue; } if(++filesSeen>=caps.maxFiles){ stopReason="files"; break; } const fs::path& p=it->path(); FacadeHit h; h.path=p.generic_string(); h.kind=classifyKind(p); h.size=fs::file_size(p,fec); if(fec){ fec.clear(); h.size=0; } if(looksText(p)) h.peek=firstLinePeek(p); size_t cost=h.path.size()+h.peek.size()+64; if(bytesHeld+cost>caps.maxBytes){ stopReason="memory"; break; } bytesHeld+=cost; hits.push_back(std::move(h)); } if(stopReason) break; } // Group by kind for a legible report. std::sort(hits.begin(),hits.end(),[](const FacadeHit&a,const FacadeHit&b){ if(a.kind!=b.kind) return a.kind to widen)\n"; } else { // Folders first -- the shape of the subject, shown implicitly. if(!folders.empty()){ o<=2)? words[1] : corpusDir(ix); return generateCorpus(ix,dir); } // Prefer a pre-generated corpus unless the caller forces live parsing with // `--live`. When forced, drop the flag and continue. bool forceLive=false; if(!words.empty() && words[0]=="--live"){ forceLive=true; words.erase(words.begin()); } std::string cdir=corpusDir(ix); bool haveCorpus = !forceLive && fs::is_directory(cdir); if(words.empty()){ listAll(o,ix); return 0; } // Two-word form: if(words.size()>=2){ const std::string& pkg=words[0]; const std::string& cls=words[1]; for(const auto& s:ix.symbols()) if(lower(s.package)==lower(pkg) && lower(s.name)==lower(cls)){ if(haveCorpus && serveCached(o,cdir,s.package,s.name)) return 0; UnitDoc u=parseUnit(s,ix.root()); if(u.classes.empty()){ o<<"(unit '"<=2?words[1]:words[0]; std::string tL=lower(topic); // 1) exact package? if(ix.hasPackage(topic) || [&]{ for(const auto& s:ix.symbols()) if(lower(s.package)==tL) return true; return false; }()){ std::string pkg; for(const auto& s:ix.symbols()) if(lower(s.package)==tL){ pkg=s.package; break; } if(pkg.empty()) pkg=topic; renderPackage(o,ix,pkg); return 0; } // 2) exact class (case-insensitive)? Collect EVERY match so a name shared // across packages disambiguates rather than silently picking one. Prefer // units that actually declare a class, and rank SL* standard-library // units ahead of the large java compat mirror. std::vector exact; std::string slForm = "sl"+tL; // standard-library classes are SL-prefixed for(const auto& s:ix.symbols()){ std::string nL=lower(s.name); if(nL==tL || nL==slForm) exact.push_back(s); } if(!exact.empty()){ auto rank=[](const library::Symbol& s){ int r=0; if(s.package=="java") r+=100; // compat mirror: deprioritise if(s.path.find("early_access")!=std::string::npos) r+=50; if(s.name.rfind("SL",0)==0) r-=10; // SL* standard library: prefer return r; }; std::sort(exact.begin(),exact.end(),[&](const library::Symbol&a,const library::Symbol&b){return rank(a)> good; for(const auto& s:exact){ UnitDoc u=parseUnit(s,ix.root()); if(!u.classes.empty()) good.push_back({s,u}); } if(good.empty()){ o<<"(no class declaration found for '"<1){ o<<"'"<