diff --git a/README.md b/README.md index ac50c5c..20f4835 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ are regenerated on each release — patches cannot be merged directly. Report problems (or send patches) to the author; accepted changes are applied to the development tree and appear in a following snapshot. -This snapshot was assembled from development commit `8173c371bba6`. +This snapshot was assembled from development commit `6594c7ff8c1f`. ## License diff --git a/com/Makefile b/com/Makefile index a326ca6..39a16b2 100644 --- a/com/Makefile +++ b/com/Makefile @@ -92,6 +92,10 @@ else $(MAKE) -C $(KS)/kutil clean $(MAKE) -C $(KS)/target clean $(MAKE) -C $(KS)/document clean + # ref/reflect.so links against libklammertext.so and is dlopened, so + # it shares the class layouts: a stale one after a header change is + # the same memory-corruption hazard as a stale sks/document.so. + $(MAKE) -C $(K)/ref clean $(MAKE) clean $(MAKE) all endif diff --git a/com/kdesc.cpp b/com/kdesc.cpp index 0f25e8f..8c15ec7 100644 --- a/com/kdesc.cpp +++ b/com/kdesc.cpp @@ -5,6 +5,7 @@ #include "file.h" #include "font_store.h" #include "ktype.h" +#include "katom_rewrite_rules.h" #include "log.h" #include "argtype_registry.h" #include "klammerset_registry.h" diff --git a/com/kdiag.cpp b/com/kdiag.cpp index 0a2a8be..4a1d8e4 100644 --- a/com/kdiag.cpp +++ b/com/kdiag.cpp @@ -11,6 +11,7 @@ #include "error.h" #include "file.h" #include "katom.h" +#include "katomize.h" #include "log.h" #include "show.h" diff --git a/doc/edit/emacs/README.md b/doc/edit/emacs/README.md index 4ce6290..4bda794 100644 --- a/doc/edit/emacs/README.md +++ b/doc/edit/emacs/README.md @@ -76,7 +76,8 @@ The name ends at the first non-name character, so a target suffix like body (between `@@name` and the closing `@@`) is highlighted like ordinary Klammertext — e.g. an `@i … @` inside it shows as a normal application. -**System/target commands** (`@@@`) — `@@@target`, `@@@argtype`, `@@@state` — get +**System/target commands** (`@@@`) — `@@@target`, `@@@argtype`, `@@@state`, +`@@@klammerset` — get a third pair of colors, so the three `@`-levels (application, definition, system) are visually distinct: diff --git a/doc/edit/emacs/klammertext-mode.el b/doc/edit/emacs/klammertext-mode.el index 3de0dea..45f0a21 100644 --- a/doc/edit/emacs/klammertext-mode.el +++ b/doc/edit/emacs/klammertext-mode.el @@ -382,7 +382,8 @@ POS and POS+1 are both @. BEFORE is the character before POS. Mirrors (defun klammertext--emit-system (pos before) "Emit a @@@ system/target delimiter token at POS (groups 8 open / 9 close). POS, POS+1 and POS+2 are all @. BEFORE is the character before POS. These -commands (@@@target, @@@argtype, @@@state) do not nest, so each delimiter is +commands (@@@target, @@@argtype, @@@state, @@@klammerset) do not nest, so +each delimiter is coloured independently, mirroring `klammertext--emit-def' with @@@ for @@." (let ((after (char-after (+ pos 3)))) ; char right after the @@@ (cond diff --git a/doc/klammertext_for_llm.md b/doc/klammertext_for_llm.md index ab77cfa..b4603a3 100644 --- a/doc/klammertext_for_llm.md +++ b/doc/klammertext_for_llm.md @@ -47,7 +47,7 @@ target coverage is wrong — including inside branches this render will not take It applies nothing, so it is safe and fast. Use it before claiming a document is correct. -**Inspect how text is being parsed:** `kdiag --type ''` shows the tokens +**Inspect how text is being parsed:** `kdiag --type ''` shows the katoms ("katoms") and their types. This settles arguments about what a piece of syntax actually means. @@ -57,7 +57,7 @@ actually means. | assumption | reality | |---|---| -| It is a macro language like TeX | No catcodes. Token structure is fixed when a file is read and cannot be changed by anything later. This is why static checking works. | +| It is a macro language like TeX | No catcodes. Katom structure — the indivisible input units, which `kdiag` displays — is fixed when a file is read and cannot be changed by anything later. This is why static checking works. | | `\command{arg}` | `@name arg @` — an `@` opens, and a **closing delimiter is required**. | | Arguments are `{a}{b}` | Positional arguments are separated by `|` — `@sup 2 \| 3 @`. | | Options are `[key=value]` | `:name value`, anywhere in the argument list — `@link URL :text Click @`. | @@ -69,7 +69,7 @@ actually means. --- -## 3. Lexical core +## 3. Core syntax Three prefix tiers, distinguished by how many `@`: @@ -110,7 +110,7 @@ klammer's name immediately before it: `@i text i@. **Abbreviated form.** `@name-arg1-arg2` is `@name arg1 | arg2 @`, valid only while every argument is letters and digits: `@i-italic`, `@sup-2-3`, `@date@`. -**Literal klammers** take their content uninterpreted. It is not scanned for +**Literal klammers** take their content uninterpreted. It is not examined for `@`, `|`, `#`, or `^`, so the machine cannot find the end by counting and **the named close is required**: diff --git a/mac/.gitignore b/mac/.gitignore index a438335..62661b6 100644 --- a/mac/.gitignore +++ b/mac/.gitignore @@ -1 +1,27 @@ +# Build products *.d + +# Editor and tooling state. These are here so a bulk "git add" cannot sweep +# them into a commit -- which happened on 2026-09-02 and put 64 of them into +# a pushed commit. Ignoring them is the structural fix; the behavioural one +# is in CLAUDE.md (never a staging verb whose scope is implicit). +.cache/ +.vscode/ +compile_commands.json +.clangd +\#*\# +.#* +*~ + +# Working files and hand backups kept beside the source +*.bu +*.bak +*.orig +*.before_* + +# Generated dependency/analysis artifacts +*.dot +headers.mk +objects.mk +sources.mk +iwyu.imp diff --git a/mac/Makefile b/mac/Makefile index 959fe37..87f2b11 100644 --- a/mac/Makefile +++ b/mac/Makefile @@ -7,7 +7,8 @@ KS := $(K)/sks include $(K)/env/makefile.env # Source files -BASENAMES := util error locator file argv character ktype katom katom_list \ +BASENAMES := util error locator file argv character ktype katom katom_process \ + katomize katom_rewrite_rules katom_matching \ log show command argument argument_set argtype argtype_registry \ state eval eval_python eval_cpp klammer klammer_registry klammerset klammerset_registry \ option_set option_set_registry deftype \ @@ -30,10 +31,11 @@ DEPFLAGS = -MMD -MP -MF $(@:.o=.d) $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(DEPFLAGS) $< -o $@ # Default target -.PHONY: all sks clean redo clang +.PHONY: all sks ref clean redo clang all : $(LIBRARY) $(MAKE) sks + $(MAKE) ref # Create lib directory $(LIBDIR): @@ -49,6 +51,15 @@ sks : $(MAKE) -C $(KS)/target $(MAKE) -C $(KS)/document +# ref/ is NOT part of a klammer set: it is the reflection library, whose +# C++ reads the engine's OWN registries and character tables and returns +# Klammertext. It is built here because, like the sks DSOs, it links +# against libklammertext.so -- but it is a separate target so it does not +# become "part of sks" by association, which is exactly the confusion the +# klammer-set naming is trying to undo. +ref : + $(MAKE) -C $(K)/ref + clean : rm -f $(OBJECTS) $(DEPFILES) $(LIBRARY) *~ diff --git a/mac/argument_set.cpp b/mac/argument_set.cpp index 6b13e9d..69848c3 100644 --- a/mac/argument_set.cpp +++ b/mac/argument_set.cpp @@ -7,6 +7,8 @@ #include "katom.h" #include "argtype_registry.h" #include "argument_set.h" +#include "katomize.h" +#include "file.h" #include "util.h" bool operator==(Parameter_set lhs, Parameter_set rhs) @@ -60,7 +62,7 @@ Parameter_set::Parameter_set(const std::string parameter_string) (void)K::log(3); Argtype_registry argtypes {}; parse_parameters( - katomize(line_split(parameter_string), Locator().str()), + katomize_lines(line_split(parameter_string), Locator().str()), argtypes); } diff --git a/mac/character.cpp b/mac/character.cpp index 1d52999..03152ec 100644 --- a/mac/character.cpp +++ b/mac/character.cpp @@ -22,6 +22,7 @@ static std::string compose(const std::string& base, const std::string& mark_cp) return base + utf8char(std::stoi(mark_cp, nullptr, 16)); } +/* inline std::string klammertext_special_characters { "@|*^#" }; @@ -33,6 +34,7 @@ std::string diacritic_symbols = "-'`h~\"cbrdwa"; inline std::string diacritic_symbols_order = "'`h~\"c-brdwa"; +*/ std::string utf8char(int cp) { @@ -122,6 +124,12 @@ std::string process_diacritics(std::string s) return result; } +// A delimited caret form left unresolved by every pass in encode(). The +// character class matches the nonascii Ktype's second alternative exactly: +// what the lexer accepts as a shape, encode() must either resolve or +// reject, or the two disagree about what is writable. +const std::regex unresolved_nonascii_re(R"(\^[A-Za-z0-9]{1,6}\^)"); + std::string extended_latin_symbol_pattern() { std::string result {}; @@ -234,6 +242,26 @@ std::string encode(const std::string& s, const Locator& loc) result = process_pinyin(result); if (dbg) std::cout << "pinyin: " << result << "\n"; + // A "^...^" form that survives every pass above resolves to no + // character: it is not in the extended-Latin table, not a code point, + // not a diacritic, not a pinyin tone. That is an ERROR here rather + // than text in the document -- the same rule as an invalid code point + // a few lines up, and for the same reason: a typo must not reach a + // finished-looking document as literal "^zz^". Only the DELIMITED + // form is checked. The two-character form (^a, ^-) is deliberately + // exempt: quoted punctuation resolves later, in hide_quoted_punctuation(). + if (std::regex_search(result, unresolved_nonascii_re)) { + std::smatch hit {}; + std::regex_search(result, hit, unresolved_nonascii_re); + throw Parsing_error("\"" + hit[0].str() + "\" is not a character code. " + "It names no extended-Latin mnemonic and is not a " + "Unicode code point. The forms are:\n\n" + " ^oe^ a mnemonic (kdesc -c lists them)\n" + " ^263A^ a code point, 4 to 6 hex digits\n" + " ^o\" a diacritic on a base letter\n", + loc); + } + result = string_replace(result, lit_start, "^'"); result = string_replace(result, lit_end, "'^"); @@ -259,6 +287,7 @@ void write_kt_example_file(std::stringstream& kt, std::string kt_filename) } +/* void diacritics_examples(const std::string& kt_filename) { if (diacritics.size() != diacritic_symbols_order.size()) { @@ -297,8 +326,9 @@ void diacritics_examples(const std::string& kt_filename) if (write_kt_file) write_kt_example_file(kt, kt_filename); } +*/ - +/* void extended_latin_examples(const std::string& kt_filename) { std::stringstream kt {}; @@ -328,7 +358,7 @@ void extended_latin_examples(const std::string& kt_filename) if (write_kt_file) write_kt_example_file(kt, kt_filename); } - +*/ void pinyin_examples(const std::string& kt_filename) { @@ -357,6 +387,11 @@ void pinyin_examples(const std::string& kt_filename) write_kt_example_file(kt, kt_filename); } +/* + Outside my window + crows build a nest in a tree... + slowly... carefully. +*/ void unicode_examples(const std::string& kt_filename) { @@ -399,7 +434,7 @@ void unicode_examples(const std::string& kt_filename) void show_special_characters() { std::cout << std::setfill(' '); - diacritics_examples(); - extended_latin_examples(); + // diacritics_examples(); + // extended_latin_examples(); pinyin_examples(); } diff --git a/mac/character.h b/mac/character.h index 2e757ce..bc069a9 100644 --- a/mac/character.h +++ b/mac/character.h @@ -13,6 +13,18 @@ #include "locator.h" +inline +std::string klammertext_special_characters { "@|*^#" }; + +inline +std::string encoding_marker { "UU" }; + +inline +std::string diacritic_symbols = "-'`h~\"cbrdwa"; + +inline +std::string diacritic_symbols_order = "'`h~\"c-brdwa"; + // ^UUUU^ — a Unicode code point in hex, 4 to 6 digits (^263A^ is the BMP, // ^13000^ EGYPTIAN HIEROGLYPH A001, ^10FFFD^ the top of the range), or a // single digit. Lengths 2-3 are NOT accepted: two- and three-letter @@ -131,7 +143,7 @@ std::string encode(const std::string& s, const Locator& loc); // UTF-8 encoding of one code point (to U+10FFFF; surrogates and values // above the range yield an "Invalid Unicode: N" text). Used by the // character tables here and by the ^UUUU UUUU ...^ sequence pass -// (process_unicode_sequences in katom_list.cpp). +// (process_unicode_sequences in katom_process.cpp). std::string utf8char(int cp); // Whether cp may be written in a ^...^ form: within the Unicode range, @@ -143,8 +155,9 @@ std::string utf8char(int cp); bool valid_codepoint(int cp); std::string invalid_codepoint_reason(int cp); -void diacritics_examples(const std::string& kt_filename = ""); -void extended_latin_examples(const std::string& kt_filename = ""); +// void diacritics_examples(const std::string& kt_filename = ""); +// void extended_latin_examples(const std::string& kt_filename = ""); + void pinyin_examples(const std::string& kt_filename = ""); void unicode_examples(const std::string& kt_filename = ""); diff --git a/mac/check.cpp b/mac/check.cpp index 4f0b199..b6676f4 100644 --- a/mac/check.cpp +++ b/mac/check.cpp @@ -6,7 +6,7 @@ #include "check.h" #include "machine.h" #include "katom.h" -#include "katom_list.h" +#include "katom_process.h" #include "util.h" namespace { diff --git a/mac/coverage.cpp b/mac/coverage.cpp index d51db04..a480cd1 100644 --- a/mac/coverage.cpp +++ b/mac/coverage.cpp @@ -4,6 +4,7 @@ #include #include +#include "file.h" #include "coverage.h" #include "machine.h" #include "katom.h" @@ -121,9 +122,28 @@ std::string short_path(const std::string& path) } const char* home = std::getenv("KLAMMERTEXT_HOME"); if (home != nullptr) { - std::string prefix = std::string(home) + "/"; - if (path.size() > prefix.size() && path.compare(0, prefix.size(), prefix) == 0) { - return path.substr(prefix.size()); + // Compare CANONICAL paths, not the spellings. The engine resolves a + // definition's file to its physical path, while KLAMMERTEXT_HOME is + // whatever the user typed -- and on macOS those routinely differ, + // because /tmp and /var are symlinks to /private/tmp and + // /private/var. A plain prefix test then fails and every row prints + // its full path instead of the tail, which is exactly what the + // report exists to avoid. Seen 2026-09-07 on the distribution leg, + // where KLAMMERTEXT_HOME was /var/folders/.../dist and the files + // resolved to /private/var/folders/.../dist. + // + // weakly_canonical rather than canonical: it does not require the + // path to exist, so a stale or generated location still shortens + // instead of throwing. Errors are swallowed -- an unshortened path + // is a cosmetic loss, never a reason to fail a report. + std::error_code ec {}; + fs::path home_real = fs::weakly_canonical(fs::path(home), ec); + fs::path path_real = fs::weakly_canonical(fs::path(path), ec); + std::string prefix = (ec ? std::string(home) : home_real.string()) + "/"; + const std::string& subject = ec ? path : path_real.string(); + if (subject.size() > prefix.size() + && subject.compare(0, prefix.size(), prefix) == 0) { + return subject.substr(prefix.size()); } } return path; diff --git a/mac/file.cpp b/mac/file.cpp index 6fe203e..53912fc 100644 --- a/mac/file.cpp +++ b/mac/file.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -697,3 +698,31 @@ void copy_preserving_basename( fs::path base_dir = is_directory(base) ? base : base.parent_path(); return fs::weakly_canonical(base_dir / relative); } + +strings_t line_split(const std::string& s) +{ + std::istringstream is(s); + strings_t result {}; + std::string line; + while (std::getline(is, line)) { + result.push_back(line); + } + result.push_back("\n"); + return result; +} + +std::pair line_split(const fs::path& pathname) +{ + std::string source {}; + strings_t lines {}; + std::ifstream infile(pathname); + std::string line; + while (std::getline(infile, line)) { + source += line + "\n"; + lines.push_back(line); + } + if (!source.empty()) { + source.pop_back(); + } + return {source, lines}; +} diff --git a/mac/file.h b/mac/file.h index 74c883c..a6a72b6 100644 --- a/mac/file.h +++ b/mac/file.h @@ -18,6 +18,16 @@ fs::path relative_to_cwd(const fs::path& input); bool file_exists(const std::string& pathname, bool error_if_not=false, bool is_directory=false); +// Text to its lines, and a file to (its whole text, its lines). These +// moved here from katom.{h,cpp} on 2026-09-02: neither produces a katom, +// and the second OPENS A FILE, so by the test in katomize.h -- does a +// character stream enter? -- they were in the wrong place. They are read +// almost exclusively as katomize_lines(line_split(text), source), which is +// what kept them next to katomization for so long; adjacency at the call +// site is not membership. +std::vector line_split(const std::string& s); +std::pair> line_split(const fs::path& pathname); + // Filenames may contain spaces. A filename LIST in a flat string or an // argument vector is separated by a standalone "/" token (whitespace on both // sides; never a legal input filename -- "/" alone is the root directory). diff --git a/mac/katom.cpp b/mac/katom.cpp index 31e141b..8bf3753 100644 --- a/mac/katom.cpp +++ b/mac/katom.cpp @@ -5,6 +5,7 @@ #include "katom.h" #include "ktype.h" +#include "katomize.h" #include "error.h" #include "log.h" #include "show.h" @@ -13,8 +14,20 @@ bool dbg = false; +// KATOMIZATION IS NOT HERE. Turning text into katoms moved to +// katomize.{h,cpp} (2026-09-02), leaving this file the KATOM: what it is, +// what its type means, and the operations on a list of them. The test that +// decided the split is in katomize.h -- does a character stream enter? +// Everything below answers no. + size_t Katom::index = 0; +// A katom holds its source text and its text, and decides nothing about +// either. It used to strip the "^" from a quoted special here -- knowledge +// of what a construct looks like IN SOURCE, inside the datum, which is the +// one thing the katom/katomization split says does not belong here. That +// moved to source_katom() in katomize.{h,cpp} on 2026-09-02, the one place +// a katom is made from matched source text. Katom::Katom(const std::string& src, katom_t type, Locator loc) : m_index(Katom::index++) , m_text(src) @@ -23,9 +36,6 @@ Katom::Katom(const std::string& src, katom_t type, Locator loc) , m_type(type) , m_initial_type(type) { - if (m_type == katom_t::special && m_text[0] == '^') { - m_text.erase(0, 1); - } } int active_count(const katom_list& katoms) @@ -45,146 +55,10 @@ bool active(const katom_list& katoms) return active_count(katoms) > 0; } -std::string expand_compound_katom(const std::string& s, std::regex rgx, const std::string& expanded) -{ - if (s.find('-') != std::string::npos) { // Hyphen shortcut for arguments that allow it - const std::regex shortcut_rgx(R"((@\w+)(-\w+)+)"); - std::smatch match {}; - if (std::regex_match(s, match, shortcut_rgx)) { - std::string modified = string_replace(s, "-", " | ") + " @"; - return std::regex_replace(modified, std::regex(R"((@\w+) \|)"), "$1"); - } - } - return std::regex_replace(s, rgx, expanded); -} -katom_list make_katoms_from_word(std::string s, const std::string& source_desc, int line, int chr) -{ - if (dbg) msg() << " make word: " << broken_bar << s << broken_bar << " [" << chr << "]\n"; - auto ktyp = std::find_if(katom_types.begin(), katom_types.end(), [&](const auto& ktype) { return ktype.match(s); }); - if (ktyp != katom_types.end()) { - return { Katom(s, ktyp->m_type, Locator(source_desc, line, chr)) }; - } else { - if (dbg) { - std::cout << "\nBREAK: |" << s << "|\n"; - } - for (const auto& [desc, rgx, expanded] : katom_rewrite_rules) { - std::string modified = expand_compound_katom(s, rgx.m_regex, expanded); - if (dbg) { - msg() << "Rewrite: " << desc << " " << rgx.m_pattern << " " << expanded << "\n"; // output-policy-ok (dbg) - } - if (modified != s) { - auto parts = word_split(modified); - if (dbg) std::cout << " Parts: " << parts << "\n"; - if (show_rewrite_rules) { - Locator loc(source_desc, line, chr); - std::cout << loc << " Rewrite (" << desc << "): " << rgx.m_pattern - << " " << right_arrow << " " << expanded << "\n"; - } - katom_list klist {}; - for (const std::string& p : parts) { - if (!p.empty()) { - auto ks = make_katoms_from_word(p, source_desc, line, chr); - std::copy(ks.begin(), ks.end(), std::back_inserter(klist)); - } - } - return klist; - } - } - Katom k(s, katom_t::word, Locator(source_desc, line, chr)); - k.m_unparsed = true; // Warning deferred to warn_unparsed_katoms() - return std::vector{ k }; - } -} -// Warn about words that matched no katom type -- but only those that -// survive processing: text removed by #, ##, or #[...]#, replaced spans, -// and literal content (definition interiors, @code bodies) never warn. -// Called at the end of Machine::process_katoms(), after those passes have -// marked the katoms. Clears the flag after warning so repeated processing -// of the same katom list does not warn twice. With warn=false (the @eval -// read-back sub-Machine, whose katoms hold machine-generated result text) -// no warning is printed and every flag is cleared, so the katoms stay -// silent after they are spliced into the calling Machine's list. -void warn_unparsed_katoms(katom_list& katoms, bool warn) -{ - for (Katom& k : katoms) { - if (!k.m_unparsed) continue; - if (!warn) { - k.m_unparsed = false; - continue; - } - if (k.m_type != katom_t::ignored - && k.m_type != katom_t::replaced - && k.m_type != katom_t::literal) { - // One of the two surviving WARNINGS (the commands otherwise treat - // an anomaly as an error and stop -- see the output policy in - // CLAUDE.md). It stays a warning because the judgment is a - // HEURISTIC, not exact: the @eval read-back legitimately fails to - // parse raw target markup, which is why m_warn_unparsed exists. A - // heuristic that halts turns every false positive into a blocker on - // a document that would otherwise render. Promote it to an error - // when the judgment becomes exact. - // - // Through warning(), not a raw std::cerr, so every warning the - // commands issue has one format: it said "[warning]" where - // everything else says "(warning)". - warning("Word not parsed:\n " + k.m_text - + "\nTo include a punctuation character literally, put " - "\"^\" before it.", - k.m_loc); - k.m_unparsed = false; - } - } -} -katom_list split_into_katoms(std::string s, const std::string& source, int source_line) -{ - if (dbg) { - msg() << "Make katoms: " << s << "<\n"; // output-policy-ok (dbg) - } - // const std::string middle_dot { "\u00B7" }; - s = string_replace(s, "\r", ""); - s = string_replace(s, "\t", " "); - - //std::regex words_regex("^\||[ ]|[\\n]|[^\\s]+|.+"); - //std::regex words_regex(R"((?:[^][|])|[ ]|[\n]|[^\s]+|.+)"); - std::regex words_regex(R"([ ]|[\n]|[^\s]+|.+)"); - auto words_begin = std::sregex_iterator(s.begin(), s.end(), words_regex); - auto words_end = std::sregex_iterator(); - if (dbg) { - std::cout << "Found " << std::distance(words_begin, words_end) << " words:\n"; - } - strings_t atoms {}; - for (std::sregex_iterator iter = words_begin; iter != words_end; ++iter) { - if (dbg) { - std::cout << middle_dot << iter->str(); - } - atoms.push_back(iter->str()); - } - if (dbg) std::cout << middle_dot << "\n"; - //std::cout << kall << ktype; - - katom_list result {}; - - int cpos = 0; - for (const auto& a : atoms) { - auto k = make_katoms_from_word(a, source, source_line, cpos); - for (auto& kk : k) { - Locator loc(source, source_line, cpos); - //kk->m_loc = loc; - kk.m_loc = loc; - //m_katoms.push_back(kk); - result.push_back(kk); - if (dbg) { - //std::cout << "LOOP: " << kk.m_text << " - " << cpos << "\n"; - } - cpos += kk.m_src.size(); - } - } - return result; -} void restore_initial_type(katom_iter begin, katom_iter end) { @@ -292,51 +166,11 @@ katom_list trim(const katom_list& katoms, bool trim_inactive) return trim(result, trim_types); } -strings_t line_split(const std::string& s) -{ - std::istringstream is(s); - strings_t result {}; - std::string line; - while (std::getline(is, line)) { - result.push_back(line); - } - result.push_back("\n"); - return result; -} - -std::pair line_split(const fs::path& pathname) -{ - std::string source {}; - strings_t lines {}; - std::ifstream infile(pathname); - std::string line; - while (std::getline(infile, line)) { - source += line + "\n"; - lines.push_back(line); - } - if (!source.empty()) { - source.pop_back(); - } - return {source, lines}; -} // I'm, like, this is // some weird shit; // what the fuck? -katom_list katomize(const strings_t& lines, const std::string& source_desc) -{ - (void)K::log(3); - katom_list katoms {}; - int i = 0; - for (std::string line : lines) { - (void)K::log(4, line); - katom_list ks = split_into_katoms(line + "\n", source_desc, ++i); - katoms.insert(katoms.end(), ks.begin(), ks.end()); - } - katoms.pop_back(); - return katoms; -} // Whitespace diff --git a/mac/katom.h b/mac/katom.h index 494256f..d057e84 100644 --- a/mac/katom.h +++ b/mac/katom.h @@ -7,8 +7,6 @@ #include "locator.h" #include "util.h" -inline bool show_rewrite_rules = false; - class Katom { public: @@ -87,7 +85,6 @@ public: bool active(const std::vector& katoms); int active_count(const std::vector& katoms); -std::vector split_into_katoms(std::string s, const std::string& source, int source_line); void restore_initial_type(std::vector::iterator begin, std::vector::iterator end); void modify_type(katom_t new_type, std::vector::iterator begin, std::vector::iterator end); void modify_type(katom_t old_type, katom_t new_type, @@ -108,11 +105,7 @@ std::vector trim(const std::vector& katoms, bool trim_inactive = f std::vector> bar_split(std::vector::iterator kbegin, std::vector::iterator kend); -std::vector line_split(const std::string& s); -std::pair> line_split(const fs::path& pathname); -std::vector katomize(const std::vector& lines, const std::string& source_desc); -void warn_unparsed_katoms(std::vector& katoms, bool warn = true); void process_whitespace_modifiers(std::vector& katoms); std::vector trim_whitespace(std::vector katoms); diff --git a/mac/katom_matching.cpp b/mac/katom_matching.cpp new file mode 100644 index 0000000..bd694c6 --- /dev/null +++ b/mac/katom_matching.cpp @@ -0,0 +1,262 @@ +// katom_matching.cpp -- finding katoms in text by matching. See +// katom_matching.h for what this is and is not. The whole of it is the +// three functions below. +// +// THE THREE RULES. Everything the direct katomizer does follows from these; there +// is nothing else to know, and no rule ordering to reconstruct. +// +// 1. LONGEST MATCH. At each position, every source katom type is tried +// anchored at that position, and the one matching the most characters +// wins. +// 2. DECLARATION ORDER BREAKS TIES. Two types matching the same length +// at the same position: the one written earlier in katom_types wins. +// This is the same precedence katomization by rewriting already uses, where +// std::find_if takes the first type that matches the whole text -- so +// the table's order keeps exactly the meaning it has today. +// 3. ALWAYS ADVANCE. Every step consumes at least one character. A +// position where nothing matches emits one character as an unparsed +// katom rather than stalling. +// +// Rule 3 is why the direct katomizer cannot have the failure that killed ktext on +// "^oe^" (2026-08-29): a rewrite fixed point can fire a rule, change the +// string, and still hand back the identical text. Here, non-termination is +// not something to guard against -- it is unavailable. +// +// WHAT IS NOT HERE, and deliberately. There is no whitespace splitting, so +// there are no rules to repair it. Roughly thirty of the thirty-four +// entries in katom_rewrite_rules say "these two katoms may abut without a +// space between them" -- "bar separator", "embedded klammer to the left", +// "trailing punctuation", "left parenthesis", "literal start to the right". +// A direct katomizer never joins two katoms into one piece of text, +// so it never needs to be told how to take one apart. Those thirty rules have no counterpart in +// this file; the four that remain are the abbreviated form, which is a +// separate pass at the bottom. + +#include +#include + +#include "katom_matching.h" +#include "katomize.h" +#include "ktype.h" +#include "util.h" + +namespace { + +// The abbreviated application @name-arg-arg. The one katom the katom_types +// table does not carry: today it lives as a special case at the top of +// expand_compound_katom() rather than as a type, which is why a trailing +// period defeats it (that regex_match is a WHOLE-text match, so "@i-arg." +// fails it) and why katom_rewrite_rules needs a rule to strip the period +// first. Here it is one pattern among the rest, and longest match keeps +// the period out of it with no rule at all. Its parts are \w+ rather than +// definition_name, matching what expand_compound_katom() accepts today. +const std::regex abbreviated_apply_rgx { R"(@\w+(?:-\w+)+)" }; + +// Processing states, not source categories -- see appears_in_source() in +// katom_matching.h. Listed by name rather than detected by an empty pattern, +// because ws_added's "pattern" is the DESCRIPTION " or \n" and would +// pass an emptiness test while matching nothing a document contains. +const std::set state_only_types { + katom_t::ws_added, + katom_t::literal, + katom_t::eval_result, + katom_t::replaced, + katom_t::ignored, +}; + +// PATTERNS THAT ASSUME A WHOLE-WORD MATCHER. This is the experiment's +// first real finding, and the only entry so far. +// +// katom_types is asked one question today: "does this WHOLE piece of text match?" +// A pattern written for that question may be sloppy about characters a +// piece of text cannot contain, because the whitespace split removed them. The +// "text" type is written [^^@#|]+ -- everything except the four structural +// characters -- and whitespace is not excluded because the whitespace +// split had already removed any. Under longest match that pattern eats spaces and newlines whole, +// and "@i arg @." is katomized as one text katom from the space to the period. +// +// So a direct katomizer needs [^^@#|\s]+ there. That is not a workaround: it is the +// pattern saying what it always meant, now that something asks it at every +// position instead of once per piece. Adopting the direct katomizer means making +// this edit in ktype.h; the override lives here so the experiment changes +// no shipped file. If more of these turn up, this table is the honest +// measure of what adoption costs. +const std::map matching_patterns { + // THIRD FINDING -- recorded, not fixed, because at this level it cannot + // be. The literal span "^'...'^" written in PROSE is katomized wrong: after + // the opening delimiter, text runs greedily to "...'", eating the + // apostrophe that literal-end ('\^) needed, and the lone caret then + // matches nothing. A lookahead stopping text before "'\^" repairs that + // one line and BREAKS SIX -- every argtype :pattern written + // "('style_code'^|\s)+", where the same two characters are an + // apostrophe ending a type reference followed by a quoted bar. + // Measured: sks/ went from 16 divergences to 22, so the lookahead is + // not here. + // + // "'^" is genuinely ambiguous and no amount of longest match decides + // it: what it means depends on whether a "^'" is OPEN, which is state, + // not lookahead. The standard answer is a katomizer MODE (flex calls them + // start conditions) -- one bool, set by literal-begin, cleared by + // literal-end, consulted here. That is a real fourth rule on top of + // the three at the top of this file, and it is left undone + // deliberately: whether the direct katomizer may carry state is a decision about + // the experiment, not a detail of it. The rewrite katomizer never meets + // the question, because word_split has already cut "^'...'^" out as its + // own piece before anything is asked. + { katom_t::text, std::regex(R"([^^@#|\s]+)") }, + + // Second finding, and a subtler one: within ONE pattern, ECMAScript + // alternation is ORDERED, not longest. The nonascii type is written + // (^Xy) | (^{1,6}^), and regex_search anchored at a caret takes + // the first alternative that succeeds -- so "^4E2D^" matched as "^4E", + // three characters, and the rest of the code point fell out as text. + // regex_match never sees this, because there both alternatives must + // consume the whole text and only the right one can. Longest match + // BETWEEN types does not rescue a pattern that is not longest-match + // WITHIN itself. The repair is to order the alternatives longest + // first; the pattern is otherwise identical. + { katom_t::nonascii, std::regex(R"((\^[A-Za-z0-9]{1,6}\^)|(\^(\w(?:.|\^))))") }, +}; + +// "@sup-2-3" -> { "@sup", "2", "3" }. Local rather than util.h's +// regex_split(), which trims its parts -- an argument's own whitespace is +// not this pass's to remove. +strings_t hyphen_split(const std::string& s) +{ + strings_t parts {}; + size_t start = 0; + for (size_t i = s.find('-'); i != std::string::npos; i = s.find('-', start)) { + parts.push_back(s.substr(start, i - start)); + start = i + 1; + } + parts.push_back(s.substr(start)); + return parts; +} + +// How many characters of s, starting at pos, this katom type matches; 0 if +// it does not match there. match_continuous is what anchors the match to +// pos -- without it regex_search would find the pattern anywhere later in +// the string, which is the search the rewrite rules do and the reason they +// need those (.*?) context groups to say where the match landed. +size_t match_length(const Ktype& ktype, const std::string& s, size_t pos) +{ + if (ktype.m_use_equal) { + return s.compare(pos, ktype.m_pattern.size(), ktype.m_pattern) == 0 + ? ktype.m_pattern.size() : 0; + } + auto override_pattern = matching_patterns.find(ktype.m_type); + const std::regex& rgx = override_pattern == matching_patterns.end() + ? ktype.m_rgx : override_pattern->second; + + std::smatch hit {}; + auto from = s.cbegin() + static_cast(pos); + if (std::regex_search(from, s.cend(), hit, rgx, + std::regex_constants::match_continuous)) { + return static_cast(hit.length(0)); + } + return 0; +} + +// Rules 1 and 2: the longest match at pos, earliest declaration winning a +// tie. Returns a length of 0 when no type matches. +std::pair longest_match(const std::string& s, size_t pos) +{ + katom_t best_type { katom_t::word }; + size_t best_length { 0 }; + + size_t abbreviated = 0; + std::smatch hit {}; + auto from = s.cbegin() + static_cast(pos); + if (std::regex_search(from, s.cend(), hit, abbreviated_apply_rgx, + std::regex_constants::match_continuous)) { + abbreviated = static_cast(hit.length(0)); + best_type = katom_t::apply_begin; + best_length = abbreviated; + } + + for (const Ktype& ktype : katom_types) { + if (!appears_in_source(ktype.m_type)) + continue; + size_t length = match_length(ktype, s, pos); + // Strictly greater: a tie leaves the earlier winner in place, which + // is rule 2. The abbreviated form is seeded above so that it wins + // a tie against apply_begin -- they can only tie when there is no + // hyphen, and then they are the same answer anyway. + if (length > best_length) { + best_length = length; + best_type = ktype.m_type; + } + } + return { best_type, best_length }; +} + +} // namespace + +bool appears_in_source(katom_t type) +{ + return !state_only_types.contains(type); +} + +katom_list katomize_by_matching(const std::string& text, const std::string& source, int source_line) +{ + // The same normalization split_into_katoms() does, so the two are + // compared on identical input rather than on a difference in what a tab + // means. + std::string s = string_replace(text, "\r", ""); + s = string_replace(s, "\t", " "); + + katom_list result {}; + size_t pos = 0; + while (pos < s.size()) { + auto [type, length] = longest_match(s, pos); + if (length == 0) { + // Rule 3. Nothing matched here, so take one character and say + // so. The rewrite katomizer reaches the same outcome by falling + // off the end of its rule list; the difference is that it can + // only do so for a whole whitespace-delimited piece, so "x@i" arrives as one + // unparsed katom where this reports the one character that is + // actually unaccounted for. + Katom k(s.substr(pos, 1), katom_t::word, + Locator(source, source_line, static_cast(pos))); + k.m_unparsed = true; + result.push_back(k); + pos += 1; + continue; + } + result.push_back(source_katom(s.substr(pos, length), type, + Locator(source, source_line, static_cast(pos)))); + pos += length; + } + return result; +} + +katom_list expand_abbreviations(const katom_list& katoms) +{ + katom_list result {}; + for (const Katom& k : katoms) { + if (k.m_type != katom_t::apply_begin + || k.m_src.find('-') == std::string::npos) { + result.push_back(k); + continue; + } + // "@sup-2-3" -> "@sup" "2" "|" "3" "@". The arguments are re-run + // through katomize_by_matching() rather than typed here, so there is one answer to + // "what is this text" and it lives in one place: an argument of + // letters becomes a word katom and an argument of digits a text + // katom, exactly as it does when the same characters are written + // out longhand. No whitespace is emitted between them, matching + // what katomization by rewriting produces today (word_split() discards the + // spaces its own rewrite inserted). + strings_t parts = hyphen_split(k.m_src); + result.push_back(Katom(parts.front(), katom_t::apply_begin, k.m_loc)); + for (size_t i = 1; i < parts.size(); ++i) { + if (i > 1) + result.push_back(Katom("|", katom_t::bar, k.m_loc)); + katom_list arg = katomize_by_matching(parts[i], k.m_loc.m_filename, k.m_loc.m_line); + result.insert(result.end(), arg.begin(), arg.end()); + } + result.push_back(Katom("@", katom_t::apply_end, k.m_loc)); + } + return result; +} + diff --git a/mac/katom_matching.h b/mac/katom_matching.h new file mode 100644 index 0000000..729176f --- /dev/null +++ b/mac/katom_matching.h @@ -0,0 +1,51 @@ +#pragma once + +#include "katom.h" + +// Finding katoms in text by MATCHING: at each +// position, the katom type that matches the most characters wins, and the +// position advances past it. No text is divided and no katom rewrite rule +// is consulted. +// +// A COMPONENT, not a katomization method in itself -- which is what the +// name says. The composition that makes a method of it, +// katomize_text_by_matching(), lives in katomize.{h,cpp} beside the entry +// point for katomization by rewriting, because naming the process is that +// file's job and this one only supplies a way of doing part of it. +// +// It is built into libklammertext.so but NOTHING IN THE ENGINE CALLS IT: +// katomization goes through katomize.{h,cpp}, which divides text by the +// rules. This is the alternative, kept so the two stay comparable +// (dbg/katomize_compare.sh --which category,matching) and so the question +// stays open rather than being settled by deletion. +// +// The claim under test is not "this is faster" or "this is more correct". +// It is that matching over the EXISTING katom_types table is easier for one +// person to hold in their head than 30 ordered katom rewrite rules. It +// diverges on 153 of 9,694 corpus lines, every one of them prose that +// MENTIONS syntax rather than using it -- so the choice is not between two +// readings of the language but between two behaviours on text that is not +// quite in it. + +// The character stream to katoms, longest match at each position. No +// rewriting, no recursion. +katom_list katomize_by_matching(const std::string& s, const std::string& source, int source_line); + +// Desugaring only: the abbreviated form @name-arg-arg becomes the katoms +// @name arg | arg @. A separate pass over the katom list, because it is a +// separate job -- it INSERTS a delimiter that was not in the source, which +// is a source-to-source transform and not a question about where katoms +// begin and end. In katomization by rewriting these two jobs share one loop. +katom_list expand_abbreviations(const katom_list& katoms); + + +// Whether a katom type is something the SOURCE can contain. Five of the +// katom_t values are processing STATES a katom is later moved into +// (replaced, ignored, literal, eval_result, ws_added), not source +// categories, and their entries in katom_types carry no usable pattern -- +// four are the empty string, which as a regex matches everywhere and would +// stall any katomizer. See notes/Klammertext_improvements.md §1: "katom_t +// mixes source categories with processing states". The rewrite katomizer +// never trips over this because it only ever asks whole-text "does this +// match", and an empty pattern matches only the empty string there. +bool appears_in_source(katom_t type); diff --git a/mac/katom_list.cpp b/mac/katom_process.cpp similarity index 99% rename from mac/katom_list.cpp rename to mac/katom_process.cpp index 375ac56..88bb383 100644 --- a/mac/katom_list.cpp +++ b/mac/katom_process.cpp @@ -1,7 +1,9 @@ +// katom_process.cpp -- the read-time passes. See katom_process.h. + // #include #include -#include "katom_list.h" +#include "katom_process.h" #include "util.h" #include "ktype.h" #include "file.h" diff --git a/mac/katom_list.h b/mac/katom_process.h similarity index 71% rename from mac/katom_list.h rename to mac/katom_process.h index 59eb14f..078608f 100644 --- a/mac/katom_list.h +++ b/mac/katom_process.h @@ -1,5 +1,21 @@ #pragma once +// THE READ-TIME PASSES: katoms in, katoms out. What Machine::process_katoms() +// runs over a katom list once it exists -- encoding the ^-character forms, +// assembling ^UUUU UUUU^ sequences, marking literal and removed regions, +// hiding the quoted specials, finding spans, and delimiting @cond and +// klammer applications. +// +// Renamed from katom_list.{h,cpp} on 2026-09-02, which said "a list of +// katoms" and so described a data structure this file barely touches. It +// is named for the function it serves, which adds no vocabulary: text is +// KATOMIZED into KATOMs (katomize.{h,cpp}), and then PROCESSED here. +// +// The neighbours, and the line between them: katom.{h,cpp} is the katom +// itself and operations on a list of them; katomize.{h,cpp} turns text into +// katoms -- the one place a character stream enters. Nothing here takes +// characters, which is why none of it belongs there. + #include "katom.h" #include "state.h" diff --git a/mac/katom_rewrite_rules.cpp b/mac/katom_rewrite_rules.cpp new file mode 100644 index 0000000..8274bac --- /dev/null +++ b/mac/katom_rewrite_rules.cpp @@ -0,0 +1,381 @@ +// katom_rewrite_rules.cpp -- the categorized rule table. See katom_rewrite_rules.h for +// what this experiment is and what was measured to justify it. +// +// The patterns and replacements are copied VERBATIM from ktype.h's +// katom_rewrite_rules -- mechanically, not retyped -- and the trailing +// index comment on each is its position there. Nothing about a rule +// changed; the only additions are the category, the never_fires mark, and +// the abbreviated-application rule, which ktype.h does not carry because it +// lives inside expand_compound_katom() instead. + +#include +#include +#include +#include +#include +#include + +#include "katom_rewrite_rules.h" +#include "file.h" +#include "log.h" +#include "show.h" +#include "util.h" + +namespace { + +// The abbreviated application: @i-a-b becomes "@i a | b @". Copied from +// expand_compound_katom() in katom.cpp, where it sits ABOVE the rule loop +// as a special case. Note the regex_match: it is a WHOLE-word test, which +// is why a trailing period defeats it and why a separate rule ("trailing +// punctuation, shortcut") exists to strip one first. +std::string abbreviated_application(const std::string& word) +{ + static const std::regex shortcut_rgx(R"((@\w+)(-\w+)+)"); + std::smatch match {}; + if (!std::regex_match(word, match, shortcut_rgx)) + return word; + std::string modified = string_replace(word, "-", " | ") + " @"; + return std::regex_replace(modified, std::regex(R"((@\w+) \|)"), "$1"); +} + +} // namespace + +const std::vector rewrite_rules { + { rule_category::caret, "bar precedence", + Rgx(R"((.*?)(\^\|)(.*))"), "$1 $2 $3", nullptr }, // 0 + { rule_category::caret, "literal without space", + Rgx(R"((\^')(.*?)('\^))"), "$1 $2 $3", nullptr }, // 1 + { rule_category::caret, "literal start to the left", + Rgx(R"((.+)(\^'))"), "$1 $2", nullptr }, // 2 + { rule_category::caret, "literal start to the right", + Rgx(R"((\^')(.+))"), "$1 $2", nullptr }, // 3 + { rule_category::caret, "literal end to the left", + Rgx(R"(('\^)(.+))"), "$1 $2", nullptr }, // 4 + { rule_category::caret, "literal end to the right", + Rgx(R"((.+)('\^))"), "$1 $2", nullptr }, // 5 + { rule_category::caret, "special character", + Rgx(R"((.*?)(\^[@|^#*:])(.*))"), "$1 $2 $3", nullptr }, // 6 + { rule_category::caret, "non-diacritic non-ascii", + Rgx(R"((.*?)(\^[a-zA-Z0-9]{5}\^)(.*?))"), "$1 $2 $3", nullptr }, // 7 + { rule_category::caret, "non-diacritic non-ascii 2", + Rgx(R"((.*?)(\^[a-zA-Z0-9]{1,4}\^)(.*?))"), "$1 $2 $3", nullptr }, // 8 + { rule_category::caret, "diacritic non-ascii", + Rgx(R"((.*?)(\^[a-zA-z][^^])(.*?))"), "$1 $2 $3", nullptr }, // 9 + { rule_category::bar, "double bar separator", + Rgx(R"(([^\s\^]+?)(\|\|)(.*))"), "$1 $2 $3", nullptr }, // 10 + { rule_category::bar, "bar separator", + Rgx(R"(([^\s\^]+?)(\|)(.*))"), "$1 $2 $3", nullptr }, // 11 + { rule_category::removal_span, "embedded ignore begin", + Rgx(R"((.*?)#\[(.*))"), "$1 #[ $2", nullptr }, // 12 + { rule_category::removal_span, "embedded ignore end", + Rgx(R"((.*?)\]#(.*))"), "$1 ]# $2", nullptr }, // 13 + { rule_category::removal_span, "embedded remove whitespace", + Rgx(R"((.*?)#-(.*))"), "$1 #- $2", nullptr }, // 14 + // RETIRED 2026-09-02, kept as a record rather than deleted. UNREACHABLE: + // its pattern requires "^" followed by one of @ | : * ^, which is exactly + // what the caret rule "special character" matches -- and caret is the + // first category tried, so it always fires first. Demonstrated: + // kdiag --rewrite '@x-^@@' -> Rewrite [caret] (special character) + // kdiag --rewrite '@x-^|@' -> Rewrite [caret] (bar precedence) + // { rule_category::desugar, "single argument for special character", + // Rgx(R"((@\w+)-(\^[@\|:\*\^])@?)"), "$1 $2 @" }, // was 15 + { rule_category::karg, "klammer body arguments", + Rgx(R"((.*?)(\*\w+\*)(.*))"), "$1 $2 $3", nullptr }, // 16 + { rule_category::punctuation, "trailing punctuation", + Rgx(R"((\w?)@([\]\}().,:;?'!]))"), "$1@ $2", nullptr }, // 17 + // RETIRED 2026-09-02, kept as a record rather than deleted. UNREACHABLE + // BY PROOF, the only one of the eight that is: the rule above has an + // OPTIONAL (\w?), so it matches everything this one does and stands + // first. Demonstrated: + // kdiag --rewrite '@.' -> Rewrite [punctuation] (trailing punctuation) + // kdiag --rewrite '@,rest' -> Rewrite [punctuation] (trailing punctuation) + // { rule_category::punctuation, "trailing punctuation", + // Rgx(R"(@([().,:;?'!].*))"), "@ $1" }, // was 18 + { rule_category::desugar, "trailing punctuation, shortcut", + Rgx(R"((@\w+)-(\w+)([\]\}().,:;?'!][^\s]*))"), "$1 $2 @ $3", nullptr }, // 19 + { rule_category::klammer, "no argument klammer", + Rgx(R"((@\w+)@)"), "$1 @", nullptr }, // 20 + // Renamed from "left parenthesis" and widened to [ and { on 2026-09-03 + // (Andy). An author writes a klammer inside brackets and expects the + // bracket not to be swallowed; only "(" was honoured, which is the + // asymmetry notes/Klammertext_improvements.md 4.3 records. The closing + // side is the punctuation classes above, which gained "]" and "}" in + // the same change. + // + // Why this is safe to whitelist: none of ( [ { ) ] } can appear in a + // definition_name ([a-zA-Z][a-zA-Z0-9_.]*) or in an abbreviated + // argument, so a delimiter meeting one has no second reading. That is + // also why LETTERS can never join the list -- "x@" is a legitimate + // token, the named closing delimiter, so "x@i" is ambiguous in the + // grammar rather than merely unhandled. + { rule_category::klammer, "opening bracket", + Rgx(R"(([(\[{])@(\w+))"), "$1 @$2", nullptr }, // 21 + { rule_category::klammer, "embedded klammer", + Rgx(R"(([^@]+)(@[^@]+@)([^@]+))"), "$1 $2 $3", nullptr }, // 22 + { rule_category::klammer, "embedded klammer to the right", + Rgx(R"(([^@]+)(@[^@]+@))"), "$1 $2", nullptr }, // 23 + { rule_category::klammer, "embedded klammer to the left", + Rgx(R"((@[^@]+@)([^@]+))"), "$1 $2", nullptr }, // 24 + { rule_category::spacing, "embedded add whitespace", + Rgx(R"(([^\s])(#\+\d*)([^\s]))"), "$1 $2 $3", nullptr }, // 25 + { rule_category::spacing, "embedded add whitespace to the left", + Rgx(R"(([^\s])(#\+\d*))"), "$1 $2", nullptr }, // 26 + { rule_category::spacing, "embedded add whitespace to the right", + Rgx(R"((#\+\d*)([^\s]))"), "$1 $2", nullptr }, // 27 + { rule_category::removal_line, "embedded ignore on the left", + Rgx(R"(#([^-+/\s]))"), "# $1", nullptr }, // 28 + { rule_category::removal_line, "embedded ignore on the right", + Rgx(R"(([^\s])#)"), "$1 #", nullptr }, // 29 + // Not in ktype.h's table: today this is the special case at the top of + // expand_compound_katom(), tried before every rule rather than as one. + { rule_category::desugar, "abbreviated application", + Rgx(R"((@\w+)(-\w+)+)"), "", abbreviated_application }, // (not in ktype.h) +}; + +std::string category_name(rule_category category) +{ + static const std::map names { + { rule_category::caret, "caret" }, + { rule_category::bar, "bar" }, + { rule_category::removal_span, "removal_span" }, + { rule_category::karg, "karg" }, + { rule_category::punctuation, "punctuation" }, + { rule_category::desugar, "desugar" }, + { rule_category::klammer, "klammer" }, + { rule_category::spacing, "spacing" }, + { rule_category::removal_line, "removal_line" }, + }; + auto found = names.find(category); + return found == names.end() ? "?" : found->second; +} + +const std::vector& category_order() +{ + // The cross-category ordering decision, entire. Every adjacency here + // is either forced by a measured preemption (see katom_rewrite_rules.h) or + // free -- and where it is free, this keeps the position the rule + // already had in ktype.h, so the two orders differ as little as + // possible. + static const std::vector order { + rule_category::caret, + rule_category::bar, + rule_category::removal_span, + rule_category::karg, + rule_category::punctuation, + rule_category::desugar, + rule_category::klammer, + rule_category::spacing, + rule_category::removal_line, + }; + return order; +} + +const std::vector& rule_order(rewrite_order order) +{ + static const std::vector declared = [] { + std::vector v(rewrite_rules.size()); + for (size_t i = 0; i < v.size(); ++i) v[i] = i; + return v; + }(); + + static const std::vector grouped = [] { + std::vector v {}; + for (rule_category category : category_order()) + for (size_t i = 0; i < rewrite_rules.size(); ++i) + if (rewrite_rules[i].category == category) + v.push_back(i); // stable: declaration order within a category + return v; + }(); + + return order == rewrite_order::as_declared ? declared : grouped; +} + +namespace { + +// Apply one rule. A rule with a special function uses it; every other is a +// plain regex_replace, exactly as the current loop does. +std::string apply_rule(const Rewrite_rule& rule, const std::string& word) +{ + return rule.special ? rule.special(word) + : std::regex_replace(word, rule.pattern.m_regex, rule.replacement); +} + +// Whether a rule's rewrite makes PROGRESS. Two conditions, and the second +// is the termination guard: the word must change, and no part of the result +// may be the whole input. A rule that only puts whitespace around a match +// spanning the entire word satisfies the first and fails the second, and +// recursing on what it returns is what killed ktext on "^\s" until +// 2026-08-30. Here the guard is part of what "matched" MEANS, rather than +// a check the caller has to remember to make. +bool rewrite_is_useful(const std::string& word, const std::string& rewritten) +{ + if (rewritten == word) + return false; + strings_t parts = word_split(rewritten); + return std::ranges::none_of(parts, [&](const std::string& p) { + return !p.empty() && p == word; }); +} + +// THE RULES ARE AUTHOR-MODE, and this is where that becomes structural +// rather than a convention (Andy, 2026-09-04). +// +// A katom rewrite rule exists to spare an AUTHOR typing -- to let "@i-x." +// and "[@i-x]" mean what they look like. A designer writing "@@" or a +// programmer writing "@@@" is already working in a language system +// deliberately, so there is no convenience to provide; and in fact those +// two tiers need none, because "@@name", "name@@" and "@@@target" are whole +// katom TYPES with nothing left for a rule to repair. +// +// Measured before the rule went in: of eleven definition- and system-mode +// forms, ten fired no rule at all. The exception was "@@w@@", where "no +// argument klammer" ((@\w+)@) found "@w@" inside it and split it -- into the +// RIGHT katoms, but by coincidence of where the boundary fell, so any change +// to that rule would have moved definition parsing with it. "@@w@@" is +// malformed anyway (a definition needs :, ::, ::: or ::::), and it is now +// reported rather than silently split. +// +// Three things this buys: no rule can reach definition text, now or in +// future; a proposed rule has a test for whether it is legitimate at all +// (does it fire on text only a designer writes?); and 1,657 of the corpus's +// 44,771 words stop walking the whole rule list to fall off the end. +// +// A WORD CAN MIX TIERS, which is why this is not simply "starts with @@". +// The first attempt was, and it broke tst/modulepath_test.sh, whose fixture +// ends a definition flush against a whitespace operator: +// +// @@desc : @eval pmod.hello() @ @@#- +// ^^^^ one word: a definition close, +// then author-mode text removal +// +// The rules must see that word -- "embedded remove whitespace" is what +// splits it. So the test is whether the word is definition-mode THROUGHOUT: +// nothing in it but delimiters and a name. Anything else -- a #, a bar, a +// caret, punctuation -- means an author-mode construct is glued on and the +// rules are needed after all. +// Named for the question it asks -- could an AUTHOR have written this? -- +// rather than for the mode, because "author_mode" would sit a few files +// from defmode_t/def_create/def_override, which is exactly where the two +// senses of "mode" collide. In prose the three are author mode, designer +// mode and programmer mode (Andy, 2026-09-04). +bool author_written(const std::string& text) +{ + if (text.size() < 2 || text[0] != '@' || text[1] != '@') + return true; + // definition_begin_name characters, plus the "@" of the delimiters + // themselves. The comma is here because a definition may name several + // targets: "@@table.html,tex". + for (char c : text) { + if (c != '@' && !std::isalnum(static_cast(c)) + && c != '_' && c != '.' && c != ',') + return true; + } + return false; +} + +} // namespace + +Rewrite_result find_rewrite(const std::string& word, rewrite_order order) +{ + if (!author_written(word)) + return {}; + for (size_t i : rule_order(order)) { + std::string rewritten = apply_rule(rewrite_rules[i], word); + if (rewrite_is_useful(word, rewritten)) + return { true, i, rewritten }; + } + return {}; +} + +std::vector matching_rules(const std::string& word, rewrite_order order) +{ + std::vector result {}; + if (!author_written(word)) + return result; + for (size_t i : rule_order(order)) { + std::string rewritten = apply_rule(rewrite_rules[i], word); + if (rewrite_is_useful(word, rewritten)) + result.push_back(i); + } + return result; +} + +bool category_order_holds(const std::string& word, std::string& why, rewrite_order order) +{ + std::vector matches = matching_rules(word, order); + if (matches.size() < 2) + return true; + const Rewrite_rule& winner = rewrite_rules[matches.front()]; + + const std::vector& sequence = category_order(); + auto position = [&](rule_category c) { + return std::find(sequence.begin(), sequence.end(), c) - sequence.begin(); }; + + for (size_t i = 1; i < matches.size(); ++i) { + const Rewrite_rule& loser = rewrite_rules[matches[i]]; + if (position(loser.category) < position(winner.category)) { + why = "\"" + word + "\": " + category_name(winner.category) + + " rule \"" + winner.description + "\" preempted the earlier category " + + category_name(loser.category) + " rule \"" + loser.description + "\""; + return false; + } + } + return true; +} + +void describe_rewrite_patterns() +{ + if (verbose_level > 0) { + std::string desc_text = "doc/katom_patterns.desc"; + std::cout << "\n" + << justify(string_from_file(klammertext_filename(desc_text))) + << "\n\n"; + } + size_t width0 = 0; + size_t width1 = 0; + size_t width2 = 0; + for (const Rewrite_rule& rule : rewrite_rules) { + width0 = std::max(width0, category_name(rule.category).size()); + width1 = std::max(width1, rule.description.size()); + width2 = std::max(width2, rule.pattern.m_pattern.size()); + } + width0 += 2; + width1 += 2; + width2 += 2; + + std::cout << boldblack << std::setfill(' ') + << std::setw(width0) << std::left << " Category" + << std::setw(width1) << std::left << "Description" + << std::setw(width2) << std::left << "Pattern" + << "Replacement" << black << "\n"; + + // Dispatch order, not declaration order: what a reader needs is the + // sequence the rules are actually TRIED in, since that is what decides + // which one fires. + rule_category shown {}; + bool first = true; + for (size_t i : rule_order(rewrite_order::by_category)) { + const Rewrite_rule& rule = rewrite_rules[i]; + std::string category = category_name(rule.category); + if (first || rule.category != shown) { + shown = rule.category; + first = false; + } else { + category = ""; // name a category once, at its head + } + std::string desc = rule.description; + desc[0] = static_cast(toupper(desc[0])); + std::cout << " " + << std::left + << std::setw(width0) << category + << std::setw(width1) << desc + << std::setw(width2) << rule.pattern.m_pattern + << (rule.special ? "(not a pattern replacement)" : rule.replacement) + << "\n"; + } + if (verbose_level > 0) { + std::string notes_text = "doc/katom_patterns.notes"; + std::cout << "\n\n" + << justify(string_from_file(klammertext_filename(notes_text))) + << "\n\n"; + } +} diff --git a/mac/katom_rewrite_rules.h b/mac/katom_rewrite_rules.h new file mode 100644 index 0000000..aa236ae --- /dev/null +++ b/mac/katom_rewrite_rules.h @@ -0,0 +1,150 @@ +#pragma once + +#include +#include +#include + +#include "ktype.h" + +// A regex kept beside the pattern it was built from, so a rule can be +// DISPLAYED as written (kdesc --rewrite) as well as applied. It lived in +// ktype.h until the rule table moved here, and nothing else ever used it. +class Rgx { + public: + explicit Rgx(std::string pattern) + : m_pattern(pattern) + , m_regex(std::regex(pattern)) {} + std::string m_pattern; + std::regex m_regex; +}; + +// THE KATOM REWRITE RULES: what divides text that is not yet a katom. +// Live since 2026-09-02. The table moved here from ktype.h, where it had +// been a flat list of tuples, and gained a category per rule; its companion +// is katomize.{h,cpp}, which does the dividing. +// +// WHY CATEGORIES. The rule list is a PRIORITY list: the first rule whose +// rewrite changes the word fires, its parts are re-katomized, and the loop +// RETURNS. Every later rule that would also have matched is preempted -- +// and that preemption is the interaction the flat list states nowhere. +// Measured over every tracked .k and .kt (dbg/rule_conflicts.cpp): 1,971 +// words reach the rules and there are 806 preemptions. +// +// Two facts from that measurement make the categories worth having: +// +// * TWO THIRDS OF THE INTERACTION IS LOCAL. 532 of the 806 preemptions +// are between rules of the SAME category. Grouping puts those pairs +// next to each other, where the reader can see them. +// +// * THE CATEGORY RELATION IS A STRICT ORDER. Cross-category preemption +// has no cycles, and -- this is the surprise -- the current declaration +// order already violates it nowhere. The order in ktype.h was arrived +// at one precedent at a time and is nevertheless a valid topological +// sort. Categories therefore DOCUMENT the existing order rather than +// changing it, which is why this file is expected to be behaviorally +// identical to the current katomizer. Any divergence the comparison +// reports is a transcription error here, not a design difference. +// +// The order between categories, and how much of it is forced: +// +// caret FORCED first -- it preempts every other category +// bar free (only "after caret" is forced) +// removal_span before removal_line +// karg before klammer +// punctuation before klammer +// desugar free (only "after caret" is forced) +// klammer before removal_line +// spacing before removal_line +// removal_line FORCED last +// +// Six constraints in place of thirty positions. Note that removal_span +// (#[, ]#, #-) and removal_line (a bare #) are one CHARACTER and two +// categories: the specific forms are decided before the general one, which +// is longest match stated as an ordering. + +enum class rule_category { + caret, // ^| ^' '^ ^X ^...^ ^Xy -- the quoting and character forms + bar, // || | -- argument separators + removal_span, // #[ ]# #- -- delimited removal and whitespace removal + karg, // *name* -- argument substitution in a body + punctuation, // @ before . , ; : ? ! ( ) ' + desugar, // @name-arg -- the abbreviated application + klammer, // @name@ and klammers embedded in a word + spacing, // #+N -- added whitespace + removal_line, // # -- remove to end of line +}; + +std::string category_name(rule_category category); + +// The category sequence the dispatch follows. This is the whole of the +// cross-category ordering decision, in one place. +const std::vector& category_order(); + +struct Rewrite_rule { + rule_category category; + std::string description; + Rgx pattern; + std::string replacement; + // Non-null for a rule whose rewrite is not a plain regex_replace. One + // rule needs it: the abbreviated application @name-a-b, which today is + // a hand-cut special case at the TOP of expand_compound_katom() rather + // than an entry in the table at all. Making it a rule in the desugar + // category is safe -- no rule of an earlier category can match a word + // made only of "@", word characters and hyphens -- and it removes the + // oddity that the shortcut is currently reported by "kdiag --rewrite" + // as the doing of whichever rule happened to be first in the list. + std::string (*special)(const std::string& word) = nullptr; +}; + +// In DECLARATION order, matching ktype.h's katom_rewrite_rules index for +// index, so the two can be diffed. Dispatch order is a separate question: +// see rule_order() below. +extern const std::vector rewrite_rules; + +enum class rewrite_order { + // The order written in ktype.h. Categories are annotation only, and + // the behavior is by construction identical to the current katomizer. + as_declared, + // Rules grouped so each category is contiguous, in category_order(). + // Differs from as_declared by the position of ONE rule -- "single + // argument for special character", which moves from 15 to just after + // the punctuation rules -- and that rule is one of the eight that never + // fire. So this too is expected to be identical, and the comparison is + // what turns "expected" into "measured". + by_category, +}; + +// Indices into rewrite_rules, in dispatch order. Computed once. +const std::vector& rule_order(rewrite_order order); + +struct Rewrite_result { + bool matched = false; + size_t rule = 0; // index into rewrite_rules + std::string text {}; // the rewritten word +}; + +// The first rule that USEFULLY rewrites word: it changes the string, and no +// part of the result is the whole input. That second condition is the +// termination guard -- a rule that only pads a word with whitespace hands +// back the word it was given, and recursing on it exhausts the stack. +Rewrite_result find_rewrite(const std::string& word, rewrite_order order = rewrite_order::by_category); + +// Every rule that would usefully rewrite word, in dispatch order. The +// first is what find_rewrite() returns; the rest are preempted by it. +std::vector matching_rules(const std::string& word, rewrite_order order = rewrite_order::by_category); + +// The executable form of the claim this file rests on: whatever fires, no +// rule of an EARLIER category also matched. When that holds for every +// word, the category order is a complete account of cross-category +// interaction and a reader need only understand one category at a time. +// Returns true when it holds; otherwise fills why with a description. +// This is what a flat priority list cannot offer -- it has no boundary to +// state an invariant across. +bool category_order_holds(const std::string& word, std::string& why, + rewrite_order order = rewrite_order::by_category); + +// The "kdesc --rewrite" listing. It moved here with the table it reads, +// and gained the CATEGORY column -- which is the point of the categories +// reaching the user rather than only the maintainer: a reader sees the nine +// groups and their order instead of thirty positions. +void describe_rewrite_patterns(); diff --git a/mac/katomize.cpp b/mac/katomize.cpp new file mode 100644 index 0000000..5374473 --- /dev/null +++ b/mac/katomize.cpp @@ -0,0 +1,162 @@ +// katomize.cpp -- the katomization pipeline over the categorized rules. +// See katomize.h for what this experiment is, and katom_rewrite_rules.h +// for what +// was measured to justify the categories. +// +// A faithful re-expression of the former make_katoms_from_word() and its +// two callers, not a redesign -- proved identical over every tracked .k and +// .kt before it replaced them. What changed is where the decisions live: +// +// * The termination guard is inside find_rewrite(), as part of what it +// means for a rule to match, instead of a condition the loop has to +// remember. A caller cannot forget it. +// * The abbreviated application is a RULE in the desugar category rather +// than a special case above the loop, so it can be named when it fires +// and counted when it does not. +// * The rule loop is gone from here entirely: this file asks +// find_rewrite() a question and acts on the answer. Which rule, and +// why that one, is katom_rewrite_rules.cpp's business. +// +// What deliberately did NOT change: the katom types are consulted first and +// unchanged, the recursion structure is the same, the character position +// handed to each katom is that of the text it came from (as in the +// original), and text no rule can divide is still an unparsed katom rather +// than an error. + +#include +#include + +#include "katomize.h" +#include "katom_matching.h" +#include "log.h" +#include "show.h" +#include "util.h" + +namespace { + +// The katom type this text IS, or nullptr. Whole-text match, exactly as +// the current katomizer does it -- the first type in declaration order +// whose pattern matches the entire text. +const Ktype* whole_text_type(const std::string& text) +{ + auto found = std::find_if(katom_types.begin(), katom_types.end(), + [&](const Ktype& k) { return k.match(text); }); + return found == katom_types.end() ? nullptr : &*found; +} + +} // namespace + +Katom source_katom(const std::string& text, katom_t type, const Locator& loc) +{ + Katom k(text, type, loc); + if (type == katom_t::special && !k.m_text.empty() && k.m_text[0] == '^') + k.m_text.erase(0, 1); + return k; +} + +katom_list katomize_by_rewriting(const std::string& text, const std::string& source, int line, int chr, + rewrite_order order) +{ + Locator loc(source, line, chr); + + if (const Ktype* type = whole_text_type(text)) + return { source_katom(text, type->m_type, loc) }; + + Rewrite_result rewrite = find_rewrite(text, order); + if (rewrite.matched) { + const Rewrite_rule& rule = rewrite_rules[rewrite.rule]; + if (show_rewrite_rules) { + std::cout << loc << " Rewrite [" << category_name(rule.category) << "] (" + << rule.description << "): " << rule.pattern.m_pattern + << " " << right_arrow << " " << rewrite.text << "\n"; + } + katom_list result {}; + for (const std::string& piece : word_split(rewrite.text)) { + if (piece.empty()) + continue; + katom_list ks = katomize_by_rewriting(piece, source, line, chr, order); + result.insert(result.end(), ks.begin(), ks.end()); + } + return result; + } + + // No katom type, no katom rewrite rule that makes progress. The + // warning is deferred to warn_unparsed_katoms(), after removal and + // literal marking, so text inside a comment never warns. + Katom unparsed(text, katom_t::word, loc); + unparsed.m_unparsed = true; + return { unparsed }; +} + +katom_list katomize_text(const std::string& text, const std::string& source, int line, + rewrite_order order) +{ + std::string s = string_replace(text, "\r", ""); + s = string_replace(s, "\t", " "); + + // Whitespace is preserved, so the split keeps each single space and + // newline as a piece in its own right -- they become katoms like any + // other text. Same regex the current katomizer splits with. + static const std::regex split_rgx(R"([ ]|[\n]|[^\s]+|.+)"); + katom_list result {}; + int position = 0; + for (auto it = std::sregex_iterator(s.begin(), s.end(), split_rgx); + it != std::sregex_iterator(); ++it) { + for (Katom& k : katomize_by_rewriting(it->str(), source, line, position, order)) { + k.m_loc = Locator(source, line, position); + result.push_back(k); + position += static_cast(k.m_src.size()); + } + } + return result; +} + +katom_list katomize_text_by_matching(const std::string& text, const std::string& source, int line) +{ + return expand_abbreviations(katomize_by_matching(text, source, line)); +} + +katom_list katomize_lines(const strings_t& lines, const std::string& source, rewrite_order order) +{ + katom_list result {}; + int number = 0; + for (const std::string& line : lines) { + katom_list ks = katomize_text(line + "\n", source, ++number, order); + result.insert(result.end(), ks.begin(), ks.end()); + } + if (!result.empty()) + result.pop_back(); // the newline added to the last line + return result; +} + +void warn_unparsed_katoms(katom_list& katoms, bool warn) +{ + for (Katom& k : katoms) { + if (!k.m_unparsed) + continue; + if (!warn) { + k.m_unparsed = false; + continue; + } + if (k.m_type != katom_t::ignored + && k.m_type != katom_t::replaced + && k.m_type != katom_t::literal) { + // One of the two surviving WARNINGS (the commands otherwise treat + // an anomaly as an error and stop -- see the output policy in + // CLAUDE.md). It stays a warning because the judgment is a + // HEURISTIC, not exact: the @eval read-back legitimately fails to + // katomize raw target markup, which is why m_warn_unparsed exists. + // A heuristic that halts turns every false positive into a blocker + // on a document that would otherwise render. Promote it to an + // error when the judgment becomes exact. + // + // Through warning(), not a raw std::cerr, so every warning the + // commands issue has one format. + warning("Word not parsed:\n " + k.m_text + + "\nTo include a punctuation character literally, put " + "\"^\" before it.", + k.m_loc); + k.m_unparsed = false; + } + } +} diff --git a/mac/katomize.h b/mac/katomize.h new file mode 100644 index 0000000..85bf4c5 --- /dev/null +++ b/mac/katomize.h @@ -0,0 +1,94 @@ +#pragma once + +#include "katom.h" +#include "katom_rewrite_rules.h" + +// KATOMIZATION: text becomes katoms, by dividing it with the katom rewrite +// rules in katom_rewrite_rules.{h,cpp}. Live since 2026-09-02, when it +// replaced make_katoms_from_word(), split_into_katoms() and katomize() in +// mac/katom.cpp -- dbg/katomize_compare.sh had shown the two identical over +// every tracked .k and .kt first, and the katoms of all 186 files are +// byte-identical across the change. +// +// THE SEPARATION. mac/katom.cpp used to hold two unrelated subjects. One is the KATOM: what it is, what its type means, +// how a list of them is trimmed, split on bars, turned back into a string. +// The other is KATOMIZATION: how a stream of characters becomes that list. +// A katom is a data structure the whole engine uses; katomization is one +// algorithm, run once per file, that nothing else calls into. Splitting +// them means the rest of the engine can include katom.h without also +// getting the katomizer, and the katomizer can be replaced -- as this file is doing +// -- without touching the type everything depends on. +// +// Done 2026-09-02: make_katoms_from_word, split_into_katoms, katomize, +// expand_compound_katom and warn_unparsed_katoms left katom.cpp for this +// file, and katom.cpp keeps Katom itself, the predicates, and +// trim/bar_split/as_string. warn_unparsed_katoms came too, because it is +// the only reader of m_unparsed, which only katomization writes -- what +// makes it LOOK like a later pass is its call site, and timing is not +// ownership. line_split() followed on 2026-09-02, to file.{h,cpp}: it +// produces no katom and one overload opens a file, so it failed this test +// in both directions. + +// Make a katom from matched source text. The one place that knows how a +// construct is WRITTEN differs from what it MEANS: a quoted special is +// written "^@" and means "@", so the caret comes off here. Both katomizers +// go through this, and nothing else constructs a katom whose type was +// decided by matching source. +// +// Note what is NOT lost: m_src keeps the text exactly as written, so the +// caret is recoverable. hide_special_katoms() replaces m_text with a KTESC +// marker shortly afterwards, precisely so the bare character cannot be +// re-read as syntax; where that pass is skipped -- inside a definition or +// an @eval/@read/@cond span -- the bare character is what the code or the +// declaration should receive. +Katom source_katom(const std::string& text, katom_t type, const Locator& loc); + +// Katomize one piece of text containing no whitespace. The replacement for +// make_katoms_from_word(). +// +// The recursion needs no name for what it divides: at every level the thing +// being katomized is TEXT, and it stops when the text IS a katom. (The +// original called it a "word", which collides with katom_t::word, a katom +// TYPE meaning letters only -- two referents, one term, in one file.) +// +// Same shape as the original: text matching a katom type IS that katom; +// otherwise the first katom rewrite rule that usefully rewrites it decides, +// and the pieces of that rewrite are katomized in turn; text no rule can +// divide becomes an unparsed katom, which warn_unparsed_katoms() reports +// later. +katom_list katomize_by_rewriting(const std::string& text, const std::string& source, int line, int chr, + rewrite_order order = rewrite_order::by_category); + +// One line (or any string) to katoms. The replacement for +// split_into_katoms(), including its \r removal and tab expansion. +katom_list katomize_text(const std::string& text, const std::string& source, int line, + rewrite_order order = rewrite_order::by_category); + +// The same, katomizing by MATCHING instead of by rewriting -- the second +// entry point, so that naming a katomization method stays this file's job. +// Its component lives in katom_matching.{h,cpp}; the desugaring of the +// abbreviated application is a separate pass there, because matching +// produces the abbreviated form as one katom and something must expand it. +katom_list katomize_text_by_matching(const std::string& text, const std::string& source, int line); + +// A file's lines to katoms. The replacement for katomize(): each line is +// katomized with its newline restored, and the final newline is dropped. +katom_list katomize_lines(const strings_t& lines, const std::string& source, + rewrite_order order = rewrite_order::by_category); + +// "kdiag --rewrite": report each rule as it fires, with its category. It +// now names the rule that ACTUALLY fired, which the old path could not do +// for the abbreviated application -- that was applied inside +// expand_compound_katom() and attributed to whichever rule the loop +// happened to be holding, so it always read as "bar precedence". +inline bool show_rewrite_rules = false; + +// Warn about text that matched no katom type and that no rule could divide +// -- but only where it survives processing: text removed by #, ##, or +// #[...]#, replaced spans, and literal content never warn. Called at the +// end of Machine::process_katoms(), after those passes have marked the +// katoms. Clears the flag afterwards so reprocessing the same list does +// not warn twice. With warn=false (the @eval read-back sub-Machine, whose +// katoms hold machine-generated result text) nothing is printed and every +// flag is cleared, so spliced result katoms stay silent. +void warn_unparsed_katoms(katom_list& katoms, bool warn = true); diff --git a/mac/klammerset_registry.cpp b/mac/klammerset_registry.cpp index e54564e..a07f570 100644 --- a/mac/klammerset_registry.cpp +++ b/mac/klammerset_registry.cpp @@ -212,6 +212,56 @@ fs::path resolve_klammerset_symbol( loc); } +// The description out of a klammerset's declaration, WITHOUT loading it. +// +// The listing below enumerates the sets a user could load, and loading is +// not an option for finding out what they are: a "@@@klammerset" +// declaration is OPERATIVE -- processing one reads its ":requires" and +// ":files" -- so describing five available sets by loading them would load +// all five. So this reads the declaring file as text and takes the second +// positional, following the declaration's own parameter spec +// ("symbol | desc :name :author :date :requires :files"): the description +// runs from the bar after the symbol to whichever comes first of an option +// (" :" and a letter -- the same boundary the real parser sees, which is +// why a description may not contain a bare ":name"), another bar, or the +// closing "@@@". +// +// It is a DISPLAY convenience and says so by failing quietly: anything it +// cannot read as a description yields an empty string and the listing +// simply shows the symbol and its file, as it did before. +static std::string klammerset_description(const fs::path& declaration) +{ + std::string text {}; + try { + text = string_from_file(declaration.string()); + } catch (...) { + return ""; + } + static const std::string tag = "@@@klammerset"; + size_t at = text.find(tag); + if (at == std::string::npos) return ""; + size_t bar = text.find('|', at + tag.size()); + if (bar == std::string::npos) return ""; + size_t pos = bar + 1; + size_t end = text.size(); + for (size_t i = pos; i + 1 < text.size(); i++) { + if (text[i] == '|' || (text.compare(i, 3, "@@@") == 0)) { + end = i; + break; + } + // An option: whitespace, ":", a letter. "Document production: + // formatting" is NOT one -- its colon ends a word and is followed + // by a space -- which is the distinction the parser makes too. + if (text[i] == ':' && i > pos + && std::isspace(static_cast(text[i - 1])) + && std::isalpha(static_cast(text[i + 1]))) { + end = i; + break; + } + } + return collapse_whitespace(text.substr(pos, end - pos)); +} + std::string describe_klammerset_search(const std::string& local_dir, int margin) { std::string tab(margin, ' '); @@ -230,13 +280,23 @@ std::string describe_klammerset_search(const std::string& local_dir, int margin) if (!is_klammerset_symbol(symbol) || !fs::is_regular_file(declaration)) { continue; } - ss << tab << symbol << sp_arrow << declaration.string(); + std::string line = tab + symbol + sp_arrow + declaration.string(); if (first_hit.count(symbol)) { - ss << " (shadowed by " << first_hit[symbol] << ")"; + line += " (shadowed by " + first_hit[symbol] + ")"; } else { first_hit[symbol] = declaration.string(); } - ss << "\n"; + // What a set IS, under where it lives: the symbol and the path + // say which file would load, and nothing at all about whether + // this is a set to load or (say) the commands' own reference + // documentation. Read from the declaration without loading it + // -- see klammerset_description() above. + std::string desc = klammerset_description(declaration); + if (!desc.empty()) { + line += "\n" + tab + std::string(symbol.size(), ' ') + + " " + desc; + } + ss << line << "\n"; } } if (first_hit.empty()) { diff --git a/mac/ktype.cpp b/mac/ktype.cpp index de4b2f3..15099a3 100644 --- a/mac/ktype.cpp +++ b/mac/ktype.cpp @@ -160,44 +160,6 @@ divides up text into katoms with the "kdiag" command.)" << "\n\n"; std::cout << "\n"; } -void describe_rewrite_patterns() -{ - if (verbose_level > 0) { - std::string desc_text = "doc/katom_patterns.desc"; - std::cout << "\n" - << justify(string_from_file(klammertext_filename(desc_text))) - << "\n\n"; - } - size_t width1 = 0; - size_t width2 = 0; - for (auto [desc, pattern, replace] : katom_rewrite_rules) { - width1 = std::max(width1, desc.size()); - width2 = std::max(width2, pattern.m_pattern.size()); - } - width1 += 2; - width2 += 2; - - std::cout << boldblack << std::setfill(' ') - << std::setw(width1) << std::left << " Description" - << std::setw(width2) << std::left << " Pattern" - << " Replacement" << black << "\n"; - - for (auto [desc, pattern, replace] : katom_rewrite_rules) { - desc[0] = toupper(desc[0]); - std::cout << " " - << std::left - << std::setw(width1) << desc - << std::setw(width2) << pattern.m_pattern - << replace - << "\n"; - } - if (verbose_level > 0) { - std::string notes_text = "doc/katom_patterns.notes"; - std::cout << "\n\n" - << justify(string_from_file(klammertext_filename(notes_text))) - << "\n\n"; - } -} std::string type_to_name(katom_t type) { diff --git a/mac/ktype.h b/mac/ktype.h index 8232195..02bcb56 100644 --- a/mac/ktype.h +++ b/mac/ktype.h @@ -37,8 +37,8 @@ inline const std::string ws_newline_s { R"(#/\d*)" }; inline const std::string definition_name = R"([a-zA-Z][a-zA-Z0-9_.]*)"; // The name at the START of a definition may carry a comma-separated list of // targets ("@@table.html,tex"), so its katom runs through commas. This is -// deliberately looser than the grammar: the katomizer's job is to delimit the -// token, and parse_name() in klammer.cpp validates the list, where a +// deliberately looser than the grammar: katomization's job is to delimit +// the katom, and parse_name() in klammer.cpp validates the list, where a // malformed one can be reported with its location. The comma stays out of // definition_name itself -- that pattern also delimits klammer APPLICATIONS, // option names and "*arg*" variables, where a comma is ordinary writer text. @@ -192,7 +192,22 @@ std::vector katom_types { Ktype(katom_t::ws_newline, "ws-newline", ws_newline_s, "Remove all whitespace, leaving newlines (default: 1)"), Ktype(katom_t::special, "special-char", R"(\^[@|#^:*])", "Klammertext special character treated as regular text"), //Ktype(katom_t::nonascii, "non-ascii-char", R"(\^\w(?:.|\^))", "Non-ASCII character"), - Ktype(katom_t::nonascii, "non-ascii-char", R"((\^(\w(?:.|\^)))|(\^[A-Fa-f0-9]{1,6}\^))", "Non-ASCII character"), + // The second alternative is the "^...^" delimited form: a code point + // (^263A^, 4-6 hex digits) OR a mnemonic (^s^, ^oe^). Its character + // class is [A-Za-z0-9], not [A-Fa-f0-9], because a mnemonic is not + // hex: "^oe^" matched NEITHER alternative and so was not a katom at + // all, fell through to the "non-diacritic non-ascii" rewrite rule, + // which pads it with whitespace and hands back the same single word -- + // and make_katoms_from_word() then recursed on the identical string + // until the stack died (SIGSEGV, no message). The two entries that + // appeared to work, ^ae^ and ^AE^, did so only because "ae" happens to + // be hex digits; they were being katomized as CODE POINTS and rendered + // correctly only because encode() runs process_extended_latin() before + // process_unicode_codepoint(). Which of the two a form means is + // decided there, by the tables, not here -- katomization's job is only to + // recognize the shape. A shape that resolves to no character at all is + // a located error, raised at the end of encode(). + Ktype(katom_t::nonascii, "non-ascii-char", R"((\^(\w(?:.|\^)))|(\^[A-Za-z0-9]{1,6}\^))", "Non-ASCII character"), // Ktype(katom_t::word, "word", R"([a-z][a-z0-9_]*)", "Lower-case letters, numbers, or underscore"), // Ktype(katom_t::text, "text", @@ -210,52 +225,7 @@ std::vector katom_types { // Ktype(katom_t::undefined, "undefined", "", "Undefined pattern") }; -class Rgx { - public: - explicit Rgx(std::string pattern) - : m_pattern(pattern) - , m_regex(std::regex(pattern)) {} - std::string m_pattern; - std::regex m_regex; -}; -inline const -std::vector> katom_rewrite_rules { - { "bar precedence", Rgx(R"((.*?)(\^\|)(.*))"), "$1 $2 $3" }, - { "literal without space", Rgx(R"((\^')(.*?)('\^))"), "$1 $2 $3" }, - { "literal start to the left", Rgx(R"((.+)(\^'))"), "$1 $2" }, - { "literal start to the right", Rgx(R"((\^')(.+))"), "$1 $2" }, - { "literal end to the left", Rgx(R"(('\^)(.+))"), "$1 $2" }, - { "literal end to the right", Rgx(R"((.+)('\^))"), "$1 $2" }, - //{ "successive non-ascii characters", Rgx(R"((.?)(\^\w[-\"^`'~hcbrdwa])(\^\w)(.?))"), "$1 $2 $3 $4" }, - // { "special character", Rgx(R"((.*?)(\^[@\|\^#\*])(.*))"), "$1 $2 $3" }, - { "special character", Rgx(R"((.*?)(\^[@|^#*:])(.*))"), "$1 $2 $3" }, - { "non-diacritic non-ascii", Rgx(R"((.*?)(\^[a-zA-Z0-9]{5}\^)(.*?))"), "$1 $2 $3" }, - { "non-diacritic non-ascii 2", Rgx(R"((.*?)(\^[a-zA-Z0-9]{1,4}\^)(.*?))"), "$1 $2 $3" }, - { "diacritic non-ascii", Rgx(R"((.*?)(\^[a-zA-z][^^])(.*?))"), "$1 $2 $3" }, - { "double bar separator", Rgx(R"(([^\s\^]+?)(\|\|)(.*))"), "$1 $2 $3" }, - { "bar separator", Rgx(R"(([^\s\^]+?)(\|)(.*))"), "$1 $2 $3" }, - { "embedded ignore begin", Rgx(R"((.*?)#\[(.*))"), "$1 #[ $2" }, - { "embedded ignore end", Rgx(R"((.*?)\]#(.*))"), "$1 ]# $2" }, - { "embedded remove whitespace", Rgx(R"((.*?)#-(.*))"), "$1 #- $2" }, - { "single argument for special character", Rgx(R"((@\w+)-(\^[@\|:\*\^])@?)"), "$1 $2 @" }, - // { "special character", Rgx(R"((.*?)(\^[@\|:\*\^])(.*))"), "$1 $2 $3" }, - { "klammer body arguments", Rgx(R"((.*?)(\*\w+\*)(.*))"), "$1 $2 $3" }, - { "trailing punctuation", Rgx(R"((\w?)@([().,:;?'!]))"), "$1@ $2" }, - { "trailing punctuation", Rgx(R"(@([().,:;?'!].*))"), "@ $1" }, - { "trailing punctuation, shortcut", Rgx(R"((@\w+)-(\w+)([().,:;?'!][^\s]*))"), "$1 $2 @ $3" }, - // { "single argument shortcut", Rgx(R"((@\w+)-([^\s@]+)@?)"), "$1 $2 @" }, - { "no argument klammer", Rgx(R"((@\w+)@)"), "$1 @" }, - { "left parenthesis", Rgx(R"(([(])@(\w+))"), "$1 @$2" }, - { "embedded klammer", Rgx(R"(([^@]+)(@[^@]+@)([^@]+))"), "$1 $2 $3" }, - { "embedded klammer to the right", Rgx(R"(([^@]+)(@[^@]+@))"), "$1 $2" }, - { "embedded klammer to the left", Rgx(R"((@[^@]+@)([^@]+))"), "$1 $2" }, - { "embedded add whitespace", Rgx(R"(([^\s])(#\+\d*)([^\s]))"), "$1 $2 $3" }, - { "embedded add whitespace to the left", Rgx(R"(([^\s])(#\+\d*))"), "$1 $2" }, - { "embedded add whitespace to the right", Rgx(R"((#\+\d*)([^\s]))"), "$1 $2" }, - { "embedded ignore on the left", Rgx(R"(#([^-+/\s]))"), "# $1" }, - { "embedded ignore on the right", Rgx(R"(([^\s])#)"), "$1 #" }, -}; inline const @@ -308,7 +278,6 @@ bool is_active(katom_t t); bool is_printable(katom_t type); void describe_katoms(bool show_regex = false); -void describe_rewrite_patterns(); bool is_deftype(katom_t type); diff --git a/mac/log.h b/mac/log.h index 4edc253..efe6b96 100644 --- a/mac/log.h +++ b/mac/log.h @@ -10,7 +10,7 @@ #include "error.h" #include "locator.h" // #include "source.h" -#include "katom_list.h" +#include "katom_process.h" extern int verbose_level; extern bool show_verbose_location; diff --git a/mac/machine.cpp b/mac/machine.cpp index 5d0c027..91816eb 100644 --- a/mac/machine.cpp +++ b/mac/machine.cpp @@ -1,6 +1,7 @@ #include #include #include "machine.h" +#include "katomize.h" #include "error.h" #include "show.h" #include "util.h" @@ -466,7 +467,7 @@ katom_list Machine::process( bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers, bool eval, bool cond, bool read) { - katom_list katoms = katomize(line_split(text), source); + katom_list katoms = katomize_lines(line_split(text), source); //katoms = process_katoms( katoms, source, @@ -543,7 +544,7 @@ void Machine::expand_read_katoms( input_filename = fs::canonical(input_filename); m_state.add_search_dir(input_filename.parent_path().string()); std::string text = trim_right(string_from_file(input_filename.string())); - katom_list ks = katomize(line_split(text), input_filename); + katom_list ks = katomize_lines(line_split(text), input_filename); // ks = process_katoms( // ks, command_pathname, @@ -675,7 +676,7 @@ void Machine::load_klammerset_files(const Klammerset& klammerset, katom_iter ins loaded_files.push_back(pathname.string()); m_state.add_search_dir(pathname.parent_path().string()); std::string text = trim_right(string_from_file(pathname.string())); - katom_list ks = katomize(line_split(text), pathname); + katom_list ks = katomize_lines(line_split(text), pathname); process_katoms(ks, pathname); loaded.insert(loaded.end(), ks.begin(), ks.end()); } @@ -904,7 +905,7 @@ std::string Machine::run_phase_functions() f = "K_phase_call(" + f + ", K_result, K)"; } f = "@eval " + f + " @"; - auto katoms = katomize(line_split(f), "phase"); + auto katoms = katomize_lines(line_split(f), "phase"); // A phase function's input and output are final target text, not // Klammertext: take the raw result string. Re-reading it as // Klammertext (Eval::eval) would misparse target characters -- diff --git a/mac/machine.h b/mac/machine.h index 5730362..f8e7d06 100644 --- a/mac/machine.h +++ b/mac/machine.h @@ -3,7 +3,7 @@ #include // #include "source.h" -#include "katom_list.h" +#include "katom_process.h" #include "klammer_registry.h" #include "klammerset_registry.h" #include "option_set_registry.h" diff --git a/mac/option_set_registry.cpp b/mac/option_set_registry.cpp index 94b9329..82f60ac 100644 --- a/mac/option_set_registry.cpp +++ b/mac/option_set_registry.cpp @@ -3,7 +3,7 @@ #include "option_set_registry.h" #include "argument_set.h" #include "error.h" -#include "katom_list.h" +#include "katom_process.h" #include "klammer.h" #include "log.h" #include "show.h" diff --git a/mac/state.cpp b/mac/state.cpp index 2243eb7..9932b27 100644 --- a/mac/state.cpp +++ b/mac/state.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include #include @@ -149,11 +151,38 @@ void State::replace(const std::string& name, const std::string& value, bool erro set(name, value, true); } +// Whether a name can be used as a state variable at all. POSIX allows only +// [A-Za-z_][A-Za-z0-9_]* in an environment variable name, and the katomizer's +// karg pattern (\*\w+\*) accepts no more, so a name outside this set could +// never be written as *NAME* in the first place. +// +// The one that matters in practice: bash EXPORTS SHELL FUNCTIONS, and they +// arrive in environ as "BASH_FUNC_name%%=() { ... }". Those were being taken +// as state variables and then emitted into the @eval Python globals as +// assignments, where "BASH_FUNC_mdpdf%%" is not a valid identifier -- so a +// single "export -f" anywhere in a user's shell profile made EVERY @eval fail +// with a Python SyntaxError, reported against whichever .k file happened to +// hold the first one (sks/date/date.k). Nothing in the message named the +// environment. Found 2026-09-04 by the dual-compiler confidence run. +static bool usable_variable_name(const std::string& name) +{ + if (name.empty()) + return false; + if (!std::isalpha(static_cast(name[0])) && name[0] != '_') + return false; + return std::all_of(name.begin(), name.end(), [](unsigned char c) { + return std::isalnum(c) != 0 || c == '_'; }); +} + void State::add_environment_frame() { open_frame(klammerstate::shell_environment_name); for (const auto& [name, value] : environment_variables()) { - set(name, value); + // Silently skipped rather than reported: the user can read it off + // their own shell profile, it changes no output, and an exported + // function would otherwise warn on every run. + if (usable_variable_name(name)) + set(name, value); } } diff --git a/mac/target_registry.h b/mac/target_registry.h index bb988be..fa032d7 100644 --- a/mac/target_registry.h +++ b/mac/target_registry.h @@ -5,6 +5,8 @@ #include "target.h" #include "argument_set.h" #include "katom.h" +#include "katomize.h" +#include "file.h" class Target_registry { @@ -53,7 +55,7 @@ public: /* std::string m_parameters_spec {"name | desc :after_apply :after_write :includes :escape :resolve | transforms.rest"}; Parameter_set m_parameters = - katomize(line_split(m_parameters_spec), Locator().str()); + katomize_lines(line_split(m_parameters_spec), Locator().str()); */ diff --git a/ref/.gitignore b/ref/.gitignore new file mode 100644 index 0000000..e257cb1 --- /dev/null +++ b/ref/.gitignore @@ -0,0 +1,2 @@ +# Build products: the dependency files the pattern rule generates. +*.d diff --git a/ref/Makefile b/ref/Makefile new file mode 100644 index 0000000..9658a6b --- /dev/null +++ b/ref/Makefile @@ -0,0 +1,56 @@ +# ref/ -- the reflection library. +# +# reflect.cpp holds C++ functions that read the engine's OWN registries and +# character tables and return Klammertext (a "@table" call and such), which +# the .kt fragments in this directory then render for whichever target is +# in force. They are reached through "@eval :cpp", so this builds a DSO +# loaded at run time, not part of libklammertext.so. +# +# Adapted from sks/document/Makefile. Two things differ and both matter: +# ref/ has no sks dependencies (a generator returns Klammertext, so it +# needs none of kutil/target's markup helpers), and it sits ONE level below +# $(KLAMMERTEXT_HOME) rather than two, which changes the rpath. + +K := $(KLAMMERTEXT_HOME) + +include $(K)/env/makefile.env + +# CPPFLAGS already carries -I$(KLAMMERTEXT_HOME)/mac, which is every header +# reflect.cpp includes (machine, character, error, show). No local -I. + +LIBDIR := $(K)/lib +LIBRARY := $(LIBDIR)/libklammertext.so + +# "../lib", not "../../lib": reflect.so sits in $(K)/ref, so one level up +# reaches $(K)/lib. (sks/document.so needs two, which is what this was +# copied from -- a wrong rpath here shows up only at dlopen time, and only +# where the loading process does not already have the library.) +REF_LDFLAGS := $(SHARED) $(EXPORT_DYNAMIC) -Wl,-rpath,'$(ORIGIN)/../lib' -L$(LIBDIR) + +OBJECTS := reflect.o +DEPFILES := $(OBJECTS:.o=.d) + +DEPFLAGS = -MMD -MP -MF $(@:.o=.d) + +.PHONY: all clean + +all : reflect.so + +%.o : %.cpp + @echo Compiling $< + $(CXX) -c $(CPPFLAGS) $(CXXFLAGS) $(DEPFLAGS) $< -o $@ + +# The objects are LINKED here, never recompiled. sks/document/Makefile +# names document.cpp on its link line as well, which is correct THERE only +# because document.o is the one object not in its LOCAL_OBJECTS; doing both +# for the same file -- compiling reflect.cpp inline AND linking reflect.o -- +# defines every symbol in it twice. Building through the pattern rule also +# gets reflect.d, so a header change rebuilds it. +reflect.so : $(OBJECTS) $(LIBRARY) + $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(REF_LDFLAGS) $(LDFLAGS) \ + -o $@ $(OBJECTS) -lklammertext $(LDLIBS) + +clean : + rm -f $(OBJECTS) $(DEPFILES) *.so *~ + +-include $(DEPFILES) diff --git a/ref/ref.k b/ref/ref.k new file mode 100644 index 0000000..a87fcb4 --- /dev/null +++ b/ref/ref.k @@ -0,0 +1,16 @@ +# Reflection Klammer set + +@@@klammerset ref | Descriptions of Klammermachine data structures in Klammertext syntax + :name Reflection Klammer Set + :author Andy Kopra + :date 2026-08-27 + :requires sks +@@@ + +@@diacritics_examples.k : Show the diacritics codes and their result in the text @@ + +@@diacritics_examples.* :: @eval :cpp *KLAMMERTEXT_HOME*/ref/reflect diacritics_examples @ @@ + +@@extended_latin_examples.k : Show the codes and the resulting extended Latin characters @@ + +@@extended_latin_examples.* :: @eval :cpp *KLAMMERTEXT_HOME*/ref/reflect extended_latin_examples @ @@ diff --git a/ref/reflect.cpp b/ref/reflect.cpp new file mode 100644 index 0000000..5f68ad3 --- /dev/null +++ b/ref/reflect.cpp @@ -0,0 +1,55 @@ +#include "machine.h" +#include "character.h" +#include "error.h" +#include "show.h" + +// extern "C" is needed for dlsym name lookup. Returning std::string from +// C-linkage functions is technically non-standard but works correctly on +// Linux (Itanium ABI) where C and C++ calling conventions are identical. +#if defined(__clang__) +#pragma clang diagnostic ignored "-Wreturn-type-c-linkage" +#endif + +extern "C" +std::string diacritics_examples([[maybe_unused]] Machine& machine) +{ + if (diacritics.size() != diacritic_symbols_order.size()) { + throw Internal_error("Mismatch between diacritics order list and their definitions"); + } + //std::cout << machine; + + std::stringstream kt {}; + kt << "@table :caption Diacritics (with typical base characters) :cell_hpos c c l |\n" + << "@i-Displayed | @i-Written | @i-Name ||\n"; + + for (char symbol : diacritic_symbols_order) { + std::string sym { symbol }; + auto [code, name] = diacritics[sym]; + std::string letter = diacritic_example_letter[sym]; + std::string written = "^" + letter + sym; + if (sym == "~") { + written = "^" + letter + "^~"; + } + std::string display = "^" + letter + sym; + kt << display << " | @t ^" << written << " @ | " << name << " ||\n"; + } + kt << "table@"; + return kt.str(); +} + +extern "C" +std::string extended_latin_examples([[maybe_unused]] Machine& machine) +{ + std::stringstream kt {}; + kt << "@table :caption Extended Latin characters and ligatures :cell_hpos c c l |\n" + << "@i-Displayed | @i-Written | @i-Name ||\n"; + for (const std::string& symbol : extended_latin_symbols) { + auto [code, name] = extended_latin[symbol]; + std::string coded = "^" + symbol + "^"; + std::string file_literal = "^^ #- " + symbol + " #- ^^"; + kt << coded << " | @t " << file_literal << " @ | " << name << " ||\n"; + } + kt << "table@"; + return kt.str(); +} + diff --git a/sks/block/block.k b/sks/block/block.k index f6cc846..0cc9672 100644 --- a/sks/block/block.k +++ b/sks/block/block.k @@ -56,8 +56,11 @@ indentation (as roughtly defined by "ex"). By default, *right* is set to the @@ @@right.txt :: - @eval import textwrap; "\n".join([line.rjust(*Target_txt_width*, "~") - for line in textwrap.wrap("""*s*""", *Target_txt_width*)]) @ + # str.rjust takes a ONE-character fill, so the padding is built by hand: + # the non-breaking space is a marker now, not a character. + @eval import textwrap, phases; + "\n".join([phases.txt_nbsp_marker() * (*Target_txt_width* - len(line)) + line + for line in textwrap.wrap("""*s*""", *Target_txt_width*)]) @ @@ @@ -102,7 +105,7 @@ indentation (as roughtly defined by "ex"). By default, *right* is set to the @@nl.k : Newline character @@ @@nl.html ::
@@ -@@nl.tex :: \newline @@ +@@nl.tex :: @eval phases.tex_nl_marker() @ @@ @@nl.txt :: @eval phases.txt_nl_marker() @ @@ diff --git a/sks/block/block.py b/sks/block/block.py index 6601f14..456a374 100644 --- a/sks/block/block.py +++ b/sks/block/block.py @@ -3,6 +3,7 @@ import textwrap import pprint import klammer_base +import phases import kutil import latex_util import html_util @@ -33,7 +34,7 @@ class Indent(klammer_base.Klammer_base): return result def txt(self): - indent = "~" * self.left # Removed in phases.justify_blocks + indent = phases.txt_nbsp_marker() * self.left # a space that survives width = int(self.Target_txt_width) - self.left - self.right + 1 # Off-by one for textwrap result = self.s result = re.sub("KK0022", '"', result) diff --git a/sks/font/font.k b/sks/font/font.k index 45dbee5..7961a19 100644 --- a/sks/font/font.k +++ b/sks/font/font.k @@ -49,12 +49,12 @@ unchanged. @@b.k text : Use a bold weight of the current font for *text* @@ @@b.html :: *text* @@ @@b.tex :: \textbf{*text*} @@ -@@b.txt :: **text** @@ +@@b.txt :: *text* @@ @@i.k text : Use an italic style of the current font for *text* @@ @@i.html :: *text* @@ @@i.tex :: \emph{*text*} @@ # \begin{emph}{*text*}\end{emph} @@ -@@i.txt :: '*text*' @@ +@@i.txt :: *text* @@ @@t.k text : Use the currently defined monospace font for *text* @@ @@t.html :: *text* @@ diff --git a/sks/table/table.py b/sks/table/table.py index 1b9ac92..85aa9bd 100644 --- a/sks/table/table.py +++ b/sks/table/table.py @@ -870,30 +870,51 @@ class Table(klammer_base.Klammer_base): # txt target def txt_column_widths(self): - def ws(lst, ext=0): - result = [len(e) for e in lst] - if ext > 0: - result += ["~"] * (ext - (len(lst))) - return result - row_count = max(ws(self.rows)) - widths = ws(self.rows[0], row_count) - for row in self.rows[1:]: - widths = list(max(widths, ws(row, row_count))) + """The visible width of the widest cell in each column. + + Measured with txt_util.visible_length, not len: a cell holding a + quoted character carries a KTESC marker, fourteen characters wide + and one glyph, so len() sizes the column to a width the reader + never sees -- and then the f-string below, padding the same + unresolved text, adds nothing because it is already "wider" than + the column. The cell came out unpadded and three columns wide in + a column sized sixteen. + """ + widths = [0] * self.row_size + for row in self.rows: + # ELEMENT-WISE, which max() of two lists is not: it compares + # them lexicographically and returns the whole larger list, so + # a later row wider in a later column was discarded entirely + # if it was narrower in an earlier one. + widths = [max(w, txt_util.visible_length(cell)) + for w, cell in zip(widths, row)] return widths def txt(self): def hpos(p): return {'l':'<', 'c':'^', 'r':'>'}[p] def if_empty(char): - return "~" if len(char) == 0 else char + return txt_util.NBSP if len(char) == 0 else char + # Padded by hand rather than by an f-string width: the format spec + # counts characters, and a cell's markers are many characters and + # few columns. visible_length is what the reader will see. + def pad(cell, width, align): + cell = if_empty(cell) + space = max(0, width - txt_util.visible_length(cell)) + if align == '>': + return txt_util.NBSP * space + cell + if align == '^': + left = space // 2 + return txt_util.NBSP * left + cell + txt_util.NBSP * (space - left) + return cell + txt_util.NBSP * space widths = self.txt_column_widths() pos = [hpos(p) for p in self.cell_hpos] result = "" - indent = "~" * int(self.Target_txt_indent) + indent = txt_util.NBSP * int(self.Target_txt_indent) colsep = " " for row in self.rows: result += indent + colsep.join( - [f"{if_empty(elt):{pos}{width}}" for elt, width, pos in + [pad(elt, width, align) for elt, width, align in list(zip(row, widths, pos))]) result += "\n" result = txt_util.add_caption( diff --git a/sks/target/phases.py b/sks/target/phases.py index 41b8baa..9c0a579 100644 --- a/sks/target/phases.py +++ b/sks/target/phases.py @@ -14,6 +14,50 @@ def txt_nl_marker(): def txt_vspace_marker(): return "__KTEXTVSPACE__" +# A space that survives to the output. "~" used to serve, and could not: +# it is also a character a writer may want, and the substitution at the end +# of txt_justify_blocks turned EVERY tilde into a space -- so a literal +# tilde could not be written in the txt target at all (a "^~" resolves +# before the phases run, and by then is indistinguishable). A marker of +# its own frees the character. SYNC: the txt target's "~ ^0020^" transform +# in sks/target/target.k is the WRITER-facing convention (a tilde means a +# non-breaking space, as in LaTeX) and is deliberately unchanged. +def txt_nbsp_marker(): + return "__KTEXTNBSP__" + +# The tex line break, carried as a marker. +# +# @nl.tex used to emit "\\newline" directly, and that is destroyed the moment +# it passes through a GENERATOR. A klammer whose @eval result still holds +# klammers is generator output, so the machine escapes its text before +# applying them -- which is what lets a writer put a "%" in generated data +# without knowing LaTeX. A backslash meets the same rule and becomes +# "\\textbackslash{}". So a row written +# +# @offer_list work done @nl@ (delivery date) | 10 || ... +# +# resolved @nl to "\\newline" while binding the rest argument, handed that to +# Python as ordinary text, and had it neutralised on the way back: +# "\\textbackslash{}newline" reached the .tex and printed as literal text. +# +# The marker has no backslash, so the escape pass has nothing to act on, and +# tex_resolve_markers puts the real command back at the very end. +# +# LETTERS AND DIGITS ONLY -- deliberately unlike the txt markers above. The +# tex target escapes "_" to "\\_", so a "__KTEXT...__" marker would arrive as +# "\\_\\_KTEXT...\\_\\_" and never match. Nothing in this marker is in tex's +# :escape list. +def tex_nl_marker(): + return "KTEXNEWLINEMARK" + +# Declared as an :after_apply phase on BOTH tex and pdf. ":includes" carries +# escapes, transforms and resolves from tex into pdf but NOT phases +# (target_registry.cpp), and pdf's own phase compiles the document, so this +# has to run before it -- hence the explicit entry in each rather than +# inheritance. +def tex_resolve_markers(text, K): + return text.replace(tex_nl_marker(), "\\newline") + def txt_justify_blocks(text, K): delim = '__DIVIDE__' text = re.sub("\n\n+", delim, text) @@ -33,27 +77,28 @@ def txt_justify_blocks(text, K): # and the one a klammer relies on to emit lines that must not # re-flow -- a @table's rows, a listing. # - # "~" counts as that whitespace, and MUST: a klammer's indent cannot - # be a real space, because Eval::eval TRIMS the result katoms, so - # the leading spaces a klammer emits never reach this pass at all - # (the "~" is the non-breaking space marker, resolved to a space at - # the end of this function -- the only space that survives). A - # writer's own "~" has become a real space long before this, in the - # target's transform table, so there is no ambiguity here. + # The nbsp MARKER counts as that whitespace, and must: a klammer's + # indent cannot be a real space, because Eval::eval TRIMS the result + # katoms, so the leading spaces a klammer emits never reach this + # pass at all. The marker is the only space that survives, and it + # becomes one at the end of this function. (It was the character + # "~" until 2026-08-27, which meant every tilde in the document + # became a space and a literal tilde could not be written at all.) # # The emission below was INSIDE this "if" until 2026-08-26, so a # verbatim paragraph was not left alone, it was DELETED: the one # branch that had nothing to do also did not copy the text through. # An indented paragraph vanished from the output with no error. - if par and par[0] not in ' ~': - label = re.match(r"~*\[\d+\] ", par) + nbsp = txt_nbsp_marker() + if par and par[0] != ' ' and not par.startswith(nbsp): + label = re.match("(?:" + re.escape(nbsp) + r")*\[\d+\] ", par) par = "\n".join(textwrap.wrap( par, width=int(K.Target_txt_width), break_long_words=False, subsequent_indent=" " * len(label.group(0)) if label else "")) result += par + "\n\n" result = re.sub(" *" + txt_vspace_marker() + " *", "\n", result) result = re.sub(r"\n? *" + txt_nl_marker() + r" *\n?", "\n", result) - result = re.sub("~", " ", result) + result = result.replace(txt_nbsp_marker(), " ") return result def txt_insert_captions(text, K): @@ -81,11 +126,11 @@ def txt_insert_captions(text, K): caption = textwrap.wrap( caption, indented_width, #initial_indent ="~"*indent, - subsequent_indent="~"*indent) + subsequent_indent=txt_nbsp_marker()*indent) caption = "\n".join(caption) caption_width = indented_width line_length = max([len(e) for e in caption.split("\n")]) - line = "~" * indent + "-" * line_length + line = txt_nbsp_marker() * indent + "-" * line_length return caption + "\n" + line result = caption_rgx.sub(replace, text) return result @@ -112,7 +157,8 @@ def txt_format_footnotes(text, K): width = len(f"[{len(footnotes)}]") for n, footnote in enumerate(footnotes, 1): label = f"[{n}]" - result += "~" * (width - len(label)) + label + " " + footnote + "\n\n" + result += (txt_nbsp_marker() * (width - len(label)) + + label + " " + footnote + "\n\n") return result def html_footnote_marker(): diff --git a/sks/target/target.k b/sks/target/target.k index 9cd9b2b..946b1fa 100644 --- a/sks/target/target.k +++ b/sks/target/target.k @@ -42,6 +42,18 @@ of this: neither the transforms nor the quoting reach it. @@@ # The txt target is not fully implemented in the SKS yet. +# +# A writer's "~" is a non-breaking space (the LaTeX convention), and it +# becomes the nbsp MARKER here rather than a space directly, for two +# reasons. A transform whose replacement is whitespace is silently +# DISCARDED -- parse_transforms takes the pair with word_split, and a +# space is not a word -- so "~ ^0020^" never did anything, and the +# substitution that made a tilde look like a space lived in the +# justification phase and caught EVERY tilde, so a literal one could not +# be written at all. The marker survives to the phase and becomes a space +# there; a quoted "^~" is a KTESC marker by then, so it is not matched +# here and reaches the reader as a tilde. +# SYNC: phases.txt_nbsp_marker() in sks/target/phases.py. @@@target txt | Plain text with formatting | -- - | --- -- | @@ -49,7 +61,7 @@ of this: neither the transforms nor the quoting reach it. '' " | ` ` | ' ' | - ~ ^0020^ + ~ __KTEXTNBSP__ :after_apply phases.txt_format_footnotes ; phases.txt_insert_captions ; @@ -81,11 +93,16 @@ of this: neither the transforms nor the quoting reach it. @@@target tex | LaTeX :escape \ \textbackslash{} & \& { \{ } \} $ \$ % \% _ \_ ^# \^# ^^ \textasciicircum{} :resolve - {-} ~ \textasciitilde{} + :after_apply phases.tex_resolve_markers @@@ @@@target pdf | PDF from LaTeX :includes tex + # Phases are NOT inherited through :includes -- escapes, transforms and + # resolves are (target_registry.cpp) -- and tex_to_pdf compiles the + # document, so the marker resolution is named here too, and first. :after_apply + phases.tex_resolve_markers ; ^:cpp *KLAMMERTEXT_HOME*/sks/document/document tex_to_pdf @@@ diff --git a/sks/target/txt_util.py b/sks/target/txt_util.py index d1433cf..caed35f 100644 --- a/sks/target/txt_util.py +++ b/sks/target/txt_util.py @@ -6,11 +6,11 @@ hang anything on: a block is set off from the prose around it by being INDENTED, and by having laid itself out before the target sees it. Two consequences run through this module: - * The indent is written with "~", the non-breaking space, never with a - real space. Eval::eval TRIMS a klammer's result, so leading spaces - never survive to the output; and the txt target's justification pass - treats a paragraph beginning with " " or "~" as already laid out and - leaves it alone (phases.txt_justify_blocks). So the same character + * The indent is written with the NBSP MARKER, never with a real space. + Eval::eval TRIMS a klammer's result, so leading spaces never survive + to the output; and the txt target's justification pass treats a + paragraph beginning with a space or that marker as already laid out + and leaves it alone (phases.txt_justify_blocks). So the same token both survives and declares "do not re-flow this". * A block takes the indent out of its own measure. The page has @@ -25,11 +25,26 @@ import sys import textwrap sys.path.append(f'{os.environ.get("KLAMMERTEXT_HOME")}/sks/kutil') import kutil +import klammer_base +import phases # The non-breaking space marker, resolved to a real space at the end of -# phases.txt_justify_blocks. Named here rather than written as "~" at -# each use, since it is the whole mechanism of this module. -NBSP = "~" +# phases.txt_justify_blocks. Stated once there, with the other txt +# markers, and referred to rather than spelled out here. +NBSP = phases.txt_nbsp_marker() + +def visible_length(s): + """How many columns s will occupy once the markers in it resolve. + + len() is the wrong measure for anything laid out: a KTESC marker is + fourteen characters and one glyph, and an nbsp marker is thirteen and + one space. Measuring the unresolved text sizes a column to a width + the reader never sees -- and then pads to it, so the padding is wrong + twice. (The same fault as abbreviate() in mac/util.cpp, which counted + escape sequences as visible width, and the reason escape resolution + runs BEFORE the txt phases.) + """ + return len(klammer_base.unescape_ktesc(s).replace(NBSP, " ")) def indent_text(text, indent): """Every line of text prefixed with `indent` columns of NBSP.""" diff --git a/tst/Makefile b/tst/Makefile index 308bca7..5483da2 100644 --- a/tst/Makefile +++ b/tst/Makefile @@ -16,6 +16,7 @@ test: ./escape_test.sh ./transform_test.sh ./character_test.sh + ./katomize_test.sh ./filename_test.sh ./alone_test.sh ./state_test.sh diff --git a/tst/character_test.sh b/tst/character_test.sh index bd76769..d80222a 100755 --- a/tst/character_test.sh +++ b/tst/character_test.sh @@ -121,6 +121,56 @@ check "17c. structural code points are re-read-safe" '@@' '^0040 0040^' check_err "17d. invalid code point in a CLOSED sequence" 'in a ^...^ sequence' '^263A FFFFFF^' check "17e. unclosed hex-like prose is untouched" 'about ^263A FFFFFF prose' 'about ^263A FFFFFF prose' +echo +echo "-- the delimited mnemonic form ^x^ (multi-letter entries, and a form" +echo " that names no character at all)" +# The two-letter mnemonics had NO lexical rule: the nonascii katom pattern +# offered "^Xy" and "^{1,6}^", and "oe" is neither. So "^oe^" matched +# no katom type, fell through to the "non-diacritic non-ascii" rewrite rule +# -- which pads the word with whitespace and hands back the SAME single +# word -- and make_katoms_from_word() recursed on it until the stack died: +# SIGSEGV, no message, exit 139. ^ae^ and ^AE^ appeared to work only +# because "ae" happens to be hex digits, so they were lexed as CODE POINTS +# and rendered correctly only because encode() runs process_extended_latin() +# before process_unicode_codepoint(); reversing those two passes would have +# turned ^ae^ into U+00AE. Cases 20-22 pin the mnemonics, 23-25 the forms +# that resolve to nothing. A crash must never read as a pass, so 20a +# distinguishes a signal death from an error exit explicitly. +check "20. two-letter mnemonic" 'œ' '^oe^' +check "21. two-letter mnemonic, capital" 'Œ' '^OE^' +check "22. hex-spelled mnemonic is the MNEMONIC, not U+00AE" 'æ' '^ae^' +check "22a. mnemonic in running text" 'cœur' 'c^oe^ur' + +# 20a: exit 139 (128+SIGSEGV) rather than a clean status is the failure +# this suite exists to catch, and check_err alone cannot tell them apart. +"$KTEXT" --klammersets none -s '^oe^' -d >/dev/null 2>&1; st=$? +if [ $st -ge 128 ]; then + echo "${red}FAIL${reset} 20a. ^oe^ dies on a signal (exit $st)" + FAIL=$((FAIL+1)) +else + echo "${green}PASS${reset} 20a. ^oe^ does not die on a signal" + PASS=$((PASS+1)) +fi + +# 25a: the katomizer's rewrite loop must terminate on a word no rule can +# decompose. "^\s" is such a word -- rule "diacritic non-ascii" matches it +# whole (its [a-zA-z] range spans ASCII 65-122 and so includes the +# backslash), pads it with spaces, and hands the same word back. It killed +# kdiag until 2026-08-30. Not a character-form case at heart, but this is +# where the class is pinned. +"$KTEXT" --klammersets none -s '^\s' -d >/dev/null 2>&1; st=$? +if [ $st -ge 128 ]; then + echo "${red}FAIL${reset} 25a. a barren rewrite rule terminates (exit $st)" + FAIL=$((FAIL+1)) +else + echo "${green}PASS${reset} 25a. a barren rewrite rule terminates" + PASS=$((PASS+1)) +fi + +check_err "23. a form naming no character is an error" 'is not a character code' '^zz^' +check_err "24. ... at any length the lexer accepts" 'is not a character code' '^hello^' +check_err "25. ... letters and digits mixed" 'is not a character code' '^g1^' + echo echo "-- direct entry and NFC convergence" check "18. pinyin tone is precomposed" 'ā' '^a1' diff --git a/tst/coverage_test.sh b/tst/coverage_test.sh index d796a30..7f5cea2 100755 --- a/tst/coverage_test.sh +++ b/tst/coverage_test.sh @@ -281,6 +281,34 @@ else echo "${red}FAIL${reset} 22. an underivable klammer still renders — got [$out]"; FAIL=$((FAIL+1)) fi +echo +echo "-- the file column survives a symlinked KLAMMERTEXT_HOME --" +# The column is shortened by stripping the KLAMMERTEXT_HOME prefix, and that +# was a plain string compare: the engine resolves a definition's file to its +# PHYSICAL path while KLAMMERTEXT_HOME is whatever the user typed. Where the +# two spellings differ the prefix never matched and every row printed its +# full path instead of the tail. +# +# macOS finds this on its own -- /tmp and /var are symlinks to /private/tmp +# and /private/var, so the distribution leg (which builds under a temp dir) +# failed five cases there on 2026-09-07 while the dev tree passed. A symlink +# reproduces it anywhere, which is what this does. +link=$(mktemp -u)/kt_home_link +mkdir -p "$(dirname "$link")" +ln -s "$K" "$link" +out=$(KLAMMERTEXT_HOME="$link" "$link/bin/kdesc" --klammersets none \ + -i "$link/tst/coverage/basic.k" --coverage all 2>&1 \ + | grep -E '^ +@ab ') +rm -f "$link" +case "$out" in + *" tst/coverage/basic.k"*) + echo "${green}PASS${reset} 58. a symlinked KLAMMERTEXT_HOME still shortens the path" + PASS=$((PASS+1));; + *) echo "${red}FAIL${reset} 58. a symlinked KLAMMERTEXT_HOME still shortens the path" + echo " got: [$out]" + FAIL=$((FAIL+1));; +esac + echo echo "=====================" echo "Results: ${PASS} passed, ${FAIL} failed" diff --git a/tst/editor/vim_feature_test.vim b/tst/editor/vim_feature_test.vim index c4121c1..5489370 100644 --- a/tst/editor/vim_feature_test.vim +++ b/tst/editor/vim_feature_test.vim @@ -9,7 +9,7 @@ " What is checked: " * ftdetect: .kt maps to the klammertext filetype, overriding the stock " runtime's .kt -> kotlin mapping -" * syntax: token classes at known positions (three @-tiers open/close, +" * syntax: token classes at known positions (three @-prefixes open/close, " removal line/block/file with markers, verbatim @code interior, " ^-escapes, abbreviated @name-arg) " * commands: KlammertextJumpToMatch (open->close, close->open, diff --git a/tst/editor_test.sh b/tst/editor_test.sh index 5437821..497d8c7 100755 --- a/tst/editor_test.sh +++ b/tst/editor_test.sh @@ -125,13 +125,52 @@ else FAIL=$((FAIL + 1)) fi +# run_bounded SECONDS CMD... -- run CMD with a wall-clock limit, returning +# 137 if it had to be killed. An external editor that is INSTALLED BUT +# NON-FUNCTIONAL is not hypothetical: Olion's Homebrew emacs blocks forever +# on "emacs --batch" with no CPU used, and because this suite had no bound it +# sat for 2h43m producing nothing (2026-09-05). A hang is worse than a +# failure -- it is indistinguishable from slowness, so nobody investigates. +# +# No "timeout(1)": macOS does not have it (nor gtimeout without coreutils), +# and this suite ships. Background the command, race a killer against it. +# "set -m" makes the job its own PROCESS GROUP so the whole group can be +# killed. Killing only the pid leaves the children: a hung editor's orphaned +# child still holds the inherited stdout, and anything reading this suite +# through a pipe then blocks anyway -- the timeout fires and the hang +# continues, which is the worst of both. Measured while writing this. +run_bounded() { + local secs="$1"; shift + set -m + "$@" & local pid=$! + set +m + ( sleep "$secs"; kill -9 -"$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null ) \ + >/dev/null 2>&1 & local killer=$! + wait "$pid" 2>/dev/null; local st=$? + kill "$killer" 2>/dev/null; wait "$killer" 2>/dev/null + return $st +} + +EDITOR_TIMEOUT=${EDITOR_TIMEOUT:-120} + # --- the Emacs units (optional) -------------------------------------------- HAVE_EMACS=0 if command -v emacs >/dev/null 2>&1; then - if emacs --batch -L "$EMACS_DIR" \ + run_bounded "$EDITOR_TIMEOUT" \ + emacs --batch -L "$EMACS_DIR" \ -l klammertext-mode -l klammertext-indent -l klammertext-align \ - -l "$FIX/editor_driver.el" "${el_args[@]}" 2>"$OUT/emacs.log"; then + -l "$FIX/editor_driver.el" "${el_args[@]}" 2>"$OUT/emacs.log" + emacs_status=$? + if [ "$emacs_status" = 0 ]; then HAVE_EMACS=1 + elif [ "$emacs_status" = 137 ]; then + # Distinguished from a failure on purpose: "it hung" and "it said no" + # need different investigations, and only one of them is about + # Klammertext. + echo "${red}FAIL${reset} the Emacs driver did not finish within" \ + "${EDITOR_TIMEOUT}s and was killed (emacs is installed but not" \ + "working on this machine)" >&2 + FAIL=$((FAIL + 1)) else echo "${red}FAIL${reset} the Emacs driver failed:" >&2 cat "$OUT/emacs.log" >&2 diff --git a/tst/katomize_test.sh b/tst/katomize_test.sh new file mode 100755 index 0000000..8b71330 --- /dev/null +++ b/tst/katomize_test.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# +# katomize_test.sh — Katomization: what abuts a klammer delimiter, and the +# katom rewrite rules that decide it. +# +# SKS-independent by construction: every case runs with --klammersets none +# and defines the one klammer it needs inline, because what is under test is +# where katom boundaries fall, not what any klammer renders. +# +# WHAT ABUTS "@" (Andy, 2026-09-03; notes/Klammertext_improvements.md §4.3). +# An author writes a klammer inside brackets, or with a sentence's +# punctuation after it, and expects the delimiter not to swallow the +# neighbour. The rule is a WHITELIST of characters that may abut, and the +# whitelist is safe for a structural reason worth stating once: +# +# None of ( [ { ) ] } . , : ; ? ! ' can appear in a definition_name +# ([a-zA-Z][a-zA-Z0-9_.]*) or in an abbreviated argument, so a delimiter +# meeting one of them has no second reading. +# +# That is also why LETTERS can never join the list, and why §4.3's "clean +# rule" (a delimiter terminates the preceding token unconditionally) cannot +# be unconditional: "x@" is a legitimate token — the named closing +# delimiter — so "x@i" is ambiguous in the grammar, not merely unhandled. +# Case 20 pins that as CURRENT BEHAVIOUR rather than as a defect. +# +# Usage: ./katomize_test.sh (needs KLAMMERTEXT_HOME set; ktext on PATH) +# Exit code: 0 if all tests pass, 1 otherwise. + +PASS=0 +FAIL=0 +KTEXT=ktext +K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set} + +red=$'\033[31m' +green=$'\033[32m' +bold=$'\033[1m' +reset=$'\033[0m' + +# An inline klammer, so nothing here depends on the SKS. It wraps its +# argument in < > so a swallowed bracket is visible in the result rather +# than merely absent. +DEF='@@w.* arg : <*arg*> @@' + +# check NAME EXPECTED INPUT — render INPUT and compare the whole output. +check() { + local name="$1" expected="$2" input="$3" + local out + # A space between the definition and the input: without it the "@@" + # ending the definition abuts the input's first character, which is + # exactly the class of thing under test and breaks the fixture instead + # of the feature. The space does not survive into the output. + out=$("$KTEXT" --klammersets none -s "$DEF $input" -d 2>/dev/null) + if [ "$out" = "$expected" ]; then + echo "${green}PASS${reset} $name"; PASS=$((PASS+1)) + else + echo "${red}FAIL${reset} $name" + echo " expected: [$expected]" + echo " got: [$out]" + FAIL=$((FAIL+1)) + fi +} + +# check_warn NAME INPUT — the render must warn "Word not parsed". Used for +# the cases that are NOT in the whitelist, so a later widening of the rule +# has to come here and say so deliberately. +check_warn() { + local name="$1" input="$2" + local err + err=$("$KTEXT" --klammersets none -s "$DEF $input" -d 2>&1 >/dev/null \ + | tr '\n' ' ' | tr -s ' ') + case "$err" in + *"Word not parsed"*) echo "${green}PASS${reset} $name"; PASS=$((PASS+1));; + *) echo "${red}FAIL${reset} $name — expected a \"Word not parsed\" warning" + echo " got: [$err]"; FAIL=$((FAIL+1));; + esac +} + +echo +echo "${bold}Katomization: what may abut a klammer delimiter${reset}" +echo "=================================================================" +echo + +echo "-- trailing punctuation after each of the four application forms --" +check " 1. abbreviated form, period" '.' '@w-x.' +check " 2. abbreviated form, comma" ',' '@w-x,' +check " 3. abbreviated form, semicolon" ';' '@w-x;' +check " 4. abbreviated form, colon" ':' '@w-x:' +check " 5. abbreviated form, question mark" '?' '@w-x?' +check " 6. abbreviated form, exclamation" '!' '@w-x!' +# The apostrophe is in the whitelist and must stay: "@w x @'s" is ordinary +# English prose and worked before the 2026-09-03 widening. Andy confirmed +# it explicitly when the bracket characters were added. +check " 7. apostrophe (possessive) is in the list" "'s" "@w-x's" +check " 8. bare close, period" '.' '@w x @.' +check " 9. named close, period" '.' '@w x w@.' +check "10. named close, comma" ',' '@w x w@,' + +echo +echo "-- brackets around an application (( was honoured; [ and { added 2026-09-03) --" +check "11. parentheses, abbreviated" '()' '(@w-x)' +check "12. square brackets, abbreviated" '[]' '[@w-x]' +check "13. braces, abbreviated" '{}' '{@w-x}' +check "14. parentheses, bare close" '()' '(@w x @)' +check "15. square brackets, bare close" '[]' '[@w x @]' +check "16. braces, bare close" '{}' '{@w x @}' +check "17. square brackets, named close" '[]' '[@w x w@]' +check "18. no spurious space is introduced" '[].' '[@w-x].' + +echo +echo "-- the boundary of the whitelist: what is NOT allowed to abut --" +# A hyphen after an OPENING name introduces an argument, so it cannot be +# text. After a CLOSING delimiter it is unambiguous and could be allowed; +# it is not, today, and this case records that rather than endorsing it. +check_warn "19. hyphen after a close is still unhandled" '@w x @-y' +# The contradiction that bounds the whole rule: "x@" is the named closing +# delimiter, so a letter before "@" has two readings and no precedence rule +# can separate them. This is the language, not a gap in the whitelist. +check_warn "20. a letter before \"@\" stays ambiguous" 'y@w x @' + +echo +echo "-- the rules are AUTHOR-MODE: they never reach @@ or @@@ text --" +# A rewrite rule exists to spare an author typing. A designer writing "@@" +# or a programmer writing "@@@" needs no such convenience, and in fact needs +# none: those forms are whole katom TYPES. Enforced in find_rewrite(), so +# no rule can reach them however it is written. + +# no_rewrite NAME INPUT -- kdiag must report no rule firing at all. +no_rewrite() { + local name="$1" input="$2" out + out=$(kdiag --rewrite "$input" 2>&1 | grep -c "Rewrite \[") + if [ "$out" = "0" ]; then + echo "${green}PASS${reset} $name"; PASS=$((PASS+1)) + else + echo "${red}FAIL${reset} $name — $out rule(s) fired on definition-mode text" + FAIL=$((FAIL+1)) + fi +} +no_rewrite "21. @@name fires no rule" '@@w.k' +no_rewrite "22. a target list fires no rule" '@@w.k,tex' +no_rewrite "23. @@@ fires no rule" '@@@target' +no_rewrite "24. a named definition close" 'w@@' +# Was the ONE violation: "no argument klammer" found "@w@" inside "@@w@@" +# and split it -- into the right katoms, but by coincidence. The form is +# malformed (a definition needs :, ::, ::: or ::::), so reporting it is +# better than silently splitting it. +no_rewrite "25. @@w@@ fires no rule either" '@@w@@' +check_warn "26. ... and is reported, not split" '@@w@@' +# A word can MIX TIERS, so the test is not "starts with @@". A definition +# closed flush against a whitespace operator is one word and the rules MUST +# see it -- this is real usage, from tst/modulepath_test.sh's fixture, and +# the first version of the guard broke it. +check "27. @@ glued to #- still splits" '' '@@v.* : @@#-@v@' + +echo +echo "=================================================================" +echo "${bold}Results: $PASS passed, $FAIL failed${reset}" +[ $FAIL -eq 0 ] && exit 0 || exit 1 diff --git a/tst/state_test.sh b/tst/state_test.sh index 2b03e42..bd93096 100755 --- a/tst/state_test.sh +++ b/tst/state_test.sh @@ -115,6 +115,41 @@ check_error " 7. ... that lists the frames it searched" \ "(searched:" \ --klammersets none -s '@@f : [*Missing*] @@ @f@' -d +# 8-9: bash EXPORTS SHELL FUNCTIONS, and they arrive in the environment as +# "BASH_FUNC_name%%=() { ... }". Those were taken as state variables and +# emitted into the @eval Python as "class K" attributes, where +# "BASH_FUNC_mdpdf%%" is not a valid identifier -- so a single "export -f" +# anywhere in a user's shell profile made EVERY @eval fail with a Python +# SyntaxError, reported against whichever .k file held the first one. +# Nothing in the message named the environment. Found 2026-09-04. +# +# Run in a subshell so the exported function does not leak into the rest of +# the suite. +probe() { # probe NAME EXPECTED ARGS... + local name="$1" expected="$2"; shift 2 + local out + out=$(bash -c ' + ktexport_fn() { echo hi; } + export -f ktexport_fn + exec "$@"' _ "$KTEXT" "$@" 2>&1) + if [ "$out" = "$expected" ]; then + echo "${green}PASS${reset} $name"; PASS=$((PASS+1)) + else + echo "${red}FAIL${reset} $name" + echo " expected: [$expected]" + echo " got: [$(printf '%s' "$out" | head -3 | tr '\n' ' ')]" + FAIL=$((FAIL+1)) + fi +} + +probe " 8. an exported shell function does not break @eval" \ + "4" \ + --klammersets none -s '@eval 2 + 2 @' -d + +probe " 9. ... and a real environment variable still reaches class K" \ + "probe-value" \ + --klammersets none -s '@eval K.KT_STATE_PROBE @' -d + echo echo "==============" echo "Results: ${green}$PASS passed${reset}, ${red}$FAIL failed${reset}"