ack
c016b3e5f3
Assembled from klammertext-dev e99530408517. The katom rewrite patterns are rendered from ref/ (a table in dispatch order, patterns through @c), the procedural listing having left mac/; a plain-text caption's rule is as long as the table or listing it captions; the editor-support comments name the tools' new dev/ location.
324 lines
15 KiB
C++
324 lines
15 KiB
C++
// katom_rewrite_rules.cpp -- the categorized rule table. See katom_rewrite_rules.h for
|
|
// what this experiment is and what was measured to justify it.
|
|
//
|
|
// The patterns and replacements are copied VERBATIM from ktype.h's
|
|
// katom_rewrite_rules -- mechanically, not retyped -- and the trailing
|
|
// index comment on each is its position there. Nothing about a rule
|
|
// changed; the only additions are the category, the never_fires mark, and
|
|
// the abbreviated-application rule, which ktype.h does not carry because it
|
|
// lives inside expand_compound_katom() instead.
|
|
|
|
#include <algorithm>
|
|
#include <cctype>
|
|
#include <iomanip>
|
|
#include <iostream>
|
|
#include <map>
|
|
#include <regex>
|
|
|
|
#include "katom_rewrite_rules.h"
|
|
#include "file.h"
|
|
#include "log.h"
|
|
#include "show.h"
|
|
#include "util.h"
|
|
|
|
namespace {
|
|
|
|
// The abbreviated application: @i-a-b becomes "@i a | b @". Copied from
|
|
// expand_compound_katom() in katom.cpp, where it sits ABOVE the rule loop
|
|
// as a special case. Note the regex_match: it is a WHOLE-word test, which
|
|
// is why a trailing period defeats it and why a separate rule ("trailing
|
|
// punctuation, shortcut") exists to strip one first.
|
|
std::string abbreviated_application(const std::string& word)
|
|
{
|
|
static const std::regex shortcut_rgx(R"((@\w+)(-\w+)+)");
|
|
std::smatch match {};
|
|
if (!std::regex_match(word, match, shortcut_rgx))
|
|
return word;
|
|
std::string modified = string_replace(word, "-", " | ") + " @";
|
|
return std::regex_replace(modified, std::regex(R"((@\w+) \|)"), "$1");
|
|
}
|
|
|
|
} // namespace
|
|
|
|
const std::vector<Rewrite_rule> rewrite_rules {
|
|
{ rule_category::caret, "bar precedence",
|
|
Rgx(R"((.*?)(\^\|)(.*))"), "$1 $2 $3", nullptr }, // 0
|
|
{ rule_category::caret, "literal without space",
|
|
Rgx(R"((\^')(.*?)('\^))"), "$1 $2 $3", nullptr }, // 1
|
|
{ rule_category::caret, "literal start to the left",
|
|
Rgx(R"((.+)(\^'))"), "$1 $2", nullptr }, // 2
|
|
{ rule_category::caret, "literal start to the right",
|
|
Rgx(R"((\^')(.+))"), "$1 $2", nullptr }, // 3
|
|
{ rule_category::caret, "literal end to the left",
|
|
Rgx(R"(('\^)(.+))"), "$1 $2", nullptr }, // 4
|
|
{ rule_category::caret, "literal end to the right",
|
|
Rgx(R"((.+)('\^))"), "$1 $2", nullptr }, // 5
|
|
{ rule_category::caret, "special character",
|
|
Rgx(R"((.*?)(\^[@|^#*:])(.*))"), "$1 $2 $3", nullptr }, // 6
|
|
{ rule_category::caret, "non-diacritic non-ascii",
|
|
Rgx(R"((.*?)(\^[a-zA-Z0-9]{5}\^)(.*?))"), "$1 $2 $3", nullptr }, // 7
|
|
{ rule_category::caret, "non-diacritic non-ascii 2",
|
|
Rgx(R"((.*?)(\^[a-zA-Z0-9]{1,4}\^)(.*?))"), "$1 $2 $3", nullptr }, // 8
|
|
{ rule_category::caret, "diacritic non-ascii",
|
|
Rgx(R"((.*?)(\^[a-zA-z][^^])(.*?))"), "$1 $2 $3", nullptr }, // 9
|
|
{ rule_category::bar, "double bar separator",
|
|
Rgx(R"(([^\s\^]+?)(\|\|)(.*))"), "$1 $2 $3", nullptr }, // 10
|
|
{ rule_category::bar, "bar separator",
|
|
Rgx(R"(([^\s\^]+?)(\|)(.*))"), "$1 $2 $3", nullptr }, // 11
|
|
{ rule_category::removal_span, "embedded ignore begin",
|
|
Rgx(R"((.*?)#\[(.*))"), "$1 #[ $2", nullptr }, // 12
|
|
{ rule_category::removal_span, "embedded ignore end",
|
|
Rgx(R"((.*?)\]#(.*))"), "$1 ]# $2", nullptr }, // 13
|
|
{ rule_category::removal_span, "embedded remove whitespace",
|
|
Rgx(R"((.*?)#-(.*))"), "$1 #- $2", nullptr }, // 14
|
|
// RETIRED 2026-09-02, kept as a record rather than deleted. UNREACHABLE:
|
|
// its pattern requires "^" followed by one of @ | : * ^, which is exactly
|
|
// what the caret rule "special character" matches -- and caret is the
|
|
// first category tried, so it always fires first. Demonstrated:
|
|
// kdiag --rewrite '@x-^@@' -> Rewrite [caret] (special character)
|
|
// kdiag --rewrite '@x-^|@' -> Rewrite [caret] (bar precedence)
|
|
// { rule_category::desugar, "single argument for special character",
|
|
// Rgx(R"((@\w+)-(\^[@\|:\*\^])@?)"), "$1 $2 @" }, // was 15
|
|
{ rule_category::karg, "klammer body arguments",
|
|
Rgx(R"((.*?)(\*\w+\*)(.*))"), "$1 $2 $3", nullptr }, // 16
|
|
{ rule_category::punctuation, "trailing punctuation",
|
|
Rgx(R"((\w?)@([\]\}().,:;?'!]))"), "$1@ $2", nullptr }, // 17
|
|
// RETIRED 2026-09-02, kept as a record rather than deleted. UNREACHABLE
|
|
// BY PROOF, the only one of the eight that is: the rule above has an
|
|
// OPTIONAL (\w?), so it matches everything this one does and stands
|
|
// first. Demonstrated:
|
|
// kdiag --rewrite '@.' -> Rewrite [punctuation] (trailing punctuation)
|
|
// kdiag --rewrite '@,rest' -> Rewrite [punctuation] (trailing punctuation)
|
|
// { rule_category::punctuation, "trailing punctuation",
|
|
// Rgx(R"(@([().,:;?'!].*))"), "@ $1" }, // was 18
|
|
{ rule_category::desugar, "trailing punctuation, shortcut",
|
|
Rgx(R"((@\w+)-(\w+)([\]\}().,:;?'!][^\s]*))"), "$1 $2 @ $3", nullptr }, // 19
|
|
{ rule_category::klammer, "no argument klammer",
|
|
Rgx(R"((@\w+)@)"), "$1 @", nullptr }, // 20
|
|
// Renamed from "left parenthesis" and widened to [ and { on 2026-09-03
|
|
// (Andy). An author writes a klammer inside brackets and expects the
|
|
// bracket not to be swallowed; only "(" was honoured, which is the
|
|
// asymmetry notes/Klammertext_improvements.md 4.3 records. The closing
|
|
// side is the punctuation classes above, which gained "]" and "}" in
|
|
// the same change.
|
|
//
|
|
// Why this is safe to whitelist: none of ( [ { ) ] } can appear in a
|
|
// definition_name ([a-zA-Z][a-zA-Z0-9_.]*) or in an abbreviated
|
|
// argument, so a delimiter meeting one has no second reading. That is
|
|
// also why LETTERS can never join the list -- "x@" is a legitimate
|
|
// token, the named closing delimiter, so "x@i" is ambiguous in the
|
|
// grammar rather than merely unhandled.
|
|
{ rule_category::klammer, "opening bracket",
|
|
Rgx(R"(([(\[{])@(\w+))"), "$1 @$2", nullptr }, // 21
|
|
{ rule_category::klammer, "embedded klammer",
|
|
Rgx(R"(([^@]+)(@[^@]+@)([^@]+))"), "$1 $2 $3", nullptr }, // 22
|
|
{ rule_category::klammer, "embedded klammer to the right",
|
|
Rgx(R"(([^@]+)(@[^@]+@))"), "$1 $2", nullptr }, // 23
|
|
{ rule_category::klammer, "embedded klammer to the left",
|
|
Rgx(R"((@[^@]+@)([^@]+))"), "$1 $2", nullptr }, // 24
|
|
{ rule_category::spacing, "embedded add whitespace",
|
|
Rgx(R"(([^\s])(#\+\d*)([^\s]))"), "$1 $2 $3", nullptr }, // 25
|
|
{ rule_category::spacing, "embedded add whitespace to the left",
|
|
Rgx(R"(([^\s])(#\+\d*))"), "$1 $2", nullptr }, // 26
|
|
{ rule_category::spacing, "embedded add whitespace to the right",
|
|
Rgx(R"((#\+\d*)([^\s]))"), "$1 $2", nullptr }, // 27
|
|
{ rule_category::removal_line, "embedded ignore on the left",
|
|
Rgx(R"(#([^-+/\s]))"), "# $1", nullptr }, // 28
|
|
{ rule_category::removal_line, "embedded ignore on the right",
|
|
Rgx(R"(([^\s])#)"), "$1 #", nullptr }, // 29
|
|
// Not in ktype.h's table: today this is the special case at the top of
|
|
// expand_compound_katom(), tried before every rule rather than as one.
|
|
{ rule_category::desugar, "abbreviated application",
|
|
Rgx(R"((@\w+)(-\w+)+)"), "", abbreviated_application }, // (not in ktype.h)
|
|
};
|
|
|
|
std::string category_name(rule_category category)
|
|
{
|
|
static const std::map<rule_category, std::string> names {
|
|
{ rule_category::caret, "caret" },
|
|
{ rule_category::bar, "bar" },
|
|
{ rule_category::removal_span, "removal_span" },
|
|
{ rule_category::karg, "karg" },
|
|
{ rule_category::punctuation, "punctuation" },
|
|
{ rule_category::desugar, "desugar" },
|
|
{ rule_category::klammer, "klammer" },
|
|
{ rule_category::spacing, "spacing" },
|
|
{ rule_category::removal_line, "removal_line" },
|
|
};
|
|
auto found = names.find(category);
|
|
return found == names.end() ? "?" : found->second;
|
|
}
|
|
|
|
const std::vector<rule_category>& category_order()
|
|
{
|
|
// The cross-category ordering decision, entire. Every adjacency here
|
|
// is either forced by a measured preemption (see katom_rewrite_rules.h) or
|
|
// free -- and where it is free, this keeps the position the rule
|
|
// already had in ktype.h, so the two orders differ as little as
|
|
// possible.
|
|
static const std::vector<rule_category> order {
|
|
rule_category::caret,
|
|
rule_category::bar,
|
|
rule_category::removal_span,
|
|
rule_category::karg,
|
|
rule_category::punctuation,
|
|
rule_category::desugar,
|
|
rule_category::klammer,
|
|
rule_category::spacing,
|
|
rule_category::removal_line,
|
|
};
|
|
return order;
|
|
}
|
|
|
|
const std::vector<size_t>& rule_order(rewrite_order order)
|
|
{
|
|
static const std::vector<size_t> declared = [] {
|
|
std::vector<size_t> v(rewrite_rules.size());
|
|
for (size_t i = 0; i < v.size(); ++i) v[i] = i;
|
|
return v;
|
|
}();
|
|
|
|
static const std::vector<size_t> grouped = [] {
|
|
std::vector<size_t> v {};
|
|
for (rule_category category : category_order())
|
|
for (size_t i = 0; i < rewrite_rules.size(); ++i)
|
|
if (rewrite_rules[i].category == category)
|
|
v.push_back(i); // stable: declaration order within a category
|
|
return v;
|
|
}();
|
|
|
|
return order == rewrite_order::as_declared ? declared : grouped;
|
|
}
|
|
|
|
namespace {
|
|
|
|
// Apply one rule. A rule with a special function uses it; every other is a
|
|
// plain regex_replace, exactly as the current loop does.
|
|
std::string apply_rule(const Rewrite_rule& rule, const std::string& word)
|
|
{
|
|
return rule.special ? rule.special(word)
|
|
: std::regex_replace(word, rule.pattern.m_regex, rule.replacement);
|
|
}
|
|
|
|
// Whether a rule's rewrite makes PROGRESS. Two conditions, and the second
|
|
// is the termination guard: the word must change, and no part of the result
|
|
// may be the whole input. A rule that only puts whitespace around a match
|
|
// spanning the entire word satisfies the first and fails the second, and
|
|
// recursing on what it returns is what killed ktext on "^\s" until
|
|
// 2026-08-30. Here the guard is part of what "matched" MEANS, rather than
|
|
// a check the caller has to remember to make.
|
|
bool rewrite_is_useful(const std::string& word, const std::string& rewritten)
|
|
{
|
|
if (rewritten == word)
|
|
return false;
|
|
strings_t parts = word_split(rewritten);
|
|
return std::ranges::none_of(parts, [&](const std::string& p) {
|
|
return !p.empty() && p == word; });
|
|
}
|
|
|
|
// THE RULES ARE AUTHOR-MODE, and this is where that becomes structural
|
|
// rather than a convention (Andy, 2026-09-04).
|
|
//
|
|
// A katom rewrite rule exists to spare an AUTHOR typing -- to let "@i-x."
|
|
// and "[@i-x]" mean what they look like. A designer writing "@@" or a
|
|
// programmer writing "@@@" is already working in a language system
|
|
// deliberately, so there is no convenience to provide; and in fact those
|
|
// two tiers need none, because "@@name", "name@@" and "@@@target" are whole
|
|
// katom TYPES with nothing left for a rule to repair.
|
|
//
|
|
// Measured before the rule went in: of eleven definition- and system-mode
|
|
// forms, ten fired no rule at all. The exception was "@@w@@", where "no
|
|
// argument klammer" ((@\w+)@) found "@w@" inside it and split it -- into the
|
|
// RIGHT katoms, but by coincidence of where the boundary fell, so any change
|
|
// to that rule would have moved definition parsing with it. "@@w@@" is
|
|
// malformed anyway (a definition needs :, ::, ::: or ::::), and it is now
|
|
// reported rather than silently split.
|
|
//
|
|
// Three things this buys: no rule can reach definition text, now or in
|
|
// future; a proposed rule has a test for whether it is legitimate at all
|
|
// (does it fire on text only a designer writes?); and 1,657 of the corpus's
|
|
// 44,771 words stop walking the whole rule list to fall off the end.
|
|
//
|
|
// A WORD CAN MIX TIERS, which is why this is not simply "starts with @@".
|
|
// The first attempt was, and it broke tst/modulepath_test.sh, whose fixture
|
|
// ends a definition flush against a whitespace operator:
|
|
//
|
|
// @@desc : @eval pmod.hello() @ @@#-
|
|
// ^^^^ one word: a definition close,
|
|
// then author-mode text removal
|
|
//
|
|
// The rules must see that word -- "embedded remove whitespace" is what
|
|
// splits it. So the test is whether the word is definition-mode THROUGHOUT:
|
|
// nothing in it but delimiters and a name. Anything else -- a #, a bar, a
|
|
// caret, punctuation -- means an author-mode construct is glued on and the
|
|
// rules are needed after all.
|
|
// Named for the question it asks -- could an AUTHOR have written this? --
|
|
// rather than for the mode, because "author_mode" would sit a few files
|
|
// from defmode_t/def_create/def_override, which is exactly where the two
|
|
// senses of "mode" collide. In prose the three are author mode, designer
|
|
// mode and programmer mode (Andy, 2026-09-04).
|
|
bool author_written(const std::string& text)
|
|
{
|
|
if (text.size() < 2 || text[0] != '@' || text[1] != '@')
|
|
return true;
|
|
// definition_begin_name characters, plus the "@" of the delimiters
|
|
// themselves. The comma is here because a definition may name several
|
|
// targets: "@@table.html,tex".
|
|
for (char c : text) {
|
|
if (c != '@' && !std::isalnum(static_cast<unsigned char>(c))
|
|
&& c != '_' && c != '.' && c != ',')
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
Rewrite_result find_rewrite(const std::string& word, rewrite_order order)
|
|
{
|
|
if (!author_written(word))
|
|
return {};
|
|
for (size_t i : rule_order(order)) {
|
|
std::string rewritten = apply_rule(rewrite_rules[i], word);
|
|
if (rewrite_is_useful(word, rewritten))
|
|
return { true, i, rewritten };
|
|
}
|
|
return {};
|
|
}
|
|
|
|
std::vector<size_t> matching_rules(const std::string& word, rewrite_order order)
|
|
{
|
|
std::vector<size_t> result {};
|
|
if (!author_written(word))
|
|
return result;
|
|
for (size_t i : rule_order(order)) {
|
|
std::string rewritten = apply_rule(rewrite_rules[i], word);
|
|
if (rewrite_is_useful(word, rewritten))
|
|
result.push_back(i);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
bool category_order_holds(const std::string& word, std::string& why, rewrite_order order)
|
|
{
|
|
std::vector<size_t> matches = matching_rules(word, order);
|
|
if (matches.size() < 2)
|
|
return true;
|
|
const Rewrite_rule& winner = rewrite_rules[matches.front()];
|
|
|
|
const std::vector<rule_category>& sequence = category_order();
|
|
auto position = [&](rule_category c) {
|
|
return std::find(sequence.begin(), sequence.end(), c) - sequence.begin(); };
|
|
|
|
for (size_t i = 1; i < matches.size(); ++i) {
|
|
const Rewrite_rule& loser = rewrite_rules[matches[i]];
|
|
if (position(loser.category) < position(winner.category)) {
|
|
why = "\"" + word + "\": " + category_name(winner.category)
|
|
+ " rule \"" + winner.description + "\" preempted the earlier category "
|
|
+ category_name(loser.category) + " rule \"" + loser.description + "\"";
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
}
|