Andy Kopra
37b6ba1c4f
Typographic transforms (---, quote pairs, ~) no longer touch verbatim text: @c/@code/@source_listing content and ^'...'^ spans show exactly the characters written. "^" before any punctuation character quotes it in every target (the apostrophe excepted: ^' opens a literal span), with the new :resolve option on @@@target declaring per-target renderings. The ^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range. The html output and transform spellings are polyglot (XML-valid), in preparation for an EPUB target. New suites: transform_test, character_test (engine), typography_test (SKS). (from dev 07ce5ea86a0a)
982 lines
44 KiB
C++
982 lines
44 KiB
C++
#include <set>
|
|
#include <utility>
|
|
#include "machine.h"
|
|
#include "error.h"
|
|
#include "show.h"
|
|
#include "util.h"
|
|
#include "file.h"
|
|
#include "log.h"
|
|
#include "eval.h"
|
|
|
|
Machine::Machine()
|
|
: m_argtypes(Argtype_registry())
|
|
, m_state(State())
|
|
, m_targets(Target_registry())
|
|
, m_klammers(Klammer_registry())
|
|
{
|
|
(void)K::log(3);
|
|
m_state.add_environment_frame();
|
|
/*
|
|
if (sks) {
|
|
fs::path sks_filename(klammertext_filename("sks/sks.k"));
|
|
// msg() << "SKS filename: " << sks_filename << "\n";
|
|
read(sks_filename);
|
|
}
|
|
*/
|
|
}
|
|
|
|
// Klammer application recursion guard.
|
|
//
|
|
// Applying a klammer expands its body, which is processed and applied in
|
|
// turn (apply_klammer -> process_katoms -> apply -> apply_klammer), so a
|
|
// klammer that reaches itself -- directly (@@f : x @f@ @@) or through a
|
|
// cycle -- descends without bound. Before this guard the descent simply
|
|
// exhausted the C++ stack: SIGSEGV, no message, no location.
|
|
//
|
|
// The counter is a translation-unit static rather than a Machine member for
|
|
// two reasons: recursion can cross Machine instances (Eval::eval builds a
|
|
// sub-Machine to re-read an @eval result, and that sub-Machine applies
|
|
// klammers on the same C++ stack), and keeping it out of Machine avoids
|
|
// changing the class layout shared with the dlopened sks/document.so.
|
|
//
|
|
// The limit bounds the C++ stack, not the language: it is far above any
|
|
// plausible nesting depth in a document, and reaching it means a klammer
|
|
// does not terminate. NOTE: not thread-safe; if input files are ever
|
|
// processed in parallel this needs to become thread_local.
|
|
namespace {
|
|
|
|
constexpr int max_apply_depth = 200;
|
|
int apply_depth = 0;
|
|
|
|
// Rounds of the top-level fixed-point loop (see Machine::apply below). The
|
|
// former limit of 5 was a silent truncation; it is now an error, so it is set
|
|
// well clear of any legitimate chain of klammers generating klammers.
|
|
constexpr int apply_round_limit = 100;
|
|
|
|
class Depth_guard
|
|
{
|
|
public:
|
|
Depth_guard(const std::string& name, const Locator& loc)
|
|
{
|
|
if (apply_depth >= max_apply_depth) {
|
|
std::stringstream ss {};
|
|
ss << "Klammer application nested more than " << max_apply_depth
|
|
<< " levels deep while applying " << q_(name) << "."
|
|
<< "A klammer that applies itself, directly or through a cycle "
|
|
<< "of klammers, does not terminate.";
|
|
throw Recursion_error(ss.str(), loc);
|
|
}
|
|
++apply_depth;
|
|
}
|
|
~Depth_guard() { --apply_depth; }
|
|
Depth_guard(const Depth_guard&) = delete;
|
|
Depth_guard& operator=(const Depth_guard&) = delete;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
void Machine::process_eval_katoms(katom_list& katoms)
|
|
{
|
|
(void)K::log(3);
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_eval) != katoms.end()) {
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "eval")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
// Inert: inside a @cond branch that has not been selected. An
|
|
// @eval in a discarded branch must not run.
|
|
if (begin->m_deferred) continue;
|
|
if (begin_eval(*begin)) {
|
|
Eval E(*this, begin->m_loc);
|
|
katom_list eval_katoms = E.eval(begin, end);
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
katoms.insert(end, eval_katoms.begin(), eval_katoms.end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Collect the bars that are direct argument separators of a @cond span:
|
|
// the bar katoms at nesting depth 0 within the span. A bar that lies
|
|
// inside a nested span — for example the "|" in an inner @frac a | b @, or
|
|
// in a nested @eval/@read/@cond — has positive depth and is excluded.
|
|
//
|
|
// This makes @cond's argument delimitation a property of the span tree
|
|
// (the operad's arity: each operator owns the bars at its own level) rather
|
|
// than of the flat katom range. Counting every bar in the range, as the
|
|
// original check did, conflated the arities of nested operators and rejected
|
|
// well-formed input such as
|
|
// @cond *bool* | @frac 1 | 2 @ | @frac 2 | 1 @ @
|
|
// because the inner @frac bars were miscounted as @cond separators.
|
|
//
|
|
// begin is the cond_begin katom; end is one past the closing apply_end, so
|
|
// *(end - 1) is the apply_end. Bars are returned in source order.
|
|
std::vector<katom_iter> cond_separator_bars(katom_iter begin, katom_iter end)
|
|
{
|
|
std::vector<katom_iter> bars {};
|
|
int depth = 0;
|
|
for (auto it = begin + 1; it != end - 1; ++it) {
|
|
if (is_bar(*it) && depth == 0) {
|
|
bars.push_back(it);
|
|
} else if (level_increase(*it)) {
|
|
++depth;
|
|
} else if (level_decrease(*it)) {
|
|
--depth;
|
|
}
|
|
}
|
|
return bars;
|
|
}
|
|
|
|
void check_bar_count(katom_iter begin, std::size_t count)
|
|
{
|
|
if (count != 1 && count != 2) {
|
|
std::stringstream ss {};
|
|
ss << "Incorrectly formatted @cond klammer. There should only be one or two bar characters:\n"
|
|
<< " @cond <predicate> | <result-if-true @\nor:\n"
|
|
<< " @cond <predicate> | <result-if-true> | <result-if-false> @";
|
|
throw Argument_error(ss.str(), begin->m_loc);
|
|
}
|
|
}
|
|
|
|
bool is_true(const std::string& s)
|
|
{
|
|
return s == "True" || s == "true" || s == "1";
|
|
}
|
|
|
|
// @cond's predicate relation is TOTAL AND STRICT (Andy, 2026-08-15, deciding
|
|
// notes/Klammertext_improvements.md §4.1): there is a defined true set, a
|
|
// defined false set, and anything else is an error at the @cond.
|
|
//
|
|
// It was partial in effect until then -- is_true() recognized three strings and
|
|
// treated EVERYTHING else as false, so a misspelled state variable, a "TRUE", a
|
|
// "yes", or a Python traceback all silently selected the false branch. The
|
|
// 2026-08-01 work made that visible with a warning while the policy was
|
|
// undecided; the warning found nothing in the SKS, which is the evidence that
|
|
// the corpus uses well-formed predicates and that the blast radius is small.
|
|
//
|
|
// Empty stays in the FALSE set, and deliberately: an optional argument that
|
|
// was not written substitutes as empty, so "absent means false" is what
|
|
// carries the "@cond *opt* | ... @" idiom. The entangled sub-question in
|
|
// §4.1 -- whether empty means false or means "not supplied" -- is answered
|
|
// "false" by that use, not left open.
|
|
bool is_recognized_predicate(const std::string& s)
|
|
{
|
|
return s.empty()
|
|
|| s == "True" || s == "true" || s == "1"
|
|
|| s == "False" || s == "false" || s == "0";
|
|
}
|
|
|
|
void check_predicate(const std::string& predicate, const Locator& loc)
|
|
{
|
|
if (is_recognized_predicate(predicate)) return;
|
|
std::stringstream ss {};
|
|
ss << "The @cond predicate " << q_(predicate) << " is not a truth value. "
|
|
<< "Recognized: true, True, 1 (true); false, False, 0, and empty (false). "
|
|
<< "A value outside these is an error rather than false, so a misspelled "
|
|
<< "variable or a failed @eval cannot silently select a branch.";
|
|
throw Argument_error(ss.str(), loc);
|
|
}
|
|
|
|
// Mark the interior of every @cond span INERT. Runs during process_katoms,
|
|
// before the passes with observable effects, so that @eval and @read inside a
|
|
// branch do nothing until a branch is selected -- which is the non-strictness
|
|
// doc/cond_evaluation_order.md already specifies ("with side-effecting
|
|
// @read/@eval, wrong ... must not read the missing file") and which @eval did
|
|
// not honour: an @eval in a discarded branch used to run, because eval swept
|
|
// the list before cond did.
|
|
//
|
|
// The delimiters themselves stay unmarked, so the span is still found later.
|
|
// Nested @cond spans are marked by the enclosing one and become live only when
|
|
// the branch holding them is selected and processed.
|
|
void Machine::mark_cond_content(katom_list& katoms)
|
|
{
|
|
(void)K::log(4);
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_cond) == katoms.end()) {
|
|
return;
|
|
}
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
if (!begin_cond(*begin)) continue;
|
|
// Only the BRANCHES are inert. The predicate is always evaluated --
|
|
// that is what a conditional is -- so marking from "begin + 1" would
|
|
// stop "@cond @eval 1==1 @ | yes | no @" from ever computing its own
|
|
// predicate. Mark from the first depth-0 bar onward.
|
|
std::vector<katom_iter> bars = cond_separator_bars(begin, end);
|
|
if (bars.empty()) continue; // malformed; reported when it resolves
|
|
for (auto k = bars[0] + 1; k < end - 1; ++k) {
|
|
k->m_deferred = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Resolve the @cond spans in `katoms`, at APPLICATION time. Returns the number
|
|
// resolved, so the caller's fixed point accounts for them.
|
|
//
|
|
// The selected branch is spliced and then processed exactly as a klammer body
|
|
// is (process_katoms + apply, the two lines apply_klammer already uses): that
|
|
// is what makes the document behave like a function body whose state variables
|
|
// are its arguments -- they are bound by the substitution at the top of
|
|
// Machine::apply, BEFORE any conditional in the document is decided. Resolving
|
|
// @cond at read time meant a top-level "@cond *Flag*" saw the literal "*Flag*".
|
|
int Machine::resolve_cond_katoms(katom_list& katoms, const std::string& target)
|
|
{
|
|
int resolved = 0;
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_cond) != katoms.end()) {
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
if (begin_cond(*begin)) {
|
|
// A @cond nested inside an unresolved outer @cond is still
|
|
// inert; the outer one will process it when its branch is
|
|
// selected. Without this an inner branch would be decided
|
|
// before it is known whether it is reached at all.
|
|
if (begin->m_deferred) continue;
|
|
// Delimit @cond's arguments by the bars at depth 0 within the
|
|
// span, so that bars belonging to nested klammers are not
|
|
// mistaken for @cond's own separators (see cond_separator_bars).
|
|
std::vector<katom_iter> bars = cond_separator_bars(begin, end);
|
|
check_bar_count(begin, bars.size());
|
|
auto bar_1 = bars[0];
|
|
std::string predicate = to_string(begin + 1, bar_1, true);
|
|
check_predicate(predicate, begin->m_loc);
|
|
katom_list true_clause {};
|
|
katom_list false_clause {};
|
|
if (bars.size() == 2) {
|
|
auto bar_2 = bars[1];
|
|
true_clause = katom_list(bar_1 + 1, bar_2);
|
|
false_clause = katom_list(bar_2 + 1, end - 1);
|
|
} else {
|
|
true_clause = katom_list(bar_1 + 1, end - 1);
|
|
}
|
|
// Splice only the selected branch. Its nested klammers remain
|
|
// unreduced here and are reduced by the outer fixed-point apply
|
|
// loop; the unselected branch is discarded without evaluation
|
|
// (@cond is a non-strict special form).
|
|
katom_list result = is_true(predicate)
|
|
? trim_whitespace(true_clause) : trim_whitespace(false_clause);
|
|
// The selected branch becomes live: clear the inert flag and
|
|
// give it the same processing a klammer body gets. The
|
|
// unselected branch is discarded still inert, so nothing in it
|
|
// ever ran.
|
|
for (Katom& k : result) {
|
|
k.m_deferred = false;
|
|
}
|
|
process_katoms(result, command_name);
|
|
apply(m_klammers, result, target);
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
katoms.insert(end, result.begin(), result.end());
|
|
++resolved;
|
|
}
|
|
}
|
|
}
|
|
return resolved;
|
|
}
|
|
|
|
|
|
// Expand the constant klammers written in a definition's BODY. A constant is
|
|
// expanded at definition time, which is what makes it a constant; the body is
|
|
// where that is meaningful.
|
|
//
|
|
// The parameter list is deliberately excluded. A klammer application there
|
|
// is an error (see expand_option_sets): parameters shared between klammers
|
|
// are declared by an option set, whose ".o" declaration is resolved as the
|
|
// parameter list is read. Expanding a constant into a parameter list used to
|
|
// be the way to share parameters, and it silently destroyed the parameter
|
|
// list of a ".k" declaration -- the spliced options AND the declared
|
|
// positionals -- surfacing only as an argument error at the first
|
|
// application, in the document rather than the declaration.
|
|
void Machine::expand_constant_klammers(katom_list& katoms, const Katom& op, const Katom& cl)
|
|
{
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
restore_initial_type(begin + 1, end - 1);
|
|
auto body_begin = std::find_if(
|
|
begin + 1, end - 1, [](const Katom& k) { return is_deftype(k.m_type); });
|
|
if (body_begin == end - 1) return;
|
|
if (std::find_if(body_begin, end - 1, begin_klammer_apply) == end - 1) return;
|
|
for (const auto& [app_op, app_cl] : find_spans(body_begin, end - 1, begin_apply, end_apply, false, "def-time")) {
|
|
auto [app_begin, app_end] = find_span_katoms(katoms, app_op, app_cl);
|
|
if (app_begin->m_type == katom_t::apply_begin) {
|
|
std::string name = trim_char(app_begin->m_text, '@');
|
|
const auto* body = m_klammers.constant_body(name);
|
|
if (body) {
|
|
// Set both m_type and m_initial_type so that
|
|
// restore_initial_type() in add() won't resurrect them
|
|
for (auto it = app_begin; it != app_end; ++it) {
|
|
it->m_type = katom_t::replaced;
|
|
it->m_initial_type = katom_t::replaced;
|
|
}
|
|
katoms.insert(app_end, body->begin(), body->end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//katom_list
|
|
void Machine::mark_literal_klammer_content(katom_list& katoms)
|
|
{
|
|
(void)K::log(4);
|
|
// Collect names of klammers that have a literal parameter
|
|
std::set<std::string> literal_names {};
|
|
for (const auto& [name, klammer] : m_klammers.m_klammers) {
|
|
if (klammer.has_literal_param())
|
|
literal_names.insert(name);
|
|
}
|
|
if (literal_names.empty()) return;
|
|
|
|
// Scan for matching @name ... name@ spans.
|
|
//
|
|
// REMOVED TEXT IS SKIPPED. This pass runs first in process_katoms(), before
|
|
// mark_ignored_katoms() takes out the "#" forms, and that ordering is not
|
|
// accidental: a literal klammer's content must be marked before anything
|
|
// else can interpret what is inside it, or removal would take a "#" that
|
|
// belongs to the literal body. The cost is that this scan sees text the
|
|
// writer has removed, so a literal klammer merely NAMED in a comment --
|
|
//
|
|
// # @image and @code share the same arguments.
|
|
//
|
|
// -- was found as an opening delimiter, went unclosed, and failed the whole
|
|
// file (TODO #40, diagnosed 2026-08-05). "##" was already handled, which is
|
|
// half the case; the fix is to teach the scan the other three removal forms
|
|
// rather than to reorder the passes.
|
|
//
|
|
// The converse still holds, structurally: this skipping happens only while
|
|
// looking for an OPENING delimiter, and finding one jumps k past the whole
|
|
// span, so a "#" inside literal content is never examined here and stays
|
|
// content.
|
|
for (auto k = katoms.begin(); k != katoms.end(); ++k) {
|
|
if (k->m_type == katom_t::ignore_rest) break; // "##": rest of the file
|
|
if (k->m_type == katom_t::ignore_line) { // "#": rest of the line
|
|
while (k + 1 != katoms.end() && (k + 1)->m_type != katom_t::newline) {
|
|
++k;
|
|
}
|
|
continue;
|
|
}
|
|
if (k->m_type == katom_t::ignore_begin) { // "#[ ... ]#", nestable
|
|
int removed = 1;
|
|
while (++k != katoms.end() && removed > 0) {
|
|
if (k->m_type == katom_t::ignore_begin) {
|
|
++removed;
|
|
} else if (k->m_type == katom_t::ignore_end) {
|
|
--removed;
|
|
}
|
|
}
|
|
if (k == katoms.end()) break; // unterminated: the rest is removed
|
|
--k; // the loop's ++k steps past the "]#"
|
|
continue;
|
|
}
|
|
if (k->m_type != katom_t::apply_begin) continue;
|
|
std::string name = trim_char(k->m_text, '@');
|
|
if (literal_names.count(name) == 0) continue;
|
|
(void)K::log(2, "Literal klammer: " + name);
|
|
|
|
// Find the matching named closing delimiter
|
|
std::string close_text = name + "@";
|
|
auto close = k + 1;
|
|
int depth = 1;
|
|
while (close != katoms.end()) {
|
|
if (close->m_type == katom_t::apply_begin &&
|
|
trim_char(close->m_text, '@') == name)
|
|
depth++;
|
|
else if (close->m_type == katom_t::apply_end &&
|
|
trim_char(close->m_text, '@') == name)
|
|
depth--;
|
|
if (depth == 0) break;
|
|
++close;
|
|
}
|
|
if (close == katoms.end()) {
|
|
throw Parsing_error(
|
|
"Klammer " + q_(name) + " has a literal parameter and must be closed with "
|
|
+ q_(close_text),
|
|
k->m_loc);
|
|
}
|
|
|
|
// Count positional parameters before the literal one.
|
|
// The literal parameter is always last. Bars separate
|
|
// the preceding positional arguments and the literal content.
|
|
const auto& klammer = m_klammers.m_klammers[name];
|
|
int bars_before_literal = 0;
|
|
for (const auto& p : klammer.m_parameters.m_positional) {
|
|
if (p.m_argtype.m_name == "literal") break;
|
|
bars_before_literal++;
|
|
}
|
|
|
|
// Find where literal content starts.
|
|
// Skip bars_before_literal bars (separating preceding positional args).
|
|
// If options are present, skip past the bar after them.
|
|
// Options are identified by :name katoms before any bar.
|
|
auto literal_start = k + 1;
|
|
bool has_options = false;
|
|
for (auto j = k + 1; j < close; ++j) {
|
|
if (j->m_type == katom_t::option_name) {
|
|
has_options = true;
|
|
}
|
|
if (j->m_type == katom_t::bar) {
|
|
if (bars_before_literal > 0) {
|
|
bars_before_literal--;
|
|
literal_start = j + 1;
|
|
} else if (has_options) {
|
|
// This bar separates options from literal content
|
|
literal_start = j + 1;
|
|
break;
|
|
} else {
|
|
// No preceding args, no options: bar is part of literal
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
std::for_each(literal_start, close, mark_as_literal);
|
|
|
|
k = close; // Skip past this span
|
|
}
|
|
}
|
|
|
|
void Machine::process_katoms(
|
|
katom_list& katoms, const std::string& source,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
|
|
bool eval, bool cond, bool read)
|
|
{
|
|
mark_literal_klammer_content(katoms);
|
|
if (literal) mark_literal_katoms(katoms);
|
|
hide_special_katoms(katoms);
|
|
if (nonascii) encode_nonascii_characters(katoms);
|
|
if (ignore) mark_ignored_katoms(katoms);
|
|
if (whitespace) process_whitespace_modifiers(katoms);
|
|
if (klammers) process_klammer_katoms(katoms);
|
|
// @cond is no longer RESOLVED here -- it is resolved in the apply fold
|
|
// (see mark_cond_content / resolve_cond_katoms). What happens here is the
|
|
// marking that makes its branches inert, and it must run BEFORE the eval
|
|
// and read passes, which are the ones with observable effects. Leaving it
|
|
// where process_cond_katoms used to sit -- after eval -- kept the old
|
|
// "eval in a discarded branch runs anyway" behaviour, since the marking
|
|
// arrived too late to stop it.
|
|
if (cond) mark_cond_content(katoms);
|
|
if (eval) process_eval_katoms(katoms);
|
|
if (read) expand_read_katoms(
|
|
katoms, source,
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
warn_unparsed_katoms(katoms, m_warn_unparsed);
|
|
// return katoms;
|
|
}
|
|
|
|
katom_list Machine::process(
|
|
std::string text, const std::string& source,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
|
|
bool eval, bool cond, bool read)
|
|
{
|
|
katom_list katoms = katomize(line_split(text), source);
|
|
//katoms =
|
|
process_katoms(
|
|
katoms, source,
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
return katoms;
|
|
}
|
|
|
|
void Machine::read(const fs::path& pathname)
|
|
{
|
|
(void)K::log(3, pathname.string());
|
|
// A file's directory joins the @eval search path (Python modules and
|
|
// :cpp libraries live next to the file that uses them).
|
|
m_state.add_search_dir(fs::absolute(pathname).parent_path().string());
|
|
m_state.open_frame("Machine state: " + pathname.string());
|
|
std::string text = m_state.subst(trim_right(string_from_file(pathname)));
|
|
katom_list katoms = process(text, pathname);
|
|
m_katoms.insert(m_katoms.end(), katoms.begin(), katoms.end());
|
|
extract_machine_definitions();
|
|
extract_klammer_definitions();
|
|
m_sources.push_back(pathname);
|
|
}
|
|
|
|
void Machine::read(const std::string& s)
|
|
{
|
|
(void)K::log(3, s);
|
|
m_state.open_frame("Machine state: " + s);
|
|
std::string text = m_state.subst(trim_right(s));
|
|
katom_list katoms = process(text, command_pathname);
|
|
m_katoms.insert(m_katoms.end(), katoms.begin(), katoms.end());
|
|
extract_machine_definitions();
|
|
extract_klammer_definitions();
|
|
m_sources.push_back(s);
|
|
}
|
|
|
|
// Read
|
|
|
|
void Machine::expand_read_katoms(
|
|
katom_list& katoms, std::string current_filename,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace,
|
|
bool klammers, bool eval, bool cond, bool read)
|
|
{
|
|
(void)K::log(3);
|
|
current_filename = resolve_relative_to(current_filename);
|
|
// msg() << "current_filename: " << current_filename << "\n";
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_read) != katoms.end()) {
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(katoms, begin_apply, end_apply, true, "read")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
// Inert: see the same guard in process_eval_katoms.
|
|
if (begin->m_deferred) continue;
|
|
if (begin_read(*begin)) {
|
|
std::string read_filename = to_string(begin + 1, end - 1, true);
|
|
|
|
// msg() << "read: " << resolve_relative_to(read_filename, current_filename) << "\n";
|
|
|
|
/*
|
|
std::string current_directory =
|
|
fs::path(current_filename).parent_path().string();
|
|
fs::path input_filename =
|
|
fs::path(current_directory + "/" + read_filename);
|
|
*/
|
|
|
|
fs::path input_filename = resolve_relative_to(read_filename, current_filename);
|
|
// msg() << "read: " << input_filename << "\n";
|
|
|
|
(void)K::log(2, input_filename.string());
|
|
if (!fs::exists(input_filename)) {
|
|
std::stringstream ss{};
|
|
ss <<"File " << input_filename << " does not exist";
|
|
throw File_error(ss.str(), begin->m_loc);
|
|
}
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
|
|
input_filename = fs::canonical(input_filename);
|
|
m_state.add_search_dir(input_filename.parent_path().string());
|
|
std::string text = trim_right(string_from_file(input_filename.string()));
|
|
katom_list ks = katomize(line_split(text), input_filename);
|
|
// ks =
|
|
process_katoms(
|
|
// ks, command_pathname,
|
|
ks, input_filename,
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
katoms.insert(end, ks.begin(), ks.end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// One @@@ declaration. Factored out so the tolerant and strict paths share
|
|
// it; `rescan` is set when a klammerset has spliced files into the stream,
|
|
// which invalidates the caller's span list.
|
|
void Machine::add_machine_definition(
|
|
const std::string& name, katom_iter begin, katom_iter end, bool& rescan)
|
|
{
|
|
if (name == "@@@target") {
|
|
m_targets.add(begin, end, m_katoms);
|
|
} else if (name == "@@@argtype") {
|
|
m_argtypes.add(begin, end, m_katoms);
|
|
} else if (name == "@@@state") {
|
|
m_state.parse_state_katoms(begin, end, m_katoms);
|
|
} else if (name == "@@@klammerset") {
|
|
if (auto klammerset = m_klammersets.add(begin, end, m_katoms)) {
|
|
load_klammerset_files(*klammerset, end);
|
|
rescan = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
void Machine::extract_machine_definitions(bool tolerant)
|
|
{
|
|
(void)K::log(3);
|
|
if (m_katoms.empty()) {
|
|
return;
|
|
}
|
|
// A @@@klammerset declaration inserts its files' katoms into the stream
|
|
// at the declaration point, invalidating the span list, so the scan
|
|
// restarts. Processed spans are marked replaced and are never found
|
|
// again, which also bounds the restarts.
|
|
bool rescan = true;
|
|
while (rescan) {
|
|
rescan = false;
|
|
for (const auto& [op, cl] : find_spans(m_katoms, begin_machine_def, end_machine_def, true, command_name)) {
|
|
auto [begin, end] = find_span_katoms(m_katoms, op, cl);
|
|
std::string name = begin->m_text;
|
|
if (tolerant) {
|
|
// kdiag: a declaration that cannot be carried out -- a
|
|
// @@@klammerset naming a file that is not there, say -- is
|
|
// skipped rather than fatal. Its span stays unreplaced, so
|
|
// the katoms remain visible and report the failure themselves.
|
|
try {
|
|
add_machine_definition(name, begin, end, rescan);
|
|
} catch (Error& e) {
|
|
K::log(1, "Machine definition not registered: " + e.m_desc);
|
|
continue;
|
|
}
|
|
} else {
|
|
add_machine_definition(name, begin, end, rescan);
|
|
}
|
|
if (rescan) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Machine::load_klammerset_files(const Klammerset& klammerset, katom_iter insert_at)
|
|
{
|
|
(void)K::log(2, "Loading klammerset \"" + klammerset.m_symbol + "\"");
|
|
// Relative names resolve against the declaring file's directory, never
|
|
// the cwd. :requires files are read before the set's own files; each
|
|
// holds its own @@@klammerset declaration, whose already-loaded guard
|
|
// makes repeated requirements a no-op.
|
|
fs::path declaring(klammerset.m_loc.m_filename);
|
|
fs::path base = fs::exists(declaring) ? declaring : fs::current_path();
|
|
std::string base_dir = (is_directory(base) ? base : base.parent_path()).string();
|
|
|
|
// A :requires entry may be a bare symbol, resolved on the klammerset
|
|
// search path with the declaring directory as the local stage; :files
|
|
// entries are always filenames (this set's own definition files).
|
|
std::vector<std::string> filenames {};
|
|
for (const auto& required : klammerset.m_requires) {
|
|
std::string path = is_klammerset_symbol(required)
|
|
? resolve_klammerset_symbol(required, base_dir, klammerset.m_loc).string()
|
|
: required;
|
|
// ALREADY LOADED? Decided BEFORE the file is opened. The guard used
|
|
// to sit in Klammerset_registry::add, which only runs once the file has
|
|
// been read and its declaration reached -- by which time the declaring
|
|
// file's OWN definitions have been re-executed. Two sets requiring a
|
|
// third therefore died on "Target ... is already defined", pointing at
|
|
// a line the author wrote once.
|
|
//
|
|
// Deciding it here is possible only because a klammerset symbol X is
|
|
// declared in X/X.k, so the symbol is a function of the path: it is
|
|
// known for a ":requires" written as a filename just as much as for one
|
|
// written as a symbol. That is what the X/X.k requirement bought.
|
|
std::string symbol = is_klammerset_symbol(required)
|
|
? required : fs::path(path).stem().string();
|
|
if (m_klammersets.has(symbol)) {
|
|
(void)K::log(1, "Klammerset \"" + symbol + "\": already loaded, skipped");
|
|
continue;
|
|
}
|
|
filenames.push_back(path);
|
|
}
|
|
filenames.insert(filenames.end(), klammerset.m_files.begin(), klammerset.m_files.end());
|
|
|
|
// Collect all files into one list and insert once: insert_at is
|
|
// invalidated by the first insertion into m_katoms.
|
|
// Recorded on the set: what it actually opened, canonical, so a display
|
|
// can tell its klammers from the input's. The declaring file counts --
|
|
// definitions may sit before or after the declaration in it.
|
|
strings_t loaded_files {};
|
|
if (fs::exists(declaring)) {
|
|
loaded_files.push_back(fs::canonical(declaring).string());
|
|
}
|
|
katom_list loaded {};
|
|
for (const auto& filename : filenames) {
|
|
fs::path pathname = resolve_relative_to(filename, base);
|
|
if (!fs::exists(pathname)) {
|
|
throw Klammerset_error(
|
|
"Klammerset \"" + klammerset.m_symbol + "\" lists the file \"" + filename
|
|
+ "\", which does not exist (resolved to \"" + pathname.string() + "\")",
|
|
klammerset.m_loc);
|
|
}
|
|
pathname = fs::canonical(pathname);
|
|
loaded_files.push_back(pathname.string());
|
|
m_state.add_search_dir(pathname.parent_path().string());
|
|
std::string text = trim_right(string_from_file(pathname.string()));
|
|
katom_list ks = katomize(line_split(text), pathname);
|
|
process_katoms(ks, pathname);
|
|
loaded.insert(loaded.end(), ks.begin(), ks.end());
|
|
}
|
|
m_katoms.insert(insert_at, loaded.begin(), loaded.end());
|
|
// The registry holds the registered copy; the parameter is a const
|
|
// reference to it, so the record goes back through the registry.
|
|
m_klammersets.set_loaded_files(klammerset.m_symbol, loaded_files);
|
|
}
|
|
|
|
// Register one "@@...@@" definition. An ".o" target declares an option set
|
|
// -- parameters shared by klammers -- and goes to its own registry: it
|
|
// defines no klammer and produces no output for any target.
|
|
void Machine::add_definition(katom_list& katoms, const Katom& op, const Katom& cl)
|
|
{
|
|
expand_constant_klammers(katoms, op, cl);
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
auto [name, target_names, general_declared] = parse_name(m_targets, *begin);
|
|
(void)general_declared; // routing only cares whether this is a ".o"
|
|
// parse_name rejects ".o" as a member of a target list, so an option set
|
|
// declaration is always the single-target form.
|
|
if (target_names.size() == 1 && target_names[0] == Target_registry::optionset_name) {
|
|
m_option_sets.add(m_argtypes, name, begin, end, katoms);
|
|
} else {
|
|
m_klammers.add(m_argtypes, m_targets, m_option_sets, begin, end, katoms);
|
|
}
|
|
}
|
|
|
|
void Machine::extract_klammer_definitions(katom_list katoms)
|
|
{
|
|
// fmsg() << katoms << "\n";
|
|
(void)K::log(3, katoms);
|
|
for (const auto& [op, cl] : find_spans(katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
|
|
add_definition(katoms, op, cl);
|
|
}
|
|
m_klammers.rationalize(m_targets);
|
|
}
|
|
|
|
void Machine::extract_klammer_definitions(bool tolerant)
|
|
{
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(m_katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
|
|
if (tolerant) {
|
|
// add_definition marks the span replaced only on success, so a
|
|
// skipped definition keeps its katoms and stays visible.
|
|
try {
|
|
add_definition(m_katoms, op, cl);
|
|
} catch (Error& e) {
|
|
K::log(1, "Definition not registered: " + e.m_desc);
|
|
}
|
|
} else {
|
|
add_definition(m_katoms, op, cl);
|
|
}
|
|
}
|
|
m_klammers.rationalize(m_targets);
|
|
}
|
|
|
|
void Machine::update_state(const std::map<std::string, std::string>& arg_map)
|
|
{
|
|
for (const auto& [k, v] : arg_map) {
|
|
m_state.set(k, v);
|
|
}
|
|
}
|
|
|
|
katom_list Machine::apply_klammer(
|
|
Klammer& klammer, const std::string& target, katom_iter arguments_begin, katom_iter arguments_end)
|
|
{
|
|
(void)K::log(3, "argument substitution", *arguments_begin, *(arguments_end - 1));
|
|
Depth_guard depth_guard(klammer.m_name, arguments_begin->m_loc);
|
|
m_state.replace("K_loc", arguments_begin->m_loc.str(), false);
|
|
auto [positional, optional, rest] =
|
|
argument_split(arguments_begin + 1, arguments_end - 1, klammer.m_parameters.m_positional.size());
|
|
auto values = klammer.m_parameters.value_map(positional, optional, rest, arguments_begin->m_loc);
|
|
|
|
// Resolve KTESC markers in argument values so that @eval code receives
|
|
// the original characters (e.g., filenames with underscores). The markers
|
|
// remain in the klammer body substitution for final target-specific output.
|
|
katom_list result(klammer.m_body[target].begin(), klammer.m_body[target].end());
|
|
auto varmap = klammer.m_varmap[target];
|
|
// Escape target-specific characters (e.g. tex "&" -> "\&") in the writer
|
|
// text of a GENERAL klammer's body. Runs BEFORE the *arg* substitution
|
|
// below: an argument value is writer text already escaped at the top
|
|
// level (KTESC markers, idempotent) plus final target markup from
|
|
// klammers the writer nested in the argument, and neither may be escaped
|
|
// here -- escaping after substitution swept both, so a general klammer
|
|
// with a klammer-bearing argument emitted \textbackslash{}emph{...}
|
|
// (fixed 2026-08-19; tst/escape_test.sh cases 26-29). A karg katom is
|
|
// skipped by the type filter; a variable inside a mixed text katom
|
|
// survives because no target declares '*' or identifier characters as
|
|
// escapes. Also runs BEFORE process_katoms/apply() expand the body, so
|
|
// that target-native markup pulled in by nested klammers (e.g. nl.tex ->
|
|
// "\newline") is left untouched -- only this klammer's own literal writer
|
|
// text is escaped here; nested klammers escape theirs when they are
|
|
// applied in turn. Bodies from target-specific definitions
|
|
// (m_body_generic[target] == false) are already in target form and
|
|
// skipped. Two kinds of body content are NOT
|
|
// writer text and must be skipped:
|
|
// * ^'...'^ literal spans -- raw target markup the writer typed directly.
|
|
// At this point they are typed literal_begin/literal_end with plain-text
|
|
// content (the literal phase runs in process_katoms, below), so track
|
|
// span depth rather than testing katom type.
|
|
// * @eval / @read / @cond argument spans -- code, filenames, and
|
|
// predicates consumed by the primitive, NOT emitted as target text.
|
|
// (Escaping an underscore in "offer.Price_list(K)" broke @eval.) The
|
|
// primitive's KLAMMERTEXT result, produced by process_katoms below, is
|
|
// klammer output and is likewise never escaped -- it is inserted after
|
|
// this pass and so is untouched, matching the top-level behavior where
|
|
// @eval is resolved before the escape pass runs.
|
|
auto gen = klammer.m_body_generic.find(target);
|
|
if (gen != klammer.m_body_generic.end() && gen->second) {
|
|
Target tgt = m_targets.get(target, Locator());
|
|
if (!tgt.m_escapes.empty()) {
|
|
int literal_depth = 0;
|
|
int code_depth = 0; // inside an @eval/@read/@cond span
|
|
std::vector<bool> apply_is_code; // one entry per open application
|
|
for (auto& k : result) {
|
|
if (k.m_type == katom_t::literal_begin) { ++literal_depth; continue; }
|
|
if (k.m_type == katom_t::literal_end) {
|
|
if (literal_depth > 0) --literal_depth;
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::eval_begin ||
|
|
k.m_type == katom_t::read_begin ||
|
|
k.m_type == katom_t::cond_begin) {
|
|
apply_is_code.push_back(true);
|
|
++code_depth;
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::apply_begin) {
|
|
apply_is_code.push_back(false);
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::apply_end) {
|
|
if (!apply_is_code.empty()) {
|
|
if (apply_is_code.back()) --code_depth;
|
|
apply_is_code.pop_back();
|
|
}
|
|
continue;
|
|
}
|
|
if (literal_depth == 0 && code_depth == 0 &&
|
|
(k.m_type == katom_t::text ||
|
|
k.m_type == katom_t::word ||
|
|
k.m_type == katom_t::newline))
|
|
k.m_text = tgt.escape_text(k.m_text);
|
|
}
|
|
}
|
|
}
|
|
m_state.open_frame("Arguments for klammer " + q_(klammer.m_name));
|
|
m_state.set(values, klammer.m_parameters);
|
|
for (const auto& [name, indices] : varmap) {
|
|
std::regex arg("\\*" + name + "\\*");
|
|
for (auto i : indices) {
|
|
result[i].m_text = std::regex_replace(
|
|
result[i].m_text, arg, m_state.value(name, true, result[i].m_loc));
|
|
result[i].m_type = katom_t::text;
|
|
}
|
|
}
|
|
process_katoms(result, klammer.m_name);
|
|
apply(m_klammers, result, target);
|
|
m_state.close_frame();
|
|
// msg() << boldblack << "APPLY: " << std::pair(arguments_begin, arguments_end) << "\n"
|
|
// << boldblack << "RESULT: " << ktype << result << black << "\n";
|
|
|
|
modify_type(katom_t::replaced, arguments_begin, arguments_end);
|
|
return result;
|
|
}
|
|
|
|
void Machine::apply_klammer_registry(
|
|
Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target, katom_iter begin, katom_iter end)
|
|
{
|
|
(void)K::log(3, "Klammer");
|
|
std::string name = trim_char(begin->m_text, '@');
|
|
katom_list applied_katoms = apply_klammer(klammer_registry.m_klammers[name], target, begin, end);
|
|
for (auto& k : applied_katoms) {
|
|
if (k.m_type == katom_t::bar || k.m_type == katom_t::double_bar || k.m_type == katom_t::option_name) {
|
|
k.m_type = katom_t::text;
|
|
}
|
|
}
|
|
katoms.insert(end, applied_katoms.begin(), applied_katoms.end());
|
|
}
|
|
|
|
int Machine::apply(
|
|
Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target)
|
|
{
|
|
(void)K::log(3, "Klammer_registry");
|
|
// @cond is a special form handled here rather than by the klammer loop
|
|
// below: its span head is a cond_begin, which begin_klammer_apply does not
|
|
// match. Resolving first means a klammer revealed by the selected branch
|
|
// is applied in this same pass. The count is returned with the klammer
|
|
// applications, so the caller's fixed point iterates while either happens.
|
|
int applied = resolve_cond_katoms(katoms, target);
|
|
for (const auto& [op, cl] : find_spans(
|
|
katoms, begin_klammer_apply, end_klammer_apply, true, command_name)) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
klammer_registry.check_klammer(
|
|
klammer_name_from_katom(begin->m_text, begin->m_loc),
|
|
target, begin->m_loc);
|
|
apply_klammer_registry(klammer_registry, katoms, target, begin, end);
|
|
++applied;
|
|
}
|
|
return applied;
|
|
}
|
|
|
|
std::string Machine::run_phase_functions()
|
|
{
|
|
Target target = m_targets.get(m_state.value("K_target"), Locator());
|
|
if (!target.m_after_apply.empty()) {
|
|
(void)K::log(2, target);
|
|
for (auto f : target.m_after_apply) {
|
|
// A mode-tagged spec (":cpp ...") names a function that receives
|
|
// the Machine itself; a bare Python function is called with the
|
|
// result text. The Eval is constructed per phase so a chained
|
|
// phase sees its predecessor's result in K_result.
|
|
Eval E(*this, Locator());
|
|
if (!f.empty() && f[0] != ':') {
|
|
// K reaches a phase only as a parameter: the state's
|
|
// "class K" exists in the EVAL's globals (python_code()
|
|
// defines it), but a phase function's body resolves names in
|
|
// its own MODULE's globals. Every phase signature has
|
|
// carried "K=None" for this since the calling convention was
|
|
// created; the call never passed it, which surfaced when
|
|
// txt_justify_blocks needed Target_txt_width (2026-08-22).
|
|
// K_phase_call (defined in Eval_python's globals) passes K
|
|
// only to a function that DECLARES a K parameter, so a
|
|
// stdlib phase (string.capwords, whose second parameter is a
|
|
// separator) keeps working.
|
|
f = "K_phase_call(" + f + ", K_result, K)";
|
|
}
|
|
f = "@eval " + f + " @";
|
|
auto katoms = katomize(line_split(f), "phase");
|
|
// A phase function's input and output are final target text, not
|
|
// Klammertext: take the raw result string. Re-reading it as
|
|
// Klammertext (Eval::eval) would misparse target characters --
|
|
// e.g. a "@" from a quoted ^@ in justified txt output.
|
|
m_result = E.eval_command(katoms.begin(), katoms.end() - 2);
|
|
}
|
|
}
|
|
return m_result;
|
|
}
|
|
|
|
|
|
void Machine::escape_target_characters(const Target& target, katom_list& katoms)
|
|
{
|
|
if (target.m_escapes.empty()) return;
|
|
for (auto& k : katoms) {
|
|
// Only escape writer content katoms — text, words, and newlines.
|
|
// Skip structural katoms (option names, bars, klammer delimiters)
|
|
// whose text is Klammertext syntax, not writer content.
|
|
if (k.m_type == katom_t::text ||
|
|
k.m_type == katom_t::word ||
|
|
k.m_type == katom_t::newline) {
|
|
k.m_text = target.escape_text(k.m_text);
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string Machine::apply(const std::string& target_name, bool final_processing, bool escape_characters)
|
|
{
|
|
(void)K::log(3, "top level");
|
|
m_state.set("K_target", target_name);
|
|
m_state.subst(m_katoms.begin(), m_katoms.end());
|
|
|
|
// Escape target-specific characters in writer text before klammer application.
|
|
// Characters produced later by klammer bodies will not be escaped.
|
|
// Skipped for sub-Machine apply() calls (e.g., from @eval), where the
|
|
// text is already in target-specific form.
|
|
auto target = m_targets.get(target_name, Locator::none());
|
|
if (escape_characters)
|
|
escape_target_characters(target, m_katoms);
|
|
|
|
// Reduce to a fixed point. A pass reports how many klammers it applied;
|
|
// the loop ends when a pass applies none. (It formerly ended when the
|
|
// katom list stopped GROWING, which is not the same thing: a klammer whose
|
|
// body expands to nothing is a reduction that adds no katoms.) Exceeding
|
|
// the round limit is now an error rather than a message followed by
|
|
// rendering the unreduced document -- silently emitting a document with
|
|
// live klammers still in it is worse than not emitting one. Runaway
|
|
// recursion is caught earlier and more precisely by the depth guard in
|
|
// apply_klammer(); this limit only bounds the number of ROUNDS, which is
|
|
// the length of a chain of klammers that generate further klammers.
|
|
int apply_count = 0;
|
|
while (apply(m_klammers, m_katoms, target_name) > 0) {
|
|
if (++apply_count > apply_round_limit) {
|
|
std::stringstream ss {};
|
|
ss << "Klammer application did not reach a fixed point after "
|
|
<< apply_round_limit << " rounds.\n\n"
|
|
<< "Each round applies every klammer present; a klammer whose "
|
|
<< "result contains further klammers starts another round.";
|
|
throw Recursion_error(ss.str(), Locator());
|
|
}
|
|
}
|
|
// Typographic transforms run on the katoms, not the joined string, so
|
|
// that ^'...'^ literal content (katom_t::literal) stays verbatim --
|
|
// Target::transform() skips literal katoms. Verbatim text produced by
|
|
// an @eval renderer (@code, @c) is plain text by the time it is spliced
|
|
// back; the SKS protects it with KTESC markers (hide_typographic() in
|
|
// klammer_base.py), which are inert here and decode in resolve_escapes.
|
|
if (final_processing) {
|
|
target.transform(m_katoms);
|
|
}
|
|
m_result = to_string(m_katoms.begin(), m_katoms.end());
|
|
|
|
if (final_processing) {
|
|
m_result = target.resolve_escapes(m_result);
|
|
m_result = run_phase_functions();
|
|
}
|
|
m_result = trim_char(m_result, '\n');
|
|
return m_result;
|
|
}
|