Assembled from klammertext-dev f344a37d772c. The commands, the library,
the Standard Klammer Set, the reflection klammerset and the Python
binding build and run natively on Windows (MSYS2 UCRT64 gcc, python.org
Python), and both test tiers pass there. doc/install/windows_source_install.md
is the fifth install guide: a source install parallel to the Linux and
macOS ones, verified by following it on a pristine Windows 11.
Behaviour on every system: paths Klammertext prints use forward slashes;
output is LF; the command line is UTF-8 on Windows; @eval :shell is a
POSIX shell command everywhere -- on Windows it runs the sh Klammertext
ships, BusyBox for Windows (win/busybox/, GPLv2, with its licence and the
source pointer). KLAMMERTEXT_FONTS and KLAMMERTEXT_KLAMMERSETS are lists in
the system's PATH convention (';' on Windows). On Linux and macOS nothing
observable changes.
1002 lines
46 KiB
C++
1002 lines
46 KiB
C++
#include <algorithm>
|
|
#include <set>
|
|
#include <utility>
|
|
#include "machine.h"
|
|
#include "katomize.h"
|
|
#include "error.h"
|
|
#include "show.h"
|
|
#include "util.h"
|
|
#include "file.h"
|
|
#include "log.h"
|
|
#include "eval.h"
|
|
|
|
Machine::Machine()
|
|
: m_argtypes(Argtype_registry())
|
|
, m_state(State())
|
|
, m_targets(Target_registry())
|
|
, m_klammers(Klammer_registry())
|
|
{
|
|
(void)K::log(3);
|
|
m_state.add_environment_frame();
|
|
/*
|
|
if (sks) {
|
|
fs::path sks_filename(klammertext_filename("sks/sks.k"));
|
|
// msg() << "SKS filename: " << sks_filename << "\n";
|
|
read(sks_filename);
|
|
}
|
|
*/
|
|
}
|
|
|
|
// Klammer application recursion guard.
|
|
//
|
|
// Applying a klammer expands its body, which is processed and applied in
|
|
// turn (apply_klammer -> process_katoms -> apply -> apply_klammer), so a
|
|
// klammer that reaches itself -- directly (@@f : x @f@ @@) or through a
|
|
// cycle -- descends without bound. Before this guard the descent simply
|
|
// exhausted the C++ stack: SIGSEGV, no message, no location.
|
|
//
|
|
// The counter is a translation-unit static rather than a Machine member for
|
|
// two reasons: recursion can cross Machine instances (Eval::eval builds a
|
|
// sub-Machine to re-read an @eval result, and that sub-Machine applies
|
|
// klammers on the same C++ stack), and keeping it out of Machine avoids
|
|
// changing the class layout shared with the dlopened sks/document.so.
|
|
//
|
|
// The limit bounds the C++ stack, not the language: it is far above any
|
|
// plausible nesting depth in a document, and reaching it means a klammer
|
|
// does not terminate. NOTE: not thread-safe; if input files are ever
|
|
// processed in parallel this needs to become thread_local.
|
|
namespace {
|
|
|
|
constexpr int max_apply_depth = 200;
|
|
int apply_depth = 0;
|
|
|
|
// Rounds of the top-level fixed-point loop (see Machine::apply below). The
|
|
// former limit of 5 was a silent truncation; it is now an error, so it is set
|
|
// well clear of any legitimate chain of klammers generating klammers.
|
|
constexpr int apply_round_limit = 100;
|
|
|
|
class Depth_guard
|
|
{
|
|
public:
|
|
Depth_guard(const std::string& name, const Locator& loc)
|
|
{
|
|
if (apply_depth >= max_apply_depth) {
|
|
std::stringstream ss {};
|
|
ss << "Klammer application nested more than " << max_apply_depth
|
|
<< " levels deep while applying " << q_(name) << "."
|
|
<< "A klammer that applies itself, directly or through a cycle "
|
|
<< "of klammers, does not terminate.";
|
|
throw Recursion_error(ss.str(), loc);
|
|
}
|
|
++apply_depth;
|
|
}
|
|
~Depth_guard() { --apply_depth; }
|
|
Depth_guard(const Depth_guard&) = delete;
|
|
Depth_guard& operator=(const Depth_guard&) = delete;
|
|
};
|
|
|
|
} // namespace
|
|
|
|
void Machine::process_eval_katoms(katom_list& katoms)
|
|
{
|
|
(void)K::log(3);
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_eval) != katoms.end()) {
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "eval")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
// Inert: inside a @cond branch that has not been selected. An
|
|
// @eval in a discarded branch must not run.
|
|
if (begin->m_deferred) continue;
|
|
if (begin_eval(*begin)) {
|
|
Eval E(*this, begin->m_loc);
|
|
katom_list eval_katoms = E.eval(begin, end);
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
katoms.insert(end, eval_katoms.begin(), eval_katoms.end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Collect the bars that are direct argument separators of a @cond span:
|
|
// the bar katoms at nesting depth 0 within the span. A bar that lies
|
|
// inside a nested span — for example the "|" in an inner @frac a | b @, or
|
|
// in a nested @eval/@read/@cond — has positive depth and is excluded.
|
|
//
|
|
// This makes @cond's argument delimitation a property of the span tree
|
|
// (the operad's arity: each operator owns the bars at its own level) rather
|
|
// than of the flat katom range. Counting every bar in the range, as the
|
|
// original check did, conflated the arities of nested operators and rejected
|
|
// well-formed input such as
|
|
// @cond *bool* | @frac 1 | 2 @ | @frac 2 | 1 @ @
|
|
// because the inner @frac bars were miscounted as @cond separators.
|
|
//
|
|
// begin is the cond_begin katom; end is one past the closing apply_end, so
|
|
// *(end - 1) is the apply_end. Bars are returned in source order.
|
|
std::vector<katom_iter> cond_separator_bars(katom_iter begin, katom_iter end)
|
|
{
|
|
std::vector<katom_iter> bars {};
|
|
int depth = 0;
|
|
for (auto it = begin + 1; it != end - 1; ++it) {
|
|
if (is_bar(*it) && depth == 0) {
|
|
bars.push_back(it);
|
|
} else if (level_increase(*it)) {
|
|
++depth;
|
|
} else if (level_decrease(*it)) {
|
|
--depth;
|
|
}
|
|
}
|
|
return bars;
|
|
}
|
|
|
|
void check_bar_count(katom_iter begin, std::size_t count)
|
|
{
|
|
if (count != 1 && count != 2) {
|
|
std::stringstream ss {};
|
|
ss << "Incorrectly formatted @cond klammer. There should only be one or two bar characters:\n"
|
|
<< " @cond <predicate> | <result-if-true @\nor:\n"
|
|
<< " @cond <predicate> | <result-if-true> | <result-if-false> @";
|
|
throw Argument_error(ss.str(), begin->m_loc);
|
|
}
|
|
}
|
|
|
|
bool is_true(const std::string& s)
|
|
{
|
|
return s == "True" || s == "true" || s == "1";
|
|
}
|
|
|
|
// @cond's predicate relation is TOTAL AND STRICT (Andy, 2026-08-15, deciding
|
|
// notes/Klammertext_improvements.md §4.1): there is a defined true set, a
|
|
// defined false set, and anything else is an error at the @cond.
|
|
//
|
|
// It was partial in effect until then -- is_true() recognized three strings and
|
|
// treated EVERYTHING else as false, so a misspelled state variable, a "TRUE", a
|
|
// "yes", or a Python traceback all silently selected the false branch. The
|
|
// 2026-08-01 work made that visible with a warning while the policy was
|
|
// undecided; the warning found nothing in the SKS, which is the evidence that
|
|
// the corpus uses well-formed predicates and that the blast radius is small.
|
|
//
|
|
// Empty stays in the FALSE set, and deliberately: an optional argument that
|
|
// was not written substitutes as empty, so "absent means false" is what
|
|
// carries the "@cond *opt* | ... @" idiom. The entangled sub-question in
|
|
// §4.1 -- whether empty means false or means "not supplied" -- is answered
|
|
// "false" by that use, not left open.
|
|
bool is_recognized_predicate(const std::string& s)
|
|
{
|
|
return s.empty()
|
|
|| s == "True" || s == "true" || s == "1"
|
|
|| s == "False" || s == "false" || s == "0";
|
|
}
|
|
|
|
void check_predicate(const std::string& predicate, const Locator& loc)
|
|
{
|
|
if (is_recognized_predicate(predicate)) return;
|
|
std::stringstream ss {};
|
|
ss << "The @cond predicate " << q_(predicate) << " is not a truth value. "
|
|
<< "Recognized: true, True, 1 (true); false, False, 0, and empty (false). "
|
|
<< "A value outside these is an error rather than false, so a misspelled "
|
|
<< "variable or a failed @eval cannot silently select a branch.";
|
|
throw Argument_error(ss.str(), loc);
|
|
}
|
|
|
|
// Mark the interior of every @cond span INERT. Runs during process_katoms,
|
|
// before the passes with observable effects, so that @eval and @read inside a
|
|
// branch do nothing until a branch is selected -- which is the non-strictness
|
|
// doc/cond_evaluation_order.md already specifies ("with side-effecting
|
|
// @read/@eval, wrong ... must not read the missing file") and which @eval did
|
|
// not honour: an @eval in a discarded branch used to run, because eval swept
|
|
// the list before cond did.
|
|
//
|
|
// The delimiters themselves stay unmarked, so the span is still found later.
|
|
// Nested @cond spans are marked by the enclosing one and become live only when
|
|
// the branch holding them is selected and processed.
|
|
void Machine::mark_cond_content(katom_list& katoms)
|
|
{
|
|
(void)K::log(4);
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_cond) == katoms.end()) {
|
|
return;
|
|
}
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
if (!begin_cond(*begin)) continue;
|
|
// Only the BRANCHES are inert. The predicate is always evaluated --
|
|
// that is what a conditional is -- so marking from "begin + 1" would
|
|
// stop "@cond @eval 1==1 @ | yes | no @" from ever computing its own
|
|
// predicate. Mark from the first depth-0 bar onward.
|
|
std::vector<katom_iter> bars = cond_separator_bars(begin, end);
|
|
if (bars.empty()) continue; // malformed; reported when it resolves
|
|
for (auto k = bars[0] + 1; k < end - 1; ++k) {
|
|
k->m_deferred = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Resolve the @cond spans in `katoms`, at APPLICATION time. Returns the number
|
|
// resolved, so the caller's fixed point accounts for them.
|
|
//
|
|
// The selected branch is spliced and then processed exactly as a klammer body
|
|
// is (process_katoms + apply, the two lines apply_klammer already uses): that
|
|
// is what makes the document behave like a function body whose state variables
|
|
// are its arguments -- they are bound by the substitution at the top of
|
|
// Machine::apply, BEFORE any conditional in the document is decided. Resolving
|
|
// @cond at read time meant a top-level "@cond *Flag*" saw the literal "*Flag*".
|
|
int Machine::resolve_cond_katoms(katom_list& katoms, const std::string& target)
|
|
{
|
|
int resolved = 0;
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_cond) != katoms.end()) {
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(katoms, level_increase, level_decrease, true, "cond")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
if (begin_cond(*begin)) {
|
|
// A @cond nested inside an unresolved outer @cond is still
|
|
// inert; the outer one will process it when its branch is
|
|
// selected. Without this an inner branch would be decided
|
|
// before it is known whether it is reached at all.
|
|
if (begin->m_deferred) continue;
|
|
// Delimit @cond's arguments by the bars at depth 0 within the
|
|
// span, so that bars belonging to nested klammers are not
|
|
// mistaken for @cond's own separators (see cond_separator_bars).
|
|
std::vector<katom_iter> bars = cond_separator_bars(begin, end);
|
|
check_bar_count(begin, bars.size());
|
|
auto bar_1 = bars[0];
|
|
std::string predicate = to_string(begin + 1, bar_1, true);
|
|
check_predicate(predicate, begin->m_loc);
|
|
katom_list true_clause {};
|
|
katom_list false_clause {};
|
|
if (bars.size() == 2) {
|
|
auto bar_2 = bars[1];
|
|
true_clause = katom_list(bar_1 + 1, bar_2);
|
|
false_clause = katom_list(bar_2 + 1, end - 1);
|
|
} else {
|
|
true_clause = katom_list(bar_1 + 1, end - 1);
|
|
}
|
|
// Splice only the selected branch. Its nested klammers remain
|
|
// unreduced here and are reduced by the outer fixed-point apply
|
|
// loop; the unselected branch is discarded without evaluation
|
|
// (@cond is a non-strict special form).
|
|
katom_list result = is_true(predicate)
|
|
? trim_whitespace(true_clause) : trim_whitespace(false_clause);
|
|
// The selected branch becomes live: clear the inert flag and
|
|
// give it the same processing a klammer body gets. The
|
|
// unselected branch is discarded still inert, so nothing in it
|
|
// ever ran.
|
|
for (Katom& k : result) {
|
|
k.m_deferred = false;
|
|
}
|
|
process_katoms(result, command_name);
|
|
apply(m_klammers, result, target);
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
katoms.insert(end, result.begin(), result.end());
|
|
++resolved;
|
|
}
|
|
}
|
|
}
|
|
return resolved;
|
|
}
|
|
|
|
|
|
// Expand the constant klammers written in a definition's BODY. A constant is
|
|
// expanded at definition time, which is what makes it a constant; the body is
|
|
// where that is meaningful.
|
|
//
|
|
// The parameter list is deliberately excluded. A klammer application there
|
|
// is an error (see expand_option_sets): parameters shared between klammers
|
|
// are declared by an option set, whose ".o" declaration is resolved as the
|
|
// parameter list is read. Expanding a constant into a parameter list used to
|
|
// be the way to share parameters, and it silently destroyed the parameter
|
|
// list of a ".k" declaration -- the spliced options AND the declared
|
|
// positionals -- surfacing only as an argument error at the first
|
|
// application, in the document rather than the declaration.
|
|
void Machine::expand_constant_klammers(katom_list& katoms, const Katom& op, const Katom& cl)
|
|
{
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
restore_initial_type(begin + 1, end - 1);
|
|
auto body_begin = std::find_if(
|
|
begin + 1, end - 1, [](const Katom& k) { return is_deftype(k.m_type); });
|
|
if (body_begin == end - 1) return;
|
|
if (std::find_if(body_begin, end - 1, begin_klammer_apply) == end - 1) return;
|
|
for (const auto& [app_op, app_cl] : find_spans(body_begin, end - 1, begin_apply, end_apply, false, "def-time")) {
|
|
auto [app_begin, app_end] = find_span_katoms(katoms, app_op, app_cl);
|
|
if (app_begin->m_type == katom_t::apply_begin) {
|
|
std::string name = trim_char(app_begin->m_text, '@');
|
|
const auto* body = m_klammers.constant_body(name);
|
|
if (body) {
|
|
// Set both m_type and m_initial_type so that
|
|
// restore_initial_type() in add() won't resurrect them
|
|
for (auto it = app_begin; it != app_end; ++it) {
|
|
it->m_type = katom_t::replaced;
|
|
it->m_initial_type = katom_t::replaced;
|
|
}
|
|
katoms.insert(app_end, body->begin(), body->end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
//katom_list
|
|
void Machine::mark_literal_klammer_content(katom_list& katoms)
|
|
{
|
|
(void)K::log(4);
|
|
// Collect names of klammers that have a literal parameter
|
|
std::set<std::string> literal_names {};
|
|
for (const auto& [name, klammer] : m_klammers.m_klammers) {
|
|
if (klammer.has_literal_param())
|
|
literal_names.insert(name);
|
|
}
|
|
if (literal_names.empty()) return;
|
|
|
|
// Scan for matching @name ... name@ spans.
|
|
//
|
|
// REMOVED TEXT IS SKIPPED. This pass runs first in process_katoms(), before
|
|
// mark_ignored_katoms() takes out the "#" forms, and that ordering is not
|
|
// accidental: a literal klammer's content must be marked before anything
|
|
// else can interpret what is inside it, or removal would take a "#" that
|
|
// belongs to the literal body. The cost is that this scan sees text the
|
|
// writer has removed, so a literal klammer merely NAMED in a comment --
|
|
//
|
|
// # @image and @code share the same arguments.
|
|
//
|
|
// -- was found as an opening delimiter, went unclosed, and failed the whole
|
|
// file (TODO #40, diagnosed 2026-08-05). "##" was already handled, which is
|
|
// half the case; the fix is to teach the scan the other three removal forms
|
|
// rather than to reorder the passes.
|
|
//
|
|
// The converse still holds, structurally: this skipping happens only while
|
|
// looking for an OPENING delimiter, and finding one jumps k past the whole
|
|
// span, so a "#" inside literal content is never examined here and stays
|
|
// content.
|
|
for (auto k = katoms.begin(); k != katoms.end(); ++k) {
|
|
if (k->m_type == katom_t::ignore_rest) break; // "##": rest of the file
|
|
if (k->m_type == katom_t::ignore_line) { // "#": rest of the line
|
|
while (k + 1 != katoms.end() && (k + 1)->m_type != katom_t::newline) {
|
|
++k;
|
|
}
|
|
continue;
|
|
}
|
|
if (k->m_type == katom_t::ignore_begin) { // "#[ ... ]#", nestable
|
|
int removed = 1;
|
|
while (++k != katoms.end() && removed > 0) {
|
|
if (k->m_type == katom_t::ignore_begin) {
|
|
++removed;
|
|
} else if (k->m_type == katom_t::ignore_end) {
|
|
--removed;
|
|
}
|
|
}
|
|
if (k == katoms.end()) break; // unterminated: the rest is removed
|
|
--k; // the loop's ++k steps past the "]#"
|
|
continue;
|
|
}
|
|
if (k->m_type != katom_t::apply_begin) continue;
|
|
std::string name = trim_char(k->m_text, '@');
|
|
if (literal_names.count(name) == 0) continue;
|
|
(void)K::log(2, "Literal klammer: " + name);
|
|
|
|
// Find the matching named closing delimiter
|
|
std::string close_text = name + "@";
|
|
auto close = k + 1;
|
|
int depth = 1;
|
|
while (close != katoms.end()) {
|
|
if (close->m_type == katom_t::apply_begin &&
|
|
trim_char(close->m_text, '@') == name)
|
|
depth++;
|
|
else if (close->m_type == katom_t::apply_end &&
|
|
trim_char(close->m_text, '@') == name)
|
|
depth--;
|
|
if (depth == 0) break;
|
|
++close;
|
|
}
|
|
if (close == katoms.end()) {
|
|
throw Parsing_error(
|
|
"Klammer " + q_(name) + " has a literal parameter and must be closed with "
|
|
+ q_(close_text),
|
|
k->m_loc);
|
|
}
|
|
|
|
// Count positional parameters before the literal one.
|
|
// The literal parameter is always last. Bars separate
|
|
// the preceding positional arguments and the literal content.
|
|
const auto& klammer = m_klammers.m_klammers[name];
|
|
int bars_before_literal = 0;
|
|
for (const auto& p : klammer.m_parameters.m_positional) {
|
|
if (p.m_argtype.m_name == "literal") break;
|
|
bars_before_literal++;
|
|
}
|
|
|
|
// Find where literal content starts.
|
|
// Skip bars_before_literal bars (separating preceding positional args).
|
|
// If options are present, skip past the bar after them.
|
|
// Options are identified by :name katoms before any bar -- but only
|
|
// a name THE KLAMMER DECLARES. Any ":word" counted until 2026-09-20,
|
|
// so code holding one before a bar ("@c ls :w | wc c@", a regex's
|
|
// "(?:a|b)") had its text up to the bar taken for options and the
|
|
// file failed with 'Optional argument ":w" not defined' -- an error
|
|
// about an option the author never wrote, in content that is
|
|
// promised to be read as nothing but text.
|
|
const auto& optional_names = klammer.m_parameters.m_optional_names;
|
|
auto literal_start = k + 1;
|
|
bool has_options = false;
|
|
for (auto j = k + 1; j < close; ++j) {
|
|
if (j->m_type == katom_t::option_name &&
|
|
std::ranges::count(optional_names, std::string(j->m_text, 1)) > 0) {
|
|
has_options = true;
|
|
}
|
|
if (j->m_type == katom_t::bar) {
|
|
if (bars_before_literal > 0) {
|
|
bars_before_literal--;
|
|
literal_start = j + 1;
|
|
} else if (has_options) {
|
|
// This bar separates options from literal content
|
|
literal_start = j + 1;
|
|
break;
|
|
} else {
|
|
// No preceding args, no options: bar is part of literal
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
std::for_each(literal_start, close, mark_as_literal);
|
|
|
|
k = close; // Skip past this span
|
|
}
|
|
}
|
|
|
|
void Machine::process_katoms(
|
|
katom_list& katoms, const std::string& source,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
|
|
bool eval, bool cond, bool read)
|
|
{
|
|
mark_literal_klammer_content(katoms);
|
|
if (literal) mark_literal_katoms(katoms);
|
|
// Sequences BEFORE the quoted-punctuation hiding: a single-character
|
|
// token such as the "-" in ^9053 - 53EF^ must reach the sequence pass
|
|
// as written; an unmatched run falls through untouched and the hiding
|
|
// then treats its ^P pairs normally.
|
|
if (nonascii) process_unicode_sequences(katoms);
|
|
hide_special_katoms(katoms);
|
|
if (nonascii) encode_nonascii_characters(katoms);
|
|
if (ignore) mark_ignored_katoms(katoms);
|
|
if (whitespace) process_whitespace_modifiers(katoms);
|
|
if (klammers) process_klammer_katoms(katoms);
|
|
// @cond is no longer RESOLVED here -- it is resolved in the apply fold
|
|
// (see mark_cond_content / resolve_cond_katoms). What happens here is the
|
|
// marking that makes its branches inert, and it must run BEFORE the eval
|
|
// and read passes, which are the ones with observable effects. Leaving it
|
|
// where process_cond_katoms used to sit -- after eval -- kept the old
|
|
// "eval in a discarded branch runs anyway" behaviour, since the marking
|
|
// arrived too late to stop it.
|
|
if (cond) mark_cond_content(katoms);
|
|
if (eval) process_eval_katoms(katoms);
|
|
if (read) expand_read_katoms(
|
|
katoms, source,
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
warn_unparsed_katoms(katoms, m_warn_unparsed);
|
|
// return katoms;
|
|
}
|
|
|
|
katom_list Machine::process(
|
|
std::string text, const std::string& source,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace, bool klammers,
|
|
bool eval, bool cond, bool read)
|
|
{
|
|
katom_list katoms = katomize_lines(line_split(text), source);
|
|
//katoms =
|
|
process_katoms(
|
|
katoms, source,
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
return katoms;
|
|
}
|
|
|
|
void Machine::read(const fs::path& pathname)
|
|
{
|
|
(void)K::log(3, pathname.string());
|
|
// A file's directory joins the @eval search path (Python modules and
|
|
// :cpp libraries live next to the file that uses them).
|
|
m_state.add_search_dir(fs::absolute(pathname).parent_path().string());
|
|
m_state.open_frame("Machine state: " + pathname.string());
|
|
input(string_from_file(pathname.string()), pathname.generic_string());
|
|
m_sources.push_back(pathname);
|
|
}
|
|
|
|
void Machine::read_string(const std::string& text)
|
|
{
|
|
(void)K::log(3, text);
|
|
m_state.open_frame("Machine state: " + text);
|
|
input(text, command_pathname);
|
|
m_sources.push_back(text);
|
|
}
|
|
|
|
void Machine::input(const std::string& text, const std::string& source)
|
|
{
|
|
katom_list katoms = process(m_state.subst(trim_right(text)), source);
|
|
m_katoms.insert(m_katoms.end(), katoms.begin(), katoms.end());
|
|
extract_machine_definitions();
|
|
extract_klammer_definitions();
|
|
}
|
|
|
|
// Read
|
|
|
|
void Machine::expand_read_katoms(
|
|
katom_list& katoms, std::string current_filename,
|
|
bool nonascii, bool literal, bool ignore, bool whitespace,
|
|
bool klammers, bool eval, bool cond, bool read)
|
|
{
|
|
(void)K::log(3);
|
|
current_filename = resolve_relative_to(current_filename).generic_string();
|
|
// msg() << "current_filename: " << current_filename << "\n";
|
|
if (std::find_if(katoms.begin(), katoms.end(), begin_read) != katoms.end()) {
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(katoms, begin_apply, end_apply, true, "read")) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
// Inert: see the same guard in process_eval_katoms.
|
|
if (begin->m_deferred) continue;
|
|
if (begin_read(*begin)) {
|
|
std::string read_filename = to_string(begin + 1, end - 1, true);
|
|
|
|
// msg() << "read: " << resolve_relative_to(read_filename, current_filename) << "\n";
|
|
|
|
/*
|
|
std::string current_directory =
|
|
fs::path(current_filename).parent_path().string();
|
|
fs::path input_filename =
|
|
fs::path(current_directory + "/" + read_filename);
|
|
*/
|
|
|
|
fs::path input_filename = resolve_relative_to(read_filename, current_filename);
|
|
// msg() << "read: " << input_filename << "\n";
|
|
|
|
(void)K::log(2, input_filename.string());
|
|
if (!fs::exists(input_filename)) {
|
|
std::stringstream ss{};
|
|
ss <<"File " << input_filename << " does not exist";
|
|
throw File_error(ss.str(), begin->m_loc);
|
|
}
|
|
std::for_each(begin, end, mark_as_replaced);
|
|
|
|
input_filename = fs::canonical(input_filename);
|
|
m_state.add_search_dir(input_filename.parent_path().string());
|
|
std::string text = trim_right(string_from_file(input_filename.string()));
|
|
katom_list ks = katomize_lines(line_split(text), input_filename.generic_string());
|
|
// ks =
|
|
process_katoms(
|
|
// ks, command_pathname,
|
|
ks, input_filename.generic_string(),
|
|
nonascii, literal, ignore, whitespace, klammers, eval, cond, read);
|
|
katoms.insert(end, ks.begin(), ks.end());
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
// One @@@ declaration. Factored out so the tolerant and strict paths share
|
|
// it; `rescan` is set when a klammerset has spliced files into the stream,
|
|
// which invalidates the caller's span list.
|
|
void Machine::add_machine_definition(
|
|
const std::string& name, katom_iter begin, katom_iter end, bool& rescan)
|
|
{
|
|
if (name == "@@@target") {
|
|
m_targets.add(begin, end, m_katoms);
|
|
} else if (name == "@@@argtype") {
|
|
m_argtypes.add(begin, end, m_katoms);
|
|
} else if (name == "@@@state") {
|
|
m_state.parse_state_katoms(begin, end, m_katoms);
|
|
} else if (name == "@@@klammerset") {
|
|
if (auto klammerset = m_klammersets.add(begin, end, m_katoms)) {
|
|
load_klammerset_files(*klammerset, end);
|
|
rescan = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
void Machine::extract_machine_definitions(bool tolerant)
|
|
{
|
|
(void)K::log(3);
|
|
if (m_katoms.empty()) {
|
|
return;
|
|
}
|
|
// A @@@klammerset declaration inserts its files' katoms into the stream
|
|
// at the declaration point, invalidating the span list, so the scan
|
|
// restarts. Processed spans are marked replaced and are never found
|
|
// again, which also bounds the restarts.
|
|
bool rescan = true;
|
|
while (rescan) {
|
|
rescan = false;
|
|
for (const auto& [op, cl] : find_spans(m_katoms, begin_machine_def, end_machine_def, true, command_name)) {
|
|
auto [begin, end] = find_span_katoms(m_katoms, op, cl);
|
|
std::string name = begin->m_text;
|
|
if (tolerant) {
|
|
// kdiag: a declaration that cannot be carried out -- a
|
|
// @@@klammerset naming a file that is not there, say -- is
|
|
// skipped rather than fatal. Its span stays unreplaced, so
|
|
// the katoms remain visible and report the failure themselves.
|
|
try {
|
|
add_machine_definition(name, begin, end, rescan);
|
|
} catch (Error& e) {
|
|
K::log(1, "Machine definition not registered: " + e.m_desc);
|
|
continue;
|
|
}
|
|
} else {
|
|
add_machine_definition(name, begin, end, rescan);
|
|
}
|
|
if (rescan) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
void Machine::load_klammerset_files(const Klammerset& klammerset, katom_iter insert_at)
|
|
{
|
|
(void)K::log(2, "Loading klammerset \"" + klammerset.m_symbol + "\"");
|
|
// Relative names resolve against the declaring file's directory, never
|
|
// the cwd. :requires files are read before the set's own files; each
|
|
// holds its own @@@klammerset declaration, whose already-loaded guard
|
|
// makes repeated requirements a no-op.
|
|
fs::path declaring(klammerset.m_loc.m_filename);
|
|
fs::path base = fs::exists(declaring) ? declaring : fs::current_path();
|
|
std::string base_dir = (is_directory(base) ? base : base.parent_path()).string();
|
|
|
|
// A :requires entry may be a bare symbol, resolved on the klammerset
|
|
// search path with the declaring directory as the local stage; :files
|
|
// entries are always filenames (this set's own definition files).
|
|
std::vector<std::string> filenames {};
|
|
for (const auto& required : klammerset.m_requires) {
|
|
std::string path = is_klammerset_symbol(required)
|
|
? resolve_klammerset_symbol(required, base_dir, klammerset.m_loc).generic_string()
|
|
: required;
|
|
// ALREADY LOADED? Decided BEFORE the file is opened. The guard used
|
|
// to sit in Klammerset_registry::add, which only runs once the file has
|
|
// been read and its declaration reached -- by which time the declaring
|
|
// file's OWN definitions have been re-executed. Two sets requiring a
|
|
// third therefore died on "Target ... is already defined", pointing at
|
|
// a line the author wrote once.
|
|
//
|
|
// Deciding it here is possible only because a klammerset symbol X is
|
|
// declared in X/X.k, so the symbol is a function of the path: it is
|
|
// known for a ":requires" written as a filename just as much as for one
|
|
// written as a symbol. That is what the X/X.k requirement bought.
|
|
std::string symbol = is_klammerset_symbol(required)
|
|
? required : fs::path(path).stem().string();
|
|
if (m_klammersets.has(symbol)) {
|
|
(void)K::log(1, "Klammerset \"" + symbol + "\": already loaded, skipped");
|
|
continue;
|
|
}
|
|
filenames.push_back(path);
|
|
}
|
|
filenames.insert(filenames.end(), klammerset.m_files.begin(), klammerset.m_files.end());
|
|
|
|
// Collect all files into one list and insert once: insert_at is
|
|
// invalidated by the first insertion into m_katoms.
|
|
// Recorded on the set: what it actually opened, canonical, so a display
|
|
// can tell its klammers from the input's. The declaring file counts --
|
|
// definitions may sit before or after the declaration in it.
|
|
strings_t loaded_files {};
|
|
if (fs::exists(declaring)) {
|
|
loaded_files.push_back(fs::canonical(declaring).string());
|
|
}
|
|
katom_list loaded {};
|
|
for (const auto& filename : filenames) {
|
|
fs::path pathname = resolve_relative_to(filename, base);
|
|
if (!fs::exists(pathname)) {
|
|
throw Klammerset_error(
|
|
"Klammerset \"" + klammerset.m_symbol + "\" lists the file \"" + filename
|
|
+ "\", which does not exist (resolved to \"" + pathname.string() + "\")",
|
|
klammerset.m_loc);
|
|
}
|
|
pathname = fs::canonical(pathname);
|
|
loaded_files.push_back(pathname.generic_string());
|
|
m_state.add_search_dir(pathname.parent_path().string());
|
|
std::string text = trim_right(string_from_file(pathname.string()));
|
|
katom_list ks = katomize_lines(line_split(text), pathname.generic_string());
|
|
process_katoms(ks, pathname.generic_string());
|
|
loaded.insert(loaded.end(), ks.begin(), ks.end());
|
|
}
|
|
m_katoms.insert(insert_at, loaded.begin(), loaded.end());
|
|
// The registry holds the registered copy; the parameter is a const
|
|
// reference to it, so the record goes back through the registry.
|
|
m_klammersets.set_loaded_files(klammerset.m_symbol, loaded_files);
|
|
}
|
|
|
|
// Register one "@@...@@" definition. An ".o" target declares an option set
|
|
// -- parameters shared by klammers -- and goes to its own registry: it
|
|
// defines no klammer and produces no output for any target.
|
|
void Machine::add_definition(katom_list& katoms, const Katom& op, const Katom& cl)
|
|
{
|
|
expand_constant_klammers(katoms, op, cl);
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
auto [name, target_names, general_declared] = parse_name(m_targets, *begin);
|
|
(void)general_declared; // routing only cares whether this is a ".o"
|
|
// parse_name rejects ".o" as a member of a target list, so an option set
|
|
// declaration is always the single-target form.
|
|
if (target_names.size() == 1 && target_names[0] == Target_registry::optionset_name) {
|
|
m_option_sets.add(m_argtypes, name, begin, end, katoms);
|
|
} else {
|
|
m_klammers.add(m_argtypes, m_targets, m_option_sets, begin, end, katoms);
|
|
}
|
|
}
|
|
|
|
void Machine::extract_klammer_definitions(katom_list katoms)
|
|
{
|
|
// fmsg() << katoms << "\n";
|
|
(void)K::log(3, katoms);
|
|
for (const auto& [op, cl] : find_spans(katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
|
|
add_definition(katoms, op, cl);
|
|
}
|
|
m_klammers.rationalize(m_targets);
|
|
}
|
|
|
|
void Machine::extract_klammer_definitions(bool tolerant)
|
|
{
|
|
(void)K::log(3);
|
|
for (const auto& [op, cl] : find_spans(m_katoms, begin_klammer_def, end_klammer_def, true, command_name)) {
|
|
if (tolerant) {
|
|
// add_definition marks the span replaced only on success, so a
|
|
// skipped definition keeps its katoms and stays visible.
|
|
try {
|
|
add_definition(m_katoms, op, cl);
|
|
} catch (Error& e) {
|
|
K::log(1, "Definition not registered: " + e.m_desc);
|
|
}
|
|
} else {
|
|
add_definition(m_katoms, op, cl);
|
|
}
|
|
}
|
|
m_klammers.rationalize(m_targets);
|
|
}
|
|
|
|
void Machine::update_state(const std::map<std::string, std::string>& arg_map)
|
|
{
|
|
for (const auto& [k, v] : arg_map) {
|
|
m_state.set(k, v);
|
|
}
|
|
}
|
|
|
|
katom_list Machine::apply_klammer(
|
|
Klammer& klammer, const std::string& target, katom_iter arguments_begin, katom_iter arguments_end)
|
|
{
|
|
(void)K::log(3, "argument substitution", *arguments_begin, *(arguments_end - 1));
|
|
Depth_guard depth_guard(klammer.m_name, arguments_begin->m_loc);
|
|
m_state.replace("K_loc", arguments_begin->m_loc.str(), false);
|
|
auto [positional, optional, rest] =
|
|
argument_split(arguments_begin + 1, arguments_end - 1, klammer.m_parameters.m_positional.size());
|
|
auto values = klammer.m_parameters.value_map(positional, optional, rest, arguments_begin->m_loc);
|
|
|
|
// Resolve KTESC markers in argument values so that @eval code receives
|
|
// the original characters (e.g., filenames with underscores). The markers
|
|
// remain in the klammer body substitution for final target-specific output.
|
|
katom_list result(klammer.m_body[target].begin(), klammer.m_body[target].end());
|
|
auto varmap = klammer.m_varmap[target];
|
|
// Escape target-specific characters (e.g. tex "&" -> "\&") in the writer
|
|
// text of a GENERAL klammer's body. Runs BEFORE the *arg* substitution
|
|
// below: an argument value is writer text already escaped at the top
|
|
// level (KTESC markers, idempotent) plus final target markup from
|
|
// klammers the writer nested in the argument, and neither may be escaped
|
|
// here -- escaping after substitution swept both, so a general klammer
|
|
// with a klammer-bearing argument emitted \textbackslash{}emph{...}
|
|
// (fixed 2026-08-19; tst/escape_test.sh cases 26-29). A karg katom is
|
|
// skipped by the type filter; a variable inside a mixed text katom
|
|
// survives because no target declares '*' or identifier characters as
|
|
// escapes. Also runs BEFORE process_katoms/apply() expand the body, so
|
|
// that target-native markup pulled in by nested klammers (e.g. nl.tex ->
|
|
// "\newline") is left untouched -- only this klammer's own literal writer
|
|
// text is escaped here; nested klammers escape theirs when they are
|
|
// applied in turn. Bodies from target-specific definitions
|
|
// (m_body_generic[target] == false) are already in target form and
|
|
// skipped. Two kinds of body content are NOT
|
|
// writer text and must be skipped:
|
|
// * ^'...'^ literal spans -- raw target markup the writer typed directly.
|
|
// At this point they are typed literal_begin/literal_end with plain-text
|
|
// content (the literal phase runs in process_katoms, below), so track
|
|
// span depth rather than testing katom type.
|
|
// * @eval / @read / @cond argument spans -- code, filenames, and
|
|
// predicates consumed by the primitive, NOT emitted as target text.
|
|
// (Escaping an underscore in "offer.Price_list(K)" broke @eval.) The
|
|
// primitive's KLAMMERTEXT result, produced by process_katoms below, is
|
|
// klammer output and is likewise never escaped -- it is inserted after
|
|
// this pass and so is untouched, matching the top-level behavior where
|
|
// @eval is resolved before the escape pass runs.
|
|
auto gen = klammer.m_body_generic.find(target);
|
|
if (gen != klammer.m_body_generic.end() && gen->second) {
|
|
Target tgt = m_targets.get(target, Locator());
|
|
if (!tgt.m_escapes.empty()) {
|
|
int literal_depth = 0;
|
|
int code_depth = 0; // inside an @eval/@read/@cond span
|
|
std::vector<bool> apply_is_code; // one entry per open application
|
|
for (auto& k : result) {
|
|
if (k.m_type == katom_t::literal_begin) { ++literal_depth; continue; }
|
|
if (k.m_type == katom_t::literal_end) {
|
|
if (literal_depth > 0) --literal_depth;
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::eval_begin ||
|
|
k.m_type == katom_t::read_begin ||
|
|
k.m_type == katom_t::cond_begin) {
|
|
apply_is_code.push_back(true);
|
|
++code_depth;
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::apply_begin) {
|
|
apply_is_code.push_back(false);
|
|
continue;
|
|
}
|
|
if (k.m_type == katom_t::apply_end) {
|
|
if (!apply_is_code.empty()) {
|
|
if (apply_is_code.back()) --code_depth;
|
|
apply_is_code.pop_back();
|
|
}
|
|
continue;
|
|
}
|
|
if (literal_depth == 0 && code_depth == 0 &&
|
|
(k.m_type == katom_t::text ||
|
|
k.m_type == katom_t::word ||
|
|
k.m_type == katom_t::newline))
|
|
k.m_text = tgt.escape_text(k.m_text);
|
|
}
|
|
}
|
|
}
|
|
m_state.open_frame("Arguments for klammer " + q_(klammer.m_name));
|
|
m_state.set(values, klammer.m_parameters);
|
|
for (const auto& [name, indices] : varmap) {
|
|
std::regex arg("\\*" + name + "\\*");
|
|
for (auto i : indices) {
|
|
result[i].m_text = std::regex_replace(
|
|
result[i].m_text, arg, m_state.value(name, true, result[i].m_loc));
|
|
result[i].m_type = katom_t::text;
|
|
}
|
|
}
|
|
process_katoms(result, klammer.m_name);
|
|
apply(m_klammers, result, target);
|
|
m_state.close_frame();
|
|
// msg() << boldblack << "APPLY: " << std::pair(arguments_begin, arguments_end) << "\n"
|
|
// << boldblack << "RESULT: " << ktype << result << black << "\n";
|
|
|
|
modify_type(katom_t::replaced, arguments_begin, arguments_end);
|
|
return result;
|
|
}
|
|
|
|
void Machine::apply_klammer_registry(
|
|
Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target, katom_iter begin, katom_iter end)
|
|
{
|
|
(void)K::log(3, "Klammer");
|
|
std::string name = trim_char(begin->m_text, '@');
|
|
katom_list applied_katoms = apply_klammer(klammer_registry.m_klammers[name], target, begin, end);
|
|
for (auto& k : applied_katoms) {
|
|
if (k.m_type == katom_t::bar || k.m_type == katom_t::double_bar || k.m_type == katom_t::option_name) {
|
|
k.m_type = katom_t::text;
|
|
}
|
|
}
|
|
katoms.insert(end, applied_katoms.begin(), applied_katoms.end());
|
|
}
|
|
|
|
int Machine::apply(
|
|
Klammer_registry& klammer_registry, katom_list& katoms, const std::string& target)
|
|
{
|
|
(void)K::log(3, "Klammer_registry");
|
|
// @cond is a special form handled here rather than by the klammer loop
|
|
// below: its span head is a cond_begin, which begin_klammer_apply does not
|
|
// match. Resolving first means a klammer revealed by the selected branch
|
|
// is applied in this same pass. The count is returned with the klammer
|
|
// applications, so the caller's fixed point iterates while either happens.
|
|
int applied = resolve_cond_katoms(katoms, target);
|
|
for (const auto& [op, cl] : find_spans(
|
|
katoms, begin_klammer_apply, end_klammer_apply, true, command_name)) {
|
|
auto [begin, end] = find_span_katoms(katoms, op, cl);
|
|
klammer_registry.check_klammer(
|
|
klammer_name_from_katom(begin->m_text, begin->m_loc),
|
|
target, begin->m_loc);
|
|
apply_klammer_registry(klammer_registry, katoms, target, begin, end);
|
|
++applied;
|
|
}
|
|
return applied;
|
|
}
|
|
|
|
std::string Machine::run_phase_functions()
|
|
{
|
|
Target target = m_targets.get(m_state.value("K_target"), Locator());
|
|
if (!target.m_after_apply.empty()) {
|
|
(void)K::log(2, target);
|
|
for (auto f : target.m_after_apply) {
|
|
// A mode-tagged spec (":cpp ...") names a function that receives
|
|
// the Machine itself; a bare Python function is called with the
|
|
// result text. The Eval is constructed per phase so a chained
|
|
// phase sees its predecessor's result in K_result.
|
|
Eval E(*this, Locator());
|
|
if (!f.empty() && f[0] != ':') {
|
|
// K reaches a phase only as a parameter: the state's
|
|
// "class K" exists in the EVAL's globals (python_code()
|
|
// defines it), but a phase function's body resolves names in
|
|
// its own MODULE's globals. Every phase signature has
|
|
// carried "K=None" for this since the calling convention was
|
|
// created; the call never passed it, which surfaced when
|
|
// txt_justify_blocks needed Target_txt_width (2026-08-22).
|
|
// K_phase_call (defined in Eval_python's globals) passes K
|
|
// only to a function that DECLARES a K parameter, so a
|
|
// stdlib phase (string.capwords, whose second parameter is a
|
|
// separator) keeps working.
|
|
f = "K_phase_call(" + f + ", K_result, K)";
|
|
}
|
|
f = "@eval " + f + " @";
|
|
auto katoms = katomize_lines(line_split(f), "phase");
|
|
// A phase function's input and output are final target text, not
|
|
// Klammertext: take the raw result string. Re-reading it as
|
|
// Klammertext (Eval::eval) would misparse target characters --
|
|
// e.g. a "@" from a quoted ^@ in justified txt output.
|
|
m_result = E.eval_command(katoms.begin(), katoms.end() - 2);
|
|
}
|
|
}
|
|
return m_result;
|
|
}
|
|
|
|
|
|
void Machine::escape_target_characters(const Target& target, katom_list& katoms)
|
|
{
|
|
if (target.m_escapes.empty()) return;
|
|
for (auto& k : katoms) {
|
|
// Only escape writer content katoms — text, words, and newlines.
|
|
// Skip structural katoms (option names, bars, klammer delimiters)
|
|
// whose text is Klammertext syntax, not writer content.
|
|
if (k.m_type == katom_t::text ||
|
|
k.m_type == katom_t::word ||
|
|
k.m_type == katom_t::newline) {
|
|
k.m_text = target.escape_text(k.m_text);
|
|
}
|
|
}
|
|
}
|
|
|
|
std::string Machine::apply(const std::string& target_name, bool final_processing, bool escape_characters)
|
|
{
|
|
(void)K::log(3, "top level");
|
|
// update=true: K_target is THIS apply's target. A command applies once,
|
|
// but the Python binding (kpy/) applies the same Machine under several
|
|
// targets -- apply("html") then apply("tex") -- and the second call
|
|
// found the first's value in the same frame and died with "Variable
|
|
// K_target is already defined" (found by tst/kpy_test.sh, 2026-09-08).
|
|
m_state.set("K_target", target_name, true);
|
|
m_state.subst(m_katoms.begin(), m_katoms.end());
|
|
|
|
// Escape target-specific characters in writer text before klammer application.
|
|
// Characters produced later by klammer bodies will not be escaped.
|
|
// Skipped for sub-Machine apply() calls (e.g., from @eval), where the
|
|
// text is already in target-specific form.
|
|
auto target = m_targets.get(target_name, Locator::none());
|
|
if (escape_characters)
|
|
escape_target_characters(target, m_katoms);
|
|
|
|
// Reduce to a fixed point. A pass reports how many klammers it applied;
|
|
// the loop ends when a pass applies none. (It formerly ended when the
|
|
// katom list stopped GROWING, which is not the same thing: a klammer whose
|
|
// body expands to nothing is a reduction that adds no katoms.) Exceeding
|
|
// the round limit is now an error rather than a message followed by
|
|
// rendering the unreduced document -- silently emitting a document with
|
|
// live klammers still in it is worse than not emitting one. Runaway
|
|
// recursion is caught earlier and more precisely by the depth guard in
|
|
// apply_klammer(); this limit only bounds the number of ROUNDS, which is
|
|
// the length of a chain of klammers that generate further klammers.
|
|
int apply_count = 0;
|
|
while (apply(m_klammers, m_katoms, target_name) > 0) {
|
|
if (++apply_count > apply_round_limit) {
|
|
std::stringstream ss {};
|
|
ss << "Klammer application did not reach a fixed point after "
|
|
<< apply_round_limit << " rounds.\n\n"
|
|
<< "Each round applies every klammer present; a klammer whose "
|
|
<< "result contains further klammers starts another round.";
|
|
throw Recursion_error(ss.str(), Locator());
|
|
}
|
|
}
|
|
// Typographic transforms run on the katoms, not the joined string, so
|
|
// that ^'...'^ literal content (katom_t::literal) stays verbatim --
|
|
// Target::transform() skips literal katoms. Verbatim text produced by
|
|
// an @eval renderer (@code, @c) is plain text by the time it is spliced
|
|
// back; the SKS protects it with KTESC markers (hide_typographic() in
|
|
// klammer_base.py), which are inert here and decode in resolve_escapes.
|
|
if (final_processing) {
|
|
target.transform(m_katoms);
|
|
}
|
|
m_result = to_string(m_katoms.begin(), m_katoms.end());
|
|
|
|
if (final_processing) {
|
|
m_result = target.resolve_escapes(m_result);
|
|
m_result = run_phase_functions();
|
|
}
|
|
m_result = trim_char(m_result, '\n');
|
|
return m_result;
|
|
}
|