Verbatim-safe typography, ^-punctuation quoting, full-range ^UUUU^, polyglot html

Typographic transforms (---, quote pairs, ~) no longer touch verbatim
text: @c/@code/@source_listing content and ^'...'^ spans show exactly the
characters written. "^" before any punctuation character quotes it in
every target (the apostrophe excepted: ^' opens a literal span), with the
new :resolve option on @@@target declaring per-target renderings. The
^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range.
The html output and transform spellings are polyglot (XML-valid), in
preparation for an EPUB target. New suites: transform_test, character_test
(engine), typography_test (SKS).

(from dev 07ce5ea86a0a)
This commit is contained in:
2026-08-23 20:48:26 +02:00
parent d982c0d6cc
commit 37b6ba1c4f
77 changed files with 1665 additions and 1608 deletions

View File

@@ -24,16 +24,21 @@ void Target::add_transforms(const string_pairs& transforms)
}
}
// The typographic transform pass, run by Machine::apply() at final
// processing. Per katom rather than over the joined result string, so
// that ^'...'^ literal content (katom_t::literal) is never transformed --
// verbatim text must show the characters the writer typed. A transform
// source therefore cannot match across a katom boundary, which is the
// correct reading: two hyphens separated by a klammer application were
// separated by the writer and are not a dash.
void Target::transform(katom_list& katoms) const
{
(void)K::log(3);
std::for_each(
katoms.begin(), katoms.end(),
[this] (Katom& k) {
// std::cout << "transform: " << k << "\n";
if (k.m_type != katom_t::literal) {
for (const auto& [a, b] : this->m_transforms) {
// std::cout << " " << a << right_arrow << b << "\n";
k.m_text = string_replace(k.m_text, a, b);
}
}
@@ -67,6 +72,14 @@ void Target::add_escapes(const std::string& escape_spec)
}
}
void Target::add_resolves(const std::string& resolve_spec)
{
auto words = word_split(resolve_spec);
for (size_t i = 0; i + 1 < words.size(); i += 2) {
m_resolves.push_back({words[i], words[i+1]});
}
}
std::string Target::escape_marker(const std::string& ch)
{
std::stringstream ss {};
@@ -94,11 +107,16 @@ std::string Target::unescape_text(std::string text) const
std::string Target::resolve_escapes(std::string text) const
{
// Target-declared escapes first (marker -> declared replacement), then
// the generic decode for the remaining markers (marker -> the character
// itself: quoted Klammertext specials and literal-span content).
// the resolution-only entries (:resolve -- how a QUOTED character
// renders here), then the generic decode for the remaining markers
// (marker -> the character itself: quoted punctuation and literal-span
// content).
for (const auto& [ch, repl] : m_escapes) {
text = string_replace(text, escape_marker(ch), repl);
}
for (const auto& [ch, repl] : m_resolves) {
text = string_replace(text, escape_marker(ch), repl);
}
return ktesc_resolve(text);
}
@@ -130,6 +148,29 @@ std::string ktesc_resolve(std::string text)
return text;
}
bool hide_quoted_punctuation(std::string& s)
{
// ASCII punctuation, EXCEPT "'" -- ^' opens a ^'...'^ literal span, the
// one documented exception to the rule (a literal apostrophe is ^0027^).
// Hand-rolled scan: no std::regex, this can run over large text.
static const std::string punct = R"pct(!"#$%&()*+,-./:;<=>?@[\]^_`{|}~)pct";
if (s.find('^') == std::string::npos) return false;
std::string result {};
bool changed = false;
for (size_t i = 0; i < s.size(); ++i) {
if (s[i] == '^' && i + 1 < s.size()
&& punct.find(s[i + 1]) != std::string::npos) {
result += Target::escape_marker(std::string(1, s[i + 1]));
++i;
changed = true;
} else {
result += s[i];
}
}
if (changed) s = result;
return changed;
}
std::string hide_structural_characters(const std::string& s)
{
std::string result {};
@@ -145,7 +186,26 @@ std::string hide_structural_characters(const std::string& s)
void Target::add_after_apply(const std::string& function_specs)
{
for (const auto& f : regex_split(function_specs, std::regex(R"(\s+;\s+)"), true)) {
// msg() << "Add " << m_name << " after-apply: " << f << "\n";
// A bare spec is ONE Python name (module.function): whitespace
// inside it means two specs were written without the " ; "
// separator, and the glued call would otherwise fail only at render
// time, as a Python SyntaxError located at "phase" rather than at
// this declaration (found 2026-08-22, the first time a target
// declared two phases). Mode-tagged specs (":cpp <library>
// <function>") are exempt: the library is a filename, and filenames
// may contain spaces -- which is exactly why the list separator is
// ";" rather than whitespace.
if (!f.empty() && f[0] != ':'
&& f.find_first_of(" \t\n") != std::string::npos) {
throw Definition_error(
"The :after_apply phase \"" + f + "\" contains whitespace. "
"A bare phase is a single Python name (module.function), and "
"several phases are separated by \" ; \":\n"
" :after_apply first.phase ; second.phase\n"
"(A mode-tagged phase -- \":cpp <library> <function>\" -- may "
"contain spaces; its library is a filename.)",
m_loc);
}
m_after_apply.push_back(f);
}
}