diff --git a/README.md b/README.md index 7668037..b10a92b 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ are regenerated on each release — patches cannot be merged directly. Report problems (or send patches) to the author; accepted changes are applied to the development tree and appear in a following snapshot. -This snapshot was assembled from development commit `a9756dac74c6`. +This snapshot was assembled from development commit `831455f36fcd`. ## License diff --git a/mac/argtype_registry.cpp b/mac/argtype_registry.cpp index 94d1386..500bb93 100644 --- a/mac/argtype_registry.cpp +++ b/mac/argtype_registry.cpp @@ -113,12 +113,12 @@ void Argtype_registry::add(const std::string& name, const std::string& desc, // which is equally unable to delimit. if (std::regex_match(std::string("one two"), m_types[name].m_regex)) { std::stringstream ss {}; - ss << "The argument type \"" << name << "\" cannot declare an :alone value" + ss << "The argument type \"" << name << "\" cannot declare an :alone value " << "because its pattern matches running text:" << " " << pattern << "\n\n" - << "An :alone value is used when an option name is written without a" - << "value. A type that matches running text cannot tell a bare option" - << "name from one whose value follows it, so the text after the name" + << "An :alone value is used when an option name is written without a " + << "value. A type that matches running text cannot tell a bare option " + << "name from one whose value follows it, so the text after the name " << "would be taken as the value instead."; throw Definition_error(ss.str(), loc); } diff --git a/sks/document/Makefile b/sks/document/Makefile index dd71b99..32a2ac4 100644 --- a/sks/document/Makefile +++ b/sks/document/Makefile @@ -23,7 +23,7 @@ DOCUMENT := document$(DSO) DOC_LDFLAGS := $(SHARED) $(EXPORT_DYNAMIC) -Wl,-rpath,'$(ORIGIN)/../../lib' -L$(LIBDIR) # Local object files -LOCAL_OBJECTS := document_class.o document_html.o document_latex.o heading.o reference.o +LOCAL_OBJECTS := document_class.o document_html.o document_latex.o heading.o reference.o section_link.o LOCAL_DEPFILES := document_class.d document_html.d document_latex.d heading.d reference.d document.d # External object files from sks/ (mac/*.o now in the shared library) diff --git a/sks/document/document_class.h b/sks/document/document_class.h index 9429d5d..1748266 100644 --- a/sks/document/document_class.h +++ b/sks/document/document_class.h @@ -72,6 +72,9 @@ public: std::map id_to_caption_number(); std::map>> html_references(); void resolve_html_references(); + void resolve_section_links(); // html, @link :section + std::string resolve_tex_section_links( + const std::vector>& pieces); // (basename, tex) std::string single_page_toc(); elements_t page(const std::string& body, const std::string& output_dir, int max_level); diff --git a/sks/document/document_html.cpp b/sks/document/document_html.cpp index f13e45d..c0b78b6 100644 --- a/sks/document/document_html.cpp +++ b/sks/document/document_html.cpp @@ -1,3 +1,4 @@ +#include "section_link.h" #include #include "util.h" #include "error.h" @@ -315,6 +316,7 @@ void Document_class::process_input_files() //if (true || !in_modification_order(inpath, cached_filename, m_no_cache)) { if (m_no_cache || cache_requires_update(m_cache_dir, inpath.string(), basename)) { Machine M = m_machine; + M.m_state.set("K_document_file", basename); // see document_latex.cpp M.read(inpath); html_text = trim(M.apply("html")); html_text = trim(html::make_paragraphs(html_text)) + "\n"; @@ -686,6 +688,54 @@ elements_t Document_class::page(const std::string& body_text, const std::string& return page; } +// @link :section (section_link.h). Every heading element that carries an +// id -- numbered or not, so @h1..@h7 and @part are linkable too, which is +// why this reads the elements rather than the headings table that +// insert_section_numbers() builds from the numbered ones -- is a target; +// the Link klammer's href marker is resolved against them. A single page +// gets href="#id". A book's pages are embedded in index.html and shown by +// go_to_section(), so a link there is the "pagelink" anchor that +// sks/link/js/link.js binds: data-page and data-id, with the app's own +// "#page#id" hash as the href so the location survives without the script. +// Before 2026-10-12 html had no resolver and the marker was the href. +void Document_class::resolve_section_links() +{ + static const std::regex heading_rgx(R"(<(h[1-7]|kt-chapter|kt-part)\b([^>]*)>(.*?))"); + static const std::regex link_rgx(R"(href="__seclink__(.*?)__KTSL__(.*?)__")"); + std::vector table {}; + strings_t files {}; + for (const auto& [basename, text] : m_file_components) { + files.push_back(basename); + for (std::sregex_iterator it(text.cbegin(), text.cend(), heading_rgx), end; it != end; ++it) { + const std::smatch& m = *it; + std::string attrs = m[2]; + std::smatch id_match {}; + if (std::regex_search(attrs, id_match, id_attr_rgx)) { + table.push_back({normalize_section_title(m[3]), basename, id_match[1]}); + } + } + } + const bool book = m_structure == docstruct_t::book; + for (auto& [basename, text] : m_file_components) { + std::string result {}; + auto begin = text.cbegin(); + for (std::sregex_iterator it(text.cbegin(), text.cend(), link_rgx), end; it != end; ++it) { + const std::smatch& m = *it; + result.append(begin, m[0].first); + const Section_target t = resolve_section_target(table, m[1], m[2], basename, files); + if (book) { + result += "href=\"#" + t.basename + "#" + t.id + "\" class=\"pagelink\" data-page=\"" + + t.basename + "\" data-id=\"" + t.id + "\""; + } else { + result += "href=\"#" + t.id + "\""; + } + begin = m[0].second; + } + result.append(begin, text.cend()); + text = result; + } +} + std::string Document_class::make_single_html_page(const std::string& output_directory) { (void)K::log(3); @@ -787,6 +837,7 @@ std::string Document_class::html() } auto headings = insert_section_numbers(); insert_section_numbers("___NUM2___", false); + resolve_section_links(); insert_html_caption_numbers(); resolve_html_references(); if (m_structure != docstruct_t::book) { diff --git a/sks/document/document_latex.cpp b/sks/document/document_latex.cpp index fee20e3..feb3968 100644 --- a/sks/document/document_latex.cpp +++ b/sks/document/document_latex.cpp @@ -1,3 +1,4 @@ +#include "section_link.h" #include "util.h" #include "kutil.h" #include "log.h" @@ -20,15 +21,54 @@ std::string Document_class::create_tex_output_directories() return output_directory; } +// @link :section (section_link.h). Section.tex() announces every heading +// in a comment line; the links are resolved against them, every link on a +// line. The resolver this replaces (2026-10-12) read ONE link per line -- +// std::regex_match once per line -- so a second @link ... :section @ on a +// line reached the pdf as a dead \hyperlink{__seclink__...}; it ignored +// :basename, and it wrote \hyperlink{} for an unknown title. +std::string Document_class::resolve_tex_section_links( + const std::vector>& pieces) +{ + static const std::regex announce_rgx( + R"(%__sectionlink__(.*?)__KTSL__(.*?)__KTSL__(.*)__\n?)"); + static const std::regex link_rgx(R"(__seclink__(.*?)__KTSL__(.*?)__)"); + std::vector table {}; + strings_t files {}; + for (const auto& [basename, text] : pieces) { + if (!basename.empty()) files.push_back(basename); + for (const std::string& line : find_all(text, announce_rgx, 0)) { + std::smatch m {}; + std::regex_match(line, m, announce_rgx); + table.push_back({normalize_section_title(m[3]), m[2], m[1]}); + } + } + std::string result {}; + for (const auto& [basename, text] : pieces) { + std::string stripped = std::regex_replace(text, announce_rgx, ""); + auto begin = stripped.cbegin(); + for (std::sregex_iterator it(stripped.cbegin(), stripped.cend(), link_rgx), end; it != end; ++it) { + const std::smatch& m = *it; + result.append(begin, m[0].first); + result += resolve_section_target(table, m[1], m[2], basename, files).id; + begin = m[0].second; + } + result.append(begin, stripped.cend()); + } + return result; +} + std::string Document_class::tex() { (void)K::log(3); create_tex_output_directories(); // :text content is already processed (klammers applied by the outer Machine). // Use it directly; don't re-process through a sub-Machine. - std::string latex_text {}; + // The pieces stay apart until the :section links are resolved, so an + // error can name the chapter that holds the link; "" is the :text. + std::vector> pieces {}; if (!m_text.empty()) { - latex_text = add_latex_caption_numbers(m_text); + pieces.push_back({"", add_latex_caption_numbers(m_text)}); } if (m_files.size() > 0) { for (auto input_filename : m_files) { @@ -39,38 +79,25 @@ std::string Document_class::tex() if (m_no_cache || cache_requires_update(m_cache_dir, inpath.string(), basename)) { Machine M = m_machine; M.m_state.set("document_landscape", landscape ? "true" : "false"); // , "Landscape mode"); + // The chapter's basename: Section.tex() builds stable + // \hypertarget names from it (a counter alone collided + // when the cache served some chapters and not others), and + // phases.check_section_links reads its presence as "inside + // a @document". Same in the html path. + M.m_state.set("K_document_file", basename); M.read(inpath); std::string processed = trim(M.apply("tex", false)); processed = add_latex_caption_numbers(processed); write_to_cache(m_cache_dir, basename, processed); - latex_text += processed; + pieces.push_back({basename, processed}); } else { - latex_text += read_from_cache(m_cache_dir, basename); + pieces.push_back({basename, read_from_cache(m_cache_dir, basename)}); } } } + std::string latex_text = resolve_tex_section_links(pieces); auto split_lines = regex_split(latex_text, std::regex(R"(\n)"), false); latex_text = resolve_caption_references("latex", split_lines, latex_line_states(split_lines)); - std::map section_to_id {}; - std::regex section_id_rgx(R"(%__sectionlink__(.*?)__(.*?)__)"); - for (std::string id_section : find_all(latex_text, section_id_rgx, 0)) { - std::smatch match {}; - std::regex_match(id_section, match, section_id_rgx); - section_to_id[match[2]] = match[1]; - } - latex_text = std::regex_replace(latex_text, section_id_rgx, ""); - std::regex seclink_rgx("(.*?)__seclink__(.*?)__(.*)"); - auto lines = regex_split(latex_text, std::regex(R"(\n)"), false); - for (auto i = 0ul; i < lines.size(); i++) { - if (lines[i].find("__seclink__") != std::string::npos) { - std::smatch match {}; - std::regex_match(lines[i], match, seclink_rgx); - std::stringstream ss {}; - ss << match[1] << section_to_id[match[2]] << match[3]; - lines[i] = ss.str(); - } - } - latex_text = join(lines, "\n"); latex_text = string_replace(latex_text, "___AT___", "^@"); // @typeface: collect what the document asked for, declare each as a // \newfontfamily switch command in the preamble, and replace every diff --git a/sks/document/section_link.cpp b/sks/document/section_link.cpp new file mode 100644 index 0000000..4ef3e52 --- /dev/null +++ b/sks/document/section_link.cpp @@ -0,0 +1,55 @@ +#include "section_link.h" +#include "error.h" +#include "locator.h" +#include "util.h" +#include +#include + +std::string normalize_section_title(const std::string& s) +{ + static const std::regex number_rgx(R"(.*?)"); + static const std::regex tag_rgx("<[^>]*>"); + static const std::regex tex_command_rgx(R"(\\[A-Za-z@]+\*?)"); + static const std::regex space_rgx(R"(\s+)"); + std::string t = std::regex_replace(s, number_rgx, ""); + t = std::regex_replace(t, tag_rgx, ""); + t = std::regex_replace(t, tex_command_rgx, ""); + t = string_replace(string_replace(t, "{", ""), "}", ""); + t = std::regex_replace(t, space_rgx, " "); + return trim(t); +} + +Section_target resolve_section_target( + const std::vector& table, const std::string& title, + const std::string& file, const std::string& where, + const std::vector& files) +{ + const std::string wanted = normalize_section_title(title); + const std::string in = file.empty() ? where : file; // the one file searched + const std::string in_name = in.empty() ? std::string("the document's :text") : in; + std::stringstream ss {}; + ss << "In " << (where.empty() ? std::string("the document's :text") : where) + << ", a :section link names the heading " << q_(title); + if (!file.empty()) { + bool known = false; + for (const auto& f : files) known = known || f == file; + if (!known) { + ss << " in " << file << ", but the document has no file " << file + << " (its files are " << join(files, ", ") << ")."; + throw Argument_error(ss.str(), Locator::none()); + } + } + std::vector hits {}; + for (const auto& t : table) { + if (t.basename == in && t.title == wanted) hits.push_back(&t); + } + if (hits.size() == 1) return *hits[0]; + if (hits.empty()) { + ss << ", which " << in_name << " does not have" + << (file.empty() ? " (a section in another file needs its basename as the value of :section)." : "."); + } else { + ss << ", which " << in_name << " has " << hits.size() + << " times: give one of them an :id and link to that."; + } + throw Argument_error(ss.str(), Locator::none()); +} diff --git a/sks/document/section_link.h b/sks/document/section_link.h new file mode 100644 index 0000000..651dbc7 --- /dev/null +++ b/sks/document/section_link.h @@ -0,0 +1,49 @@ +// @link :section -- a link to a section by the TEXT of its heading. +// +// Decided by Andy, 2026-10-12. :section (argtype section_link) written +// alone says the heading is in the CURRENT file; written with a value, the +// value is the basename (no directory, no extension) of the file of the +// same @document that has it. So a link names exactly one file, and a +// title shared between files is never ambiguous. Titles match with +// klammer markup removed and whitespace collapsed to one space, +// case-sensitive. +// +// Resolution happens here, when @document assembles its files -- the only +// place every heading is known -- from two markers the klammers emit: +// Link __seclink____KTSL__<file>__ (in the href / \hyperlink; +// <file> empty for the current one) +// Section %__sectionlink__<id>__KTSL__<basename>__KTSL__<title>__ (tex only; +// the html pass reads the heading elements themselves, which +// carry their ids after insert_ids()) +// KTSL separates the fields because a title or a basename may contain a +// single underscore (md_to_sks_test). A file the document does not have, a +// title the file does not have, and a title it has more than once are +// errors, never fallbacks (the commands do not warn and continue). +#ifndef SECTION_LINK_H +#define SECTION_LINK_H + +#include <string> +#include <vector> + +struct Section_target { + std::string title {}; // normalized + std::string basename {}; // the file's basename, "" for :text content + std::string id {}; // the anchor: html id, or tex \hypertarget name +}; + +// The comparison form of a heading's text: tags and TeX control sequences +// removed, braces dropped, whitespace collapsed, trimmed. +std::string normalize_section_title(const std::string& s); + +// The target a :section link names, or an Argument_error that says why +// there is none. FILE is the :section value ("" for the current file), +// WHERE the basename of the file holding the link ("" for the document's +// :text) and FILES the document's basenames. At assembly the link's own +// position is gone, so the file is the location an error can give +// (Locator::none(), never the C++ site). +Section_target resolve_section_target( + const std::vector<Section_target>& table, const std::string& title, + const std::string& file, const std::string& where, + const std::vector<std::string>& files); + +#endif diff --git a/sks/link/js/link.js b/sks/link/js/link.js index d6a5989..cae5802 100644 --- a/sks/link/js/link.js +++ b/sks/link/js/link.js @@ -1,8 +1,19 @@ +// In-text links to a section on another (or the same) page of a book: +// <a class="pagelink" data-page="BASENAME" data-id="ID">, which @document +// emits for @link ... :section @ (document_html.cpp, resolve_section_links). +// go_to_section() swaps in the embedded page and scrolls to the id, and +// calls this again for the links of the page it just showed. Each anchor +// is bound once: a page shown twice is the same nodes bound twice otherwise. function set_link_listeners() { Array.from(document.getElementsByClassName("pagelink")).forEach( function (elt) { + if (elt.getAttribute("data-bound")) { + return; + } + elt.setAttribute("data-bound", "true"); elt.addEventListener("click", function(event) { + event.preventDefault(); let basename = elt.getAttribute("data-page"); let id = elt.getAttribute("data-id"); go_to_section(elt, basename, id); @@ -10,3 +21,4 @@ function set_link_listeners() { }); } +window.addEventListener("load", set_link_listeners); diff --git a/sks/link/link.k b/sks/link/link.k index e6fa90d..9d9a83d 100755 --- a/sks/link/link.k +++ b/sks/link/link.k @@ -1,15 +1,39 @@ +@@@argtype section_link | + treat the link target as a section title, not an id. + + If ^:section is present but does not have a value, the section title is in + the current file. + + If ^:section has a value, then it is the basename (no directory; no + extension) of the file in the same document that contains the section with + that title. + :pattern [A-Za-z0-9_.-]+ + :default false + :alone true +@@@ + @@link.k target :text :footnote.bool false -:section.bool false +:section.section_link :nq.bool false -:basename # xxx -- Deprecated? :code # xxx :phrase # xxx :dox # xxx :doxanchor # xxx :id # xxx -: Links within documents and to external URLs. +: + +Links within documents and to external URLs. + +With ^:section the target is a section title, matched with klammer markup +removed and whitespace collapsed to one space. Written alone, ^:section +means the section is in the current file; written with a value, the value is +the basename of the file of the same ^@document that has the section. A +section link is resolved when ^@document assembles its files, so one outside +a ^@document is an error, as is a title the file does not have, or has more +than once (give one heading an ^:id and link to that). + @@ @@link.html,tex,txt :: @eval link.Link(K) eval@ @@ diff --git a/sks/link/link.py b/sks/link/link.py index 0f01d82..38bac01 100755 --- a/sks/link/link.py +++ b/sks/link/link.py @@ -6,8 +6,25 @@ import klammer_base import urllib import kutil -def html_link(target, link_text, is_section, no_quotation_marks): - # [__link__target__text__] +# :section (argtype section_link, Andy 2026-10-12) is absent ("false"), +# written alone ("true": the section is in the current file) or written +# with a basename (the file that has the section). A section link is +# resolved by @document when it assembles its files (sks/document/ +# section_link.h), from this marker: the heading's text and the file, "" +# meaning the current one, separated by KTSL because either may contain an +# underscore. +def section_file(section): + """The file a :section value names: "" for the current file, None when + the link is not a section link at all.""" + s = str(section) + if s in ("", "false", "False"): + return None + return "" if s in ("true", "True") else s + +def section_marker(title, file): + return f"__seclink__{title}__KTSL__{file}__" + +def html_link(target, link_text, is_section, no_quotation_marks, file=""): if not link_text: if is_section: link_text = target if no_quotation_marks else f"“{target}”" @@ -16,14 +33,11 @@ def html_link(target, link_text, is_section, no_quotation_marks): link_text = re.sub('/', '/<wbr/>', target) link_text = re.sub('/<wbr/>/<wbr/>', '//', link_text) link_text = f'<tt>{link_text}</tt>' - target = f"[{target}]" if is_section else target - #result = f"[__link__{target}__{link_text}__]" - #return result - if target.startswith("http") or ".html" in target: + if is_section: + url = section_marker(target, file) + elif target.startswith("http") or ".html" in target: url = target - elif is_section: - url = f"__sectionlink__{target}__" else: url = f"^#{target}" result = f'<a href="{url}">{link_text}</a>' @@ -38,7 +52,7 @@ def add_latex_footnote_if_necessary(url, link_text): result += f'\\footnote{{\\\rule{{0pt}}{{12pt}}\\texttt{{{escaped_url}}}}}' return result -def latex_link(target, link_text, is_section, no_quotation_marks, add_footnote): +def latex_link(target, link_text, is_section, no_quotation_marks, add_footnote, file=""): if target.startswith("http"): if link_text and link_text != "URL": result = f"#/ \\href{{{target}}}{{{link_text}}} #/" @@ -48,7 +62,7 @@ def latex_link(target, link_text, is_section, no_quotation_marks, add_footnote): elif is_section: if not link_text: link_text = target if no_quotation_marks else f"``{target}''" - result = f"\\hyperlink{{__seclink__{target}__}}{{{link_text}}}" + result = f"\\hyperlink{{{section_marker(target, file)}}}{{{link_text}}}" else: result = f"\\hyperlink{{{target}}}{{{link_text}}}" # if add_footnote: @@ -63,10 +77,12 @@ class Link(klammer_base.Klammer_base): # self.show("Link") def html(self): - return html_link(self.target, self.text, self.section, self.nq) + file = section_file(self.section) + return html_link(self.target, self.text, file is not None, self.nq, file or "") def tex(self): - return latex_link(self.target, self.text, self.section, self.nq, self.footnote) + file = section_file(self.section) + return latex_link(self.target, self.text, file is not None, self.nq, self.footnote, file or "") def txt(self): text = f'{self.text} ' if self.text else '' diff --git a/sks/section/section.py b/sks/section/section.py index b28a8e8..e413179 100644 --- a/sks/section/section.py +++ b/sks/section/section.py @@ -49,10 +49,15 @@ class Section(klammer_base.Klammer_base): # names = "part chapter section subsection subsubsection paragraph subparagraph subsubparagraph subsubsubparagraph".split() names = "part sA sB sC sD sE".split() + # The \hypertarget name. K_document_file is the chapter's basename, + # set by @document (document_latex.cpp): with it the name is stable + # across the page cache, where a counter alone collided when some + # chapters were served from cache and others reprocessed. + basename = getattr(self, "K_document_file", "") or "" if self.id: id = self.id else: - id = f"-s{Section.label_number}" + id = f"{basename}-s{Section.label_number}" Section.label_number += 1 result = '' @@ -64,7 +69,10 @@ class Section(klammer_base.Klammer_base): # result = kutil.protect_klammertext_special_characters(result) - result = f"%__sectionlink__{id}__{self.title}__" + "\n" + result + # The announcement @document's section-link resolver reads (fields + # separated by KTSL; see sks/document/section_link.h). Title last: + # it may hold anything. + result = f"%__sectionlink__{id}__KTSL__{basename}__KTSL__{self.title}__" + "\n" + result return result def txt(self): diff --git a/sks/target/phases.py b/sks/target/phases.py index 3bb721b..ef6a969 100644 --- a/sks/target/phases.py +++ b/sks/target/phases.py @@ -417,3 +417,20 @@ def html_format_footnotes(text, K=None): # Temporarily add room for footnotes to move to the top of the window for the test: result += '<div style="height: 100lh"></div>\n' return result + + + +# @link :section outside a @document (2026-10-12). A section link is +# resolved when @document assembles its chapters, so one that is applied +# outside any document can never be resolved: its marker would reach the +# output. @document sets K_document_file on each chapter's Machine; a +# Machine without it that still holds the marker after its own apply is +# that case. (Inside a document the outer Machine's apply ends with the +# resolved text -- html writes files and returns nothing -- so the marker +# is gone by the time this runs there.) +def check_section_links(text, K): + if "__seclink__" in text and not getattr(K, "K_document_file", ""): + raise ValueError("A @link with :section names a section by its heading, " + "which only a @document can resolve, and this link is " + "outside any @document") + return text diff --git a/sks/target/target.k b/sks/target/target.k index fd56109..dba8f01 100644 --- a/sks/target/target.k +++ b/sks/target/target.k @@ -102,7 +102,8 @@ of this: neither the transforms nor the quoting reach it. ' ’ | ~ &^#160; :after_apply - phases.html_format_footnotes + phases.html_format_footnotes ; + phases.check_section_links @@@ # No transforms: LaTeX applies the input conventions itself. The :resolve @@ -116,7 +117,7 @@ of this: neither the transforms nor the quoting reach it. @@@target tex | LaTeX :escape \ \textbackslash{} & \& { \{ } \} $ \$ % \% _ \_ ^# \^# ^^ \textasciicircum{} :resolve - {-} ~ \textasciitilde{} - :after_apply phases.tex_resolve_markers + :after_apply phases.tex_resolve_markers ; phases.check_section_links @@@ @@@target pdf | PDF from LaTeX diff --git a/sks/tns/md_to_sks.py b/sks/tns/md_to_sks.py index 67cdfb7..162020d 100644 --- a/sks/tns/md_to_sks.py +++ b/sks/tns/md_to_sks.py @@ -11,9 +11,13 @@ correspondence between a Markdown construct and a klammer is a property of the klammer set. That is why this lives under sks/ rather than in mac/. What cannot be converted exactly is recorded in the output as a -#[MD <kind>: ... ]# marker -- lossy, gap, or judgment -- so a converted -document carries its own worklist and "how far from done is this file" is a -grep. A file with no markers is inside the convertible subset. +#[MD <kind>: ... ]# marker -- lossy, gap, or judgment; with --chapter also +section -- so a converted document carries its own worklist and "how far +from done is this file" is a grep. A file with no markers is inside the +convertible subset. The marker is the contract: a construct this program +does not handle must leave one, never pass through as prose. It does not +hold yet -- see "What the reference implementation does not do" in the +spec for the constructs that still pass through unmarked. First run: the Rectify user guide (1,805 lines), 2026-08-07. It rendered to html and pdf without error or warning, with 99 markers -- 91 of them internal @@ -22,20 +26,100 @@ vocabulary lists (4,200 lines, 152 tables), 2026-09-30, which rendered without an error and with two silently wrong results -- an emphasis that wrapped and a table's empty header corner -- so "renders clean" is not "converted"; the fixes are in the spec's "What the second conversion -taught". +taught". A review against a probe document (2026-10-11) found twelve more +ways the output was silently wrong and eight constructs passed through +with no marker; four were fixed first (a ragged nested list dropped items, +a loose list became one @ol per item, the "@@" separator ran over @c +literals, an escaped bar split a table cell), and the probe became the +fixture sks/tst/md_to_sks_test.sh: the conversion, in both modes, must +equal a golden file line for line and then render. The golden files pin +only what is right; the remaining defects are listed in the spec and join +the probe as they are fixed. -Usage: python3 md_to_sks.py <input.md> <output.kt> [sibling.md ...] +Usage: python3 md_to_sks.py [--chapter] <input.md> <output.kt> [sibling.md ...] -The siblings are Markdown files converted alongside this one: a link to -one of them becomes a link to its html page (../<stem>/index.html, where -ktext writes it); a link to any other .md file is a gap. +The siblings are Markdown files converted alongside this one. Without +--chapter each file is a document of its own, and a link to a sibling +becomes a link to its html page (../<stem>/index.html, where ktext writes +it); a link to any other .md file is a gap. + +With --chapter (Andy, 2026-10-12) the file is one CHAPTER of a book whose +@document is written by hand and names the chapters through :files: no +@document wrapper is emitted, no :title is lifted, and every heading maps +one level deeper -- "#" is @s1, the chapter's own heading. A link to a +sibling is then a link to another chapter of the same book, written +"@link <its title> :section @" (:section finds a section by its heading; +implemented 2026-10-12), the title being the sibling's first "#" heading +read from the file. A Markdown link targets a FILE, so the title is a +guess, and a "section" marker records it. The same mechanism converts a +Markdown anchor, "#some-heading" or "other.md#some-heading": the slug is +matched against the headings of the file it points into, and the link +names that heading's text -- again marked, since the match is a guess. +The motivating case was ~/projects/kokkoja/docs/, five topic files +cross-linked as "[A host's Kokkoja](host.md)". """ -import re, sys, os, datetime +import re, sys, os, datetime, argparse from collections import Counter SPECIALS = "^@#|*" # ^ first: the other quotings introduce ^ marks = Counter() -SIBLINGS = set() # stems of the .md files converted together +SIBLINGS = {} # stem -> path of the .md files converted together +CHAPTER = False # --chapter: a chapter of a hand-written book +STEM = "" # the file being converted +HEADINGS = {} # stem -> [(slug, level, title as Klammertext)], lazily + +def slug(text): + """GitHub's anchor for a heading: lower case, punctuation other than + hyphen and underscore removed, every whitespace a hyphen. A duplicate + heading gets -1, -2 ... appended, which lookups retry without.""" + return re.sub(r'\s', '-', re.sub(r'[^\w\s-]', '', text.lower()).strip()) + +def headings_of(stem): + """The headings of a file converted alongside (or this one): the anchor + each would have in Markdown, its level, and the title AS THE CONVERTER + EMITS IT -- manual number stripped, inline Markdown converted -- which + is the text a :section link must name. Headings inside fenced code + are not headings.""" + if stem in HEADINGS: + return HEADINGS[stem] + found = [] + try: + with open(SIBLINGS[stem], encoding="utf-8") as f: + fenced = False + for line in f: + line = line.rstrip("\n") + if FENCE.match(line): + fenced = not fenced + continue + m = HEADING.match(line) + if m and not fenced: + text = m.group(2) + title = MANUAL_NUMBER.sub("", text) if MANUAL_NUMBER.match(text) else text + found.append((slug(text), len(m.group(1)), title)) + except (OSError, KeyError): + pass + HEADINGS[stem] = found + return found + +def find_heading(stem, anchor): + """The (level, title) a Markdown anchor names in STEM's file, or None. + An anchor made unique with -N is retried as its base.""" + for s_, level, title in headings_of(stem): + if s_ == anchor: + return level, title + m = re.match(r'^(.*)-\d+$', anchor) + if m: + for s_, level, title in headings_of(stem): + if s_ == m.group(1): + return level, title + return None + +def sibling_title(stem, fallback): + """A sibling chapter's title: its first "#" heading.""" + for _, level, title in headings_of(stem): + if level == 1: + return title + return fallback def mark(kind, text): """A #[MD ... ]# marker. @@ -115,17 +199,48 @@ def inline(text): # loss. The marker goes BEFORE the words: a link is usually # followed by punctuation, and a marker between the words and the # period leaves the space before the marker standing ("words ."). + def section_link(title, stem, note): + """A link to a section by its title (@link :section, resolved by + @document; 2026-10-12). :section alone means this file; a + heading in another chapter names that file as the value. The + label is kept as :text only where it differs from the title. + The title is a GUESS from a Markdown anchor or a file, so the + marker stays.""" + where = "" if stem == STEM else " " + stem + text = "" if label == title else " :text " + emphasis(label) + return (keep(mark("section", note)) + " " + + opening("@link " + inline(title) + " :section" + where + text) + closing()) + if target.startswith("#"): - # An anchor into this document: no klammer gives an arbitrary - # heading an anchor. - return keep(mark("gap", f"link to {target}")) + " " + emphasis(label) - if target.endswith(".md") and "/" not in target: - # Another Markdown file. "@link X.md" would be read as an - # anchor (href="#X.md"). Converted alongside this one, its html - # page is X/index.html beside ours; otherwise it is a gap. - stem = target[:-3] + # An anchor into this file: Markdown's slug of a heading. The + # heading it names becomes a :section link to that heading's + # text; an anchor matching no heading is the gap it always was. + found = find_heading(STEM, target[1:]) + if found: + return section_link(found[1], STEM, f'link to {target}, read as the heading "{found[1]}"') + return keep(mark("gap", f"link to {target}, no such heading")) + " " + emphasis(label) + m_file = re.match(r'^([^/#]+)\.md(?:#(.+))?$', target) + if m_file: + # Another Markdown file, with or without an anchor into it. + # "@link X.md" would be read as an anchor (href="#X.md"). + stem, anchor = m_file.group(1), m_file.group(2) if stem not in SIBLINGS: return keep(mark("gap", f"link to {target}, not converted")) + " " + emphasis(label) + if CHAPTER: + # Another chapter of the same book: a link to a section by + # its title -- the heading the anchor names, or the file's + # own title (its first "#") when there is no anchor. + if anchor: + found = find_heading(stem, anchor) + if not found: + return keep(mark("gap", f"link to {target}, no such heading")) + " " + emphasis(label) + return section_link(found[1], stem, f'link to {target}, read as the heading "{found[1]}"') + title = sibling_title(stem, label) + return section_link(title, stem, f'link to {target}, read as its title "{title}"') + if anchor: + # Converted as separate documents, the other page's ids are + # not known here. + return keep(mark("gap", f"link to {target}: a heading on another page")) + " " + emphasis(label) target = f"../{stem}/index.html" return opening("@link " + quote(target) + " :text ") + emphasis(label) + closing() @@ -155,6 +270,34 @@ def inline(text): text = "".join(p if re.match(r'^[\x00\x01]', p) else quote(p) for p in parts) text = re.sub(r'\x01(\d+)\x01', lambda m: verbatim[int(m.group(1))], text) + # The code spans stay placeholders through the two fixes below, so that + # neither touches their content: a code span is LITERAL, and the "@@" + # rule once rewrote a Klammertext example inside one ("`@@name.k`" + # rendered as "@ @name.k"). Only the delimiter sentinels go on now; the + # content comes back last. + text = re.sub(r'(\x00\d+\x00)', OPEN + r'\1' + CLOSE, text) + # Klammertext delimiters need whitespace around them, but Markdown + # emphasis abuts its neighbours: "un**bold**ed" and "*name*_rectified" + # both put a word character hard against a delimiter, which the + # katomizer then reads as an unparsable word. A space makes it parse, + # and "#-" removes that space again from the OUTPUT, so intra-word + # emphasis converts exactly rather than approximately. + # EXCEPT where the katomizer splits the neighbour off itself (kdesc + # --rewrite, the "punctuation" and "klammer" categories): trailing + # punctuation after a closing "@" -- "It is @i italicized @." -- and + # an opening bracket before an opening "@name". Those are legal as + # written, and the "#-" was noise (Andy, 2026-10-12). Everything else + # that is not a space needs the fix: a word character, a quote, an + # underscore ("*name*_x"), a digit. + text = re.sub(r'(?<=[^\s(\[{])' + OPEN, ' #-', text).replace(OPEN, "") + text = re.sub(CLOSE + r"(?=[^\s\]})(.,:;?'!])", '#- ', text).replace(CLOSE, "") + # A closing "@" abutting the next klammer's opening "@" spells "@@", + # which is a DEFINITION delimiter -- the engine then reports a span that + # ends without a beginning, pointing at text the writer never wrote. + # Adjacent klammers are ordinary in converted prose (*a*_b_ produces + # two), so separate them. The code spans are still placeholders here, + # on purpose (see above): a "@@" inside one is the author's. + text = re.sub(r'@(?=@)', '@ ', text) # Inline code goes in RAW, and is closed with the named delimiter. "@c" # took an ordinary string parameter until 2026-08-16, so every Klammertext # special inside a code span had to be quoted -- and about 40% of the @@ -164,24 +307,8 @@ def inline(text): # source reads as the author wrote it. A literal klammer cannot be closed # with a bare "@" -- the machine cannot find the end by counting -- so the # close is "c@". - text = re.sub(r'\x00(\d+)\x00', - lambda m: OPEN + "@c " + spans[int(m.group(1))] + " c@" + CLOSE, text) - # Klammertext delimiters need whitespace around them, but Markdown - # emphasis abuts its neighbours: "un**bold**ed" and "*name*_rectified" - # both put a word character hard against a delimiter, which the - # katomizer then reads as an unparsable word. A space makes it parse, - # and "#-" removes that space again from the OUTPUT, so intra-word - # emphasis converts exactly rather than approximately. - # Any non-space neighbour, not just a word character: "*name*_x" and - # "_1.*ext*" put an underscore or a period against the delimiter. - text = re.sub(r'(?<=\S)' + OPEN, ' #-', text).replace(OPEN, "") - text = re.sub(CLOSE + r'(?=[^\s])', '#- ', text).replace(CLOSE, "") - # A closing "@" abutting the next klammer's opening "@" spells "@@", - # which is a DEFINITION delimiter -- the engine then reports a span that - # ends without a beginning, pointing at text the writer never wrote. - # Adjacent klammers are ordinary in converted prose (*a*_b_ produces - # two), so separate them. - return re.sub(r'@(?=@)', '@ ', text) + return re.sub(r'\x00(\d+)\x00', + lambda m: "@c " + spans[int(m.group(1))] + " c@", text) # ---- blocks --------------------------------------------------------------- @@ -193,8 +320,30 @@ ORDERED = re.compile(r'^(\s*)(\d+)[.)]\s+(.*)$') TABLEROW = re.compile(r'^\s*\|(.*)\|\s*$') ALIGNROW = re.compile(r'^\s*\|[\s:|-]+\|\s*$') +def list_marker(line): + """The marker of an item line -- its bullet character, or the delimiter + after its number ("." or ")") -- and None for a line that is no item. + CommonMark starts a NEW list when the marker changes.""" + m = BULLET.match(line) + if m: + return line[len(m.group(1))] + m = ORDERED.match(line) + if m: + return line[len(m.group(1)) + len(m.group(2))] + return None + def cells(line): - return [c.strip() for c in TABLEROW.match(line).group(1).split("|")] + """The cells of one table row. + + A bar escaped as "\\|" is cell CONTENT -- GFM requires the escape even + inside a code span -- so the split is on the unescaped bars only, and + the backslash is then dropped: inline() quotes a prose bar to "^|", and + a bar inside a code span goes into the @c literal as it is. Splitting + on every bar shifted the header of any table that documents Klammertext + (whose own tables are full of "\\|") one column right, silently. + """ + inner = TABLEROW.match(line).group(1) + return [c.strip().replace("\\|", "|") for c in re.split(r'(?<!\\)\|', inner)] def convert(path): src = open(path, encoding="utf-8").read().split("\n") @@ -224,8 +373,18 @@ def convert(path): m = HEADING.match(line) if m: level, text = len(m.group(1)), m.group(2) - if level == 1 and title is None: + if CHAPTER: + level += 1 # "#" is the chapter's heading, @s1 + elif level == 1 and title is None: title = inline(text); i += 1; continue + elif level == 1: + # A second "#": not a title. Probably a part, or the file + # is two documents; neither is the converter's call, so it + # is a top-level section and the marker says so. (It was + # "@s0", which does not exist, and failed at render.) + out.append(mark("judgment", "a second top-level heading was " + "made @s1; it may be a @part, or a second document")) + level = 2 if MANUAL_NUMBER.match(text): text = MANUAL_NUMBER.sub("", text); stripped_numbers += 1 if re.match(r'^contents$', text, re.I): @@ -284,10 +443,32 @@ def convert(path): continue if BULLET.match(line) or ORDERED.match(line): - block, base = [], None - while i < n and (BULLET.match(src[i]) or ORDERED.match(src[i]) or - (src[i].strip() and src[i].startswith((" ", "\t")))): - block.append(src[i]); i += 1 + block = [] + while i < n: + if (BULLET.match(src[i]) or ORDERED.match(src[i]) or + (src[i].strip() and src[i].startswith((" ", "\t")))): + block.append(src[i]); i += 1 + elif not src[i].strip(): + # A blank line ends the list UNLESS the next non-blank + # line is another item of it: a loose list (items + # separated by blank lines) is one list, and ending it + # here made every item its own @ol, each numbered 1. + # "Of it" is CommonMark's rule: an indented item (a + # nested one) or one with the same marker; a different + # marker starts a new list. An indented non-item after + # the blank (a code block inside the item) still ends + # the list, as it always did. + j = i + while j < n and not src[j].strip(): + j += 1 + if j < n and list_marker(src[j]) and ( + src[j].startswith((" ", "\t")) or + list_marker(src[j]) == list_marker(block[0])): + i = j + else: + break + else: + break out.append(convert_list(block)) out.append("") continue @@ -329,39 +510,70 @@ def convert_list(block): parts.append(body) return klammer + " " + " | ".join(parts) + " @" - def parse(lines, indent): + def parse(lines): + """The items at one level, each with its children. + + An item's children are the item lines that begin at or past its + CONTENT column (where the text after the marker starts, as in + CommonMark); an item line that begins before that is the next item + at THIS level, however it is indented. The loop never stops on an + indent: the caller has already bounded the lines, so there is + nothing an indent could hand back to. The earlier rule took the + first child's indent as the sublist's and broke off at anything + shallower, so " - a1" followed by " - a2" DROPPED a2 -- and the + document rendered without a word about it. + """ items, i = [], 0 ordered = bool(ORDERED.match(lines[0])) if lines else False while i < len(lines): m = BULLET.match(lines[i]) or ORDERED.match(lines[i]) - if not m or len(m.group(1)) < indent: - break + if not m: # a stray wrapped line: previous item's + if items: + text, children = items[-1] + items[-1] = (text + " " + lines[i].strip(), children) + i += 1 + continue text = m.groups()[-1] + content_col = m.start(m.lastindex) i += 1 child_lines = [] while i < len(lines): m2 = BULLET.match(lines[i]) or ORDERED.match(lines[i]) - if m2 and len(m2.group(1)) > indent: + if m2 and len(m2.group(1)) >= content_col: child_lines.append(lines[i]); i += 1 - elif not m2 and lines[i].strip(): - text += " " + lines[i].strip(); i += 1 - else: + elif m2: break - children = None - if child_lines: - sub, sub_ordered = parse(child_lines, len(BULLET.match(child_lines[0]).group(1)) - if BULLET.match(child_lines[0]) - else len(ORDERED.match(child_lines[0]).group(1))) - children = (sub, sub_ordered) + elif child_lines: # a wrapped line of the last child + child_lines.append(lines[i]); i += 1 + else: # a wrapped line of this item + text += " " + lines[i].strip(); i += 1 + children = parse(child_lines) if child_lines else None items.append((text, children)) return items, ordered - items, ordered = parse([l for l in block if l.strip()], 0) + items, ordered = parse([l for l in block if l.strip()]) return emit(items, ordered, 0) def main(): - src_path, dst_path = sys.argv[1], sys.argv[2] - SIBLINGS.update(os.path.splitext(os.path.basename(a))[0] for a in sys.argv[3:]) + global CHAPTER + ap = argparse.ArgumentParser( + description="Convert a Markdown file to the Standard Klammer Set.", + epilog="Siblings are the .md files converted alongside; a link to one " + "is converted as a link to its page (or, with --chapter, to " + "its section), a link to any other .md file is a gap.") + ap.add_argument("--chapter", action="store_true", + help="the file is a chapter of a hand-written @document: " + "no wrapper, '#' maps to @s1, sibling links are :section links") + ap.add_argument("input", metavar="input.md") + ap.add_argument("output", metavar="output.kt") + ap.add_argument("siblings", metavar="sibling.md", nargs="*") + args = ap.parse_args() + global STEM + CHAPTER = args.chapter + src_path, dst_path = args.input, args.output + STEM = os.path.splitext(os.path.basename(src_path))[0] + SIBLINGS.update((os.path.splitext(os.path.basename(a))[0], a) for a in args.siblings) + SIBLINGS[STEM] = src_path # its own headings resolve "#anchor" links body, title, stripped, dropped_toc = convert(src_path) today = datetime.date.today().isoformat() head = [f"#[MD source: {os.path.abspath(src_path)}", @@ -372,15 +584,20 @@ def main(): '("2.1 ..."); the numbers were removed because @s1/@s2 ' "number the headings themselves")) head.append("") - head += ["@document", ":structure plain"] - if title: - head.append(f":title {title}") - head += [":text", ""] - text = "\n".join(head + body + ["", "@", ""]) + if CHAPTER: + tail = [""] + else: + head += ["@document", ":structure plain"] + if title: + head.append(f":title {title}") + head += [":text", ""] + tail = ["", "@", ""] + text = "\n".join(head + body + tail) text = re.sub(r'\n{3,}', "\n\n", text) open(dst_path, "w", encoding="utf-8").write(text) print(f"{dst_path}: {len(text.splitlines())} lines") - for kind in ("source", "lossy", "gap", "judgment"): - print(f" {kind:9} {marks[kind]}") + for kind in ("lossy", "gap", "judgment", "section"): + if kind != "section" or CHAPTER: + print(f" {kind:9} {marks[kind]}") main()