ack
bc2b3d8461
- A rest argument (a list's or a table's items) now strips exactly ONE leading bar -- the separator from what precedes it -- so a second one is an empty first element: a table with an empty header corner is written `@table |` then ` | Maskulinum | ...`. Before, every leading bar was dropped and such a header row shifted one column left. New shipped suite tst/rest_test.sh. - Text written hard against a closing ]# (a period, a letter) is no longer removed with the block; a following space or line break still is. - sks/tns/md_to_sks.py converts inline Markdown per paragraph, so emphasis that wraps is converted and not mis-paired; bold may contain italic; tables are unnumbered; links to sibling .md files converted alongside become links to their html pages; the arrow becomes @rightarrow@. - doc/install/windows_source_install.md: Smart App Control must be off -- it blocks MSYS2's unsigned toolchain, and its Evaluation state turns itself on later.
387 lines
18 KiB
Python
387 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Markdown -> Standard Klammer Set converter (first implementation).
|
|
|
|
SPECIFICATION: doc/markdown_to_klammertext.md. That document is authoritative
|
|
and is written to be sufficient to rebuild this program; this program is its
|
|
reference implementation, not the other way round. Its "The algorithm"
|
|
section states the pass order, which is forced, and the traps that make it so.
|
|
|
|
The conversion is to the SKS, not to Klammertext the language: the
|
|
correspondence between a Markdown construct and a klammer is a property of
|
|
the klammer set. That is why this lives under sks/ rather than in mac/.
|
|
|
|
What cannot be converted exactly is recorded in the output as a
|
|
#[MD <kind>: ... ]# marker -- lossy, gap, or judgment -- so a converted
|
|
document carries its own worklist and "how far from done is this file" is a
|
|
grep. A file with no markers is inside the convertible subset.
|
|
|
|
First run: the Rectify user guide (1,805 lines), 2026-08-07. It rendered to
|
|
html and pdf without error or warning, with 99 markers -- 91 of them internal
|
|
anchor links, the one gap that appears at volume. Second run: three German
|
|
vocabulary lists (4,200 lines, 152 tables), 2026-09-30, which rendered
|
|
without an error and with two silently wrong results -- an emphasis that
|
|
wrapped and a table's empty header corner -- so "renders clean" is not
|
|
"converted"; the fixes are in the spec's "What the second conversion
|
|
taught".
|
|
|
|
Usage: python3 md_to_sks.py <input.md> <output.kt> [sibling.md ...]
|
|
|
|
The siblings are Markdown files converted alongside this one: a link to
|
|
one of them becomes a link to its html page (../<stem>/index.html, where
|
|
ktext writes it); a link to any other .md file is a gap.
|
|
"""
|
|
import re, sys, os, datetime
|
|
from collections import Counter
|
|
|
|
SPECIALS = "^@#|*" # ^ first: the other quotings introduce ^
|
|
marks = Counter()
|
|
SIBLINGS = set() # stems of the .md files converted together
|
|
|
|
def mark(kind, text):
|
|
"""A #[MD ... ]# marker.
|
|
|
|
Two things have to be neutralised in the text, both of which cost a
|
|
session's debugging if they are not:
|
|
|
|
* "]#" would close the block early and "#[" would open one that must then
|
|
be balanced (the nesting hazard);
|
|
* a klammer name written with its "@" is READ before the block is
|
|
removed -- mark_literal_klammer_content() runs first in
|
|
process_katoms() -- so "@code has no :language" makes the engine demand
|
|
a "code@" close from inside removed text. Quoting the @ makes it a
|
|
special katom, which the literal scan does not see. (TODO #39.)
|
|
"""
|
|
marks[kind] += 1
|
|
text = text.replace("]#", "] #").replace("#[", "# [")
|
|
text = text.replace("@", "^@").replace("|", "^|")
|
|
return f"#[MD {kind}: {text} ]#"
|
|
|
|
def quote(s):
|
|
"""Quote every Klammertext special in writer text."""
|
|
for ch in SPECIALS:
|
|
s = s.replace(ch, "^" + ch)
|
|
return s
|
|
|
|
# ---- inline ---------------------------------------------------------------
|
|
|
|
CODE = re.compile(r'`([^`\n]+)`')
|
|
LINK = re.compile(r'(?<!\!)\[([^\]]*)\]\(([^)]+)\)')
|
|
# A bold run may hold italic runs ("**only for *sein* and *haben***"): the
|
|
# group admits a balanced *...* inside, so the closing ** is found past it.
|
|
BOLD = re.compile(r'\*\*((?:[^*]|\*[^*\s][^*]*\*)+)\*\*|__([^_]+)__')
|
|
# Markdown's underscore emphasis does not open or close INSIDE a word, and
|
|
# technical prose is full of identifiers like *name*_rectified_1.*ext* where
|
|
# the underscores are part of the name. So the closing _ must not be
|
|
# followed by a word character -- without that, the converter invents
|
|
# emphasis in the middle of filenames.
|
|
ITAL = re.compile(r'(?<![\*\w])\*([^*\s][^*]*)\*(?!\*)|(?<![_\w])_([^_\s][^_]*)_(?![\w_])')
|
|
STRIKE = re.compile(r'~~([^~]+)~~')
|
|
|
|
def inline(text):
|
|
"""Convert one run of inline Markdown.
|
|
|
|
Order matters: code spans are protected first (their content must not be
|
|
read as emphasis), links and emphasis consume their markers next, and only
|
|
then are the remaining specials quoted -- quoting first would hide the
|
|
** and * that emphasis is recognised by.
|
|
"""
|
|
spans = []
|
|
def stash(m):
|
|
spans.append(m.group(1))
|
|
return f"\x00{len(spans)-1}\x00"
|
|
text = CODE.sub(stash, text)
|
|
|
|
# Markers are Klammertext, not writer text: they must survive the quoting
|
|
# pass below untouched, or the "#[" that makes them a removal block is
|
|
# itself quoted and the marker renders into the output instead of
|
|
# vanishing from it. Stash them like code spans.
|
|
verbatim = []
|
|
def keep(s):
|
|
verbatim.append(s)
|
|
return f"\x01{len(verbatim)-1}\x01"
|
|
|
|
# OPEN and CLOSE wrap the delimiters this function emits, so the
|
|
# whitespace fix below can find exactly its own delimiters. Matching
|
|
# them by regex on the finished text does not work: " @" is both the
|
|
# closing delimiter and the space before an opening "@i", and a rule
|
|
# keyed on it rewrites "@i" into "@#- i".
|
|
OPEN, CLOSE = "\x03", "\x02"
|
|
def opening(s): return keep(OPEN + s)
|
|
def closing(): return keep(" @" + CLOSE)
|
|
|
|
def link(m):
|
|
label, target = m.group(1), m.group(2)
|
|
# A link that cannot be converted keeps its words and records the
|
|
# loss. The marker goes BEFORE the words: a link is usually
|
|
# followed by punctuation, and a marker between the words and the
|
|
# period leaves the space before the marker standing ("words .").
|
|
if target.startswith("#"):
|
|
# An anchor into this document: no klammer gives an arbitrary
|
|
# heading an anchor.
|
|
return keep(mark("gap", f"link to {target}")) + " " + emphasis(label)
|
|
if target.endswith(".md") and "/" not in target:
|
|
# Another Markdown file. "@link X.md" would be read as an
|
|
# anchor (href="#X.md"). Converted alongside this one, its html
|
|
# page is X/index.html beside ours; otherwise it is a gap.
|
|
stem = target[:-3]
|
|
if stem not in SIBLINGS:
|
|
return keep(mark("gap", f"link to {target}, not converted")) + " " + emphasis(label)
|
|
target = f"../{stem}/index.html"
|
|
return opening("@link " + quote(target) + " :text ") + emphasis(label) + closing()
|
|
|
|
def emphasis(t):
|
|
"""The marker-consuming transforms, sharing THIS call's stashes.
|
|
|
|
A link label or a struck-through run is itself inline Markdown and
|
|
must be converted -- but not by re-entering inline(), which would
|
|
start empty stashes while the text already holds this call's
|
|
placeholders. A code span inside a link label then restores against
|
|
the wrong list: IndexError, found by converting a second document.
|
|
"""
|
|
t = LINK.sub(link, t)
|
|
# An arrow is the SKS klammer (Andy, 2026-10-01): U+2192, and the
|
|
# halfwidth U+FFEB his documents use.
|
|
t = re.sub(r'[\u2192\uffeb]', lambda m: keep(OPEN + "@rightarrow@" + CLOSE), t)
|
|
t = STRIKE.sub(lambda m: emphasis(m.group(1)) + " " +
|
|
keep(mark("gap", "strikethrough")), t)
|
|
t = BOLD.sub(lambda m: opening("@b ") + (m.group(1) or m.group(2)) + closing(), t)
|
|
return ITAL.sub(lambda m: opening("@i ") + (m.group(1) or m.group(2)) + closing(), t)
|
|
|
|
text = emphasis(text)
|
|
|
|
# Everything that is still writer text gets quoted; the stashed fragments
|
|
# are Klammertext already.
|
|
parts = re.split(r'(\x00\d+\x00|\x01\d+\x01)', text)
|
|
text = "".join(p if re.match(r'^[\x00\x01]', p) else quote(p) for p in parts)
|
|
|
|
text = re.sub(r'\x01(\d+)\x01', lambda m: verbatim[int(m.group(1))], text)
|
|
# Inline code goes in RAW, and is closed with the named delimiter. "@c"
|
|
# took an ordinary string parameter until 2026-08-16, so every Klammertext
|
|
# special inside a code span had to be quoted -- and about 40% of the
|
|
# inline code in the measured corpus contains one (see
|
|
# doc/markdown_to_klammertext.md). It is now a LITERAL parameter, like
|
|
# "@code": nothing inside is interpreted, so nothing needs quoting, and the
|
|
# source reads as the author wrote it. A literal klammer cannot be closed
|
|
# with a bare "@" -- the machine cannot find the end by counting -- so the
|
|
# close is "c@".
|
|
text = re.sub(r'\x00(\d+)\x00',
|
|
lambda m: OPEN + "@c " + spans[int(m.group(1))] + " c@" + CLOSE, text)
|
|
# Klammertext delimiters need whitespace around them, but Markdown
|
|
# emphasis abuts its neighbours: "un**bold**ed" and "*name*_rectified"
|
|
# both put a word character hard against a delimiter, which the
|
|
# katomizer then reads as an unparsable word. A space makes it parse,
|
|
# and "#-" removes that space again from the OUTPUT, so intra-word
|
|
# emphasis converts exactly rather than approximately.
|
|
# Any non-space neighbour, not just a word character: "*name*_x" and
|
|
# "_1.*ext*" put an underscore or a period against the delimiter.
|
|
text = re.sub(r'(?<=\S)' + OPEN, ' #-', text).replace(OPEN, "")
|
|
text = re.sub(CLOSE + r'(?=[^\s])', '#- ', text).replace(CLOSE, "")
|
|
# A closing "@" abutting the next klammer's opening "@" spells "@@",
|
|
# which is a DEFINITION delimiter -- the engine then reports a span that
|
|
# ends without a beginning, pointing at text the writer never wrote.
|
|
# Adjacent klammers are ordinary in converted prose (*a*_b_ produces
|
|
# two), so separate them.
|
|
return re.sub(r'@(?=@)', '@ ', text)
|
|
|
|
# ---- blocks ---------------------------------------------------------------
|
|
|
|
HEADING = re.compile(r'^(#{1,6})\s+(.*?)\s*$')
|
|
MANUAL_NUMBER = re.compile(r'^\d+(\.\d+)*\.?\s+')
|
|
FENCE = re.compile(r'^\s*```(\w*)\s*$')
|
|
BULLET = re.compile(r'^(\s*)[-*+]\s+(.*)$')
|
|
ORDERED = re.compile(r'^(\s*)(\d+)[.)]\s+(.*)$')
|
|
TABLEROW = re.compile(r'^\s*\|(.*)\|\s*$')
|
|
ALIGNROW = re.compile(r'^\s*\|[\s:|-]+\|\s*$')
|
|
|
|
def cells(line):
|
|
return [c.strip() for c in TABLEROW.match(line).group(1).split("|")]
|
|
|
|
def convert(path):
|
|
src = open(path, encoding="utf-8").read().split("\n")
|
|
out, i, n = [], 0, len(src)
|
|
title = None
|
|
stripped_numbers = 0
|
|
dropped_toc = False
|
|
|
|
while i < n:
|
|
line = src[i]
|
|
|
|
m = FENCE.match(line)
|
|
if m: # fenced code
|
|
lang, body, i = m.group(1), [], i + 1
|
|
while i < n and not FENCE.match(src[i]):
|
|
body.append(src[i]); i += 1
|
|
i += 1
|
|
if lang:
|
|
out.append(mark("lossy", f'fenced language "{lang}" dropped '
|
|
"-- @code has no :language"))
|
|
out.append("@code |")
|
|
out.extend(body) # literal: nothing to quote
|
|
out.append("code@")
|
|
out.append("")
|
|
continue
|
|
|
|
m = HEADING.match(line)
|
|
if m:
|
|
level, text = len(m.group(1)), m.group(2)
|
|
if level == 1 and title is None:
|
|
title = inline(text); i += 1; continue
|
|
if MANUAL_NUMBER.match(text):
|
|
text = MANUAL_NUMBER.sub("", text); stripped_numbers += 1
|
|
if re.match(r'^contents$', text, re.I):
|
|
# Klammertext generates a table of contents; a hand-written
|
|
# one would duplicate it. Skip to the next heading.
|
|
j = i + 1
|
|
while j < n and not HEADING.match(src[j]):
|
|
j += 1
|
|
out.append(mark("judgment",
|
|
"a hand-written Contents section was dropped; "
|
|
":structure article generates one"))
|
|
out.append("")
|
|
dropped_toc = True
|
|
i = j
|
|
continue
|
|
out.append(f"@s{level-1} {inline(text)} @")
|
|
out.append("")
|
|
i += 1
|
|
continue
|
|
|
|
if TABLEROW.match(line): # table
|
|
rows, align = [], None
|
|
while i < n and TABLEROW.match(src[i]):
|
|
if ALIGNROW.match(src[i]):
|
|
align = [("r" if c.endswith(":") and c.startswith(":") is False
|
|
else "c" if c.startswith(":") and c.endswith(":")
|
|
else "l") for c in cells(src[i])]
|
|
else:
|
|
rows.append([inline(c) for c in cells(src[i])])
|
|
i += 1
|
|
body = " ||\n".join(" | ".join(r) for r in rows)
|
|
# A Markdown table has no caption and no number.
|
|
opts = " :number false"
|
|
opts += f" :cell_hpos {' '.join(align)}" if align and set(align) != {"l"} else ""
|
|
# A Markdown table says nothing about column widths, and
|
|
# Klammertext's default "fit" never wraps -- a prose column then
|
|
# runs off the page (the pdf target warns). Give the widest
|
|
# column "fill", which takes the remaining width but no more than
|
|
# its own widest line.
|
|
widths = [max((len(r[c]) for r in rows if c < len(r)), default=0)
|
|
for c in range(max(len(r) for r in rows))]
|
|
if max(widths) > 40:
|
|
widest = widths.index(max(widths))
|
|
spec = " ".join("fill" if c == widest else "fit"
|
|
for c in range(len(widths)))
|
|
opts += f" :column_width {spec}"
|
|
out.append(mark("judgment",
|
|
"column widths are not in the Markdown; the "
|
|
f"widest column ({max(widths)} characters) was "
|
|
"given fill so the table wraps instead of "
|
|
"overflowing"))
|
|
out.append(f"@table{opts} |")
|
|
out.append(body)
|
|
out.append("table@")
|
|
out.append("")
|
|
continue
|
|
|
|
if BULLET.match(line) or ORDERED.match(line):
|
|
block, base = [], None
|
|
while i < n and (BULLET.match(src[i]) or ORDERED.match(src[i]) or
|
|
(src[i].strip() and src[i].startswith((" ", "\t")))):
|
|
block.append(src[i]); i += 1
|
|
out.append(convert_list(block))
|
|
out.append("")
|
|
continue
|
|
|
|
if not line.strip():
|
|
out.append(""); i += 1
|
|
continue
|
|
|
|
# A PARAGRAPH goes through inline() whole, its line breaks kept.
|
|
# Converting line by line left a literal "*" on each line of an
|
|
# emphasis that wrapped, and the leftovers then paired with EACH
|
|
# OTHER, so the wrong words came out italic -- 33 paragraphs of the
|
|
# vocabulary lists, visible only in a diff.
|
|
para = [line]; i += 1
|
|
while i < n and src[i].strip() and not (HEADING.match(src[i]) or
|
|
TABLEROW.match(src[i]) or FENCE.match(src[i]) or
|
|
BULLET.match(src[i])):
|
|
if ORDERED.match(src[i]):
|
|
# A numbered line hard against the line above is a wrapped
|
|
# line that happens to begin with a number ("1. and 3.
|
|
# Sg."). CommonMark reads it as a list; the writer did not
|
|
# mean one. Kept in the paragraph, and said.
|
|
out.append(mark("judgment", "a line beginning with a number "
|
|
"was kept in its paragraph, not read as a list"))
|
|
para.append(src[i]); i += 1
|
|
out.append(inline("\n".join(para)))
|
|
|
|
return out, title, stripped_numbers, dropped_toc
|
|
|
|
def convert_list(block):
|
|
"""One list, possibly nested, as @ul/@ol with bar-separated items."""
|
|
def emit(items, ordered, depth):
|
|
klammer = "@ol" if ordered else "@ul"
|
|
parts = []
|
|
for text, children in items:
|
|
body = inline(text)
|
|
if children:
|
|
body += " " + emit(children[0], children[1], depth + 1)
|
|
parts.append(body)
|
|
return klammer + " " + " | ".join(parts) + " @"
|
|
|
|
def parse(lines, indent):
|
|
items, i = [], 0
|
|
ordered = bool(ORDERED.match(lines[0])) if lines else False
|
|
while i < len(lines):
|
|
m = BULLET.match(lines[i]) or ORDERED.match(lines[i])
|
|
if not m or len(m.group(1)) < indent:
|
|
break
|
|
text = m.groups()[-1]
|
|
i += 1
|
|
child_lines = []
|
|
while i < len(lines):
|
|
m2 = BULLET.match(lines[i]) or ORDERED.match(lines[i])
|
|
if m2 and len(m2.group(1)) > indent:
|
|
child_lines.append(lines[i]); i += 1
|
|
elif not m2 and lines[i].strip():
|
|
text += " " + lines[i].strip(); i += 1
|
|
else:
|
|
break
|
|
children = None
|
|
if child_lines:
|
|
sub, sub_ordered = parse(child_lines, len(BULLET.match(child_lines[0]).group(1))
|
|
if BULLET.match(child_lines[0])
|
|
else len(ORDERED.match(child_lines[0]).group(1)))
|
|
children = (sub, sub_ordered)
|
|
items.append((text, children))
|
|
return items, ordered
|
|
|
|
items, ordered = parse([l for l in block if l.strip()], 0)
|
|
return emit(items, ordered, 0)
|
|
|
|
def main():
|
|
src_path, dst_path = sys.argv[1], sys.argv[2]
|
|
SIBLINGS.update(os.path.splitext(os.path.basename(a))[0] for a in sys.argv[3:])
|
|
body, title, stripped, dropped_toc = convert(src_path)
|
|
today = datetime.date.today().isoformat()
|
|
head = [f"#[MD source: {os.path.abspath(src_path)}",
|
|
f" converted {today} ]#", ""]
|
|
if stripped:
|
|
head.append(mark("judgment",
|
|
f"{stripped} headings carried a manual number "
|
|
'("2.1 ..."); the numbers were removed because @s1/@s2 '
|
|
"number the headings themselves"))
|
|
head.append("")
|
|
head += ["@document", ":structure plain"]
|
|
if title:
|
|
head.append(f":title {title}")
|
|
head += [":text", ""]
|
|
text = "\n".join(head + body + ["", "@", ""])
|
|
text = re.sub(r'\n{3,}', "\n\n", text)
|
|
open(dst_path, "w", encoding="utf-8").write(text)
|
|
print(f"{dst_path}: {len(text.splitlines())} lines")
|
|
for kind in ("source", "lossy", "gap", "judgment"):
|
|
print(f" {kind:9} {marks[kind]}")
|
|
|
|
main()
|