klammertext/sks/tns/md_to_sks.py

#!/usr/bin/env python3
"""Markdown -> Standard Klammer Set converter (first implementation).

SPECIFICATION: doc/markdown_to_klammertext.md.  That document is authoritative
and is written to be sufficient to rebuild this program; this program is its
reference implementation, not the other way round.  Its "The algorithm"
section states the pass order, which is forced, and the traps that make it so.

The conversion is to the SKS, not to Klammertext the language: the
correspondence between a Markdown construct and a klammer is a property of
the klammer set.  That is why this lives under sks/ rather than in mac/.

What cannot be converted exactly is recorded in the output as a
#[MD <kind>: ... ]# marker -- lossy, gap, or judgment -- so a converted
document carries its own worklist and "how far from done is this file" is a
grep.  A file with no markers is inside the convertible subset.

First run: the Rectify user guide (1,805 lines), 2026-08-07.  It rendered to
html and pdf without error or warning, with 99 markers -- 91 of them internal
anchor links, the one gap that appears at volume.

Usage: python3 md_to_sks.py <input.md> <output.kt>
"""
import re, sys, os, datetime
from collections import Counter

SPECIALS = "^@#|*"          # ^ first: the other quotings introduce ^
marks = Counter()

def mark(kind, text):
    """A #[MD ... ]# marker.

    Two things have to be neutralised in the text, both of which cost a
    session's debugging if they are not:

    * "]#" would close the block early and "#[" would open one that must then
      be balanced (the nesting hazard);
    * a klammer name written with its "@" is READ before the block is
      removed -- mark_literal_klammer_content() runs first in
      process_katoms() -- so "@code has no :language" makes the engine demand
      a "code@" close from inside removed text.  Quoting the @ makes it a
      special katom, which the literal scan does not see.  (TODO #39.)
    """
    marks[kind] += 1
    text = text.replace("]#", "] #").replace("#[", "# [")
    text = text.replace("@", "^@").replace("|", "^|")
    return f"#[MD {kind}: {text} ]#"

def quote(s):
    """Quote every Klammertext special in writer text."""
    for ch in SPECIALS:
        s = s.replace(ch, "^" + ch)
    return s

# ---- inline ---------------------------------------------------------------

CODE = re.compile(r'`([^`\n]+)`')
LINK = re.compile(r'(?<!\!)\[([^\]]*)\]\(([^)]+)\)')
BOLD = re.compile(r'\*\*([^*]+)\*\*|__([^_]+)__')
# Markdown's underscore emphasis does not open or close INSIDE a word, and
# technical prose is full of identifiers like *name*_rectified_1.*ext* where
# the underscores are part of the name.  So the closing _ must not be
# followed by a word character -- without that, the converter invents
# emphasis in the middle of filenames.
ITAL = re.compile(r'(?<![\*\w])\*([^*\s][^*]*)\*(?!\*)|(?<![_\w])_([^_\s][^_]*)_(?![\w_])')
STRIKE = re.compile(r'~~([^~]+)~~')

def inline(text):
    """Convert one run of inline Markdown.

    Order matters: code spans are protected first (their content must not be
    read as emphasis), links and emphasis consume their markers next, and only
    then are the remaining specials quoted -- quoting first would hide the
    ** and * that emphasis is recognised by.
    """
    spans = []
    def stash(m):
        spans.append(m.group(1))
        return f"\x00{len(spans)-1}\x00"
    text = CODE.sub(stash, text)

    # Markers are Klammertext, not writer text: they must survive the quoting
    # pass below untouched, or the "#[" that makes them a removal block is
    # itself quoted and the marker renders into the output instead of
    # vanishing from it.  Stash them like code spans.
    verbatim = []
    def keep(s):
        verbatim.append(s)
        return f"\x01{len(verbatim)-1}\x01"

    # OPEN and CLOSE wrap the delimiters this function emits, so the
    # whitespace fix below can find exactly its own delimiters.  Matching
    # them by regex on the finished text does not work: " @" is both the
    # closing delimiter and the space before an opening "@i", and a rule
    # keyed on it rewrites "@i" into "@#- i".
    OPEN, CLOSE = "\x03", "\x02"
    def opening(s): return keep(OPEN + s)
    def closing():  return keep(" @" + CLOSE)

    def link(m):
        label, target = m.group(1), m.group(2)
        if target.startswith("#"):
            # An anchor into this document: no klammer gives an arbitrary
            # heading an anchor, so keep the words and record the loss.
            return emphasis(label) + " " + keep(mark("gap", f"link to {target}"))
        return opening("@link " + quote(target) + " :text ") + emphasis(label) + closing()

    def emphasis(t):
        """The marker-consuming transforms, sharing THIS call's stashes.

        A link label or a struck-through run is itself inline Markdown and
        must be converted -- but not by re-entering inline(), which would
        start empty stashes while the text already holds this call's
        placeholders.  A code span inside a link label then restores against
        the wrong list: IndexError, found by converting a second document.
        """
        t = LINK.sub(link, t)
        t = STRIKE.sub(lambda m: emphasis(m.group(1)) + " " +
                       keep(mark("gap", "strikethrough")), t)
        t = BOLD.sub(lambda m: opening("@b ") + (m.group(1) or m.group(2)) + closing(), t)
        return ITAL.sub(lambda m: opening("@i ") + (m.group(1) or m.group(2)) + closing(), t)

    text = emphasis(text)

    # Everything that is still writer text gets quoted; the stashed fragments
    # are Klammertext already.
    parts = re.split(r'(\x00\d+\x00|\x01\d+\x01)', text)
    text = "".join(p if re.match(r'^[\x00\x01]', p) else quote(p) for p in parts)

    text = re.sub(r'\x01(\d+)\x01', lambda m: verbatim[int(m.group(1))], text)
    # Inline code goes in RAW, and is closed with the named delimiter.  "@c"
    # took an ordinary string parameter until 2026-08-16, so every Klammertext
    # special inside a code span had to be quoted -- and about 40% of the
    # inline code in the measured corpus contains one (see
    # doc/markdown_to_klammertext.md).  It is now a LITERAL parameter, like
    # "@code": nothing inside is interpreted, so nothing needs quoting, and the
    # source reads as the author wrote it.  A literal klammer cannot be closed
    # with a bare "@" -- the machine cannot find the end by counting -- so the
    # close is "c@".
    text = re.sub(r'\x00(\d+)\x00',
                  lambda m: OPEN + "@c " + spans[int(m.group(1))] + " c@" + CLOSE, text)
    # Klammertext delimiters need whitespace around them, but Markdown
    # emphasis abuts its neighbours: "un**bold**ed" and "*name*_rectified"
    # both put a word character hard against a delimiter, which the
    # katomizer then reads as an unparsable word.  A space makes it parse,
    # and "#-" removes that space again from the OUTPUT, so intra-word
    # emphasis converts exactly rather than approximately.
    # Any non-space neighbour, not just a word character: "*name*_x" and
    # "_1.*ext*" put an underscore or a period against the delimiter.
    text = re.sub(r'(?<=\S)' + OPEN, ' #-', text).replace(OPEN, "")
    text = re.sub(CLOSE + r'(?=[^\s])', '#- ', text).replace(CLOSE, "")
    # A closing "@" abutting the next klammer's opening "@" spells "@@",
    # which is a DEFINITION delimiter -- the engine then reports a span that
    # ends without a beginning, pointing at text the writer never wrote.
    # Adjacent klammers are ordinary in converted prose (*a*_b_ produces
    # two), so separate them.
    return re.sub(r'@(?=@)', '@ ', text)

# ---- blocks ---------------------------------------------------------------

HEADING = re.compile(r'^(#{1,6})\s+(.*?)\s*$')
MANUAL_NUMBER = re.compile(r'^\d+(\.\d+)*\.?\s+')
FENCE = re.compile(r'^\s*```(\w*)\s*$')
BULLET = re.compile(r'^(\s*)[-*+]\s+(.*)$')
ORDERED = re.compile(r'^(\s*)(\d+)[.)]\s+(.*)$')
TABLEROW = re.compile(r'^\s*\|(.*)\|\s*$')
ALIGNROW = re.compile(r'^\s*\|[\s:|-]+\|\s*$')

def cells(line):
    return [c.strip() for c in TABLEROW.match(line).group(1).split("|")]

def convert(path):
    src = open(path, encoding="utf-8").read().split("\n")
    out, i, n = [], 0, len(src)
    title = None
    stripped_numbers = 0
    dropped_toc = False

    while i < n:
        line = src[i]

        m = FENCE.match(line)
        if m:                                   # fenced code
            lang, body, i = m.group(1), [], i + 1
            while i < n and not FENCE.match(src[i]):
                body.append(src[i]); i += 1
            i += 1
            if lang:
                out.append(mark("lossy", f'fenced language "{lang}" dropped '
                                         "-- @code has no :language"))
            out.append("@code |")
            out.extend(body)                    # literal: nothing to quote
            out.append("code@")
            out.append("")
            continue

        m = HEADING.match(line)
        if m:
            level, text = len(m.group(1)), m.group(2)
            if level == 1 and title is None:
                title = inline(text); i += 1; continue
            if MANUAL_NUMBER.match(text):
                text = MANUAL_NUMBER.sub("", text); stripped_numbers += 1
            if re.match(r'^contents$', text, re.I):
                # Klammertext generates a table of contents; a hand-written
                # one would duplicate it.  Skip to the next heading.
                j = i + 1
                while j < n and not HEADING.match(src[j]):
                    j += 1
                out.append(mark("judgment",
                                "a hand-written Contents section was dropped; "
                                ":structure article generates one"))
                out.append("")
                dropped_toc = True
                i = j
                continue
            out.append(f"@s{level-1} {inline(text)} @")
            out.append("")
            i += 1
            continue

        if TABLEROW.match(line):                # table
            rows, align = [], None
            while i < n and TABLEROW.match(src[i]):
                if ALIGNROW.match(src[i]):
                    align = [("r" if c.endswith(":") and c.startswith(":") is False
                              else "c" if c.startswith(":") and c.endswith(":")
                              else "l") for c in cells(src[i])]
                else:
                    rows.append([inline(c) for c in cells(src[i])])
                i += 1
            body = " ||\n".join(" | ".join(r) for r in rows)
            opts = f" :cell_hpos {' '.join(align)}" if align and set(align) != {"l"} else ""
            # A Markdown table says nothing about column widths, and
            # Klammertext's default "fit" never wraps -- a prose column then
            # runs off the page (the pdf target warns).  Give the widest
            # column "fill", which takes the remaining width but no more than
            # its own widest line.
            widths = [max((len(r[c]) for r in rows if c < len(r)), default=0)
                      for c in range(max(len(r) for r in rows))]
            if max(widths) > 40:
                widest = widths.index(max(widths))
                spec = " ".join("fill" if c == widest else "fit"
                                for c in range(len(widths)))
                opts += f" :column_width {spec}"
                out.append(mark("judgment",
                                "column widths are not in the Markdown; the "
                                f"widest column ({max(widths)} characters) was "
                                "given fill so the table wraps instead of "
                                "overflowing"))
            out.append(f"@table{opts} |")
            out.append(body)
            out.append("table@")
            out.append("")
            continue

        if BULLET.match(line) or ORDERED.match(line):
            block, base = [], None
            while i < n and (BULLET.match(src[i]) or ORDERED.match(src[i]) or
                             (src[i].strip() and src[i].startswith((" ", "\t")))):
                block.append(src[i]); i += 1
            out.append(convert_list(block))
            out.append("")
            continue

        out.append(inline(line) if line.strip() else "")
        i += 1

    return out, title, stripped_numbers, dropped_toc

def convert_list(block):
    """One list, possibly nested, as @ul/@ol with bar-separated items."""
    def emit(items, ordered, depth):
        klammer = "@ol" if ordered else "@ul"
        parts = []
        for text, children in items:
            body = inline(text)
            if children:
                body += " " + emit(children[0], children[1], depth + 1)
            parts.append(body)
        return klammer + " " + " | ".join(parts) + " @"

    def parse(lines, indent):
        items, i = [], 0
        ordered = bool(ORDERED.match(lines[0])) if lines else False
        while i < len(lines):
            m = BULLET.match(lines[i]) or ORDERED.match(lines[i])
            if not m or len(m.group(1)) < indent:
                break
            text = m.groups()[-1]
            i += 1
            child_lines = []
            while i < len(lines):
                m2 = BULLET.match(lines[i]) or ORDERED.match(lines[i])
                if m2 and len(m2.group(1)) > indent:
                    child_lines.append(lines[i]); i += 1
                elif not m2 and lines[i].strip():
                    text += " " + lines[i].strip(); i += 1
                else:
                    break
            children = None
            if child_lines:
                sub, sub_ordered = parse(child_lines, len(BULLET.match(child_lines[0]).group(1))
                                         if BULLET.match(child_lines[0])
                                         else len(ORDERED.match(child_lines[0]).group(1)))
                children = (sub, sub_ordered)
            items.append((text, children))
        return items, ordered

    items, ordered = parse([l for l in block if l.strip()], 0)
    return emit(items, ordered, 0)

def main():
    src_path, dst_path = sys.argv[1], sys.argv[2]
    body, title, stripped, dropped_toc = convert(src_path)
    today = datetime.date.today().isoformat()
    head = [f"#[MD source: {os.path.abspath(src_path)}",
            f"   converted {today} ]#", ""]
    if stripped:
        head.append(mark("judgment",
                         f"{stripped} headings carried a manual number "
                         '("2.1 ..."); the numbers were removed because @s1/@s2 '
                         "number the headings themselves"))
        head.append("")
    head += ["@document", ":structure plain"]
    if title:
        head.append(f":title {title}")
    head += [":text", ""]
    text = "\n".join(head + body + ["", "@", ""])
    text = re.sub(r'\n{3,}', "\n\n", text)
    open(dst_path, "w", encoding="utf-8").write(text)
    print(f"{dst_path}: {len(text.splitlines())} lines")
    for kind in ("source", "lossy", "gap", "judgment"):
        print(f"   {kind:9} {marks[kind]}")

main()