Verbatim-safe typography, ^-punctuation quoting, full-range ^UUUU^, polyglot html

Typographic transforms (---, quote pairs, ~) no longer touch verbatim
text: @c/@code/@source_listing content and ^'...'^ spans show exactly the
characters written. "^" before any punctuation character quotes it in
every target (the apostrophe excepted: ^' opens a literal span), with the
new :resolve option on @@@target declaring per-target renderings. The
^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range.
The html output and transform spellings are polyglot (XML-valid), in
preparation for an EPUB target. New suites: transform_test, character_test
(engine), typography_test (SKS).

(from dev 07ce5ea86a0a)

sks/code/code_block.py

@@ -1,30 +1,20 @@
-if __name__ == "__main__":
-    import sys
-    sys.path.append("../kutil")
-    sys.path.append("../target")
+# if __name__ == "__main__":
+#     import sys
+#     sys.path.append("../kutil")
+#     sys.path.append("../target")
 
-import sys
 import re
 import klammer_base
-import kutil
 import html_util
 from html_util import E
 import latex_util as L
-import pprint
-import phases
+
 
 def escape_newlines(s):
     # A newline becomes the marker the html paragraph pass turns back into a
     # line break, so a verbatim source file keeps its lines.  Used by Source.
     return re.sub("\n", " ___NL___ ", s)
 
-def undash(s):
-    # Verbatim text must show the hyphens the writer typed: the target's
-    # "--"/"---" transforms have already run, so put them back.  Used by
-    # Code_fragment.
-    result = re.sub("__MDASH__", "---", s)
-    return re.sub("__NDASH__", "--", result)
-
 def is_comment(s):
     return s.strip().startswith("//")
 
@@ -168,9 +158,17 @@ def html_line(text):
     tex_line -- a raw # in "#include" would start a text removal).  The
     html entities introduce no Klammertext special, so the two passes
     cannot interfere.  No target :escape entries apply here, so each
-    quoted special decodes back to its own character."""
-    return quote_specials(html_escape_rgx.sub(
-        lambda m: html_escapes[m.group()], text))
+    quoted special decodes back to its own character.
+
+    Last, the typographically active characters are hidden as KTESC
+    markers (klammer_base.hide_typographic): the html target's transforms
+    run over the final result, and without this a "--check" in a listing
+    became an en-dash.  The markers decode after the transform pass.
+    tex_line needs no such step -- the tex target declares no typographic
+    transforms (LaTeX applies its input conventions itself), and its ~ is
+    this code's own markup for a preserved space."""
+    return klammer_base.hide_typographic(quote_specials(html_escape_rgx.sub(
+        lambda m: html_escapes[m.group()], text)))
 
 def html_block(code, comment):
     r"""One block: its lines beside its comment.
@@ -192,11 +190,25 @@ def html_block(code, comment):
     return f'<div class="code_block">{result}</div>\n'
 
 
+def expand_whitespace_markers(text, K=None):
+    def count(count_match):
+        count = int(count_match) if count_match else 1
+        return count
+    def replace_spaces(match):
+        return " " * count(match.group(1))
+    def replace_newlines(match):
+        return "\n" * count(match.group(1))
+    result = text
+    result = re.compile(r" *#- *", re.S).sub("", result)
+    result = re.compile(r" *#\+(\d*) *", re.S).sub(replace_spaces, result)
+    result = re.compile(r"\s*\#\/(\d*)\s*", re.S).sub(replace_newlines, result)
+    return result
+
 class Code(klammer_base.Klammer_base):
     id = 0
     def __init__(self, K):
         super().__init__(K)
-        self.text = phases.expand_whitespace_markers(self.text)
+        self.text = expand_whitespace_markers(self.text)
 
     def annotated(self, pairs):
         """Does any block of this listing carry a comment?"""
@@ -332,7 +344,7 @@ class Code_fragment(klammer_base.Klammer_base):
     # ordinary string and the machine did the escaping; html() got away with
     # handling "<" by hand and tex() with nothing at all.
     def html(self):
-        return f'<span class="code">{html_line(undash(self.code_text.strip()))}</span>'
+        return f'<span class="code">{html_line(self.code_text.strip())}</span>'
 
     def tex(self):
         return f"{{\\tt {tex_line(self.code_text.strip())}}}"
@@ -373,7 +385,7 @@ def extract_marked_region(src, marker, filename):
 
 
 class Source(Code):
-    """@source_listing -- a Code listing whose text comes from a FILE.
+    r"""@source_listing -- a Code listing whose text comes from a FILE.
 
     It IS a Code: @source_listing and @code differ only in where the text
     comes from, so they must render identically, and subclassing is what
@@ -406,7 +418,7 @@ class Source(Code):
         except OSError as e:
             raise Exception(
                 f'Cannot read the source listing "{self.filename}": {e.strerror}.\n'
-                f'  A relative name resolves against the DOCUMENT\'s directory.')
+                f'  A relative name resolves against the document\'s directory.')
         if self.marker:
             text = extract_marked_region(text, self.marker, self.filename)
         self.text = text