Verbatim-safe typography, ^-punctuation quoting, full-range ^UUUU^, polyglot html

Typographic transforms (---, quote pairs, ~) no longer touch verbatim
text: @c/@code/@source_listing content and ^'...'^ spans show exactly the
characters written. "^" before any punctuation character quotes it in
every target (the apostrophe excepted: ^' opens a literal span), with the
new :resolve option on @@@target declaring per-target renderings. The
^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range.
The html output and transform spellings are polyglot (XML-valid), in
preparation for an EPUB target. New suites: transform_test, character_test
(engine), typography_test (SKS).

(from dev 07ce5ea86a0a)
This commit is contained in:
2026-08-23 20:48:26 +02:00
parent d982c0d6cc
commit 37b6ba1c4f
77 changed files with 1665 additions and 1608 deletions

View File

@@ -15,11 +15,6 @@
vertical-align: middle;
}
.center {
display: flex;
justify-content: center;
}
/* -------------------------------------------------------------------------------- */
.bgap {
@@ -179,3 +174,20 @@
margin: 0 1em 0 1em;
}
*/
.footnote_in_text {
vertical-align: baseline;
position: relative;
top: -0.4em;
font-size: 70%;
line-height: 0;
padding-left: 1px;
font-weight: bold;
}
.footnote_rule {
width: 40%;
margin: 3lh 0 .5lh 0;
border: none;
border-top: 1px solid black;
}

View File

@@ -8,7 +8,7 @@ import kutil
_novalue = '__no_value__'
class E:
void_elements = set("area base br col embed hr img input link meta param source track wbr".split())
void_elements = "area base br col embed hr img input link meta source track wbr".split()
def __init__(self, tag):
self._tag = tag
self._sty = []
@@ -17,6 +17,7 @@ class E:
self._attr = []
self.no_value = '__novalue__'
def sty(self, name, value=_novalue):
if name and value is _novalue:
self._sty.append(name.strip(';'))
@@ -49,6 +50,7 @@ class E:
self._attr.append(attr)
return self
def __str__(self):
def label(name, s):
return ' {}="{}"'.format(name, s) if s else ''
@@ -60,7 +62,16 @@ class E:
b = "\n".join(self._body)
tag = '{}{}{}{}'.format(self._tag, a, c, s).strip()
end_tag = '</{}>'.format(self._tag) if self._tag not in E.void_elements else ""
return '<{}>{}{}\n'.format(tag, b, end_tag)
no_body = self._tag in E.void_elements
if b and no_body:
raise Exception(f"Tag {self._tag} incorrectly defines a body.")
if no_body:
result = f'<{tag}/>'
else:
result = f'<{tag}>{b}{end_tag}\n' # .format(tag, b, end_tag)
if no_body:
print(result)
return result
def __repr__(self):
return self.__str__()
@@ -80,6 +91,9 @@ class E:
result = result.rstrip() + "\n"
return result
def color(rgb):
return "rgb(" + ",".join([f"{float(e)*100}%" for e in rgb.split(",")]) + ")"
def html_indent(filename):
command = "(progn (setq make-backup-files nil) (mark-whole-buffer) "
command += "(indent-region (point-min) (point-max) nil) (save-buffer))"
@@ -99,10 +113,10 @@ def page(head_elt, body, js_files=[], load_jquery=True):
def head(title, js_files=[], js_code="", css_files=[], css_code="",
include_fonts=True, google_font=[], favicon=None, load_jquery=True):
result = '<meta name="viewport" content="width=device-width, initial-scale=1">\n'
result += '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">\n'
result = '<meta name="viewport" content="width=device-width, initial-scale=1"/>\n'
result += '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>\n'
if favicon:
result += f'<link rel="icon" type="image/x-icon" href="{favicon}">\n'
result += f'<link rel="icon" type="image/x-icon" href="{favicon}"/>\n'
result += "".join([E("link").attr("href", e).attr("rel", "stylesheet").str() for e in css_files])
fonts = None
css = ''

View File

@@ -35,7 +35,7 @@ def block(material):
required to know that a target distinguishes horizontal from
vertical mode. Asserted by sks/tst/paragraph_test.sh.
"""
return "\\par\n" + material.strip("\n") + "\n\\par\n"
return "^'\\par'^\n" + material.strip("\n") + "\n^'\\par'^\n"
def environment(name, body, required=None, optional=None):
req = f"{{{required}}}" if required else ""
@@ -164,3 +164,35 @@ def add_caption(element, caption_label, number, caption_text, latex_width,
element = minipage(caption + "\\rule[-0.75\\baselineskip]{0pt}{0pt}\n" + element,
latex_width, vertical="t")
return caption_wrapper(element, hpos, offset=offset)
def define_color(name, rgb_or_hex):
if "," in rgb_or_hex:
r, g, b = [float(e) for e in rgb_or_hex.split(",")]
else:
def c(h):
return float(eval('0x' + h)) / 255.0
r = c(rgb_or_hex[:2])
g = c(rgb_or_hex[2:4])
b = c(rgb_or_hex[4:])
return r'\\definecolor{{{}}}{{rgb}}{{{:.3f},{:.3f},{:.3f}}}'.format(name, r, g, b)
COLOR_BOX_TEMPLATE = r'''
^'\hfill\begingroup
COLOR
\setlength{\fboxsep}{8pt}
\fcolorbox{bordercolor}{localcolor}{
\parbox{WIDTH\linewidth}{\raggedright\setlength{\parskip}{8pt}'^
TEXT
^'}}\endgroup'^
'''.strip()
def color_box(text, width=1, box_color="1,1,1", border_color="0,0,0"):
color_definitions = define_color("localcolor", box_color) + "\n" \
+ define_color("bordercolor", border_color)
result = COLOR_BOX_TEMPLATE
for old, new in [['TEXT', re.sub(r'\\', r'\\\\', text)],
['COLOR', color_definitions],
['WIDTH', str(width)]]:
result = re.sub(old, new, result)
result = block(result)
return result

View File

@@ -1,331 +1,93 @@
import sys, os, re, textwrap, pprint
basedir = f'{os.environ["KLAMMERTEXT_HOME"]}/sks'
sys.path = [f"{basedir}/kutil"] + sys.path
sys.path = [f"{basedir}/target"] + sys.path
import kutil
import html_util
def tex_to_pdf(text):
print("in tex_to_pdf")
def expand_whitespace_markers(text, K=None):
def count(count_match):
count = int(count_match) if count_match else 1
return count
def replace_spaces(match):
return " " * count(match.group(1))
def replace_newlines(match):
return "\n" * count(match.group(1))
result = text
# result = re.sub(r"\s*#-\s*", "", result)
# space_pat = re.compile(r" *#\+(\d*) *", re.S)
# print("hits:", space_pat.findall(text))
# for p in space_pat.findall(text):
# print("FOUND:", p)
#
#result = re.sub(" ", "SPACE", result)
#result = re.sub("#/", "\n", result)
#result = re.sub("X", " ", result)
result = re.compile(r" *#- *", re.S).sub("", result)
result = re.compile(r" *#\+(\d*) *", re.S).sub(replace_spaces, result)
result = re.compile(r"\s*\#\/(\d*)\s*", re.S).sub(replace_newlines, result)
#print(result)
#sys.exit(0)
#print("expand_whitespace_markers", text, result)
return result
def handle_dashes(html_text, K=None):
def replace(match):
tag_start, text, tag_end = match.groups()
text = re.sub("__MDASH__", "---", text)
text = re.sub("__NDASH__", "--", text)
result = f"{tag_start}{text}{tag_end}"
return result
result = html_text
result = re.compile('(<span class="monospace">)(.*?)(</span>)', re.S).sub(replace, result)
result = re.sub("__MDASH__", "&#x2014;", result);
result = re.sub("__NDASH__", "&#x2013;", result);
return result
def add_tex_caption_numbers(tex_text, K=None):
chapter_pat = re.compile(r"\\section\\{")
index = {}
result = ""
chapter_number = 1
for line in tex_text.split("\n"):
if chapter_pat.search(line):
chapter_number += 1
reset_indices(index)
if kutil.caption_delimiter() in line:
before, caption_type, caption, after = line.split(kutil.caption_delimiter())
if index.get(caption_type) is None:
index[caption_type] = 1
n = index[caption_type]
number = f"{chapter_number}.{n}" if chapter_number != 0 else n
line = f"{before}{caption_type} {number} {caption}{after}"
index[caption_type] += 1
result += line + "\n"
return result
def remove_redundant_vspace(tex_text, K=None):
#print("remove_redundant_vspace")
rgx = re.compile(r"(\\vspace\*\{-[^}]+\})\s*\\vspace\*\{-[^}]+\}", re.S)
return rgx.sub(r"\1", tex_text)
def restore_backslash(tex_text, K=None):
#print("restore backslash")
#return re.compile(r"\^/").sub(r"\\", tex_text)
return re.sub("\6", "", tex_text)
#--------------------------------------------------------------------------------
def levels_to_section(levels):
result = ".".join([str(e) for e in levels])
result = re.sub(r"\.0", "", result)
return result
def chapter_title_span():
return '<span class="chapter_title">'
def add_html_section_numbers(html_text, K=None, start=0):
depth = 6
levels = [start-1] + ([0] * (depth-1))
section_pat = re.compile(r"(.*?)<h(\d)(.*?)>(.*?)</h\2>")
id_pat = re.compile(r'.*?id="([-\w]+)".*', re.S)
result = ""
id_number = 1
id_map = []
for line in html_text.split('\n'):
match = section_pat.match(line)
if match:
pre, level, attr, text = match.groups()
id_match = id_pat.fullmatch(attr)
if id_match:
id = id_match.group(1)
else:
id = f"id{id_number}"
attr = " " + f'id="{id}" {attr}'.strip()
id_number += 1
level = int(level)
levels[level-1] = levels[level-1] + 1
for i in range(level, depth):
levels[i] = 0
section = levels_to_section(levels)
line = f'{pre}<h{level}{attr}>{chapter_title_span()}{section}&#160;&#160;</span>{text}</h{level}>'
id_map.append([level, section, id, text])
result += line + "\n"
[print(e) for e in id_map]
return result, id_map
def make_html_table_of_contents(titles):
result = '<div id="_toc" class="toc-title">Table of contents</div>\n'
for level, section, id, text in titles:
result += f'<div class="level{level}"><a href="#{id}" class="level">{section} {text}</a></div>\n'
return result
def process_html_sections(html_text, K=None, start=1):
html, id_map = add_html_section_numbers(html_text)
toc = make_html_table_of_contents(id_map)
result = re.sub(r'(<div id="middle">)', rf"\1\n{toc}", html)
return result
def reset_indices(indices):
for key in indices.keys():
indices[key] = 1
def add_html_caption_numbers(html_text, K=None):
chapter_pat = re.compile(rf"{chapter_title_span()}(\d+)</span>")
index = {}
result = ""
chapter_number = ""
for line in html_text.split("\n"):
if '<span' in line:
match = chapter_pat.search(line)
print(match.groups())
if match and match.group(1) != "0":
chapter_number = f"{match.group(1)}."
reset_indices(index)
if kutil.caption_delimiter() in line:
before, caption_type, caption, after = line.split(kutil.caption_delimiter())
if index.get(caption_type) is None:
index[caption_type] = 1
n = index[caption_type]
line = f"{before}{caption_type} {chapter_number}{n} {caption}{after}"
index[caption_type] += 1
result += line + "\n"
return result
def escape_pre_angle_brackets(code, K=None):
def replace(match):
code = re.sub(r"<", "&#x003C;", match.group(1))
return f"<pre>{code}</pre>"
pat = re.compile("<pre>(.*?)</pre>", re.S)
return pat.sub(replace, code)
def block_elements():
return """
address article aside blockquote details dialog dd div dl dt fieldset
figcaption figure footer form h0 h1 h2 h3 h4 h5 h6 header hgroup hr li
main nav ol p section table ul td tr pre
""".strip().split()
def not_a_block(par):
if par.strip()[0] != "<":
return True
pat = re.compile(r'^</?([^\s>]+).*?>', re.S|re.M)
match = pat.match(par.strip())
return match.group(1) not in block_elements()
def make_paragraphs(html_text, K):
def replace(match):
before, content, after = match.groups()
result = ""
for par in [e.strip() for e in
re.compile(r"\n *(\n *)+", re.S).split(content)]:
if par.strip() and not_a_block(par):
#jpar = "\n".join(textwrap.wrap(par, width=80))
#result += f"\n<p>\n{jpar}\n</p>\n"
jpar = "\n".join(textwrap.wrap(f"<p>{par}</p>", width=80))
result += f"\n{jpar}\n"
else:
result += f"\n{par}\n"
result = re.compile(r"\s*</(\w+)>\n<\1>").sub(r"\n</\1>\n\n<\1>", result)
return f"{before}{result}{after}"
pat = re.compile(r'(.*?<div id="middle">)(.*?)(</div>\s*<div id="bottom">.*)', re.S)
match = pat.match(html_text)
result = pat.sub(replace, html_text)
result = re.compile(r"</td>\s+<td>", re.S).sub("</td>\n<td>", result)
result = re.compile(r"\n *(\n *)+", re.S).sub("\n\n", result)
return result
def indent_html(filename, K):
#print('indent_html')
html_util.html_indent(filename)
def copy_html_resources(filename, K):
output_dir = os.path.dirname(K._output_filename) or "."
css_dir = f"{output_dir}/{html_util.css_reldir()}"
kutil.make_dir_if_necessary(css_dir, delete_contents=True)
for basename, source in kutil.sks_files_of_type("css"):
command = f"cp {source} {css_dir}/{basename}.css"
os.system(command)
# def latex_escapes(latex_text):
# return latex_text
# result = latex_text
# result = re.sub('#', '\\#', result)
# result = re.sub('\^', '\\^', result)
# result += "LATEX"
# return result
# def restore_backslash(latex_text, K): # ?
# return latex_text
# print(latex_text)
# print("restore_backslash")
# result = latex_text
# result = re.sub('\b', r'\\b', result)
# result = re.sub(r'\\t', r'\\b', result)
# return result
def make_pdf_from_tex(filename, K, twice=True):
#debug_mode = int(K.K_verbose_level) > 1
verbose_level = int(K.K_verbose_level)
pathname = os.path.abspath(filename)
dirname, filename = os.path.split(pathname)
basename, ext = os.path.splitext(filename)
command = f"mv {pathname} {dirname}/{basename}.tex"
log_file = f"{dirname}/{basename}.log"
pdf_file = f"{dirname}/{basename}.pdf"
result = os.system(command)
#debug_mode = True
if verbose_level == 3:
remove_log = ''
else:
dbg_log = "/dev/null"
remove_log = ' >{} 2>&1'.format(dbg_log)
env_var = "KLAMMERTEXT_TEXLIVE_BIN"
texbin = os.environ.get(env_var)
if texbin is None:
msg = f"The directory of TeX Live commands must be defined by ${env_var}"
raise Exception(msg)
latex_command = f"{texbin}/xelatex"
#latex_command = f"{texbin}/pdflatex"
if not os.path.exists(latex_command):
raise Exception(f"LaTeX command not found: {latex_command}")
flags = "--halt-on-error"
#flags = "-file-line-error --shell-escape -halt-on-error -interaction nonstopmode -output-directory"
env = "export max_print_line=1000 ; export TEXINPUTS=${KLAMMERTEXT_HOME}/sks//: ;"
command = f"{env} cd {dirname} ; {latex_command} {flags} {basename}.tex {remove_log}"
#print(command)
result = os.system(command)
if result != 0:
os.system(f"tail -n 20 {log_file}")
msg = f"Error in LaTeX processing. See file {os.path.relpath(log_file)}"
#print(f"Error in LaTeX processing. See file {os.path.relpath(log_file)}")
raise Exception(msg)
# No error; is repetition necessary for links, etc? Check log for this.
# result = os.system(command)
# print(f"Wrote {os.path.relpath(pdf_file)}")
if twice:
result = os.system(command)
aux_files = 'aux out toc'.split()
if verbose_level < 2:
aux_files += 'tex log'.split()
for unused_ext in aux_files:
os.system('rm -rf {}/{}.{}'.format(dirname, basename, unused_ext))
import re
import textwrap
# Emitted by @vspace.txt (sks/block/block.k), one marker per line of space.
# An @eval result consisting only of whitespace is trimmed away by the
# Klammermachine, so vertical space must travel as markers and become
# newlines here, after blank-line runs have been normalized.
vspace_marker = "__VSPACE__"
def justify_blocks(text, K=None):
def txt_nl_marker():
return "__KTEXTNEWLINE__"
def txt_vspace_marker():
return "__KTEXTVSPACE__"
def txt_justify_blocks(text, K):
delim = '__DIVIDE__'
text = re.sub("\n\n+", delim, text)
result = ""
for par in text.split(delim):
stripped = par.strip()
n = stripped.count(vspace_marker)
if n and stripped == vspace_marker * n:
n = stripped.count(txt_vspace_marker())
if n and stripped == txt_vspace_marker() * n:
# A paragraph of only @vspace markers: n blank lines in addition
# to the normal paragraph separation.
result += "\n" * n
continue
if par and par[0] not in {' ', '['}:
par = "\n".join(textwrap.wrap(par, width=80))
result += par + "\n\n"
return result.replace(vspace_marker, "\n")
if par and par[0] != ' ':
label = re.match(r"~*\[\d+\] ", par)
par = "\n".join(textwrap.wrap(
par, width=int(K.Target_txt_width), break_long_words=False,
subsequent_indent=" " * len(label.group(0)) if label else ""))
# if par and par[0] != ' ':
# par = "\n".join(textwrap.wrap(
# par, width=int(K.Target_txt_width), break_long_words=False))
result += par + "\n\n"
result = re.sub(" *" + txt_vspace_marker() + " *", "\n", result)
result = re.sub(r"\n? *" + txt_nl_marker() + r" *\n?", "\n", result)
result = re.sub("~", " ", result)
return result
def txt_footnote_marker():
return "KTEXTFOOTNOTE"
def txt_format_footnotes(text, K):
footnote_number = 0
footnotes = []
def replace(m):
nonlocal footnote_number, footnotes
footnote_number += 1
footnotes.append(m.group(1))
return f"[{footnote_number}]"
start_mark = "__" + txt_footnote_marker()
end_mark = txt_footnote_marker() + "__"
footnote_rgx = re.compile(rf"\s*{start_mark}\s+(.*?)\s+{end_mark}", re.S)
result = footnote_rgx.sub(replace, text).rstrip() + "\n\n"
if footnotes:
rule_count = int(round(float(K.Target_txt_width) * 0.4))
result += "-" * rule_count + "\n\n"
# Calculate maximum width for right-justified numbers:
width = len(f"[{len(footnotes)}]")
for n, footnote in enumerate(footnotes, 1):
label = f"[{n}]"
result += "~" * (width - len(label)) + label + " " + footnote + "\n\n"
return result
def html_footnote_marker():
return "kt-footnote"
def html_format_footnotes(text, K=None):
fnum = 0
footnotes = []
def replace(m):
nonlocal fnum, footnotes
fnum += 1
footnotes.append(m.group(1))
href = f'href ="#_footnote_{fnum}"'
id = f'id="_footnote_src_{fnum}"'
return f'<a {id} {href}><sup class="footnote_in_text">{fnum}</sup></a>'
#return f'<a href="#_footnote_{fnum}" id="_footnote_src_{fnum}"><sup>{fnum}</sup></a>'
start_mark = f"<{html_footnote_marker()}>"
end_mark = f"</{html_footnote_marker()}>"
footnote_rgx = re.compile(rf"\s*{start_mark}\s*(.*?)\s*{end_mark}", re.S)
result = footnote_rgx.sub(replace, text).rstrip() + "\n\n"
if footnotes:
result += '<hr class="footnote_rule"/>\n'
for n, footnote in enumerate(footnotes, 1):
result += f'<a href="#_footnote_src_{n}" id="_footnote_{n}"><sup>{n} </sup></a>{footnote}<br/>\n'
# Temporarily add room for footnotes to move to the top of the window for the test:
result += '<div style="height: 100lh"></div>\n'
return result

View File

@@ -15,10 +15,30 @@ The LaTeX transformations supported by the SKS are:
'' -> close double quote
~ -> non-breaking space
"^" before any punctuation character quotes it (an engine rule, not an SKS
one): the character reaches the output literally, taking no part in the
transforms above -- ^-^- writes two plain hyphens (a --flag in prose), ^~ a
plain tilde. The one exception is the apostrophe: "^" before it opens a
literal span instead (a literal apostrophe is ^0027^), and writing that
pair even in THIS comment would open one -- the span sentinels are placed
before text removal. A target where the raw character would
still be re-interpreted declares its rendering with :resolve -- the tex
target maps a quoted hyphen to {-} (a bare -- in .tex re-forms TeX's dash
ligature) and a quoted tilde to \textasciitilde{} (a raw ~ is TeX's
non-breaking space). Verbatim text (@c, @code, ^'...'^) never needs any
of this: neither the transforms nor the quoting reach it.
]#
@@@target txt | Plain text with formatting | # This txt target is not full implemented in the SKS yet.
@@@state Target_txt_width
:desc Width of the text in the plain text target (txt)
:value 72
@@@
# The txt target is not fully implemented in the SKS yet.
@@@target txt | Plain text with formatting |
-- - |
--- -- |
`` " |
@@ -27,24 +47,35 @@ The LaTeX transformations supported by the SKS are:
' ' |
~ ^0020^
:after_apply
phases.justify_blocks
phases.txt_format_footnotes ;
phases.txt_justify_blocks
@@@
@@@target html | HTML page
# :escape < &lt; & &amp; |
|
--- &mdash; |
-- &ndash; |
--- &^#x2014; |
-- &^#x2013; |
`` “ |
'' ” |
` |
' |
^^~ &tilde; |
~ &nbsp;
~ &^#160;
:after_apply
phases.html_format_footnotes
@@@
# No transforms: LaTeX applies the input conventions itself. The :resolve
# entries say how a QUOTED character renders where the raw character would
# still be re-interpreted by TeX: ^- resolves to {-}, not a bare -, or the
# hyphens re-form a run and the dash ligature turns ^-^- into an en-dash;
# ^~ resolves to \textasciitilde{}, since a raw ~ is TeX's non-breaking
# space. Other quoted punctuation either has an :escape entry (its marker
# takes the escape replacement: ^$ renders as \$) or decodes to the raw
# character.
@@@target tex | LaTeX
:escape \ \textbackslash{} & \& { \{ } \} $ \$ % \% _ \_ ^# \^# ^^ \textasciicircum{}
:escape \ \textbackslash{} & \& { \{ } \} $ \$ % \% _ \_ ^# \^# ^^ \textasciicircum{}
:resolve - {-} ~ \textasciitilde{}
@@@
@@@target pdf | PDF from LaTeX