ack/klammertext
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
37b6ba1c4f42fcc9f41186d28d9ed07227a385cb
klammertext/doc/edit/shared/klammertext_edit.py
Andy Kopra 37b6ba1c4f Verbatim-safe typography, ^-punctuation quoting, full-range ^UUUU^, polyglot html 
Typographic transforms (---, quote pairs, ~) no longer touch verbatim
text: @c/@code/@source_listing content and ^'...'^ spans show exactly the
characters written. "^" before any punctuation character quotes it in
every target (the apostrophe excepted: ^' opens a literal span), with the
new :resolve option on @@@target declaring per-target renderings. The
^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range.
The html output and transform spellings are polyglot (XML-valid), in
preparation for an EPUB target. New suites: transform_test, character_test
(engine), typography_test (SKS).

(from dev 07ce5ea86a0a)
2026-08-23 20:48:26 +02:00

1099 lines
41 KiB
Python
Raw Blame History

#!/usr/bin/env python3
# klammertext_edit.py
#
# The SHARED implementation of Klammertext's editor-support algorithms:
# everything about the language's *structural* layer — @-run tiers, bar-run
# dimension, ^-escapes, # removal, literal spans, nesting depth — that more
# than one editor integration needs. One algorithm, one file:
#
# * scanner helpers name_char_p, escaped_p, block_end, at_run_end
# * delimiter matching app_delim_info, app_match, ... (jump / highlight)
# * indentation reindent_lines, target_column, ...
# * table alignment enclosing_span, compute_edits, ...
# * diagnostics diagnostics(): unclosed / mismatched delimiters
# * a CLI indent | align | match | check over stdin/stdout
#
# Consumers:
# * doc/edit/sublime/ the Sublime Text plugins import this module
# directly (Sublime's plugin host is Python)
# * doc/edit/vim/ the Vim plugin shells out to the CLI
# * doc/edit/shared/klammertext_ls.py
# the language server (used by VS Code, and by any
# LSP client: Neovim, Emacs eglot, Sublime LSP)
# * tst/editor_test.sh the regression suite drives the fixtures through
# this module and through the CLI
# * doc/edit/emacs/ NOT a consumer: the Emacs mode is an independent
# elisp implementation of the same algorithms,
# held equal to this file by the byte-equality
# checks in tst/editor_test.sh
#
# This file is the SOURCE OF TRUTH for the policy lists (LITERAL_KLAMMERS,
# TRANSPARENT_KLAMMERS, CODE_KLAMMERS, ALIGN_KLAMMERS) and the numeric limits
# (INDENT_OFFSET, CELL_MAX, ROW_MAX). It replaces the former per-editor
# copies in Klammertext.py / Klammertext_indent.py / Klammertext_align.py.
# SYNC: the following per-editor artifacts cannot import Python and must be
# kept in step by hand:
# * the Emacs defcustoms (klammertext-literal-klammers, -transparent-,
# -code-, -align-klammers, -indent-offset, -align-cell-max, -align-row-max)
# in doc/edit/emacs/klammertext-mode.el / -indent.el / -align.el
# * the '@code'/'@c' rules + literal_code/literal_c contexts in
# doc/edit/sublime/Klammertext.sublime-syntax
# * the '@code'/'@c' verbatim regions in doc/edit/vim/syntax/klammertext.vim
# * the '@code'/'@c' rules in doc/edit/vscode/syntaxes/klammertext.tmLanguage.json
#
# Installation note: editors locate this file either next to their own plugin
# files (a vendored copy, placed there by doc/make_editing_zip.sh), as
# ../shared/klammertext_edit.py relative to the plugin directory (the layout
# of this repository), or under $KLAMMERTEXT_HOME/doc/edit/shared/.
#
# Python floor: 3.8 (the Sublime Text 4 plugin host) — no 3.9+ syntax here.
#
# Known limitation (inherited by every consumer): a raw @ or # inside a
# ^'...'^ literal region confuses the scanners; use ^@ / ^# there instead.
import sys
# --- policy ----------------------------------------------------------------
# Klammer names whose content is a literal argument (verbatim interior,
# closed by a named NAME@ delimiter).
LITERAL_KLAMMERS = set(["code", "c"])
# Klammers that contribute no indentation level (a @document's paragraphs
# stay at the left margin).
TRANSPARENT_KLAMMERS = set(["document"])
# Klammers whose argument span holds code (inline Python is
# indentation-sensitive): lines inside are never reindented.
CODE_KLAMMERS = set(["eval"])
# Klammers whose rows the table-alignment command aligns.
ALIGN_KLAMMERS = set(["table"])
# Spaces per nesting level.
INDENT_OFFSET = 2
# Alignment limits: a row with a cell longer than CELL_MAX characters (or
# spanning lines) is left untouched; if the aligned rows would exceed ROW_MAX
# columns, nothing is changed.
CELL_MAX = 30
ROW_MAX = 100
# --- scanner helpers -------------------------------------------------------
def name_char_p(ch):
"""True if CH can be part of a klammer name (letter, digit or _).
A hyphen is NOT a name char: @name-arg1 ends the name at the first hyphen."""
if ch is None:
return False
return (('a' <= ch <= 'z') or ('A' <= ch <= 'Z')
or ('0' <= ch <= '9') or ch == '_')
def find_literal_close(s, name, start):
"""Index of the exact close token NAME@ at or after START, or -1.
The engine's close is a whole katom, so NAME@ preceded by a name
character is content, not a close -- "basic@" does not close @c, and
"barcode@" does not close @code. Single-letter literal names (@c) make
this guard essential rather than theoretical."""
close = name + '@'
idx = s.find(close, start)
while idx > 0 and name_char_p(s[idx - 1]):
idx = s.find(close, idx + 1)
return idx
def escaped_p(s, pos):
"""True if the char at POS is escaped by an odd run of ^ before it.
In Klammertext ^# and ^@ are literal, so such a char is not a delimiter."""
n = 0
i = pos - 1
while i >= 0 and s[i] == '^':
n += 1
i -= 1
return (n % 2) == 1
def block_end(s, frm):
"""Index just after the ]# that closes a #[ block opened at FROM (the index
just after the opening #[). Counts nested #[ ... ]#; len(s) if unclosed."""
depth = 1
i = frm
n = len(s)
while depth > 0:
a = s.find('#[', i)
b = s.find(']#', i)
if a == -1 and b == -1:
return n
if b == -1 or (a != -1 and a < b):
depth += 1
i = a + 2
else:
depth -= 1
i = b + 2
return i
def at_run_end(s, pos):
"""Index just after the run of @ that begins at POS."""
p = pos
n = len(s)
while p < n and s[p] == '@':
p += 1
return p
def _name_forward(s, pos):
"""Index just past the run of name chars starting at POS."""
n = len(s)
k = pos
while k < n and name_char_p(s[k]):
k += 1
return k
# --- delimiter matching (klammer APPLICATIONS, the single-@ tier) ----------
#
# Matching is context-dependent (the same @ is both open and close, decided
# by its neighbors), so no editor's built-in bracket matching can express it;
# every editor integration routes its jump-to-match and live match
# highlighting through these functions (Emacs excepted; see the header).
# A literal klammer's own delimiters are matched BY NAME (@code <-> code@,
# content opaque); all others match by depth.
def next_app_delim(s, i, limit):
"""From index I, find the next single-@ application delimiter before LIMIT.
Step over @@/@@@ runs, removed text, literal spans, escaped ^@, and the
abbreviated @name-arg form. Return (pos, kind, next_i) with kind 'open' or
'close' and next_i the index to resume from, or None when none is found."""
n = len(s)
if limit is None:
limit = n
while i < limit:
j = i
while j < limit and s[j] != '@' and s[j] != '#':
j += 1
if j >= limit:
return None
hit = j
i = hit + 1 # default: advance past the hit
if escaped_p(s, hit): # ^@ / ^# : keep going
continue
nxt = s[hit + 1] if hit + 1 < n else None
if s[hit] == '#': # removal: step over it
if nxt == '#':
i = n
elif nxt == '[':
i = block_end(s, hit + 2)
elif nxt in ('+', '/', '-'):
i = hit + 1
else: # to end of line
eol = s.find('\n', hit)
i = n if eol == -1 else eol
continue
# s[hit] == '@'
if nxt == '@': # @@ / @@@ : step over the run
i = at_run_end(s, hit)
continue
if name_char_p(nxt): # @name : opening?
k = _name_forward(s, hit + 1)
name = s[hit + 1:k]
after = s[k] if k < n else None
if name in LITERAL_KLAMMERS: # literal span: skip to its close
idx = find_literal_close(s, name, k)
i = n if idx == -1 else idx + len(name) + 1
continue
elif after == '-': # @name-arg : opens no span
i = k
continue
else:
return (hit, 'open', k)
else: # name@ / bare @ : closing
return (hit, 'close', hit + 1)
return None
def match_forward(s, open_pos):
"""OPEN_POS is the @ of an opening application. Return the matching close @
index, or None if unbalanced."""
i = _name_forward(s, open_pos + 1) # past the opening name
depth = 1
while depth > 0:
d = next_app_delim(s, i, None)
if d is None:
return None
pos, kind, nxt = d
i = nxt
if kind == 'open':
depth += 1
else:
depth -= 1
if depth == 0:
return pos
return None
def match_backward(s, close_pos):
"""CLOSE_POS is the @ of a closing application. Return the matching open @
index, or None if unbalanced. Scans forward from 0 with a stack."""
stack = []
i = 0
limit = close_pos + 1
while True:
d = next_app_delim(s, i, limit)
if d is None:
return None
pos, kind, nxt = d
i = nxt
if kind == 'open':
stack.append(pos)
else:
open_pos = stack.pop() if stack else None
if pos == close_pos:
return open_pos
def app_delim_info(s, pos):
"""If the char at POS is a single-@ application delimiter, return
(pos, kind) with kind 'open' or 'close'; else None. The abbreviated
@name-arg form (which opens no span) returns None."""
n = len(s)
if not (0 <= pos < n):
return None
if s[pos] != '@':
return None
if pos > 0 and s[pos - 1] == '@':
return None
if pos + 1 < n and s[pos + 1] == '@':
return None
if escaped_p(s, pos):
return None
nxt = s[pos + 1] if pos + 1 < n else None
if name_char_p(nxt):
k = _name_forward(s, pos + 1)
after = s[k] if k < n else None
if after == '-':
return None
return (pos, 'open')
return (pos, 'close')
def open_name(s, open_pos):
"""Name of the opening @name whose @ is at OPEN_POS."""
return s[open_pos + 1:_name_forward(s, open_pos + 1)]
def close_name(s, close_pos):
"""Name of a named close NAME@ whose @ is at CLOSE_POS, or None for a bare @
(including the compact @name@ form, whose name belongs to the opening)."""
ns = close_pos
while ns > 0 and name_char_p(s[ns - 1]):
ns -= 1
if ns < close_pos and (ns == 0 or s[ns - 1] != '@'):
return s[ns:close_pos]
return None
def paren_mismatch(s, open_pos, close_pos):
"""True if the pair is unbalanced (either side None) or the named close
disagrees with the opening name."""
if open_pos is None or close_pos is None:
return True
cname = close_name(s, close_pos)
return cname is not None and cname != open_name(s, open_pos)
def token_region(s, pos, kind):
"""(start, end) of the whole delimiter token whose @ is at POS.
Opening: @ plus its name. Named close: the name plus @. Bare @: just @."""
if kind == 'open':
return (pos, _name_forward(s, pos + 1))
ns = pos
while ns > 0 and name_char_p(s[ns - 1]):
ns -= 1
if ns < pos and (ns == 0 or s[ns - 1] != '@'):
return (ns, pos + 1) # named close NAME@
return (pos, pos + 1) # bare @ (or @name@)
def literal_delim_name(s, pos, kind):
"""If the application delimiter at POS (kind 'open'/'close') belongs to a
literal klammer (name in LITERAL_KLAMMERS), return its name; else None."""
name = open_name(s, pos) if kind == 'open' else close_name(s, pos)
if name and name in LITERAL_KLAMMERS:
return name
return None
def literal_match_forward(s, open_pos, name):
"""Index of the @ of the NAME@ that closes the literal @NAME at OPEN_POS, or
None. The content is opaque, so search for the literal close string."""
start = open_pos + 1 + len(name)
idx = find_literal_close(s, name, start)
return idx + len(name) if idx != -1 else None
def literal_match_backward(s, close_pos, name):
"""Index of the @ of the @NAME that opens the literal NAME@ whose @ is at
CLOSE_POS, or None. Literal spans do not nest, so the nearest preceding
real @NAME is the opener (not @@NAME, and not escaped)."""
open_str = '@' + name
end = close_pos
while True:
idx = s.rfind(open_str, 0, end)
if idx == -1:
return None
before = s[idx - 1] if idx > 0 else None
# The name must end where the token ends: "@c" found inside
# "@caption" is not an opener of @c.
follower = s[idx + len(open_str)] if idx + len(open_str) < len(s) else None
if before != '@' and not name_char_p(follower) and not escaped_p(s, idx):
return idx
end = idx
def app_match(s, pos, kind):
"""Matching application delimiter for the delimiter at POS of KIND
('open'/'close'), or None. A literal klammer matches by name (@NAME <->
NAME@) with content opaque; other klammers match by depth."""
lit = literal_delim_name(s, pos, kind)
if lit is not None:
return (literal_match_forward(s, pos, lit) if kind == 'open'
else literal_match_backward(s, pos, lit))
return match_forward(s, pos) if kind == 'open' else match_backward(s, pos)
def match_at(s, pos):
"""The full matching story for a caret at POS (used by jump-to-match and
the live highlighters). The delimiter is taken at POS or, failing that,
just before POS (the on-or-just-after rule). Return None when POS is not
at an application delimiter; else a dict:
{'pos', 'kind', 'token': (start, end),
'match': int or None, 'match_token': (start, end) or None,
'mismatch': bool, 'message': str or None}"""
info = app_delim_info(s, pos)
if info is None and pos > 0:
info = app_delim_info(s, pos - 1)
if info is None:
return None
dpos, kind = info
match = app_match(s, dpos, kind)
open_pos = dpos if kind == 'open' else match
close_pos = match if kind == 'open' else dpos
mism = paren_mismatch(s, open_pos, close_pos)
message = None
if mism:
if match is None:
if kind == 'open':
message = ("opening @%s has no matching close"
% open_name(s, open_pos))
else:
message = "closing delimiter has no matching open"
else:
message = ("closing %s@ does not match opening @%s"
% (close_name(s, close_pos) or '?',
open_name(s, open_pos)))
other_kind = 'close' if kind == 'open' else 'open'
return {'pos': dpos, 'kind': kind,
'token': token_region(s, dpos, kind),
'match': match,
'match_token': (token_region(s, match, other_kind)
if match is not None else None),
'mismatch': mism, 'message': message}
# --- indentation -----------------------------------------------------------
#
# The convention (2026-07-27): a line indents to INDENT_OFFSET x (effective
# depth); a line that BEGINS with a bar run (|, ||, ...) or a closing
# delimiter sits at its opener's column. Effective depth counts every
# enclosing span uniformly — applications (@), definitions (@@), system
# commands (@@@) — except that TRANSPARENT_KLAMMERS contribute no level, and
# lines inside literal/verbatim content, CODE_KLAMMERS argument spans, or
# removed regions are never touched. Reindentation is explicit-only in every
# editor: whitespace is content in Klammertext, so nothing reformats as a
# side effect of typing.
def state_at(s, pos):
"""Scan s[0:POS] (POS a line beginning). Return (stack, opaque): STACK is
the list of names of the klammer applications, @@ definitions and @@@
commands open at POS, outermost first; OPAQUE is True when POS lies inside
content that indentation must not touch (removed text, a literal klammer's
verbatim span, or a code klammer's argument span)."""
stack = []
n = len(s)
i = 0
while i < pos:
j = i
while j < pos and s[j] != '@' and s[j] != '#':
j += 1
if j >= pos:
break
hit = j
i = hit + 1
if escaped_p(s, hit): # ^@ / ^# : plain text
continue
nxt = s[hit + 1] if hit + 1 < n else None
if s[hit] == '#':
if nxt == '#': # ## removes to end of buffer
return (stack, True)
elif nxt == '[': # #[ ... ]# (nestable)
end = block_end(s, hit + 2)
if pos < end:
return (stack, True)
i = end
elif nxt in ('+', '/', '-'): # whitespace operators
pass
else: # # to end of line
eol = s.find('\n', hit)
i = n if eol == -1 else eol
continue
# an @-run
run_end = at_run_end(s, hit)
run_len = run_end - hit
after = s[run_end] if run_end < n else None
if name_char_p(after):
# @name / @@name / @@@name : an opener (or, for a literal
# klammer, a verbatim span to step over).
k = _name_forward(s, run_end)
name = s[run_end:k]
i = k
if run_len == 1 and name in LITERAL_KLAMMERS:
# Verbatim interior: find the closing NAME@ by name.
idx = find_literal_close(s, name, k)
if idx == -1: # never closed
return (stack, True)
close_end = idx + len(name) + 1
if pos < close_end:
return (stack, True)
i = close_end
elif run_len == 1 and k < n and s[k] == '-':
pass # @name-arg : opens no span
else:
stack.append(name)
else:
# a bare @-run, or the run of a named close: a close.
if stack:
stack.pop()
i = run_end
opaque = any(name in CODE_KLAMMERS for name in stack)
return (stack, opaque)
def depth(stack):
"""Number of indentation levels STACK contributes.
Transparent klammers contribute none."""
return sum(1 for name in stack if name not in TRANSPARENT_KLAMMERS)
def dedent_line_p(s, bol):
"""True when the line starting at BOL begins with a token that sits at its
owner's opening column: a bar run (|, ||, ...), a bare close run (@, @@,
@@@), or a named close (name@, name@@, name@@@). A line beginning with an
opener (@name, @@name, @@@name) is content-level."""
n = len(s)
i = bol
while i < n and s[i] in ' \t':
i += 1
if i >= n:
return False
c = s[i]
if c == '|':
return True
if c == '@':
run_end = at_run_end(s, i)
return not name_char_p(s[run_end] if run_end < n else None)
if name_char_p(c):
# A named close: name chars followed by an @-run (an unescaped @ can
# only be a delimiter).
k = _name_forward(s, i)
return k < n and s[k] == '@'
return False
def target_column(s, bol):
"""Column for the line starting at BOL, or None for lines that must not
be touched (verbatim, code, or removed content)."""
stack, opaque = state_at(s, bol)
if opaque:
return None
if dedent_line_p(s, bol) and stack:
stack = stack[:-1]
return INDENT_OFFSET * depth(stack)
def line_beginnings(s):
"""Offsets of every line beginning in S."""
return [0] + [i + 1 for i, ch in enumerate(s)
if ch == '\n' and i + 1 < len(s)]
def reindent_lines(s, bols):
"""Compute the edits that reindent the lines whose beginnings are BOLS.
Return a list of (start, end, replacement) triples over S, in ascending
order, replacing each line's leading whitespace; untouchable lines and
already-correct lines produce no edit. Pure function."""
n = len(s)
edits = []
for bol in bols:
tgt = target_column(s, bol)
if tgt is None:
continue
i = bol
while i < n and s[i] in ' \t':
i += 1
if s[bol:i] != ' ' * tgt:
edits.append((bol, i, ' ' * tgt))
return edits
# --- table alignment -------------------------------------------------------
#
# Pads the cells of the enclosing @table's rows so the depth-0 | separators
# line up. Alignment is for SMALL data items: a row with a cell longer than
# CELL_MAX or spanning lines is untouched (and contributes no width); if the
# aligned rows would exceed ROW_MAX columns nothing changes. No whitespace
# is ever inserted inside a bar run (|| is a row separator; | | is an empty
# cell — the load-bearing-whitespace trap). Bars inside a nested klammer
# belong to that klammer: only depth-0 bars count, the same rule the
# Klammermachine applies to @cond.
def enclosing_span(s, pos, names):
"""Innermost span of a klammer named in NAMES that contains POS.
Return (name, content_start, content_end) with content_start just after
the opening @name token and content_end at the start of the closing
delimiter token, or None. Scans S from the start with a position stack,
stepping over removed text, literal spans, escaped characters, and the
abbreviated @name-arg form."""
stack = [] # (name, open_token_start, content_start)
n = len(s)
i = 0
while i < n:
j = i
while j < n and s[j] != '@' and s[j] != '#':
j += 1
if j >= n:
break
hit = j
i = hit + 1
if escaped_p(s, hit):
continue
nxt = s[hit + 1] if hit + 1 < n else None
if s[hit] == '#':
if nxt == '#':
break # ## removes the rest of the buffer
elif nxt == '[':
i = block_end(s, hit + 2)
elif nxt in ('+', '/', '-'):
pass
else:
eol = s.find('\n', hit)
i = n if eol == -1 else eol
continue
run_end = at_run_end(s, hit)
run_len = run_end - hit
after = s[run_end] if run_end < n else None
if name_char_p(after):
k = _name_forward(s, run_end)
name = s[run_end:k]
i = k
if run_len == 1 and name in LITERAL_KLAMMERS:
idx = find_literal_close(s, name, k)
if idx == -1:
break
i = idx + len(name) + 1
elif run_len == 1 and k < n and s[k] == '-':
pass # @name-arg : opens no span
else:
stack.append((name, hit, k))
else:
# a close: the token starts at the preceding name run, if any
ns = hit
while ns > 0 and name_char_p(s[ns - 1]):
ns -= 1
tok_start = ns if (ns < hit and (ns == 0 or s[ns - 1] != '@')) else hit
if stack:
name, open_start, content_start = stack.pop()
if name in names and open_start <= pos <= run_end:
return (name, content_start, tok_start)
i = run_end
return None
def scan_lines(content):
"""Scan CONTENT (the text of a table span). Return a list of line
records, one per line: dicts with start, end (offsets into CONTENT, end
excludes the newline), start_depth, end_depth (klammer nesting relative
to the span), bars (list of (pos, runlen) for unescaped depth-0 bar
runs), blocked (inside removed/verbatim content), comment (a # removes
the rest of the line)."""
import bisect
n = len(content)
line_starts = [0]
for idx, ch in enumerate(content):
if ch == '\n':
line_starts.append(idx + 1)
nlines = len(line_starts)
def line_index(p):
return bisect.bisect_right(line_starts, p) - 1
lines = [{'start': line_starts[k],
'end': (line_starts[k + 1] - 1 if k + 1 < nlines else n),
'bars': [], 'blocked': False, 'comment': False,
'start_depth': None, 'end_depth': None}
for k in range(nlines)]
lines[0]['start_depth'] = 0
def block_range(a, b):
"""Mark every line touched by [a, b) as blocked."""
last = max(a, b - 1)
for k in range(line_index(a), line_index(min(last, n - 1)) + 1):
lines[k]['blocked'] = True
depth = 0
i = 0
while i < n:
j = i
while j < n and content[j] not in '@#|\n':
j += 1
if j >= n:
break
hit = j
i = hit + 1
c = content[hit]
if c == '\n':
k = line_index(hit)
lines[k]['end_depth'] = depth
if k + 1 < nlines:
lines[k + 1]['start_depth'] = depth
continue
if escaped_p(content, hit):
continue
nxt = content[hit + 1] if hit + 1 < n else None
if c == '#':
if nxt == '#':
block_range(hit, n)
break
elif nxt == '[':
e = block_end(content, hit + 2)
if line_index(max(hit, e - 1)) != line_index(hit):
block_range(hit, e)
i = e
elif nxt in ('+', '/', '-'):
pass
else: # # to end of line
lines[line_index(hit)]['comment'] = True
eol = content.find('\n', hit)
i = n if eol == -1 else eol
continue
if c == '|':
if hit > 0 and content[hit - 1] == '|':
continue # mid-run (after an escaped ^|)
k = hit
while k < n and content[k] == '|':
k += 1
if depth == 0:
lines[line_index(hit)]['bars'].append((hit, k - hit))
i = k
continue
# '@'
run_end = at_run_end(content, hit)
run_len = run_end - hit
after = content[run_end] if run_end < n else None
if name_char_p(after):
k = _name_forward(content, run_end)
name = content[run_end:k]
i = k
if run_len == 1 and name in LITERAL_KLAMMERS:
idx = find_literal_close(content, name, k)
e = n if idx == -1 else idx + len(name) + 1
if line_index(max(hit, e - 1)) != line_index(hit):
block_range(hit, e)
i = e
elif run_len == 1 and k < n and content[k] == '-':
pass
else:
depth += 1
else:
depth = max(0, depth - 1)
i = run_end
for ln in lines:
if ln['start_depth'] is None:
ln['blocked'] = True
if ln['end_depth'] is None:
ln['end_depth'] = depth
return lines
def compute_edits(content):
"""Compute the alignment edits for CONTENT (a table span's text).
Return (edits, message): edits is a list of (start, end, new_text)
triples relative to CONTENT, ascending; message is a status string (a
reason when edits is empty)."""
lines = scan_lines(content)
# The last line holding actual content (a row there may omit its ||).
last_content = None
for k in range(len(lines) - 1, 0, -1):
if content[lines[k]['start']:lines[k]['end']].strip():
last_content = k
break
rows = [] # (line record, cells, trailing_p)
chain_ok = True # a row must START a row: the previous
for k in range(1, len(lines)): # content line ended with || (or was the
ln = lines[k] # opener / an option line / blank)
text = content[ln['start']:ln['end']]
stripped = text.strip()
if not stripped:
continue # blank line: chain unchanged
if stripped.startswith(':') and not ln['bars']:
continue # option line: chain unchanged
row = _parse_row(content, ln, text, chain_ok, k == last_content)
trailing = _trailing_rowsep(content, ln)
chain_ok = trailing
if row is not None:
rows.append(row)
if not rows:
return ([], "no alignable rows found")
widths = []
for _ln, cells, _tr in rows:
for c_idx, cell in enumerate(cells):
if c_idx >= len(widths):
widths.append(0)
widths[c_idx] = max(widths[c_idx], len(cell))
indent = ' ' * _indent_width(content, rows[0][0])
if ROW_MAX is not None:
longest = 0
for _ln, cells, trailing in rows:
m = len(cells)
w = (len(indent) + sum(widths[:m]) + 3 * (m - 1)
+ (3 if trailing else 0))
longest = max(longest, w)
if longest > ROW_MAX:
return ([], "aligned rows would be %d characters (limit %d); "
"not aligning" % (longest, ROW_MAX))
edits = []
for ln, cells, trailing in rows:
parts = [cells[c].ljust(widths[c]) for c in range(len(cells) - 1)]
last = cells[-1]
if trailing:
last = last.ljust(widths[len(cells) - 1])
parts.append(last)
new = indent + ' | '.join(parts) + (' ||' if trailing else '')
if new != content[ln['start']:ln['end']]:
edits.append((ln['start'], ln['end'], new))
msg = ("aligned %d rows" % len(rows)) if edits else "already aligned"
return (edits, msg)
def _indent_width(content, ln):
i = ln['start']
while i < ln['end'] and content[i] in ' \t':
i += 1
return i - ln['start']
def _trailing_rowsep(content, ln):
"""True when the line's LAST depth-0 bar run is a || sitting at the end of
the line (only whitespace after it)."""
if not ln['bars']:
return False
pos, runlen = ln['bars'][-1]
return (runlen == 2
and content[pos + 2:ln['end']].strip() == '')
def _parse_row(content, ln, text, chain_ok, is_last_content):
"""If the line is an alignable row, return (ln, cells, trailing_p);
else None."""
if ln['blocked'] or ln['comment'] or not chain_ok:
return None
if ln['start_depth'] != 0 or ln['end_depth'] != 0:
return None
if not ln['bars']:
return None
trailing = _trailing_rowsep(content, ln)
singles = ln['bars'][:-1] if trailing else ln['bars']
for _pos, runlen in singles:
if runlen != 1:
return None # a mid-line || (or |||): not one row
if not trailing and not is_last_content:
return None # row continues onto the next line
cell_start = ln['start'] + _indent_width(content, ln)
cell_end = ln['bars'][-1][0] if trailing else ln['end']
bounds = [cell_start] + [p for p, _r in singles] + [cell_end]
cells = []
for b_idx in range(len(bounds) - 1):
a = bounds[b_idx] + (1 if b_idx > 0 else 0) # skip the | itself
cell = content[a:bounds[b_idx + 1]].strip()
if len(cell) > CELL_MAX:
return None
cells.append(cell)
return (ln, cells, trailing)
# --- diagnostics -----------------------------------------------------------
#
# A whole-buffer balance check over all three @-tiers, for editor problem
# panels (LSP publishDiagnostics, Vim quickfix). The scan is the uniform
# one the indentation uses — every @-run is an opener when a name follows it
# and a close otherwise — extended with positions and names so problems can
# be reported where they are:
#
# * a closing delimiter with no opening to match
# * a named close whose name disagrees with its opening
# * a close whose @-run length differs from its opening's (@name ... @@)
# * an opening never closed (reported at the opening, at end of scan)
# * a literal klammer never closed (@code without code@)
# * an unclosed #[ removal block
#
# Content removed by ## is not scanned (it is not part of the document).
def diagnostics(s):
"""Scan S and return a list of problems, each a dict:
{'start': int, 'end': int, 'message': str, 'severity': 'error'|'warning'}.
Positions are character offsets into S (token start/end)."""
probs = []
stack = [] # (name, run_len, tok_start, tok_end)
n = len(s)
i = 0
while i < n:
j = i
while j < n and s[j] != '@' and s[j] != '#':
j += 1
if j >= n:
break
hit = j
i = hit + 1
if escaped_p(s, hit):
continue
nxt = s[hit + 1] if hit + 1 < n else None
if s[hit] == '#':
if nxt == '#': # rest of file removed: stop scanning
break
elif nxt == '[':
end = block_end(s, hit + 2)
if end >= n and not s.endswith(']#'):
probs.append({'start': hit, 'end': hit + 2,
'message': "#[ has no closing ]#",
'severity': 'warning'})
i = end
elif nxt in ('+', '/', '-'):
pass
else:
eol = s.find('\n', hit)
i = n if eol == -1 else eol
continue
# an @-run
run_end = at_run_end(s, hit)
run_len = run_end - hit
after = s[run_end] if run_end < n else None
if name_char_p(after):
k = _name_forward(s, run_end)
name = s[run_end:k]
i = k
if run_len == 1 and name in LITERAL_KLAMMERS:
idx = find_literal_close(s, name, k)
if idx == -1:
probs.append({'start': hit, 'end': k,
'message': ("literal klammer @%s has no "
"closing %s@" % (name, name)),
'severity': 'error'})
break # everything after is verbatim
i = idx + len(name) + 1
elif run_len == 1 and k < n and s[k] == '-':
pass # @name-arg : opens no span
else:
stack.append((name, run_len, hit, k))
else:
# a close; find the token (a preceding name run makes it named)
ns = hit
while ns > 0 and name_char_p(s[ns - 1]):
ns -= 1
named = ns < hit and (ns == 0 or s[ns - 1] != '@')
cname = s[ns:hit] if named else None
tok_start = ns if named else hit
if not stack:
probs.append({'start': tok_start, 'end': run_end,
'message': ("closing delimiter %s has no "
"matching opening"
% s[tok_start:run_end]),
'severity': 'error'})
else:
oname, orun, ostart, oend = stack.pop()
if cname is not None and cname != oname:
probs.append({'start': tok_start, 'end': run_end,
'message': ("closing %s does not match "
"opening %s"
% (s[tok_start:run_end],
s[ostart:oend])),
'severity': 'error'})
elif orun != run_len:
probs.append({'start': tok_start, 'end': run_end,
'message': ("closing %s does not match "
"opening %s (%d-@ close for "
"a %d-@ opening)"
% (s[tok_start:run_end],
s[ostart:oend],
run_len, orun)),
'severity': 'error'})
i = run_end
for name, run_len, ostart, oend in stack:
probs.append({'start': ostart, 'end': oend,
'message': "opening %s is never closed" % s[ostart:oend],
'severity': 'error'})
probs.sort(key=lambda p: p['start'])
return probs
# --- whole-buffer conveniences (used by the CLI, the LSP server, tests) ----
def apply_edits(s, edits):
"""Apply (start, end, new) EDITS (ascending, non-overlapping) to S."""
out = []
last = 0
for a, b, new in edits:
out.append(s[last:a])
out.append(new)
last = b
out.append(s[last:])
return ''.join(out)
def indent_text(s, first=None, last=None):
"""Reindent S (whole buffer, or 1-based inclusive line range FIRST..LAST).
Return the new text."""
bols = line_beginnings(s)
if first is not None:
lo = max(first, 1)
hi = len(bols) if last is None else min(last, len(bols))
bols = bols[lo - 1:hi]
return apply_edits(s, reindent_lines(s, bols))
def align_text(s, pos):
"""Align the table klammer enclosing POS. Return (new_text, message)."""
span = enclosing_span(s, pos, ALIGN_KLAMMERS)
if span is None:
return (s, "not inside a table klammer (%s)"
% ", ".join("@" + name for name in sorted(ALIGN_KLAMMERS)))
_name, cs, ce = span
edits, msg = compute_edits(s[cs:ce])
shifted = [(cs + a, cs + b, new) for a, b, new in edits]
return (apply_edits(s, shifted), msg)
def offset_of(s, line, col):
"""Character offset of 1-based LINE, 1-based character column COL."""
bols = line_beginnings(s)
line = max(1, min(line, len(bols)))
bol = bols[line - 1]
eol = s.find('\n', bol)
if eol == -1:
eol = len(s)
return min(bol + max(col - 1, 0), eol)
def line_col(s, offset):
"""(1-based line, 1-based character column) of character OFFSET."""
offset = max(0, min(offset, len(s)))
line = s.count('\n', 0, offset) + 1
bol = s.rfind('\n', 0, offset) + 1
return (line, offset - bol + 1)
# --- CLI -------------------------------------------------------------------
#
# The shell-out interface for editors that are neither Python-hosted nor LSP
# clients (the Vim plugin). Reads the buffer on stdin (UTF-8), writes the
# transformed buffer on stdout; status messages go to stderr.
#
# klammertext_edit.py indent [FIRST[-LAST]] reindent all / a line range
# klammertext_edit.py align LINE COL align the enclosing table
# klammertext_edit.py match LINE COL print "match L C" | "none MSG"
# (with "mismatch" when the
# pair disagrees); no buffer
# klammertext_edit.py check print "L:C: message" lines;
# no buffer output
#
# LINE and COL are 1-based; COL counts characters. Exit code: 0 (including
# "nothing to do"), 2 on a usage error.
def _cli(argv):
def usage():
sys.stderr.write(
"usage: klammertext_edit.py indent [FIRST[-LAST]] |"
" align LINE COL | match LINE COL | check\n")
return 2
if len(argv) < 1:
return usage()
mode = argv[0]
s = sys.stdin.read()
if mode == 'indent':
first = last = None
if len(argv) > 1:
rng = argv[1]
try:
if '-' in rng:
a, b = rng.split('-', 1)
first, last = int(a), int(b)
else:
first = last = int(rng)
except ValueError:
return usage()
sys.stdout.write(indent_text(s, first, last))
return 0
if mode == 'align':
if len(argv) != 3:
return usage()
try:
pos = offset_of(s, int(argv[1]), int(argv[2]))
except ValueError:
return usage()
out, msg = align_text(s, pos)
sys.stdout.write(out)
sys.stderr.write(msg + "\n")
return 0
if mode == 'match':
if len(argv) != 3:
return usage()
try:
pos = offset_of(s, int(argv[1]), int(argv[2]))
except ValueError:
return usage()
m = match_at(s, pos)
if m is None:
print("none point is not on a klammer application delimiter (@)")
elif m['match'] is None:
print("none " + (m['message'] or "no matching delimiter"))
else:
line, col = line_col(s, m['match'])
kind = "mismatch" if m['mismatch'] else "match"
msg = (" " + m['message']) if m['mismatch'] and m['message'] else ""
print("%s %d %d%s" % (kind, line, col, msg))
return 0
if mode == 'check':
for p in diagnostics(s):
line, col = line_col(s, p['start'])
print("%d:%d: %s" % (line, col, p['message']))
return 0
return usage()
if __name__ == '__main__':
sys.exit(_cli(sys.argv[1:]))
Reference in New Issue View Git Blame Copy Permalink