ack
af29bbf7ac
A run of Unicode code points now needs one pair of carets, not one pair
each: ^9053 53EF 9053^ renders the characters adjacently, and a single
character inside such a run stands for itself. An invalid code point is
an error naming the position that wrote it, rather than an apology
rendered into the finished document; noncharacters are refused alongside
surrogates; and a code point that lands on a Klammertext special is safe
through the re-reads that assemble a document. The character tables now
produce the precomposed form, so a diacritic written as ^a" and one typed
directly are the same text to a search or a copy-paste.
"kdesc -u" describes how to write a Unicode character, with tables of
glyphs beside the codes that produce them. It is new and not yet
complete: it predates the sequence form and the typeface klammer below.
A typeface and a style are no longer both called "font". A typeface is a
family resolved by name; a style code is the one letter naming a type
style. So @document takes :serif_typeface and its siblings, a table takes
:style, captions take :caption_style and :caption_size, the uniform scale
is :text_scale, and the command is "kdesc --typeface". This renames
arguments a document may already use; an old name is a located error
rather than a silent omission.
@typeface sets a passage in another typeface for the length of that
passage:
@typeface Noto Sans Egyptian Hieroglyphs | ^132AA 133CF^ :scale 1.5 @
Writing the klammer is the whole declaration -- the document resolves the
typeface, embeds it in what it produces, and carries its license along.
Two typefaces for scripts no text font covers now ship with Klammertext:
Noto Sans Egyptian Hieroglyphs and Noto Sans Cuneiform. Every typeface in
the distribution now carries its license file.
(from dev 61e598698891)
134 lines
6.5 KiB
Bash
Executable File
134 lines
6.5 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# character_test.sh — Regression tests for the ^-character forms the engine
|
|
# resolves at read time: the ^UUUU^ Unicode code point (4-6 hex digits,
|
|
# since 2026-08-23 covering the full range to U+10FFFF), the
|
|
# ^UUUU UUUU ...^ SEQUENCE form (added 2026-08-25: one caret pair around
|
|
# whitespace-delimited tokens, assembled at katom level by
|
|
# process_unicode_sequences — so, unlike the string-level single form, it
|
|
# is skipped inside @eval/@read/@cond spans and for literal content), and
|
|
# the character tables (mnemonics, diacritics) it must not disturb.
|
|
#
|
|
# SKS-independent: character handling is katomizer/engine machinery
|
|
# (mac/ktype.h nonascii katom, mac/character.cpp), so everything runs with
|
|
# --klammersets none under the default (general) target.
|
|
#
|
|
# The 2026-08-23 changes these cases pin:
|
|
# - a 5-digit code point converts WHOLE: ^13000^ (EGYPTIAN HIEROGLYPH
|
|
# A001) rendered as U+1300 followed by a literal "0" before, because
|
|
# process_unicode_codepoint() re-scanned its capture at a fixed 4-digit
|
|
# width;
|
|
# - 6 digits are accepted (^10FFFD^ is the top of the range; the katom
|
|
# regex capped at {1,5});
|
|
# - a value above U+10FFFF renders as the "Invalid Unicode" text, the
|
|
# same treatment as a surrogate.
|
|
#
|
|
# Note: expected strings use $'\xHH' UTF-8 BYTE escapes, not $'\U...' code
|
|
# points -- macOS ships bash 3.2, which does not expand \U (it failed there
|
|
# exactly as a naive check would).
|
|
#
|
|
# Usage: ./character_test.sh (needs KLAMMERTEXT_HOME set; ktext on PATH)
|
|
# Exit code: 0 if all tests pass, 1 otherwise.
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
KTEXT=ktext
|
|
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
|
|
ERR=$(mktemp)
|
|
|
|
red=$'\033[31m'
|
|
green=$'\033[32m'
|
|
bold=$'\033[1m'
|
|
reset=$'\033[0m'
|
|
|
|
trim() { awk '{ sub(/[ \t\r]+$/, "") } { line[NR]=$0 } END { f=1; while (f<=NR && line[f]=="") f++; l=NR; while (l>=1 && line[l]=="") l--; for (i=f;i<=l;i++) print line[i] }'; }
|
|
|
|
# check_err NAME PHRASE INPUT — the render must FAIL (nonzero exit) with
|
|
# PHRASE on stderr. Matched wrap-insensitively: error messages are
|
|
# justified to 80 columns and may wrap anywhere.
|
|
check_err() {
|
|
local name="$1" phrase="$2" input="$3"
|
|
local err status
|
|
err=$("$KTEXT" --klammersets none -s "$input" -d 2>&1 >/dev/null); status=$?
|
|
err=$(printf '%s' "$err" | tr '\n' ' ' | tr -s ' ')
|
|
if [ $status -eq 0 ]; then
|
|
echo "${red}FAIL${reset} $name — expected an error, got exit 0"
|
|
FAIL=$((FAIL+1)); return
|
|
fi
|
|
case "$err" in
|
|
*"$phrase"*) echo "${green}PASS${reset} $name"; PASS=$((PASS+1));;
|
|
*) echo "${red}FAIL${reset} $name"
|
|
echo " wanted phrase: [$phrase]"
|
|
echo " stderr: $(echo "$err" | head -c 200)"; FAIL=$((FAIL+1));;
|
|
esac
|
|
}
|
|
|
|
# check NAME EXPECTED INPUT — render INPUT (no klammer set, default
|
|
# target); exit 0 and stdout == EXPECTED.
|
|
check() {
|
|
local name="$1" expected="$2" input="$3"
|
|
local out status
|
|
out=$("$KTEXT" --klammersets none -s "$input" -d 2>"$ERR"); status=$?
|
|
out=$(printf '%s' "$out" | trim)
|
|
if [ $status -ne 0 ]; then
|
|
echo "${red}FAIL${reset} $name — ktext exited $status"
|
|
head -3 "$ERR" | sed 's/^/ /'; FAIL=$((FAIL+1)); return
|
|
fi
|
|
if [ "$out" = "$expected" ]; then
|
|
echo "${green}PASS${reset} $name"; PASS=$((PASS+1))
|
|
else
|
|
echo "${red}FAIL${reset} $name"
|
|
echo " expected: [$expected]"; echo " got: [$out]"; FAIL=$((FAIL+1))
|
|
fi
|
|
}
|
|
|
|
echo "${bold}Character form tests (engine: ^UUUU^ code points and the tables)${reset}"
|
|
echo "================================================================="
|
|
echo
|
|
|
|
check " 1. 4-digit BMP code point" $'A ☺ B' 'A ^263A^ B'
|
|
check " 2. 5-digit code point, whole" $'A \xf0\x93\x80\x80 B' 'A ^13000^ B'
|
|
check " 3. 5-digit emoji" $'\xf0\x9f\x98\x80' '^1F600^'
|
|
check " 4. 6-digit top of the range" $'\xf4\x8f\xbf\xbd' '^10FFFD^'
|
|
# An invalid code point is a located ERROR (2026-08-25, the
|
|
# no-warn-and-continue policy) -- it was in-band "Invalid Unicode" text in
|
|
# the document before, which is the wrong channel for a generator's typo.
|
|
check_err " 5. above U+10FFFF is an error" 'beyond U+10FFFF' '^FFFFFF^'
|
|
check_err " 6. surrogate is an error" 'a surrogate' '^D800^'
|
|
check_err " 6a. noncharacter is an error" 'a noncharacter' '^FDD0^'
|
|
check " 7. mnemonic undisturbed" 'ß' '^s^'
|
|
# The tables emit the PRECOMPOSED (NFC) character where one exists (since
|
|
# 2026-08-25; they formerly emitted base + combining mark) -- the same
|
|
# bytes an editor's direct entry produces, so ^a" and a typed ä compare
|
|
# equal in search and copy-paste. The literal ä below is precomposed.
|
|
check " 8. diacritic is the precomposed character" 'Mädchen' 'M^a"dchen'
|
|
check " 9. code point beside quoted punctuation" $'☺ --' '^263A^ ^-^-'
|
|
|
|
echo
|
|
echo "-- sequences: ^UUUU UUUU ...^ (one caret pair, tokens 4-6 hex digits,"
|
|
echo " no space at the elbows, one line; process_unicode_sequences)"
|
|
check "10. CJK sequence" $'X \xe9\x81\x93\xe5\x8f\xaf\xe9\x81\x93\xe9\x9d\x9e\xe5\xb8\xb8\xe9\x81\x93 Y' 'X ^9053 53EF 9053 975E 5E38 9053^ Y'
|
|
check "11. mixed widths" $'\xe2\x98\xba\xf0\x93\x80\x80\xf0\x9f\x98\x80' '^263A 13000 1F600^'
|
|
check "12. space after opening caret: not a sequence" 'A ^ 263A 263A^ B' 'A ^ 263A 263A^ B'
|
|
check "13. space before closing caret: not a sequence" 'A ^263A 263A ^ B' 'A ^263A 263A ^ B'
|
|
check "14. newline aborts (one line only)" $'A ^263A\n263A^ B' $'A ^263A\n263A^ B'
|
|
check "15. short token aborts whole sequence" 'A ^263A 263^ B' 'A ^263A 263^ B'
|
|
check "16. @eval code keeps its carets" '11' '@eval len("^263A 263A^") @'
|
|
check "17. literal span: sequence stays verbatim" 'x ^263A 263A^ y' "^'x ^263A 263A^ y'^"
|
|
check "17a. single-character token stands for itself" $'\xe9\x81\x93x\xe5\x8f\xaf' '^9053 x 53EF^'
|
|
check "17b. non-alnum single token behaves as quoted" '☺-☺' '^263A - 263A^'
|
|
check "17c. structural code points are re-read-safe" '@@' '^0040 0040^'
|
|
check_err "17d. invalid code point in a CLOSED sequence" 'in a ^...^ sequence' '^263A FFFFFF^'
|
|
check "17e. unclosed hex-like prose is untouched" 'about ^263A FFFFFF prose' 'about ^263A FFFFFF prose'
|
|
|
|
echo
|
|
echo "-- direct entry and NFC convergence"
|
|
check "18. pinyin tone is precomposed" 'ā' '^a1'
|
|
check "19. typed UTF-8 passes through byte-identical" 'Mädchen naïveté ü — done' 'Mädchen naïveté ü — done'
|
|
|
|
echo
|
|
echo "================================================================="
|
|
echo "${bold}Results: $PASS passed, $FAIL failed${reset}"
|
|
rm -f "$ERR"
|
|
[ $FAIL -eq 0 ] && exit 0 || exit 1
|