diff --git a/README.md b/README.md index 3c2df5c..7cf2638 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ are regenerated on each release — patches cannot be merged directly. Report problems (or send patches) to the author; accepted changes are applied to the development tree and appear in a following snapshot. -This snapshot was assembled from development commit `1ce461b48a7d`. +This snapshot was assembled from development commit `1083db946f05`. ## License diff --git a/doc/install/windows_source_install.md b/doc/install/windows_source_install.md index b7ad9dc..9dce94d 100644 --- a/doc/install/windows_source_install.md +++ b/doc/install/windows_source_install.md @@ -28,6 +28,24 @@ special rights. Allow an hour or two the first time: the downloads are about a gigabyte in all, TeX Live alone takes twenty to thirty minutes, and the build a few minutes. Nothing here needs to be done twice. +**Smart App Control must be off.** A clean installation of Windows 11 +runs Smart App Control, a security feature that blocks any program +Microsoft has not seen signed and widespread — and the MSYS2 compiler, +`make` and the `bash` shell on which this page depends are not signed, so +with it on the MSYS2 shell does not open and the build cannot start +("An Application Control policy has blocked this file"). Check it first: +Windows Security ￫ **App & browser control** ￫ **Smart App Control +settings**. It is in one of three states. *Off* is what you need. *On* +blocks MSYS2; switch it off. *Evaluation* lets everything run for now, +but Windows turns it on by itself when it decides the machine is used +"safely", at which point a working installation stops without warning — +so switch it off at once rather than wait. Switching it off cannot be +undone short of reinstalling Windows, which is why Microsoft asks you to +confirm; the rest of Windows Security (Defender, SmartScreen for +downloads) stays on and is not affected. A machine upgraded from Windows +10, or one on which an unsigned program has already been installed, has +it off already. + ## Prerequisites @@ -403,6 +421,11 @@ it (a new UCRT64 shell does, once the line is in `~/.bashrc`) and run `MSYS2_PATH_TYPE` is not set to `inherit`, or the shell was opened before you set it. Set it and open a new UCRT64 shell. +**"An Application Control policy has blocked this file", or the MSYS2 +shell does not open.** Smart App Control is on; see *Before you start*. +It can turn itself on after the installation, in which case the +installation is intact and only the setting needs changing. + **The build is slow.** Windows Defender scans every file the compiler writes. Excluding the repository folder from real-time scanning speeds the build up considerably; it is not required. diff --git a/mac/argument_set.cpp b/mac/argument_set.cpp index 5411e16..3d0c7e6 100644 --- a/mac/argument_set.cpp +++ b/mac/argument_set.cpp @@ -225,6 +225,17 @@ katom_list trim_part(katom_list part) return trim(part, {katom_t::space, katom_t::newline, katom_t::bar}); } +// The trailing half of trim_part(): the leading side of a rest argument is +// decided bar by bar in argument_split(). +static katom_list trim_trailing_part(katom_list part) +{ + const std::set trim_types {katom_t::space, katom_t::newline, katom_t::bar}; + while (!part.empty() && trim_types.contains(part.back().m_type)) { + part.pop_back(); + } + return part; +} + std::tuple parameter_split(katom_list::const_iterator kbegin, katom_list::const_iterator kend) { @@ -330,24 +341,51 @@ argument_split(katom_list::const_iterator kbegin, katom_list::const_iterator ken katom_lists optional {}; katom_list rest {}; - for (auto p : parts) { + // function_symbol_parts() opens the first part with a bar of its own + // when the arguments do not begin with an option. Without positional + // parameters that bar is the first katom of the rest, and it is not + // the writer's. + bool rest_opens_synthetic = false; + for (std::size_t i = 0; i < parts.size(); i++) { + const auto& p = parts[i]; if (p[0].m_type == katom_t::option_name) { optional.push_back(p); } else if (positional.size() < positional_limit) { positional.push_back(trim_part(p)); } else { - // Parts were trimmed, so adjacent parts would abut their bar - // katoms (a row separator "||" next to an empty cell's "|" - // would serialize as "|||"). A space keeps the writer's - // bar/double-bar distinction parseable. - if (!rest.empty()) { + if (rest.empty()) { + rest_opens_synthetic = (i == 0) && positional_limit == 0; + } else { + // Parts were trimmed, so adjacent parts would abut their bar + // katoms (a row separator "||" next to an empty cell's "|" + // would serialize as "|||"). A space keeps the writer's + // bar/double-bar distinction parseable. rest.push_back(Katom(" ", katom_t::space, p[0].m_loc)); } rest.insert(rest.end(), p.begin(), p.end()); } } - return {positional, optional, trim_part(rest)}; -} + + // Exactly ONE written leading bar is stripped from the rest (decided + // 2026-10-01): it is the bar that separates the rest from what + // precedes it -- the last positional, the options, or the klammer name + // in the opening-bar idiom `@table |` with the rows on the lines below. + // Every further leading bar delimits an EMPTY FIRST ELEMENT the writer + // meant (`@table | | M | F ||` has an empty corner cell), mirroring + // kutil.rest_split, which strips one trailing delimiter and keeps all + // other empty elements. Trailing bars and whitespace are trimmed as + // before. + rest = trim_whitespace(rest); + if (rest_opens_synthetic && !rest.empty() && rest[0].m_type == katom_t::bar) { + rest.erase(rest.begin()); + rest = trim_whitespace(rest); + } + if (!rest.empty() && rest[0].m_type == katom_t::bar && rest[0].m_text == "|") { + rest.erase(rest.begin()); + rest = trim_whitespace(rest); + } + return {positional, optional, trim_trailing_part(rest)}; +} // Parameter/argument mapping diff --git a/mac/katom_process.cpp b/mac/katom_process.cpp index 88bb383..0f841c9 100644 --- a/mac/katom_process.cpp +++ b/mac/katom_process.cpp @@ -554,13 +554,16 @@ void mark_ignored_katoms(katom_list& katoms) //auto is_ignore_begin = [](const Katom& k) { return k.m_type == katom_t::ignore_begin; }; //auto is_ignore_end = [](const Katom& k) { return k.m_type == katom_t::ignore_end; }; for (auto [op,cl] : find_spans(katoms, begin_ignore, end_ignore, true, "ignored")) { + // find_span_katoms() returns a half-open range: `end` is the katom + // AFTER the closing "]#", as every other caller reads it. This one + // read it as inclusive until 2026-10-02 and so removed the katom + // abutting the close: "#[...]#." lost its period, "#[...]#x" its x. + // A following space or newline is removed by design (the loop + // below, and here), which is what hid it. auto [begin, end] = find_span_katoms(katoms, op, cl); - // std::for_each(begin, end + 1, [](Katom& k) { k.m_type = katom_t::ignored; }); - std::for_each(begin, end + 1, mark_as_ignored); - //if (end + 1 != katoms.end() && (end+1)->m_type == katom_t::newline) { - if (end + 1 != katoms.end() && is_newline(*(end + 1))) { - //(end+1)->m_type = katom_t::ignored; - mark_as_ignored(*(end + 1)); + std::for_each(begin, end, mark_as_ignored); + if (end != katoms.end() && is_newline(*end)) { + mark_as_ignored(*end); } } bool ignore_line = false; diff --git a/sks/tns/md_to_sks.py b/sks/tns/md_to_sks.py index 43c2680..67cdfb7 100644 --- a/sks/tns/md_to_sks.py +++ b/sks/tns/md_to_sks.py @@ -17,15 +17,25 @@ grep. A file with no markers is inside the convertible subset. First run: the Rectify user guide (1,805 lines), 2026-08-07. It rendered to html and pdf without error or warning, with 99 markers -- 91 of them internal -anchor links, the one gap that appears at volume. +anchor links, the one gap that appears at volume. Second run: three German +vocabulary lists (4,200 lines, 152 tables), 2026-09-30, which rendered +without an error and with two silently wrong results -- an emphasis that +wrapped and a table's empty header corner -- so "renders clean" is not +"converted"; the fixes are in the spec's "What the second conversion +taught". -Usage: python3 md_to_sks.py +Usage: python3 md_to_sks.py [sibling.md ...] + +The siblings are Markdown files converted alongside this one: a link to +one of them becomes a link to its html page (..//index.html, where +ktext writes it); a link to any other .md file is a gap. """ import re, sys, os, datetime from collections import Counter SPECIALS = "^@#|*" # ^ first: the other quotings introduce ^ marks = Counter() +SIBLINGS = set() # stems of the .md files converted together def mark(kind, text): """A #[MD ... ]# marker. @@ -56,7 +66,9 @@ def quote(s): CODE = re.compile(r'`([^`\n]+)`') LINK = re.compile(r'(?' '@@v.* : @@#-@v@' +echo +echo "-- what abuts a closing ]# --" +# A removal block ends at its "]#"; what follows is the writer's. Until +# 2026-10-02 mark_ignored_katoms() read find_span_katoms()'s half-open end +# as inclusive and removed the katom abutting the close as well: a sentence +# ending "#[ note ]#." lost its period. A following space or newline is +# removed by design, which is what hid it. +check "28. punctuation after ]# survives" 'a . c' 'a #[gone]#. c' +check "29. a letter after ]# survives" 'a x c' 'a #[gone]#x c' +check "30. a space after ]# is removed" 'a . c' 'a #[gone]# . c' +check "31. nested blocks, punctuation after" 'a . c' 'a #[ #[in]# out ]#. c' + echo echo "=================================================================" echo "${bold}Results: $PASS passed, $FAIL failed${reset}" diff --git a/tst/rest_test.sh b/tst/rest_test.sh new file mode 100755 index 0000000..279b9e6 --- /dev/null +++ b/tst/rest_test.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# rest_test.sh — Regression tests for the bar structure of a rest argument. +# +# A rest argument collects everything after the declared positional +# arguments, bars included, and hands the structure on; a klammer reads the +# bar count as dimension (| elements, || rows). The one thing the ENGINE +# decides is where the structure begins and ends: +# +# * exactly ONE written leading bar is stripped — the bar that separates +# the rest from what precedes it: the last positional argument, the +# options, or the klammer name in the opening-bar idiom (`@table |` with +# the rows on the lines below). Every further leading bar delimits an +# EMPTY FIRST ELEMENT the writer meant: `@table | | M | F ||` has an +# empty corner cell. Decided 2026-10-01; before it every leading bar +# was trimmed, so an empty first element could not be written at all, +# and a Markdown table's empty header corner shifted the header row one +# column left. +# * trailing bars and whitespace are trimmed (the customary dangling +# separator before a closing @), as before. +# * an empty element between two bars is kept wherever it stands. +# +# The rest(N) cast (kutil.rest_split) is SKS Python and is tested at that +# tier; here the value is read back raw through *name* substitution. +# +# Engine tier: --klammersets none, fixtures inline. +# +# Usage: ./rest_test.sh (LSan suppressions come from env/runtime.env) +# Exit code: 0 if all tests pass, 1 otherwise. + +PASS=0 +FAIL=0 +KTEXT=ktext +K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set} + +red=$'\033[31m' +green=$'\033[32m' +reset=$'\033[0m' + +trim() { awk '{ sub(/[ \t\r]+$/, "") } { line[NR]=$0 } END { f=1; while (f<=NR && line[f]=="") f++; l=NR; while (l>=1 && line[l]=="") l--; for (i=f;i<=l;i++) print line[i] }'; } + +# check_eq TEST_NAME EXPECTED INPUT +# The fixtures: @zz takes only a rest, @yy one positional then a rest. +# Each prints its arguments in brackets, so the bar structure is visible. +check_eq() { + local test_name="$1" expected="$2" input="$3" + local output status + output=$("$KTEXT" --klammersets none -d -s "@@zz items.rest :x 0 : [*items*] @@ @@yy first | second.rest : [*first*][*second*] @@ $input" 2>/tmp/rest_test_err.$$) + status=$? + output=$(printf '%s' "$output" | trim) + if [ $status -ne 0 ]; then + echo "${red}FAIL${reset} $test_name — ktext exited $status" + echo " stderr: $(head -3 /tmp/rest_test_err.$$)" + FAIL=$((FAIL + 1)) + return + fi + if [ "$output" = "$expected" ]; then + echo "${green}PASS${reset} $test_name" + PASS=$((PASS + 1)) + else + echo "${red}FAIL${reset} $test_name" + echo " expected: [$expected]" + echo " got: [$output]" + FAIL=$((FAIL + 1)) + fi +} + +echo "-- the leading bar --" +check_eq " 1. no leading bar: the elements as written" '[M | F]' '@zz M | F @' +check_eq " 2. the opening-bar idiom: one bar stripped" '[M | F]' '@zz | M | F @' +check_eq " 3. a second leading bar is an empty first element" '[| M | F]' '@zz | | M | F @' +check_eq " 4. the idiom with the rows below, empty corner cell" '[| M | F || +Nom | der]' '@zz | + | M | F || +Nom | der @' +check_eq " 5. options first, then the idiom bar" '[M | F]' '@zz :x 1 | M | F @' +check_eq " 6. options first, then an empty first element" '[| M | F]' '@zz :x 1 | | M | F @' + +echo +echo "-- after a positional argument --" +check_eq " 7. the bar after the positional is the separator" '[a][b | c]' '@yy a | b | c @' +check_eq " 8. a second bar there is an empty first element" '[a][| b | c]' '@yy a | | b | c @' + +echo +echo "-- the middle and the end --" +check_eq " 9. an empty element between bars is kept" '[M | | F]' '@zz M | | F @' +check_eq "10. one trailing bar is trimmed" '[M | F]' '@zz M | F | @' +check_eq "11. a row separator is never the stripped bar" '[|| M | F]' '@zz || M | F @' + +echo +echo "================================" +echo "Passed: $PASS Failed: $FAIL" +rm -f /tmp/rest_test_err.$$ +[ $FAIL -eq 0 ]