Andy Kopra
37b6ba1c4f
Typographic transforms (---, quote pairs, ~) no longer touch verbatim text: @c/@code/@source_listing content and ^'...'^ spans show exactly the characters written. "^" before any punctuation character quotes it in every target (the apostrophe excepted: ^' opens a literal span), with the new :resolve option on @@@target declaring per-target renderings. The ^UUUU^ code-point form accepts 4-6 hex digits, the full Unicode range. The html output and transform spellings are polyglot (XML-valid), in preparation for an EPUB target. New suites: transform_test, character_test (engine), typography_test (SKS). (from dev 07ce5ea86a0a)
249 lines
9.4 KiB
Bash
Executable File
249 lines
9.4 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# eval_test.sh — the @eval primitive's contract with the outside world.
|
|
#
|
|
# @eval is the one primitive that reaches OUT of Klammertext, and until
|
|
# 2026-08-15 nothing tested what it did with what came back. ":shell" is
|
|
# covered here; the other modes (Python, :cpp, :haskell) are exercised
|
|
# incidentally by other suites and can grow into this one.
|
|
#
|
|
# The two defects this suite exists to hold shut, both of them the same shape
|
|
# as a msg() on the wrong stream -- output nobody chose to see, and a failure
|
|
# nobody was told about:
|
|
#
|
|
# * The command's STDERR went straight to the user's terminal, unattributed
|
|
# and unsuppressable. It is not the command's output in any of the three
|
|
# policy categories (CLAUDE.md, "Command output policy"): it belongs to a
|
|
# subprocess a klammer invoked, at a location the Locator can name. It is
|
|
# now captured and reported at "-v 1".
|
|
# * The EXIT STATUS was discarded, so a command that failed contributed its
|
|
# partial output (or nothing) to the document and said nothing at all.
|
|
#
|
|
# NOT to be confused with "eval_test", the C++ diagnostic program built from
|
|
# eval_test.cpp in this directory: that one constructs engine objects and prints
|
|
# what it gets, for a person to read, and asserts nothing (see the `smoke`
|
|
# target in tst/Makefile). This is the regression suite. The ".sh" is the only
|
|
# thing distinguishing them, and it is the first such collision in tst/.
|
|
#
|
|
# Usage: ./eval_test.sh (needs KLAMMERTEXT_HOME set; ktext on PATH)
|
|
# Exit code: 0 if all tests pass, 1 otherwise.
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
KTEXT=ktext
|
|
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
|
|
|
|
red=$'\033[31m'
|
|
green=$'\033[32m'
|
|
bold=$'\033[1m'
|
|
reset=$'\033[0m'
|
|
|
|
OUTF=$(mktemp /tmp/eval_out.XXXXXX)
|
|
ERRF=$(mktemp /tmp/eval_err.XXXXXX)
|
|
trap 'rm -f "$OUTF" "$ERRF"' EXIT
|
|
|
|
plain() { sed 's/\x1b\[[0-9;]*m//g'; }
|
|
|
|
# run SOURCE [EXTRA...] — sets STATUS, and fills OUTF/ERRF.
|
|
STATUS=0
|
|
run() {
|
|
local src="$1"; shift
|
|
"$KTEXT" --klammersets none -s "$src" -d "$@" >"$OUTF" 2>"$ERRF"
|
|
STATUS=$?
|
|
}
|
|
|
|
pass() { echo "${green}PASS${reset} $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "${red}FAIL${reset} $1"; [ -n "$2" ] && echo " $2"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "${bold}@eval tests${reset}"
|
|
echo "==========="
|
|
echo
|
|
|
|
echo "-- :shell, the ordinary case --"
|
|
run '@eval :shell echo hello @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "hello" ]; then
|
|
pass " 1. the command's stdout becomes document text"
|
|
else
|
|
fail " 1. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- stderr belongs to the command, not to the terminal --"
|
|
# A signal death is not an error exit, and a crash must never read as a pass:
|
|
# every case here checks the status numerically.
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "OUT" ]; then
|
|
pass " 2. stdout is the document; stderr is not in it"
|
|
else
|
|
fail " 2. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
if [ ! -s "$ERRF" ]; then
|
|
pass " 3. ... and nothing leaks to the terminal at the default verbosity"
|
|
else
|
|
fail " 3. stderr leaked: $(head -1 "$ERRF")"
|
|
fi
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @' -v 1
|
|
if grep -q "NOISE" "$ERRF"; then
|
|
pass " 4. ... while -v 1 reports what the command said"
|
|
else
|
|
fail " 4. -v 1 did not report the command's stderr" "$(head -2 "$ERRF")"
|
|
fi
|
|
if grep -qi "stderr" "$ERRF"; then
|
|
pass " 5. ... and says that is what it is"
|
|
else
|
|
fail " 5. the -v 1 report does not identify the stream"
|
|
fi
|
|
|
|
echo
|
|
echo "-- a failing command is an error, not silence --"
|
|
run '@eval :shell exit 3 @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass " 6. a nonzero exit status fails the run"
|
|
else
|
|
fail " 6. exit $STATUS (128+ would be a signal death, 0 a silent pass)"
|
|
fi
|
|
for want in "exit status 3" "The shell command failed"; do
|
|
if grep -qF "$want" "$ERRF"; then
|
|
pass " 7. the error says [$want]"
|
|
else
|
|
fail " 7. the error does not say [$want]" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
done
|
|
# What the command itself reported is the useful half of the diagnosis.
|
|
run '@eval :shell echo WHY-IT-FAILED >&2; exit 1 @'
|
|
if grep -qF "WHY-IT-FAILED" "$ERRF"; then
|
|
pass " 8. ... and includes what the command wrote to stderr"
|
|
else
|
|
fail " 8. the command's own message was dropped" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# An error is category 2: stderr, and nothing on stdout to confuse a pipe.
|
|
if [ ! -s "$OUTF" ]; then
|
|
pass " 9. ... and leaves stdout empty"
|
|
else
|
|
fail " 9. stdout carried [$(head -c 60 "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the escape hatch, because some commands exit nonzero on purpose --"
|
|
# "grep" finding no match is the usual one. Strictness with an explicit way to
|
|
# say "I meant that" is the same shape as the @cond predicate rule.
|
|
run '@eval :shell exit 3 @'
|
|
if grep -qF "|| true" "$ERRF"; then
|
|
pass "10. the error names the way to say a nonzero status was intended"
|
|
else
|
|
fail "10. the error does not offer the remedy" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# NOT "exit N || true": exit terminates the shell before "||" is reached, so
|
|
# that spelling cannot work and is not what the message suggests. A command
|
|
# that merely RETURNS nonzero is the case the remedy is for.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null || true @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "kept" ]; then
|
|
pass "11. ... and it works"
|
|
else
|
|
fail "11. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
# The same command without the remedy is an error, or case 11 proves nothing.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass "11a. ... and without it the same command fails"
|
|
else
|
|
fail "11a. exit $STATUS — expected a nonzero, non-signal exit"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the command may contain its own pipeline --"
|
|
# The redirection that captures stderr must not disturb the writer's command.
|
|
run '@eval :shell echo one two three | tr " " "-" @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "one-two-three" ]; then
|
|
pass "12. a pipeline inside the command still works"
|
|
else
|
|
fail "12. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run '@eval :shell echo a > /dev/null; echo b @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "b" ]; then
|
|
pass "13. ... and so does a redirection of its own"
|
|
else
|
|
fail "13. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the other modes still work --"
|
|
run '@eval 6*7 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "42" ]; then
|
|
pass "14. a Python expression"
|
|
else
|
|
fail "14. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- :haskell, the same contract --"
|
|
# Skipped where GHC is absent: runghc is an optional dependency (the
|
|
# akopra/klammertext:haskell image, or a local GHCup install), and a suite that
|
|
# fails for its absence would be reporting the machine, not the code.
|
|
if ! command -v runghc >/dev/null 2>&1; then
|
|
echo "SKIP 15-18. :haskell (runghc not installed)"
|
|
else
|
|
# The defect: runghc ran with "2>&1", so on a SUCCESSFUL run everything the
|
|
# program or GHC wrote to stderr was merged into the result and became part
|
|
# of the document.
|
|
HS='@eval :haskell import System.IO
|
|
main = hPutStrLn stderr "HS-NOISE" >> putStrLn "HS-OUT" @'
|
|
run "$HS"
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "HS-OUT" ]; then
|
|
pass "15. the program's stdout is the document; its stderr is not"
|
|
else
|
|
fail "15. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run "$HS" -v 1
|
|
if grep -q "HS-NOISE" "$ERRF"; then
|
|
pass "16. ... and -v 1 reports what it wrote to stderr"
|
|
else
|
|
fail "16. -v 1 did not report it" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# A compile error was ALREADY reported rather than swallowed -- the exit
|
|
# status was checked -- so this pins behaviour that was right, and that the
|
|
# detail now comes from the captured stderr rather than a merged stream.
|
|
run '@eval :haskell main = putStrLn (1 + "x") @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ] && grep -qi "error" "$ERRF"; then
|
|
pass "17. a compile error fails the run and shows what runghc said"
|
|
else
|
|
fail "17. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# A program that compiles, runs, and then exits nonzero is the other half.
|
|
run '@eval :haskell import System.Exit
|
|
main = putStrLn "partial" >> exitWith (ExitFailure 3) @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "exit status 3" "$ERRF"; then
|
|
pass "18. a nonzero exit from the program itself is reported too"
|
|
else
|
|
fail "18. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "-- the environment can be missing: the errors name what and how --"
|
|
# Added 2026-08-22 by the error-gallery coverage review: neither message had
|
|
# ever been printed by a test.
|
|
run '@eval :cpp nonexistent_gallery_lib @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "Cannot open library" "$ERRF"; then
|
|
pass "19. :cpp names the library it could not open"
|
|
else
|
|
fail "19. exit $STATUS" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# :haskell without runghc: strip PATH so the case is deterministic whether or
|
|
# not GHC is installed; ktext itself is invoked by absolute path (which works
|
|
# since the 2026-08-21 construct_command_pathname fix).
|
|
env PATH=/nonexistent "$K/bin/ktext" --klammersets none \
|
|
-s '@eval :haskell main = putStrLn "x" @' -d >"$OUTF" 2>"$ERRF"
|
|
hstatus=$?
|
|
if [ $hstatus -ne 0 ] && tr '\n' ' ' < "$ERRF" | grep -qF "requires runghc"; then
|
|
pass "20. :haskell without runghc says what to install"
|
|
else
|
|
fail "20. exit $hstatus" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
|
|
echo
|
|
echo "==========="
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
|
exit 0
|