ack
fc2e848e91
Assembled from development commit d5f9c20b94b3. Since 5c29da5: a print() in
@eval, or in any Python a klammer set runs, is a log line -- on the
error stream under the command's name at -v 1, silent otherwise, never
in the document -- and K_log(level, text) is the builtin behind it; a
Python script using the binding keeps its own stdout. Previously such
output was lost, or on a terminal landed in the document.
307 lines
13 KiB
Bash
Executable File
307 lines
13 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
# eval_test.sh — the @eval primitive's contract with the outside world.
|
|
#
|
|
# @eval is the one primitive that reaches OUT of Klammertext, and until
|
|
# 2026-08-15 nothing tested what it did with what came back. ":shell" is
|
|
# covered here; the other modes (Python, :cpp, :haskell) are exercised
|
|
# incidentally by other suites and can grow into this one.
|
|
#
|
|
# The two defects this suite exists to hold shut, both of them the same shape
|
|
# as a msg() on the wrong stream -- output nobody chose to see, and a failure
|
|
# nobody was told about:
|
|
#
|
|
# * The command's STDERR went straight to the user's terminal, unattributed
|
|
# and unsuppressable. It is not the command's output in any of the three
|
|
# policy categories (CLAUDE.md, "Command output policy"): it belongs to a
|
|
# subprocess a klammer invoked, at a location the Locator can name. It is
|
|
# now captured and reported at "-v 1".
|
|
# * The EXIT STATUS was discarded, so a command that failed contributed its
|
|
# partial output (or nothing) to the document and said nothing at all.
|
|
#
|
|
# NOT to be confused with "eval_test", the C++ diagnostic program built from
|
|
# eval_test.cpp in this directory: that one constructs engine objects and prints
|
|
# what it gets, for a person to read, and asserts nothing (see the `smoke`
|
|
# target in tst/Makefile). This is the regression suite. The ".sh" is the only
|
|
# thing distinguishing them, and it is the first such collision in tst/.
|
|
#
|
|
# Usage: ./eval_test.sh (needs KLAMMERTEXT_HOME set; ktext on PATH)
|
|
# Exit code: 0 if all tests pass, 1 otherwise.
|
|
|
|
PASS=0
|
|
FAIL=0
|
|
KTEXT=ktext
|
|
K=${KLAMMERTEXT_HOME:?KLAMMERTEXT_HOME must be set}
|
|
|
|
red=$'\033[31m'
|
|
green=$'\033[32m'
|
|
bold=$'\033[1m'
|
|
reset=$'\033[0m'
|
|
|
|
OUTF=$(mktemp "${TMPDIR:-/tmp}"/eval_out.XXXXXX)
|
|
ERRF=$(mktemp "${TMPDIR:-/tmp}"/eval_err.XXXXXX)
|
|
trap 'rm -f "$OUTF" "$ERRF"' EXIT
|
|
|
|
plain() { sed 's/\x1b\[[0-9;]*m//g'; }
|
|
|
|
# run SOURCE [EXTRA...] — sets STATUS, and fills OUTF/ERRF.
|
|
STATUS=0
|
|
run() {
|
|
local src="$1"; shift
|
|
"$KTEXT" --klammersets none -s "$src" -d "$@" >"$OUTF" 2>"$ERRF"
|
|
STATUS=$?
|
|
}
|
|
|
|
pass() { echo "${green}PASS${reset} $1"; PASS=$((PASS + 1)); }
|
|
fail() { echo "${red}FAIL${reset} $1"; [ -n "$2" ] && echo " $2"; FAIL=$((FAIL + 1)); }
|
|
|
|
echo "${bold}@eval tests${reset}"
|
|
echo "==========="
|
|
echo
|
|
|
|
echo "-- :shell, the ordinary case --"
|
|
run '@eval :shell echo hello @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "hello" ]; then
|
|
pass " 1. the command's stdout becomes document text"
|
|
else
|
|
fail " 1. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- stderr belongs to the command, not to the terminal --"
|
|
# A signal death is not an error exit, and a crash must never read as a pass:
|
|
# every case here checks the status numerically.
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "OUT" ]; then
|
|
pass " 2. stdout is the document; stderr is not in it"
|
|
else
|
|
fail " 2. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
if [ ! -s "$ERRF" ]; then
|
|
pass " 3. ... and nothing leaks to the terminal at the default verbosity"
|
|
else
|
|
fail " 3. stderr leaked: $(head -1 "$ERRF")"
|
|
fi
|
|
run '@eval :shell echo OUT; echo NOISE >&2 @' -v 1
|
|
if grep -q "NOISE" "$ERRF"; then
|
|
pass " 4. ... while -v 1 reports what the command said"
|
|
else
|
|
fail " 4. -v 1 did not report the command's stderr" "$(head -2 "$ERRF")"
|
|
fi
|
|
if grep -qi "stderr" "$ERRF"; then
|
|
pass " 5. ... and says that is what it is"
|
|
else
|
|
fail " 5. the -v 1 report does not identify the stream"
|
|
fi
|
|
|
|
echo
|
|
echo "-- a failing command is an error, not silence --"
|
|
run '@eval :shell exit 3 @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass " 6. a nonzero exit status fails the run"
|
|
else
|
|
fail " 6. exit $STATUS (128+ would be a signal death, 0 a silent pass)"
|
|
fi
|
|
for want in "exit status 3" "The shell command failed"; do
|
|
if grep -qF "$want" "$ERRF"; then
|
|
pass " 7. the error says [$want]"
|
|
else
|
|
fail " 7. the error does not say [$want]" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
done
|
|
# What the command itself reported is the useful half of the diagnosis.
|
|
run '@eval :shell echo WHY-IT-FAILED >&2; exit 1 @'
|
|
if grep -qF "WHY-IT-FAILED" "$ERRF"; then
|
|
pass " 8. ... and includes what the command wrote to stderr"
|
|
else
|
|
fail " 8. the command's own message was dropped" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# An error is category 2: stderr, and nothing on stdout to confuse a pipe.
|
|
if [ ! -s "$OUTF" ]; then
|
|
pass " 9. ... and leaves stdout empty"
|
|
else
|
|
fail " 9. stdout carried [$(head -c 60 "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the escape hatch, because some commands exit nonzero on purpose --"
|
|
# "grep" finding no match is the usual one. Strictness with an explicit way to
|
|
# say "I meant that" is the same shape as the @cond predicate rule.
|
|
run '@eval :shell exit 3 @'
|
|
if grep -qF "|| true" "$ERRF"; then
|
|
pass "10. the error names the way to say a nonzero status was intended"
|
|
else
|
|
fail "10. the error does not offer the remedy" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# NOT "exit N || true": exit terminates the shell before "||" is reached, so
|
|
# that spelling cannot work and is not what the message suggests. A command
|
|
# that merely RETURNS nonzero is the case the remedy is for.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null || true @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "kept" ]; then
|
|
pass "11. ... and it works"
|
|
else
|
|
fail "11. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
# The same command without the remedy is an error, or case 11 proves nothing.
|
|
run '@eval :shell echo kept; grep -q zzz /dev/null @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ]; then
|
|
pass "11a. ... and without it the same command fails"
|
|
else
|
|
fail "11a. exit $STATUS — expected a nonzero, non-signal exit"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the command may contain its own pipeline --"
|
|
# The redirection that captures stderr must not disturb the writer's command.
|
|
run '@eval :shell echo one two three | tr " " "-" @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "one-two-three" ]; then
|
|
pass "12. a pipeline inside the command still works"
|
|
else
|
|
fail "12. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run '@eval :shell echo a > /dev/null; echo b @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "b" ]; then
|
|
pass "13. ... and so does a redirection of its own"
|
|
else
|
|
fail "13. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- the other modes still work --"
|
|
run '@eval 6*7 @'
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "42" ]; then
|
|
pass "14. a Python expression"
|
|
else
|
|
fail "14. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
|
|
echo
|
|
echo "-- :haskell, the same contract --"
|
|
# Skipped where GHC is absent: runghc is an optional dependency (the
|
|
# akopra/klammertext:haskell image, or a local GHCup install), and a suite that
|
|
# fails for its absence would be reporting the machine, not the code.
|
|
if ! command -v runghc >/dev/null 2>&1; then
|
|
echo "SKIP 15-18. :haskell (runghc not installed)"
|
|
else
|
|
# The defect: runghc ran with "2>&1", so on a SUCCESSFUL run everything the
|
|
# program or GHC wrote to stderr was merged into the result and became part
|
|
# of the document.
|
|
HS='@eval :haskell import System.IO
|
|
main = hPutStrLn stderr "HS-NOISE" >> putStrLn "HS-OUT" @'
|
|
run "$HS"
|
|
if [ $STATUS -eq 0 ] && [ "$(cat "$OUTF")" = "HS-OUT" ]; then
|
|
pass "15. the program's stdout is the document; its stderr is not"
|
|
else
|
|
fail "15. exit $STATUS, stdout [$(cat "$OUTF")]"
|
|
fi
|
|
run "$HS" -v 1
|
|
if grep -q "HS-NOISE" "$ERRF"; then
|
|
pass "16. ... and -v 1 reports what it wrote to stderr"
|
|
else
|
|
fail "16. -v 1 did not report it" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# A compile error was ALREADY reported rather than swallowed -- the exit
|
|
# status was checked -- so this pins behaviour that was right, and that the
|
|
# detail now comes from the captured stderr rather than a merged stream.
|
|
run '@eval :haskell main = putStrLn (1 + "x") @'
|
|
if [ $STATUS -ne 0 ] && [ $STATUS -lt 128 ] && grep -qi "error" "$ERRF"; then
|
|
pass "17. a compile error fails the run and shows what runghc said"
|
|
else
|
|
fail "17. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
# A program that compiles, runs, and then exits nonzero is the other half.
|
|
run '@eval :haskell import System.Exit
|
|
main = putStrLn "partial" >> exitWith (ExitFailure 3) @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "exit status 3" "$ERRF"; then
|
|
pass "18. a nonzero exit from the program itself is reported too"
|
|
else
|
|
fail "18. exit $STATUS" "$(plain < "$ERRF" | head -3)"
|
|
fi
|
|
fi
|
|
|
|
echo
|
|
echo "-- the environment can be missing: the errors name what and how --"
|
|
# Added 2026-08-22 by the error-gallery coverage review: neither message had
|
|
# ever been printed by a test.
|
|
run '@eval :cpp nonexistent_gallery_lib @'
|
|
if [ $STATUS -ne 0 ] && grep -qF "Cannot open library" "$ERRF"; then
|
|
pass "19. :cpp names the library it could not open"
|
|
else
|
|
fail "19. exit $STATUS" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
# :haskell without runghc: strip PATH so the case is deterministic whether or
|
|
# not GHC is installed; ktext itself is invoked by absolute path (which works
|
|
# since the 2026-08-21 construct_command_pathname fix).
|
|
# A PATH without runghc. On Windows a program finds its DLLs through PATH,
|
|
# so there it keeps the directories of libklammertext.so and python312.dll
|
|
# -- neither holds a runghc, which is all the case needs.
|
|
NORUNGHC_PATH=/nonexistent
|
|
if command -v cygpath >/dev/null 2>&1; then
|
|
NORUNGHC_PATH="/nonexistent:$(cygpath -u "$K")/bin:$(dirname "$(command -v python)")"
|
|
fi
|
|
env PATH="$NORUNGHC_PATH" "$K/bin/ktext" --klammersets none \
|
|
-s '@eval :haskell main = putStrLn "x" @' -d >"$OUTF" 2>"$ERRF"
|
|
hstatus=$?
|
|
if [ $hstatus -ne 0 ] && tr '\n' ' ' < "$ERRF" | grep -qF "requires runghc"; then
|
|
pass "20. :haskell without runghc says what to install"
|
|
else
|
|
fail "20. exit $hstatus" "$(plain < "$ERRF" | head -2)"
|
|
fi
|
|
|
|
# An error raised INSIDE a klammer's body (here an @eval that divides by
|
|
# zero) is reported at the application in the user's document -- the
|
|
# headline -- with the chain of applications and the raising place below
|
|
# (2026-09-25; before, the headline was the @eval's own line, which for an
|
|
# SKS klammer is a line of a .k file the user never wrote). Wrap-insensitive
|
|
# greps: the report is justified to 80 columns.
|
|
CTX=$(mktemp -d "${TMPDIR:-/tmp}/eval_ctx.XXXXXX")
|
|
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@bad 0 @\n' > "$CTX/ctx.kt"
|
|
printf '@@bad x : @eval 1/*x* @ @@\n@@outer y : @bad *y* @ @@\ntext\n@outer 0 @\n' > "$CTX/nested.kt"
|
|
"$K/bin/ktext" "$CTX/ctx.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
|
|
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
|
|
if [ $cstatus -ne 0 ] && case "$ctext" in *"ctx.kt, line 4, character 1:"*"applying @bad"*"at "*"ctx.kt, line 1, character "*"ZeroDivisionError"*) true;; *) false;; esac; then
|
|
pass "21. an error inside a klammer body is headlined at the application, with the chain"
|
|
else
|
|
fail "21. exit $cstatus" "$(plain < "$ERRF" | head -6 | tr '\n' '|')"
|
|
fi
|
|
"$K/bin/ktext" "$CTX/nested.kt" --klammersets none -d >"$OUTF" 2>"$ERRF"; cstatus=$?
|
|
ctext=$(tr '\n' ' ' < "$ERRF" | tr -s ' ')
|
|
if [ $cstatus -ne 0 ] && case "$ctext" in *"nested.kt, line 4, character 1:"*"applying @outer"*"applying @bad at "*"nested.kt, line 2, character "*"at "*"nested.kt, line 1, character "*) true;; *) false;; esac; then
|
|
pass "22. ... nested applications are listed outermost first, the raising place last"
|
|
else
|
|
fail "22. exit $cstatus" "$(plain < "$ERRF" | head -7 | tr '\n' '|')"
|
|
fi
|
|
rm -rf "$CTX"
|
|
|
|
# A klammer set's Python speaks through the log (2026-09-25, TODO 69): a
|
|
# print() -- embedded Python's stdout, which was never flushed and so never
|
|
# seen, or on a terminal landed in the document -- goes to stderr at -v 1
|
|
# under the command's name, and never into the result; K_log(level, text)
|
|
# is the builtin behind it.
|
|
"$K/bin/ktext" --klammersets none -s '@eval print("from print"); K_log(2, "from K_log"); 42 @' -d >"$OUTF" 2>"$ERRF"
|
|
if [ "$(cat "$OUTF")" = "42" ] && [ ! -s "$ERRF" ]; then
|
|
pass "23. at -v 0 a print() in @eval is silent and the result is the result alone"
|
|
else
|
|
fail "23. stdout [$(cat "$OUTF")], stderr [$(plain < "$ERRF" | head -2 | tr '\n' '|')]"
|
|
fi
|
|
"$K/bin/ktext" --klammersets none -s '@eval print("from print"); K_log(2, "from K_log"); 42 @' -d -v 1 >"$OUTF" 2>"$ERRF"
|
|
# A log line is "<prefix>: <text> " (K::log ends with a space); the -v 1
|
|
# argument dump also echoes the -s text, so the checks want the log form.
|
|
if [ "$(cat "$OUTF")" = "42" ] && grep -qE 'ktext(\.exe)?: from print $' "$ERRF" && ! grep -qE ': from K_log $' "$ERRF"; then
|
|
pass "24. at -v 1 the print() is a log line on stderr; a level-2 K_log is not"
|
|
else
|
|
fail "24. stdout [$(cat "$OUTF")], stderr [$(plain < "$ERRF" | grep -v '^ ' | tr '\n' '|')]"
|
|
fi
|
|
"$K/bin/ktext" --klammersets none -s '@eval K_log(2, "from K_log"); 42 @' -d -v 2 >"$OUTF" 2>"$ERRF"
|
|
if [ "$(cat "$OUTF")" = "42" ] && grep -qE ': from K_log $' "$ERRF"; then
|
|
pass "25. at -v 2 the level-2 K_log line appears"
|
|
else
|
|
fail "25. stdout [$(cat "$OUTF")]" "$(plain < "$ERRF" | tail -3 | tr '\n' '|')"
|
|
fi
|
|
|
|
echo
|
|
echo "==========="
|
|
echo "Results: ${PASS} passed, ${FAIL} failed"
|
|
[ "$FAIL" -eq 0 ] || exit 1
|
|
exit 0
|