What a Windows editor saves, and a standard user's Windows install (from dev 9d6f66456a3d)
- CRLF line endings are read as newlines on every system (the reader used to delete the pair, collapsing a Notepad++ file into one line on Linux and macOS); a leading UTF-8 byte-order mark is ignored; a UTF-16 or Windows-1252 source is refused with the encoding, the place and the remedy named. .gitattributes pins LF in every clone. - Windows source install: the repository path must hold no space (GNU Make), accented letters are fine (every linker input is relative now); python.org's Python is selected ahead of an MSYS2 one that poppler pulls in; TeX Live goes to its Windows default C:\texlive\2026. - The Python binding needs an ASCII installation path on Windows, and its suite says so there. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
fc2e848e91
commit
715d939727
15 files changed
+236
-41
No files matched your search
+75
-1
@@ -325,7 +325,20 @@ std::string string_from_file(const std::string& pathname, bool strip_surrounding
|
||||
} else {
|
||||
result = buffer.str();
|
||||
|
||||
result = string_replace(result, "\r\n", ""); // Urgh.
|
||||
// A leading UTF-8 byte-order mark is not text: Windows tools
|
||||
// (PowerShell's Set-Content, Notepad's "UTF-8 with BOM") put
|
||||
// one at the start of a file, and without this a .kt beginning
|
||||
// "@document" was reported as "Word not parsed" at 1:1, the
|
||||
// offending word invisible (decided 2026-09-26).
|
||||
if (result.compare(0, 3, "\xEF\xBB\xBF") == 0)
|
||||
result.erase(0, 3);
|
||||
|
||||
// Windows line endings become newlines. (This line DELETED the
|
||||
// pair until 2026-09-26, so on Linux and macOS a file written by
|
||||
// Notepad++ or a DOS-mode Emacs collapsed into one line and its
|
||||
// spans no longer matched; on Windows the C runtime's text mode
|
||||
// had already translated the pair, which hid the bug there.)
|
||||
result = string_replace(result, "\r\n", "\n");
|
||||
|
||||
|
||||
if (std::regex_match(pathname, klammertext_filename_re)) {
|
||||
@@ -575,6 +588,67 @@ void write_to_cache(const std::string& cache_dir, const std::string& basename, c
|
||||
string_to_file(output_filename, text);
|
||||
}
|
||||
|
||||
// The offset of the first byte that is not part of well-formed UTF-8, or
|
||||
// npos. Overlong forms, surrogates and code points above 10FFFF are rejected
|
||||
// like a decoder would (RFC 3629, table 3-7 of the Unicode standard).
|
||||
static std::size_t first_invalid_utf8(const std::string& s)
|
||||
{
|
||||
const auto n = s.size();
|
||||
for (std::size_t i = 0; i < n;) {
|
||||
const unsigned char b = static_cast<unsigned char>(s[i]);
|
||||
std::size_t len; unsigned char lo = 0x80, hi = 0xBF;
|
||||
if (b < 0x80) { i += 1; continue; }
|
||||
else if (b >= 0xC2 && b <= 0xDF) len = 2;
|
||||
else if (b == 0xE0) { len = 3; lo = 0xA0; }
|
||||
else if (b >= 0xE1 && b <= 0xEC) len = 3;
|
||||
else if (b == 0xED) { len = 3; hi = 0x9F; }
|
||||
else if (b >= 0xEE && b <= 0xEF) len = 3;
|
||||
else if (b == 0xF0) { len = 4; lo = 0x90; }
|
||||
else if (b >= 0xF1 && b <= 0xF3) len = 4;
|
||||
else if (b == 0xF4) { len = 4; hi = 0x8F; }
|
||||
else return i;
|
||||
if (i + len > n) return i;
|
||||
for (std::size_t k = 1; k < len; ++k) {
|
||||
const unsigned char c = static_cast<unsigned char>(s[i + k]);
|
||||
const unsigned char l = (k == 1) ? lo : 0x80, h = (k == 1) ? hi : 0xBF;
|
||||
if (c < l || c > h) return i;
|
||||
}
|
||||
i += len;
|
||||
}
|
||||
return std::string::npos;
|
||||
}
|
||||
|
||||
std::string klammertext_from_file(const std::string& pathname)
|
||||
{
|
||||
std::string text = string_from_file(pathname);
|
||||
// A UTF-16 file (Notepad++ offers "UCS-2 LE BOM") begins with FF FE or
|
||||
// FE FF; read as bytes it is NUL-interleaved and every word is "not
|
||||
// parsed". Name the encoding and the remedy instead.
|
||||
if (text.size() >= 2 && ((static_cast<unsigned char>(text[0]) == 0xFF && static_cast<unsigned char>(text[1]) == 0xFE)
|
||||
|| (static_cast<unsigned char>(text[0]) == 0xFE && static_cast<unsigned char>(text[1]) == 0xFF))) {
|
||||
throw File_error("File \"" + pathname + "\" is UTF-16 (it begins with a byte-order mark), and Klammertext reads UTF-8. "
|
||||
"Save it as UTF-8 in your editor (Notepad++: Encoding, Convert to UTF-8) and run the command again.",
|
||||
Locator(fs::path(pathname), 1, 0)); // the Locator's character counts from 0
|
||||
}
|
||||
// A file saved as ANSI (Windows-1252, the default of some editors) is
|
||||
// not UTF-8 from its first accented letter on; the bytes otherwise
|
||||
// reach the embedded Python and fail there with a message about
|
||||
// something else entirely.
|
||||
if (const std::size_t bad = first_invalid_utf8(text); bad != std::string::npos) {
|
||||
int line = 1, chr = 1;
|
||||
for (std::size_t i = 0; i < bad; ++i) {
|
||||
if (text[i] == '\n') { ++line; chr = 1; }
|
||||
else if ((static_cast<unsigned char>(text[i]) & 0xC0) != 0x80) ++chr; // count characters, not bytes
|
||||
}
|
||||
char hex[8]; std::snprintf(hex, sizeof hex, "%02X", static_cast<unsigned char>(text[bad]));
|
||||
throw File_error("File \"" + pathname + "\" is not UTF-8 at line " + std::to_string(line) + ", character "
|
||||
+ std::to_string(chr) + " (byte " + hex + "), and Klammertext reads UTF-8. "
|
||||
"Save it as UTF-8 in your editor (Notepad++: Encoding, Convert to UTF-8) and run the command again.",
|
||||
Locator(fs::path(pathname), line, chr - 1));
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
std::string read_from_cache(const std::string& cache_dir, const std::string& basename)
|
||||
{
|
||||
std::string input_filename = cache_dir + "/" + basename;
|
||||
|
||||
Reference in new issue
Block a user