klammertext/mac/katom.cpp

#include <iostream>
#include <fstream>
#include <ranges>
#include <algorithm>

#include "katom.h"
#include "ktype.h"
#include "error.h"
#include "log.h"
#include "show.h"
#include "util.h"
#include "file.h"

bool dbg = false;

size_t Katom::index = 0;

Katom::Katom(const std::string& src, katom_t type, Locator loc)
    : m_index(Katom::index++)
    , m_text(src)
    , m_src(src)
    , m_loc(loc)
    , m_type(type)
    , m_initial_type(type)
{
    if (m_type == katom_t::special && m_text[0] == '^') {
        m_text.erase(0, 1);
    }
}

int active_count(const katom_list& katoms)
{
    return std::ranges::count_if(
        katoms,
        [] (const Katom& k) {
            return k.m_type != katom_t::replaced
                && k.m_type != katom_t::ignored
                && k.m_type != katom_t::space
                && k.m_type != katom_t::newline;
        });
}     

bool active(const katom_list& katoms)
{
    return active_count(katoms) > 0;
}

std::string expand_compound_katom(const std::string& s, std::regex rgx, const std::string& expanded)
{
    if (s.find('-') != std::string::npos) { // Hyphen shortcut for arguments that allow it
        const std::regex shortcut_rgx(R"((@\w+)(-\w+)+)");
        std::smatch match {};
        if (std::regex_match(s, match, shortcut_rgx)) {
            std::string modified = string_replace(s, "-", " | ") + " @";
            return std::regex_replace(modified, std::regex(R"((@\w+) \|)"), "$1");
        }
    }  
    return std::regex_replace(s, rgx, expanded);
}


katom_list make_katoms_from_word(std::string s, const std::string& source_desc, int line, int chr)
{
    if (dbg) msg() << " make word: " << broken_bar << s << broken_bar << "  [" << chr << "]\n";
    auto ktyp = std::find_if(katom_types.begin(), katom_types.end(), [&](const auto& ktype) { return ktype.match(s); });
    if (ktyp != katom_types.end()) {
        return { Katom(s, ktyp->m_type, Locator(source_desc, line, chr)) };
    } else {
        if (dbg) {
            std::cout << "\nBREAK: |" << s << "|\n";
        }
        for (const auto& [desc, rgx, expanded] : katom_rewrite_rules) {
            std::string modified = expand_compound_katom(s, rgx.m_regex, expanded);
            if (dbg) {
                msg() << "Rewrite:  " << desc << " " << rgx.m_pattern << " " << expanded << "\n";  // output-policy-ok (dbg)
            }
            if (modified != s) {
                auto parts = word_split(modified);
                if (dbg) std::cout << "  Parts: " << parts << "\n";
                if (show_rewrite_rules) {
                    Locator loc(source_desc, line, chr);
                    std::cout << loc << " Rewrite (" << desc << "): " << rgx.m_pattern
                              << " " << right_arrow << " " << expanded << "\n";
                }
                katom_list klist {};
                for (const std::string& p : parts) {
                    if (!p.empty()) {
                        auto ks = make_katoms_from_word(p, source_desc, line, chr); 
                        std::copy(ks.begin(), ks.end(), std::back_inserter(klist));
                    }
                }
                return klist;
            }
        }
        Katom k(s, katom_t::word, Locator(source_desc, line, chr));
        k.m_unparsed = true; // Warning deferred to warn_unparsed_katoms()
        return std::vector{ k };
    }
}

// Warn about words that matched no katom type -- but only those that
// survive processing: text removed by #, ##, or #[...]#, replaced spans,
// and literal content (definition interiors, @code bodies) never warn.
// Called at the end of Machine::process_katoms(), after those passes have
// marked the katoms.  Clears the flag after warning so repeated processing
// of the same katom list does not warn twice.  With warn=false (the @eval
// read-back sub-Machine, whose katoms hold machine-generated result text)
// no warning is printed and every flag is cleared, so the katoms stay
// silent after they are spliced into the calling Machine's list.
void warn_unparsed_katoms(katom_list& katoms, bool warn)
{
    for (Katom& k : katoms) {
        if (!k.m_unparsed) continue;
        if (!warn) {
            k.m_unparsed = false;
            continue;
        }
        if (k.m_type != katom_t::ignored
            && k.m_type != katom_t::replaced
            && k.m_type != katom_t::literal) {
            // One of the two surviving WARNINGS (the commands otherwise treat
            // an anomaly as an error and stop -- see the output policy in
            // CLAUDE.md).  It stays a warning because the judgment is a
            // HEURISTIC, not exact: the @eval read-back legitimately fails to
            // parse raw target markup, which is why m_warn_unparsed exists.  A
            // heuristic that halts turns every false positive into a blocker on
            // a document that would otherwise render.  Promote it to an error
            // when the judgment becomes exact.
            //
            // Through warning(), not a raw std::cerr, so every warning the
            // commands issue has one format: it said "[warning]" where
            // everything else says "(warning)".
            warning("Word not parsed:\n      " + k.m_text
                    + "\nTo include a special character (@, |, #, and ^), put "
                      "\"^\" before it.",
                    k.m_loc);
            k.m_unparsed = false;
        }
    }
}

katom_list split_into_katoms(std::string s, const std::string& source, int source_line)
{
    if (dbg) {
        msg() << "Make katoms: " << s << "<\n";  // output-policy-ok (dbg)
    }
    // const std::string middle_dot { "\u00B7" };
    s = string_replace(s, "\r", "");
    s = string_replace(s, "\t", "        ");

    //std::regex words_regex("^\||[ ]|[\\n]|[^\\s]+|.+");
    //std::regex words_regex(R"((?:[^][|])|[ ]|[\n]|[^\s]+|.+)");
    std::regex words_regex(R"([ ]|[\n]|[^\s]+|.+)");
    auto words_begin = std::sregex_iterator(s.begin(), s.end(), words_regex);
    auto words_end = std::sregex_iterator();
    if (dbg) {
        std::cout << "Found " << std::distance(words_begin, words_end) << " words:\n";
    }
    strings_t atoms {};
    for (std::sregex_iterator iter = words_begin; iter != words_end; ++iter) {
        if (dbg) {
            std::cout << middle_dot << iter->str();
        }
        atoms.push_back(iter->str());
    }
    if (dbg) std::cout << middle_dot << "\n";
    //std::cout << kall << ktype;

    katom_list result {};

    int cpos = 0;
    for (const auto& a : atoms) {
        auto k = make_katoms_from_word(a, source, source_line, cpos);
        for (auto& kk : k) {
            Locator loc(source, source_line, cpos);
            //kk->m_loc = loc;
            kk.m_loc = loc;
            //m_katoms.push_back(kk);
            result.push_back(kk);
            if (dbg) {
                //std::cout << "LOOP: " << kk.m_text << " - " << cpos << "\n";
            }
            cpos += kk.m_src.size();
        }
    }
    return result;
}

void restore_initial_type(katom_iter begin, katom_iter end)
{
    std::for_each(
        begin, end, [](Katom& k) { k.m_type = k.m_initial_type; });
}

void modify_type(katom_t new_type, katom_iter begin, katom_iter end)
{
    std::for_each(
        begin, end, [&new_type](Katom& k) { k.m_type = new_type; });
}

void modify_type(katom_t old_type, katom_t new_type, katom_iter begin, katom_iter end)
{
    std::for_each(
        begin, end, [&](Katom& k) { if (k.m_type == old_type) k.m_type = new_type; });
}

void ignore_whitespace(katom_iter& begin, katom_list& katoms)
{
    constexpr bool _dbg = false;
    if (begin < katoms.end()) {
        katom_iter ki = begin;
        if (_dbg) std::cout << "IGNORE_WHITESPACE: ";
        auto end = katoms.end();
        while (ki < end && ki->is_whitespace()) {
            if (_dbg) std::cout << kall << ktype << *ki << sp_arrow;
            ki->m_type = katom_t::ignored;
            if (_dbg) std::cout << kignored << kall << ktype << *ki << " " << "\n";
            ++ki;
        }
        if (_dbg) std::cout << black << kreset;
    }
}

katom_iter after_whitespace(katom_iter begin)
{
    katom_iter result = begin;
    while (result->is_whitespace()) {
        result++;
    }
    return result;
}

std::vector<Katom> text_katoms(katom_iter& begin, katom_iter& end)
{
    katom_list result {};
    for (auto ki = begin; ki < end; ki++) {
        if (!ki->is_whitespace()) {
            auto k = *ki;
            //msg() << "  push: " << kindex << ktype << kall << kws << k << "\n";
            result.push_back(k);
        }
    }
    return result;
}

std::string as_string(std::vector<Katom>::const_iterator begin, std::vector<Katom>::const_iterator end, bool strip_whitespace)
{
    auto include =
        [&](katom_t t)
            { return t != katom_t::replaced and t != katom_t::ignored; };
    std::string result {};
    for (auto ki = begin; ki < end; ki++) {
        katom_t t = ki->m_type;
        if (include(t)) {
            std::string text = !ki->m_display.empty() ? ki->m_display : ki->m_text;
            result += text;
        }
    }
    if (strip_whitespace)
        result = trim(result);
    return result;
}

std::string as_string(const katom_list& ks, bool strip_whitespace)
{
    return as_string(ks.begin(), ks.end(), strip_whitespace);
}

katom_list trim(katom_list& katoms, std::set<katom_t> trim_types)
{
    katom_iter begin =
        std::find_if(katoms.begin(), katoms.end(),
                     [&trim_types](const Katom& k) { return !trim_types.contains(k.m_type); });
    if (begin == katoms.end()) {
        return katom_list{};
    }
    katom_iter end = katoms.end() - 1;
    while (trim_types.contains(end->m_type)) {
        --end;
    }
    return katom_list(begin, end+1);
}

katom_list trim(const katom_list& katoms, bool trim_inactive)
{
    katom_list result = katoms;
    std::set<katom_t> trim_types {katom_t::space, katom_t::newline};
    if (trim_inactive) {
        trim_types.insert(katom_t::replaced);
        trim_types.insert(katom_t::ignored);
    }
    return trim(result, trim_types);
}

strings_t line_split(const std::string& s)
{
    std::istringstream is(s);
    strings_t result {};
    std::string line;
    while (std::getline(is, line)) {
        result.push_back(line);
    }
    result.push_back("\n");
    return result;
}

std::pair<std::string, strings_t> line_split(const fs::path& pathname)
{
    std::string source {};
    strings_t lines {};
    std::ifstream infile(pathname);
    std::string line;
    while (std::getline(infile, line)) {
        source += line + "\n";
        lines.push_back(line);
    }
    if (!source.empty()) {
        source.pop_back();
    }
    return {source, lines};
}

//  I'm, like, this is
//   some weird shit;
//    what the fuck? 

katom_list katomize(const strings_t& lines, const std::string& source_desc)
{
    (void)K::log(3);
    katom_list katoms {};
    int i = 0;
    for (std::string line : lines) {
        (void)K::log(4, line);
        katom_list ks = split_into_katoms(line + "\n", source_desc, ++i);
        katoms.insert(katoms.end(), ks.begin(), ks.end());
    }
    katoms.pop_back();
    return katoms;
}

// Whitespace

std::pair<katom_iter,katom_iter> whitespace_span(const katom_list& katoms, const katom_iter& start, katom_t start_type)
{
    katom_iter ki = start;
    if (ki != katoms.begin()) {
        --ki;
        while (ki != katoms.begin() && ki->is_whitespace()) {
            --ki;
        }
        if (!ki->is_whitespace()) {
            ++ki;
        }
    }
    katom_iter begin = ki;
    while (ki != katoms.end() && (ki->is_whitespace() || ki->m_type == start_type
                                  || ki->m_type == katom_t::ignored || ki->m_type == katom_t::replaced)) {
        ++ki;
    }
    //katom_iter end = ki;
    return {begin, ki}; //end};
}
        
std::vector<Katom> find_katoms_of_type(const katom_list& katoms, katom_t type)
{
    auto result = katoms | std::views::filter([type](const Katom& k) { return k.m_type == type; });
    return std::vector(result.begin(), result.end());
}

katom_iter find_katom_of_type(katom_iter begin, katom_iter end, katom_t type)
{
    return std::find_if(begin, end, [&](const Katom& k) { return k.m_type == type; });
}

// #-

void remove_whitespace(katom_list& katoms) // Lint error
{
    (void)K::log(3);
    auto begin = katoms.begin();
    while (begin < katoms.end()) {
        auto ki = find_katom_of_type(begin, katoms.end(), katom_t::ws_remove);
        if (ki == katoms.end()) break;
        auto [b, e] = whitespace_span(katoms, ki, katom_t::ws_remove);
        std::for_each(b, e, [](Katom& k) { k.m_type = katom_t::ignored; });
        begin = e;
    }
}

// #+n and #/n

int whitespace_arg(Katom& k)
{
    int result = 1;
    std::smatch match{};
    if (std::regex_match(k.m_text, match, std::regex(R"(#[+/](\d*))"))) {
        if (!match[1].str().empty()) {
            result = stoi(match[1]);
        }
    }
    return result;
}

void insert_whitespace(katom_list& katoms, katom_t type, const std::string& c)
{
    (void)K::log(3);
    for (Katom k : find_katoms_of_type(katoms, type)) {
        katom_iter ki = find_katom(katoms.begin(), katoms.end(), k.m_index);
        auto count = whitespace_arg(k);
        auto [b, e] = whitespace_span(katoms, ki, type);
        std::for_each(b, e, [](Katom& ka) { ka.m_type = katom_t::ignored; });
        katom_list inserted = std::vector<Katom>{};
        for (int i = 0; i < count; ++i) {
            inserted.push_back(Katom(c, katom_t::ws_added, k.m_loc));
        }
        katoms.insert(e, inserted.begin(), inserted.end());
    }
}

void process_whitespace_modifiers(katom_list& katoms)
{
    (void)K::log(4);
    remove_whitespace(katoms);
    insert_whitespace(katoms, katom_t::ws_space, " ");
    insert_whitespace(katoms, katom_t::ws_newline, "\n");
}

katom_list trim_whitespace(katom_list katoms)
{
    return trim(katoms, {katom_t::space, katom_t::newline});
}