/*
    This file is part of Msc-generator.
    Copyright (C) 2008-2024 Zoltan Turanyi
    Distributed under GNU Affero General Public License.

    Msc-generator is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    Msc-generator is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with Msc-generator.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file cgen_progress.h The declaration of the progress tracker base and its generic descendant.
 * @ingroup libcgencommon_files */
#if !defined(CGEN_PROGRESS_H)
#define CGEN_PROGRESS_H

#include <ctime>
#include <string>
#include <vector>
#include <numeric>

//These below and a few functions of ProgressBase are defined in chartbase.cpp
void DeserializeWhiteSpace(std::string_view &s);
bool DeserializeNewline(std::string_view &s);
bool DeserializeToken(std::string_view &s, std::string_view token);
std::pair<bool, int64_t> DeserializeInt(std::string_view &s);
std::pair<bool, double> DeserializeDouble(std::string_view &s);
std::vector<int64_t> DeserializeInts(std::string_view &s, size_t maxnum = std::numeric_limits<size_t>::max());
std::vector<double> DeserializeDoubles(std::string_view &s, size_t maxnum = std::numeric_limits<size_t>::max());
std::pair<bool, std::string_view> DeserializeLine(std::string_view &s, bool newline_mandatory = false);

/** An exception thrown during compilation by the progress function,
 * to indicate that compilation needs to be aborted.*/
class AbortCompilingException : public std::exception {};


/** The base class for all progress tracking facility*/
class ProgressBase
{
public:
    /** Lists the ways the abortion of compilation can be signalled to the chart.*/
    enum EPreferredAbortMethod
    {
        NONE,     ///<This chart cannot abort compilation
        RETVAL,   ///<If the callback should return true to abort compilation
        EXCEPTION ///<The callback shall throw an exception to stop the compilatoon (will not be caught by the chart)
    };
    using clock = std::chrono::steady_clock;
protected:
    /** How are we preferred to be aborted. */
    EPreferredAbortMethod myPreferredAbortMethod;
    /** The current percentage (not necessarily reported). */
    mutable double current_percent;
    /**Last percentage value reported*/
    mutable double last_reported;
    /** When the callback was called the last time */
    mutable clock::time_point last_time = clock::time_point::min();
    /**Call the callback if p is larger than granularity demands*/
    bool DoCallback(double p) const;

public:
    /** User provided callback to display progress.
     * For 'percent' values outside [0..100], the progressbar shall
     * be erased (relevant for commandline).
     * 'a' Can be used by the chart to signal to the progress function,
     * how it wishes to receive a termination signal. Note that if
     * the progress function cannot signal a break, it can safely
     * ignore 'a'.*/
    using ProgressCallback = bool(const ProgressBase*progress, void*data, EPreferredAbortMethod a);
    /** In percentage, the granularity of calling the callback.*/
    const double granularity;
    /** We aim to report at least this frequently. */
    const clock::duration time_granularity;
    /** The callback function to call */
    ProgressCallback *callback;
    /** The supplemental value to send to the callback */
    void * data;
    /** Clear this to ignore any progress reporting. */
    bool active = true;
    explicit ProgressBase(EPreferredAbortMethod a = NONE,
                          ProgressCallback *cb = nullptr, void *d = nullptr,
                          double g = 1, clock::duration tg = std::chrono::seconds(1)) :
        myPreferredAbortMethod(a), last_reported(-101),
        granularity(g), time_granularity(tg),
        callback(cb), data(d) {}

    bool Busy() const { return DoCallback(last_reported); }

    double GetPercentage() const { return current_percent; }

    /** Returns true if Callback will ever be called. */
    virtual bool DoIdoSomething() { return false; }

    /** Call this when parsing has started .*/
    virtual void StartParse() {}
    /** Call this to register a file to parse. */
    virtual void RegisterParse(size_t) {}
    /** Call this when you have parsed this much of the input. */
    virtual void DoneParse(size_t) {}
    /** Call this when done.*/
    virtual void Done() { active = false; } //No more calls to Progress methods will have any effect (possible to re-draw)
    /** Call this when reset. Keep load data, but delete all collected data*/
    virtual void Reset() { active = true; }

    /** Write the learned load distribution to a string. */
    virtual std::string WriteLoadData() const { return std::string(); }
    /** Load previously learned load distribution from the string.
     * This function must tolerate empty or null input.
     * @returns True if OK, false if not. In the latter case load data
     *          is substituted with the defaults.*/
    virtual bool ReadLoadData(std::string_view) { return false; }

    /** Return the number of unified sections we have (bulk and arc based in Progress).
     * The term unified section is used to hide differences between any kind of
     * progress sections.
     * Zero, if we dont have sectioning. (E.g., just one big section)*/
    virtual unsigned GetNumOfUnifiedSections() const { return 0; }
    /** Return the number of the current unified section we are in.*/
    virtual std::optional<unsigned> GetCurrentUnifiedSection() const { return {}; }
    /** return the user friendly name of a given section (if any).*/
    virtual std::string_view GetUnifiedSectionName(unsigned) const { return {}; }
    /** Returns the percentage spent in each unified section so far. Should contain at most as many
     * values as GetSectionNum() - missing values assumed to be zero.*/
    virtual std::vector<double> GetUnifiedSectionPercentage() const { return {}; }
};


/** Generic Progress Tracker supporting bulk and per category progress.
 * EBulkSection is an enumeration (convertible to size_t) listing all the bulk sections.
 * There is always at least one bulk section: parsing. We assume this is numbered 0.
 * EItemizedSection is an enumeration  (convertible to size_t) listing all the itemized sections,
 * these are operands that take time depending on which category of object they are done on.
 * EItemCategory is an enumeration (convertible to size_t) listing the individual item categories
 * for itemized sections. We assume item zero is the remainder item of which we have exactly one in each section.
 * Sections and categories are numbered from 0.
 * You can register items only for all sections at present.
 * You can register additional items or bulk tasks at any time (decreases percentage).
 * You can open a section - then we _ASSERT if an item is done from any other section. But this
 * is only for debugging, if you do not open a section, you can report anything any time.
 * Closing a section, OTOH deregisters any not yet done items/bulk tasks in that section.
 * Reporting an item/bulk task done over what have been registered is ignored (that will be time
 * spent with non-moving progress bar).
 * We use the term 'tick' to refer to the time unit of std::chrono::steady_clock. When saving to
 * files, we always save in nanoseconds.*/
template <typename EBulkSection, EBulkSection num_bulk_sections,
          typename EItemizedSection = size_t, EItemizedSection num_itemized_sections = 0,
          typename EItemCategory = size_t, EItemCategory num_item_categories =  0>
class Progress : public ProgressBase
{
public:
    struct UnifiedSection
    {
        bool is_bulk;       //True, if this is a bulk section, false if this is an itemized section
        size_t section_num; //The number of bulk or itemized section.
        std::string name;   //The user friendly name (if any)
        UnifiedSection(bool b, size_t s, std::string_view n) :is_bulk(b), section_num(s), name(n) {}
        UnifiedSection(EBulkSection s, std::string_view n) :is_bulk(true), section_num(size_t(s)), name(n) {}
        UnifiedSection(EItemizedSection s, std::string_view n) :is_bulk(false), section_num(size_t(s)), name(n) {}
    };
protected:

    /** Shows the ticks needed to do one section for one item of a given category
    * (loaded from file)*/
    std::array<std::array<std::chrono::nanoseconds, (size_t)num_item_categories>, (size_t)num_itemized_sections> loaded_item_ns;
    /** Shows the ticks needed to do one section for items of a given category
    * (counted). The number of items needed this much ticks is in 'done_items'*/
    std::array<std::array<clock::duration, (size_t)num_item_categories>, (size_t)num_itemized_sections> counted_item_ticks;
    /** Shows how many items have been processed (counted)*/
    std::array<std::array<size_t, (size_t)num_item_categories>, (size_t)num_itemized_sections> done_items;
    /** The number of items registered (same for all sections)*/
    std::array<size_t, (size_t)num_item_categories> registered_items;

    /** Shows the relative effort of processing a task in a bulk section.
    * (loaded from file)*/
    std::array<std::chrono::nanoseconds, (size_t)num_bulk_sections> loaded_bulk_ns;
    /** During progress, counts the number ticks spent in doing bulk
    * sections (cumulative)*/
    std::array<clock::duration, (size_t)num_bulk_sections> counted_bulk_ticks;
    /** The number of items processed in bulk sections.*/
    std::array<size_t, (size_t)num_bulk_sections> done_bulk;
    /** The number of items registered.*/
    std::array<size_t, (size_t)num_bulk_sections> registered_bulk;

    /** True if we have a current section. If false, we did not start any sections or have closed the last one.*/
    bool has_current_section;
    /** The current section, either bulk or item. */
    unsigned current_section;
    /** Tells if the current section is a bulk section or an itemized section.*/
    bool current_section_is_bulk;
    /** When the current item or bulk task has started */
    clock::time_point started;
   /** Calculate and report load. If force is false, we dont
    * do anything if we dont have a callback function. Else we
    * also update current_percent.*/
    void Callback(bool force=false) const
    {
        using namespace std::chrono;
        if (!callback && !force) return;
        //Aim to calculate how much ticks we will need
        clock::duration total_estimated_ticks = clock::duration::zero();
        clock::duration total_ticks_so_far = clock::duration::zero();
        if constexpr ((bool)(size_t)num_itemized_sections && (bool)(size_t)num_item_categories) {
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++)
                for (unsigned c = 0; c<(size_t)num_item_categories; c++)
                    if (registered_items[c]) {
                        if (done_items[s][c]) {
                            //Make a balance between loaded and calculated versions of
                            //how much tick one item takes. The ratio of balance is: how much
                            //did we complete of our registered item/bulk tasks.
                            const double ratio = double(done_items[s][c])/registered_items[c];
                            const nanoseconds counted_item_ns = duration_cast<nanoseconds>(counted_item_ticks[s][c]);
                            const nanoseconds calculated_ns_per_item = counted_item_ns/done_items[s][c];
                            const nanoseconds balanced_ns_per_item = duration_cast<nanoseconds>(calculated_ns_per_item*ratio + loaded_item_ns[s][c]*(1-ratio));
                            total_estimated_ticks += registered_items[c]*balanced_ns_per_item;
                            total_ticks_so_far += counted_item_ticks[s][c];
                        } else {
                            total_estimated_ticks += registered_items[c]*loaded_item_ns[s][c];
                        }
                    }
        }
        if constexpr ((bool)(size_t)num_bulk_sections) {
            for (unsigned s = 0; s<(size_t)num_bulk_sections; s++)
                if (registered_bulk[s]) {
                    if (done_bulk[s]) {
                        const double ratio = double(done_bulk[s])/registered_bulk[s];
                        const nanoseconds calculated_ns_per_item = duration_cast<nanoseconds>(counted_bulk_ticks[s])/done_bulk[s];
                        const nanoseconds balanced_ns_per_item = duration_cast<nanoseconds>(calculated_ns_per_item*ratio + loaded_bulk_ns[s]*(1-ratio));
                        total_estimated_ticks += registered_bulk[s]*balanced_ns_per_item;
                        total_ticks_so_far += counted_bulk_ticks[s];
                    } else {
                        total_estimated_ticks += registered_bulk[s]*loaded_bulk_ns[s];
                    }
                }
        }
        DoCallback(total_estimated_ticks.count() ? std::min(100., 100.*total_ticks_so_far/total_estimated_ticks) : 0);
    }

public:
    const std::vector<UnifiedSection> unified_sections;
    const EBulkSection section_parse = EBulkSection(0);      ///<The 'PARSE' bulk section is assumed to be Section #0.
    const EItemCategory category_remainder = EItemCategory(0); ///<The 'remainder' item category of which we have exactly 1 in each section.

    /** Construct a progress object.
     * @param [in] uni An array containing the sections in their order of processing.
     * @param [in] abort The preferred abort mechanism.
     * @param [in] cb The callback function. May be null initially.
     * @param [in] d The data to pass to the callback.
     * @param [in] g The granularity of reporting in percent.*/
    Progress(std::vector<UnifiedSection> &&uni, EPreferredAbortMethod abort,
             ProgressCallback cb = nullptr, void *d = nullptr, double g = 1) :
        ProgressBase(abort, cb, d, g),  unified_sections(std::move(uni))
    {
        Reset(); Callback();
    }
    bool DoIdoSomething() override { return true; }
    void StartParse() override { if (active) StartSection(section_parse); }
    void RegisterParse(size_t len) override { if (active) RegisterBulk(section_parse, len); }
    void DoneParse(size_t len) override { if (active) DoneBulk(section_parse, len); }

    /** Register a certain amount from a bulk section to be done in the future*/
    void RegisterBulk(EBulkSection section, size_t len=1) {
        if (active)
            registered_bulk[unsigned(section)] += len;
    }
    /** Register an item to be done in the future, in all categories.*/
    void RegisterItem(EItemCategory category, unsigned len=1) {
        if (active)
            registered_items[unsigned(category)] += len;
    }

    /**Unregister a previously registered item that will not have to be done in the future*/
    void UnRegisterItem(EItemCategory category)
    {
        if (active) {
            _ASSERT(registered_items[unsigned(category)] > 0);
            if (registered_items[unsigned(category)])
                registered_items[unsigned(category)]--;
            //If we removed all such items, zero out the time already spent in them.
            if (registered_items[unsigned(category)]==0)
                for (auto &a:counted_item_ticks)
                    a[unsigned(category)] = clock::duration::zero();
        }
    }
    /** Close the current section. This unregisters any unprocessed bulk items.
     * If this is an item category, we report the 'remainder' item done.
     * No operand if we have no open section.
     * If 'cancel' is true, we know we may have not done all items...
     * We silently unregister any undone items. Else we _ASSERT in debug mode.*/
    void CloseSection(bool cancel = false) {
        if (active) {
            const clock::time_point now = clock::now();
            //Close previous section (if any)
            if (has_current_section) {
                if (current_section_is_bulk) {
                    _ASSERT(cancel || registered_bulk[current_section] == done_bulk[current_section]);
                    registered_bulk[current_section] = done_bulk[current_section];
                } else {
                    DoneItem(EItemizedSection(current_section), category_remainder);
                    if (!cancel)
                        if constexpr (bool(num_item_categories))
                            for (unsigned c = 0; c<unsigned(num_item_categories); c++) {
                                _ASSERT(done_items[current_section][c]==registered_items[c]);
                                done_items[current_section][c] = registered_items[c];
                            }
                }
                has_current_section = false;
            }
            started = now;
            Callback();
        }
    }

    void Start()
    {
        if (active)
            started = clock::now();
    }

    void StartSection(EBulkSection section) {
        if (active) {
            has_current_section = true;
            current_section_is_bulk = true;
            current_section = unsigned(section);
            Start();
        }
    }
    void StartSection(EItemizedSection section)
    {
        if (active) {
            has_current_section = true;
            current_section_is_bulk = false;
            current_section = unsigned(section);
            Start();
        }
    }

    /** Signal that an item in the current itemized section is completed. */
    void DoneItem(EItemizedSection section, EItemCategory category, size_t num = 1)
    {
        if (active) {
            const clock::time_point now = clock::now();
            if (has_current_section) {
                _ASSERT(!current_section_is_bulk);
                //When processing a section, we sometimes generate a new Element
                //and need to mark prior sections for that Element as Done.
                //So it is not the case that when starting a section we only report
                //work from that section, hence this check sometimes fail.
                //_ASSERT(unsigned(section)== current_section);
            };
            //We need to be very flexible in bookkeeping items done:
            //We may do any items multiple times (e.g., draw)
            //We may delete items (e.g., content of a signalling Box) already processed, due to gouped entity hiding
            //Plus signalling Notes are also exempted. Cant remember, why.
            //Plus we only check if we report something in the current section
            //(At autopaginate we may add autoheaders and do width, etc., when we
            //are already beyond WIDTH section.)
            if (done_items[unsigned(section)][unsigned(category)] == registered_items[unsigned(category)]) {
                //We have already reported the processing of all registered items.
                //Do nothing now, the time spent processing this extra item will go
                //unaccounted for.
//                _ASSERT(0);
                started = now;
                return;
            } else if (done_items[unsigned(section)][unsigned(category)]+num <= registered_items[unsigned(category)]) {
                done_items[unsigned(section)][unsigned(category)] += num;
                if (done_items[unsigned(section)][unsigned(category)] > registered_items[unsigned(category)])
                    done_items[unsigned(section)][unsigned(category)] = registered_items[unsigned(category)];
                counted_item_ticks[unsigned(section)][unsigned(category)] += now - started;
            } else {
                done_items[unsigned(section)][unsigned(category)] = registered_items[unsigned(category)];
//                _ASSERT(0);
            }
            started = now;
            Callback();
        }
    }

    void DoneBulk(EBulkSection section, size_t number = 1) {
        if (active) {
        //Bulk item reporting is usually precise, thus we do
        //do not cater for overreporting here.
            const clock::time_point now = clock::now();
            if (has_current_section) {
                _ASSERT(current_section_is_bulk);
                _ASSERT(unsigned(section) == current_section);
            }
            done_bulk[unsigned(section)] += number;
            if (done_bulk[unsigned(section)] > registered_bulk[unsigned(section)]) {
                _ASSERT(0);
            }
            counted_bulk_ticks[unsigned(section)] += now - started;
            started = now;
            Callback();
        }
    }

    /** Called from the constructor to initialize members. */
    void Reset()
    {
        if constexpr (bool(num_itemized_sections))
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++) {
                loaded_item_ns[s].fill(clock::duration(1)); //by default everything takes one tick
                counted_item_ticks[s].fill(clock::duration::zero());
                done_items[s].fill(0);
            }
        registered_items.fill(0);
        loaded_bulk_ns.fill(clock::duration(1));
        counted_bulk_ticks.fill(clock::duration::zero());
        done_bulk.fill(0);
        registered_bulk.fill(0);
        has_current_section = false;
        current_percent = 0;
        active = true;
        Start(); //to have some meaningful value
    };

    /** Calculate and write the current statistics to a string*/
    std::string WriteLoadData() const override
    {
        using namespace std::chrono;
        constexpr double ratio = 0.1;  //for ewma
        std::array<std::array<nanoseconds, (size_t)num_item_categories>, (size_t)num_itemized_sections> updated_item_ticks;
        std::array<nanoseconds, (size_t)num_bulk_sections> updated_bulk_ticks;
        if constexpr ((bool)(size_t)num_itemized_sections && (bool)(size_t)num_item_categories) {
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++)
                for (unsigned c = 0; c<(size_t)num_item_categories; c++)
                    if (registered_items[c] && done_items[s][c]) {
                        //Make a balance between loaded and calculated versions of
                        //how much tick one item takes. The ratio of balance is: how much
                        //did we complete of our registered item/bulk tasks.
                        const nanoseconds calculated_ticks_per_item = duration_cast<nanoseconds>(counted_item_ticks[s][c])/done_items[s][c];
                        updated_item_ticks[s][c] = duration_cast<nanoseconds>(calculated_ticks_per_item*ratio + loaded_item_ns[s][c]*(1-ratio));
                    } else {
                        updated_item_ticks[s][c] = loaded_item_ns[s][c];
                    }
        }
        if constexpr ((bool)(size_t)num_bulk_sections) {
            for (unsigned s = 0; s<(size_t)num_bulk_sections; s++)
                if (registered_bulk[s] && done_bulk[s]) {
                    const double ratio = double(done_bulk[s])/registered_bulk[s];
                    const nanoseconds calculated_ticks_per_item = duration_cast<nanoseconds>(counted_bulk_ticks[s])/done_bulk[s];
                    updated_bulk_ticks[s] = duration_cast<nanoseconds>(calculated_ticks_per_item*ratio + loaded_bulk_ns[s]*(1-ratio));
                } else {
                    updated_bulk_ticks[s] = loaded_bulk_ns[s];
                }
        }
        std::stringstream out;
        out << unsigned(1) << " "; //version
        if constexpr ((bool)(size_t)num_bulk_sections)
            for (unsigned s = 0; s<(size_t)num_bulk_sections; s++)
                out << updated_bulk_ticks[s].count() << " ";
        out << std::endl;
        if constexpr ((bool)(size_t)num_itemized_sections)
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++) {
                if constexpr ((bool)(size_t)num_item_categories)
                    for (unsigned c = 0; c<(size_t)num_item_categories; c++)
                        out <<  updated_item_ticks[s][c].count() << " ";
                out << std::endl;
            }
        return out.str();
    }

    /** Load the number of ticks required for stuff from the string.
    * @returns True if OK, false if not. In the latter case load data
    *          is substituted with the defaults.*/
    bool ReadLoadData(std::string_view inp) override
    {
        if (inp.length()==0 || inp.front()==0) return false;
        auto ver = DeserializeInt(inp);
        if (!ver.first || ver.second != 1) return false; //too advanced or too old version
        auto v = DeserializeInts(inp, (size_t)num_bulk_sections);
        if (v.size()!=(size_t)num_bulk_sections) return false;
        if constexpr ((bool)(size_t)num_bulk_sections)
            for (size_t i = 0; i<(size_t)num_bulk_sections; i++)
                loaded_bulk_ns[i] = std::chrono::nanoseconds(v[i]);

        if constexpr ((bool)(size_t)num_itemized_sections)
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++) {
                v = DeserializeInts(inp, (size_t)num_item_categories);
                if (v.size()!=(size_t)num_item_categories) return false;
                if constexpr ((bool)(size_t)num_item_categories)
                    for (size_t i = 0; i<(size_t)num_item_categories; i++)
                        loaded_item_ns[s][i] = std::chrono::nanoseconds(v[i]);
            }
        return true;
    }


    unsigned GetNumOfUnifiedSections() const override { return (size_t)num_bulk_sections+(size_t)num_itemized_sections; }
    std::optional<unsigned> GetCurrentUnifiedSection() const override
    {
        if (!has_current_section) return 0;
        auto i = std::find_if(unified_sections.begin(), unified_sections.end(), [this](const UnifiedSection &u) {return u.is_bulk == current_section_is_bulk && u.section_num == current_section; });
        _ASSERT(i!=unified_sections.end());
        return i!=unified_sections.end() ? std::optional<unsigned>(unsigned(i-unified_sections.begin())) : std::optional<unsigned>{};
    }

    std::string_view GetUnifiedSectionName(unsigned section) const override
    {
        return section<unified_sections.size() ? unified_sections[section].name : std::string_view{};
    }
    std::vector<double> GetUnifiedSectionPercentage() const override
    {
        Callback(true);
        std::vector<double> ret;
        ret.reserve(GetNumOfUnifiedSections());
        std::array<clock::duration, (size_t)num_itemized_sections> itemized_ticks;
        if constexpr (bool(num_itemized_sections))
            for (unsigned s = 0; s<(size_t)num_itemized_sections; s++)
                itemized_ticks[s] = std::accumulate(counted_item_ticks[s].begin(), counted_item_ticks[s].end(), clock::duration::zero());
        const clock::duration total_ticks
            = std::accumulate(itemized_ticks.begin(), itemized_ticks.end(), clock::duration::zero())
            + std::accumulate(counted_bulk_ticks.begin(), counted_bulk_ticks.end(), clock::duration::zero());
        for (auto &u : unified_sections)
            ret.push_back((u.is_bulk ? counted_bulk_ticks[u.section_num] : itemized_ticks[u.section_num])*
                          GetPercentage()/total_ticks);
        return ret;
    }
};


#endif //CGEN_PROGRESS_H
