/*
    This file is part of Msc-generator.
	Copyright (C) 2008-2024 Zoltan Turanyi
	Distributed under GNU Affero General Public License.

    Msc-generator is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    Msc-generator is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with Msc-generator.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file blockchart.cpp The definition for the BlockChart class and all of the "block" language.
* @ingroup libblock_files */

/** @defgroup libblock The engine of the block library
@ingroup libs

 More documentation is coming.

*/

/** @defgroup libblock_files Files for the block library.
* @ingroup libblock*/

#include <glpk.h>
#include <cmath>
#include "canvas.h"
#include "blockchart.h"
#include "blockcsh.h"
#include "block_parser_compile.h"

using namespace block;

std::string LayoutStat::Dump(std::chrono::steady_clock::duration time, size_t no_blocks, size_t from, size_t to) const {
    const char * const what = from==0 && to==0 ? "X dir" : from==1 && to==1 ? "Y dir" : "X+Y dirs";
    char buff[1024];
    snprintf(buff, sizeof(buff), "%zu layout passes for %zu blocks in the %s (%ldms). "
                                 "(%zu constraints of which mandatory=%zu, removed=%zu, pruned=%zu, rejected=%zu) "
                                 "(%zu variables of which removed=%zu)",
             no_passes, no_blocks, what, (long int)std::chrono::duration_cast<std::chrono::milliseconds>(time).count(),
             no_constraints, no_must_constraints, no_removed_constraints, no_pruned_constraints, no_rejected_constraints,
             no_variables, no_removed_variables);
    return buff;
}


std::unique_ptr<Csh> BlockChart::CshFactory(Csh::FileListProc proc, const LanguageCollection* languages) const
{
    return std::make_unique<BlockCsh>(proc, languages);
}

std::map<std::string, std::string> BlockChart::RegisterLibraries() const
{
    return {{"glpk v" + std::string(glp_version()), {} }};
}

/** Exception thrown before glpk terminates us.*/
class MyGLPKException
{

};

static void my_glp_error_hook_function(void*)
{
    throw MyGLPKException();
}

BlockChart::BlockChart(FileReadProcedure *p, void *param, const LanguageCollection* languages) : ChartBase(p, param, languages),
    parent_style(EBlockStyleType::Content),
    Blocks(*this),
    conflict_report(EAlignPrio::Explicit) //Do not report default style conflicts
{
    Contexts.emplace_back(true, EContextParse::NORMAL, EContextCreate::PLAIN, FileLineCol());  //creates new, plain context
    Designs.emplace("plain", Contexts.back());
    Blocks.SetContent(nullptr); //to set style
    glp_error_hook(&my_glp_error_hook_function, nullptr);
    //Do these before the first input file is added
    parent_style.Empty();
    current_parent = nullptr;
    block_seq = 0;
}

BlockChart::~BlockChart()
{
    //Your own destructor
}


std::unique_ptr<BlockInstrList> BlockChart::PopContext()
{
    std::unique_ptr<BlockInstrList> ret;
    if (Contexts.size()<2) return ret;
    const bool full = MyCurrentContext().IsFull();
    const size_t old_size = full ? MyCurrentContext().numberingStyle.Size() : 0;
    ChartBase::PopContext();
    //if numbering continues with the same amount of levels (or if this was a partial
    //context used to specify a design), no action will be needed
    //in PostParseProcess, so we do not generate any Command arcs.
    if (!full || old_size == MyCurrentContext().numberingStyle.Size())
        return ret;
    //if number of levels is less in the outer context, we will need to trim the Numbering list
    //during PostParseProcess after processing all arcs in the inner context, so we insert
    //a SetNumbering, which first trims the Numbering list to the new size and then increments
    //the last number, so after 1.2.2 comes 1.3
    ret = std::make_unique<BlockInstrList>();
    if (old_size > MyCurrentContext().numberingStyle.Size())
        ret->Append(std::make_unique<SetNumbering>(*this, MyCurrentContext().numberingStyle.Size(),
                                                   MyCurrentContext().numberingStyle.Last().increment));
    else
        //We should never get here, but if the length is increasing, we just expand the Numbering list
        ret->Append(std::make_unique<SetNumbering>(*this, MyCurrentContext().numberingStyle.Size(), 0));
    return ret;
}


bool BlockChart::AddCommandLineArg(const std::string & /*arg*/)
{
    return false;
}

void BlockChart::AddCommandLineOption(const Attribute & a)
{
    AddAttribute(a);
}

/** Adds a chart option. */
bool BlockChart::AddAttribute(const Attribute &a)
{
    if (a.Is("numbering.append")) {
        std::vector<NumberingStyleFragment> nsfs;
        if (NumberingStyleFragment::Parse(this, a.linenum_value.start, a.value.c_str(), nsfs))
            MyCurrentContext().numberingStyle.Push(nsfs);
        return true;
    }
    if (a.Is("numbering")) {
        MyCurrentContext().running_style_blocks.write().AddAttribute(Attribute("number", a), this);
        MyCurrentContext().running_style_arrows.write().AddAttribute(Attribute("number", a), this);
        return true;
    }
    if (a.Is("conflict_report")) {
        if (!a.EnsureNotClear(Error, EStyleType::OPTION))
            return true;
        if (CaseInsensitiveEqual(a.value, "full"))
            conflict_report = EAlignPrio::Default;
        else if (CaseInsensitiveEqual(a.value, "default"))
            conflict_report = EAlignPrio::Default_style;
        else if (CaseInsensitiveEqual(a.value, "style"))
            conflict_report = EAlignPrio::Running_style;
        else if (CaseInsensitiveEqual(a.value, "off"))
            conflict_report = EAlignPrio::Explicit;
        else
            Error.Error(a, false, "Unrecognized value. Use one of 'full', 'default', 'style' or 'off'. Ignoring option.");
        return true;
    }
    if (AddDesignAttribute(a)) return true;
    //handle 'text.*' as the default text format
    if (a.StartsWith("text"))
        if (MyCurrentContext().text.AddAttribute(a, this, EStyleType::OPTION))
            return true;

    if (a.Is("file.url")) {
        if (!a.EnsureNotClear(Error, EStyleType::OPTION)) return true;
        file_url = a.value;
        return true;
    }
    if (a.Is("file.info")) {
        if (!a.EnsureNotClear(Error, EStyleType::OPTION)) return true;
        if (file_info.length())
            file_info.append("\n").append(a.value);
        else
            file_info = a.value;
        return true;
    }
    Error.Error(a.linenum_attr.start, "Unrecognized chart option. Ignoring it.");
    return false;
}


/** Adds an attribute to the running style.
 * Incoming e==0 means an error already reported, so we no-op silently.*/
bool BlockChart::AddRunningStyleAttribute(const Attribute &a, EUseKeywords e)
{
    if (e==0) return true;
    bool ret = false;
    if (USE_KEYWORD_BLOCKS & e)
        ret |= MyCurrentContext().running_style_blocks.write().AddAttribute(a, this);
    if (USE_KEYWORD_ARROWS & e)
        ret |= MyCurrentContext().running_style_arrows.write().AddAttribute(a, this);
    return ret;
}

bool BlockChart::AddDesignAttribute(const Attribute &a)
{
    if (a.StartsWith("background")) {
        if (MyCurrentContext().background.AddAttribute(a, this, EStyleType::OPTION))
            return true;
    }
    if (a.Is("numbering.pre")) {
        MyCurrentContext().numberingStyle.pre = a.value;
        StringFormat::ExpandReferences(MyCurrentContext().numberingStyle.pre.value(), this,
                                       a.linenum_value.start, nullptr,
                                       false, true, StringFormat::LABEL, true);
        return true;
    }
    if (a.Is("numbering.post")) {
        MyCurrentContext().numberingStyle.post = a.value;
        StringFormat::ExpandReferences(MyCurrentContext().numberingStyle.post.value(), this,
                                       a.linenum_value.start, nullptr,
                                       false, true, StringFormat::LABEL, true);
        return true;
    }
    if (a.Is("numbering.format")) {
        std::vector<NumberingStyleFragment> nsfs;
        if (NumberingStyleFragment::Parse(this, a.linenum_value.start, a.value.c_str(), nsfs)) {
            int off = MyCurrentContext().numberingStyle.Apply(nsfs);
            if (off > 0) {
                string msg = StrCat("Numbering here is ", std::to_string(off),
                                    " levels deep, and you specified more (", std::to_string(nsfs.size()),
                                    ") levels of formatting. Ignoring option.");
                Error.Error(a, true, msg);
            }
        }
        return true;
    }
    if (a.Is("numbering.increment")) {
        if (!a.CheckType(EAttrType::NUMBER, Error)) return true;
        if (a.value.find('.') != std::string::npos || a.number == 0 || abs(a.number)>1000000)
            Error.Error(a, true, "The numbering increment must be a nonzero integer, between plus and minus a million.");
        else
            MyCurrentContext().numberingStyle.Last().increment = int(a.number);
        return true;
    }
    if (a.Is("pedantic")) {
		if (!a.EnsureNotClear(Error, EStyleType::OPTION))
			return true;
		if (a.CheckType(EAttrType::BOOL, Error))
			MyCurrentContext().pedantic = a.yes;
		return true;
	}
	return false; //return true if you have recognized & handled the chart option
}

void BlockChart::AttributeNames(Csh &csh, bool designOnly)
{
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRNAME)+"conflict_report",
                           "Specify if and what level of alignment conflicts are reported.",
                           EHintType::ATTR_NAME, true, CshHintGraphicCallbackForAttributeNames));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.color",
                           "Set the color of the background.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.color2",
                           "Set the second color of the background (useful for two-color gradients).",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.gradient",
                           "Set the gradient type of the background.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.*",
                           "Set the background fill.",
                           EHintType::ATTR_NAME));
    StringFormat::AttributeNames(csh, "text.");
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "text.*",
                           "Set the default text attributes.",
                           EHintType::ATTR_NAME));
	csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "pedantic",
						   "When set to 'no' block names mentioned in arrows get automatically defined. "
                           "When set to 'yes', all blocks must be defined with the 'box' or 'shape' "
                           "keywords or the asterisk shortcut before being referenced.",
						   EHintType::ATTR_NAME));
	if (designOnly) return;
    //arrow and shape has all the elements of a style
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering",
                           "Setting it 'yes' will make all labels have an auto-increasing number until the next closing brace  ('}').",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.pre",
                           "Set the text to prepend to label numbers. E.g., use 'Step ' to achieve 'Step 1', 'Step 2', etc.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.post",
                           "Set the text to append to label numbers. E.g., use ':' to achieve '1:', '2:', etc.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.format",
                           "Set the format of auto-numbering, including text format, number type (like roman numbers) and the number of numbering levels.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.append",
                           "Append a new numbering level, to have, e.g., 2.1, which lasts until the next closing brace ('}').",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.increment",
                           "Sets by how much the currently deepest numbering level shall be incremented at each label. Defaults to 1.",
                           EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.*",
                           "Auto numbering related options, like format or the number of levels.",
                           EHintType::ATTR_NAME));

}

bool BlockChart::AttributeValues(std::string_view attr, Csh &csh)
{
    if (csh.AttributeValuesForStyle(std::string(attr), "arrow")) return true;
    if (csh.AttributeValuesForStyle(std::string(attr), "container")) return true;
    if (CaseInsensitiveEqual(attr, "conflict_report")) {
        csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE)+"off",
                               "Report conflicts only if an alignment explicitly specified for a block needs to be removed.",
                               EHintType::ATTR_NAME));
        csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE)+"full",
                               "Report all conflicts even if only a default alignment rule value needs to be violated.",
                               EHintType::ATTR_NAME));
        csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE)+"default",
                               "Report all conflicts even if an alignment attribute coming from a default style needs to be removed.",
                               EHintType::ATTR_NAME));
        csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE)+"style",
                               "Report all conflicts only if an alignment attribute coming from a user style needs to be removed.",
                               EHintType::ATTR_NAME));
        return true;
    }
    if (CaseInsensitiveBeginsWith(attr, "background")) {
        FillAttr::AttributeValues(attr, csh);
        return true;
    }
    if (CaseInsensitiveEqual(attr, "numbering")  ||
		CaseInsensitiveEqual(attr, "pedantic")) {
        csh.AddYesNoToHints();
        return true;
    }
    if (CaseInsensitiveEqual(attr, "numbering.pre")||
        CaseInsensitiveEqual(attr, "numbering.post")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"text\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    if (CaseInsensitiveEqual(attr, "numbering.format")||
        CaseInsensitiveEqual(attr, "numbering.append")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"numbering format\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    return false;
}


bool BlockChart::DeserializeGUIState(std::string_view s)
{
    GUIState state;
    while (s.size()) {
        size_t p = s.find('*');
        if (p == 0) {
            s.remove_prefix(1);
            break;
        }
        if (p == s.npos) return false;
        if (s.size()<p+3) return false;
        if (s[p+2]!='*') return false;
        state.collapse[std::string(s.substr(0, p))].collapsed = s[p+1]=='1';
        s.remove_prefix(p+3);
    }
    while (s.size()) {
        const size_t p = s.find('*');
        if (p == 0 || p == s.npos) return false;
        std::string unique_name{ s.substr(0, p) };
        s.remove_prefix(p + 1);
        size_t len = 0;
        while (s.size() && '0' <= s.front() && s.front() <= '9') {
            len = len * 10 + s.front() - '0';
            s.remove_prefix(1);
        }
        if (s.length() < 1 + len || s.front() != '*') return false;
        if (len)
            state.inlined[std::move(unique_name)].gui_state.assign(s.substr(1, len));
        s.remove_prefix(len + 1);
    }
    GUI_State = std::move(state);
    return true;
}

std::string BlockChart::SerializeGUIState() const
{
    std::string ret;
    for (auto& [n, c]: GUI_State.collapse)
        ret.append(n).append("*").append(c.collapsed ? "1" : "0").push_back('*');
    ret.push_back('*');
    for (auto& [n, i] : GUI_State.inlined)
        ret.append(n).append("*").append(std::to_string(i.gui_state.length())).append("*").append(i.gui_state);
    return ret;
}

bool BlockChart::ControlClicked(Element *arc, EGUIControlType t)
{
    if (!arc) return false;
    const auto c = arc->GetControls();
    if (std::ranges::find(c, t)==c.end()) return false;
    const BlockBlock* const b = dynamic_cast<BlockBlock*>(arc);
    _ASSERT(b); //only BlockBlocks shall have controls
    if (!b) return false;
    switch (t) {
    case EGUIControlType::EXPAND: GUI_State.collapse[b->name_gui_collapse].collapsed = false; return true;
    case EGUIControlType::COLLAPSE: GUI_State.collapse[b->name_gui_collapse].collapsed = true; return true;
    default: return false;
    }
}

bool BlockChart::ApplyForcedDesign(const string& name)
{
    //this is called before parsing
    auto i = Designs.find(name);
    if (i==Designs.end())
        return false;
    ignore_designs = true;
    Contexts.back().ApplyContextContent(i->second);
    return Chart::ApplyForcedDesign(name);
}

void BlockChart::AddAttributeListToStyleListWithFilePos(gsl::owner<AttributeList *> al,
                                                        const StringWithPosList* styles)
{
    if (!styles || styles->size()==0 || !al)
        return;
    //Warn the user if using directional styles
    for (auto &sp : *styles)
        if (sp.name=="<" || sp.name == "<>") Error.Warning(sp.file_pos.start, StrCat("Style '", sp.name, "' will not modify the dotted arrows. Did you mean '>'?"));
        else if (sp.name=="<<" || sp.name == "<<>>") Error.Warning(sp.file_pos.start, StrCat("Style '", sp.name, "' will not modify the dashed arrows. Did you mean '>>'?"));
        else if (sp.name=="<-" || sp.name == "<->") Error.Warning(sp.file_pos.start, StrCat("Style '", sp.name, "' will not modify the solid arrows. Did you mean '->'?"));
        else if (sp.name=="<=" || sp.name == "<=>") Error.Warning(sp.file_pos.start, StrCat("Style '", sp.name, "' will not modify the double lined arrows. Did you mean '=>'?"));
    std::list<std::string> list;
    auto forbidden = {"col", "row", "invis"};
    for (auto &bn : *styles) {
        if (bn.name.length()==0) continue;
        auto i = std::find_if(forbidden.begin(), forbidden.end(), [&bn](auto &s) {return bn.name==s; });
        if (i==forbidden.end())
            list.push_back(bn.name);
        else
            Error.Error(bn.file_pos.start, "This style cannot be modified. Ignoring it.");
    }
    if (al->size())
        for (auto &a : *al) {
            std::list<string> problem;
            bool had_generic = false;
            for (auto &s : list) {
                auto style = MyCurrentContext().styles.GetStyle(s); //may be default style
                if (style.write().AddAttribute(*a, this)) {
                    MyCurrentContext().styles[s] = style;
                } else {
                    problem.push_back(s);
                    had_generic |= (style.read().type == EStyleType::STYLE);
                }
            }
            if (problem.size()==0) continue;
            string msg;
            if (problem.size()==1) {
                if (had_generic)
                    msg = "Attribute '" + a->name + "' is not applicable to styles. Ignoring it.";
                else
                    msg = "Attribute '" + a->name + "' is not applicable to style '" + *problem.begin() + "'. Ignoring it.";
            } else if (problem.size() == styles->size()) {
                if (had_generic)
                    msg = "Attribute '" + a->name + "' is not applicable to styles. Ignoring it.";
                else
                    msg = "Attribute '" + a->name + "' is not applicable to any of these styles. Ignoring it.";
            } else {
                msg = "Attribute '" + a->name + "' is not applicable to styles '" + *problem.begin();
                for (auto p = ++problem.begin(); p!=--problem.end(); p++)
                    msg.append("', '").append(*p);
                msg.append("' and '").append(*--problem.end()).append("'. Ignoring it.");
                _ASSERT(!had_generic);
            }
            Error.Error(*a, false, msg); //we set a->error, but the copies of 'a' in 'attrs' are left unmarked.
        }
    else //no attributes, we are defining empty styles
        for (auto &s : list)
            MyCurrentContext().styles[s] = MyCurrentContext().styles.GetStyle(s); //may be default style
    delete al;
}

void BlockChart::AddAttributeListToRunningStyle(gsl::owner<AttributeList*> al, EUseKeywords e)
{
    if (al && e)
        for (auto &a : *al)
            if (!AddRunningStyleAttribute(*a, e))
                Error.Error(*a, false, "This attribute is not applicable to any of the specified running styles. Ignoring it.");
    delete al;
}

void BlockChart::ParseText(std::string_view input, const FileLineCol& first_char)
{
    Progress.RegisterParse(input.length());
    procedure_parse_helper proc_helper;
    sv_reader<false> reader(input, first_char);
    block_compile_parse(*this, proc_helper, reader); //return value ignored
}

/** Appends a single instruction to the end of an instruction list.
 * If the instruction list does not exist, we create it. */
gsl::owner<BlockInstrList *>
BlockChart::AppendInstructionToList(gsl::owner<BlockInstrList *> al, gsl::owner<BlockInstruction *> b)
{
    if (!al) al = new BlockInstrList;
    al->Append(b);
    return al;
}

void BlockChart::MoveArrowLabels(BlockInstrList &al)
{
    for (auto i = al.begin(); i!=al.end(); /*nope*/) {
        auto lab = dynamic_cast<ArrowLabel*>(i->get());
        if (lab) {
            auto prev = std::prev(i);
            if (prev==al.end()) {
                Error.Error(lab->file_pos.start, "This arrow label is the first in a list. No arrow before to attach to. Ignoring it.");
            } else {
                auto arr = dynamic_cast<Arrow*>(prev->get());
                if (arr) {
                    arr->labels.Append(lab);
                    i->release();
                } else {
                    Error.Error(lab->file_pos.start, "This arrow label is not following an arrow to attach it to. Ignoring it.");
                    Error.Error((*prev)->file_pos.start, lab->file_pos.start, "Here is the previous instruction, which is not an arrow.");
                }
            }
            al.erase(i++);
        } else
            i++;
    }
}




/** Create an Attribute object on the stack.
 * This is needed since the value may contain position escapes.
 * These are required if we have a series of blocks separated by the plus sign
 * (so that we can give accurate errors, if needed).
 * But if we simply have a number or token as value, position escapes
 * would just confuse matching code.
 * So, if the name of the attribute is NOT an alignment attribute, we remove the
 * position escapes.*/
gsl::owner<Attribute*>
BlockChart::CreateAttribute(std::string_view name, std::string_view value,
                            const FileLineColRange & ln, const FileLineColRange & lv) const
{
    const std::string str = StringFormat::RemovePosEscapesCopy(value);
    return new Attribute(name, AlignmentAttr::AttributeNameDir(name) ? value : str,
                         ln, lv);
}

/** This checks if among the things possible for an arrow_end
 * (coords, blocks with ports, etc.) is there a real block name
 * that we can auto-create. We update 'name' to contain just that part.
 * Thus we search the string for a name - before any + or @ character.
 * If the name, we find contains a dot '.' or is one of the special
 * names, we return false, since it is not possible to auto create those.
 * In case of coordinates or if we don't find an entity_string, we
 * return false.
 * If we return false, 'name' is guaranteed to remain unchanged.*/
bool BlockChart::BlockNameToCreatePart(std::string_view &name)
{
    std::string_view ret;
    const auto pos = name.find_first_of("@,().+-");
    if (pos==std::string::npos) {
        ret = name;
    } else {
        if (name[pos]=='.' || name[pos]==',' || name[pos]=='(' || name[pos]==')')
            return false;
        ret = name.substr(0, pos);
    }
    if (BlockElement::spec_block_names.end() !=
        std::find(BlockElement::spec_block_names.begin(), BlockElement::spec_block_names.end(),
                  ret))
        return false;
    name = ret;
    return true;
}

std::unique_ptr<BlockBlock>
BlockChart::AutoCreateBlock(std::string_view name, const FileLineColRange & l,
							const AttributeList *al, const AlignmentAttr * align)
{
	if (SkipContent() || !BlockNameToCreatePart(name)) return {};
	const int shape = MyCurrentContext().running_style_blocks.write().shape.value_or(-1);
	auto b = shape<0 ?
		std::make_unique<BlockBlock>(*this, EBlockType::Box, l, name) :
		std::make_unique<BlockBlock>(*this, shape, l, name);
    if (align)
        b->style.alignment.MergeByDimension(*align);
    if (al)
        b->AddAttributeList(Duplicate(*al).release());
	else
        b->AddAttributeList(nullptr);
	b->SetContent(nullptr);
	return b;
}

/** Creates an arrow instruction.
 * @param [in] n1 The name of the block as start. May be a coordinate starting with '('
 * @param [in] t The type of the arrow symbol.
 * @param [in] n2 The name of the block as end. May be a coordinate starting with '('
 * @param [in] ah The attribute and label lists to apply to the arrow. Destroy after use.
 * @param [in] l The location of the arrow symbol in the file.
 * @param [in] l1 The location of the start block name in the input file.
 * @param [in] l2 The location of the end block name in the input file.
 * @returns an owning pointer to the arrow created.*/
Arrow *BlockChart::CreateArrow(std::string_view n1, ArrowType t, std::string_view n2,
                               const ArrowAttrHelper &ah, const FileLineColRange &l,
                               const FileLineColRange &l1, const FileLineColRange &l2)
{
    std::unique_ptr<Arrow> ret;
    //If only the source side has an arrowhead, swap sides to make that the end side.
    if (t.start && !t.end)
        ret = std::make_unique<Arrow>(*this, n2, t.swap(), n1, l, l2, l1);
    else
        ret = std::make_unique<Arrow>(*this, n1, t, n2, l, l1, l2);
    if (ah.al)
        ret->AddAttributeList(Duplicate(*ah.al).release());
    ret->AddLabels(ah.bl); //must be called even if ah.bl==nullptr
    return ret.release();
}


/** Add several arrows to an instruction list. Used during parsing with
 * arrow specifications like a,b->c,d. This is in fact 4 separate arrows.
 * @param [in] l1 A list of names on the left side of the arrow symbol (e.g., 'a' and 'b').
 * @param [in] t The type of the arrow symbol (e.g., ->)
 * @param [in] l2 A list of names on the right side of the arrow symbol (e.g., 'c' and 'd').
 * @param [in] ah The attribute and label lists to apply to the arrows. Destroy after use.
 * @param [in] l The location of the arrow symbol in the file.*/
void BlockChart::StashArrows(const StringWithPosList* l1, ArrowType t, const StringWithPosList* l2,
                             ArrowAttrHelper &&ah, const FileLineColRange &l)
{
    if (ah.bl) {
        for (auto &pInstr : *ah.bl) {
            auto pLabel = dynamic_cast<ArrowLabel*>(pInstr.get());
            if (pLabel==nullptr) {
                Error.Error(pInstr->file_pos.start,
                            "You can only have arrow labels attached to an arrow. Ignoring this instruction.");
                pInstr.reset();
            }
        }
        ah.bl->remove(nullptr);
    }
	//If auto define is allowed, we create a block using the default running style (and the shape in it)
    const bool auto_create = !MyCurrentContext().pedantic.value_or(false);
	for (auto l :{ l1, l2 })
		if (l) {
            for (auto &bn: *l)
                if (std::string_view name = bn.name; name.empty())
                    Error.Error(bn.file_pos.start, "Block name missing.");
                else if (auto_create && BlockNameToCreatePart(name) &&
                         GetBlockByName(std::string(name), MyCurrentContext().parent, true, false)[0] == nullptr) {
                    auto b = AutoCreateBlock(name, bn.file_pos);
                    StashedElements.push_back(std::move(b));
                }
        }
    if (l1 && l2)
        for (auto &bn1 : *l1)
            if (bn1.name.size())
                for (auto& bn2 : *l2)
                    if (bn2.name.size())
                        StashedElements.Append(CreateArrow(bn1.name, t, bn2.name,
                                                           ah, l, bn1.file_pos, bn2.file_pos));
    std::erase_if(StashedElements, [](const std::unique_ptr<BlockInstruction>& i) { return !i->valid; });
    ah.Free();
}

BlockInstrList *BlockChart::ExtractStashed()
{
    auto ret = std::make_unique<BlockInstrList>(std::move(StashedElements));
    StashedElements.clear();
    return ret.release();
}

/** Takes a list of blocks and creates a multiple version of them.
 * @param [in] num How many to create.
 * @param b The list of blocks to multiply. They must have all its attributes and content set.
 * @param l The location of the whole command from the multi keyword up to and
 *          including the block to use. Usable only if b is of size 1.*/

BlockBlockList *BlockChart::CreateMulti(int num, gsl::owner<BlockBlockList*> b,
                                        const FileLineColRange &l)
{
    if (b==nullptr || b->size()==0) return b;
    for (auto &pBlock : *b)
        if (pBlock->CanBeMulti())
            pBlock = std::make_unique<MultiBlock>(num, pBlock.release(),
                                                  b->size()==1 ? l : pBlock->file_pos);
        else {
            Error.Error(pBlock->file_pos.start, "You cannot create a multi-version of this block. Ignoring 'multi' clause.");
            pBlock.release();
        }
    b->remove(nullptr);
    return b;
}

/** Creates a Join Command.
 * @param [in] l The location of the command.
 * @param [in] blks The blocks to join
 * @param [in] al The attributes attached to the join command.*/
BlockInstruction *BlockChart::CreateJoin(const FileLineColRange &l, gsl::owner<StringWithPosList*> blks,
                                         gsl::owner<AttributeList*> al)
{
    if (blks==nullptr) return nullptr;
    remove_if(*blks, [](auto &a) {return a.name.length()==0; });
    std::unique_ptr<JoinCommand> ret;
    if (blks->size()) {
        ret = std::make_unique<JoinCommand>(*this, l, std::move(*blks));
        if (ret)
            ret->AddAttributeList(al);
    } else if (al)
        delete al;
    delete blks;
    return ret.release();
}

/** Take a copy helper with a potentially filled file_pos, block_name and block_pos.
 * Search for block name and resolve it to 'block'. Also set the parent_style and
 * parent name for any potential content. This is called after parsing the copy
 * command header up to 'copy x as y'. We need to do this here as we do not yet
 * construct the copied block and all these settings are normally done in the constructor.
 * It is not const because we may emit errors.*/
gsl::owner<CopyParseHelper*> BlockChart::ResolveCopyHelper(gsl::owner<CopyParseHelper*> helper)
{
    if (helper && !helper->block_name.had_error && !helper->block_name.empty()) {
        //first try to look up among templates
        auto i = Templates.find(helper->block_name.view());
        if (i==Templates.end())
            helper->block = GetBlockByNameWithError(helper->block_name, helper->block_pos,
                                                    MyCurrentContext().parent,
                                                    "block copy attempt");
        else
            helper->block = i->second.get();
        if (helper->block) {
            //Set chart.parent_style so that content has good defaults
            const auto s = helper->block->ContentStyle();
            if (s) {
                parent_style = *s;
                parent_style.type = EStyleType::STYLE;
                parent_style.block_style_type = EBlockStyleType::Content;
                parent_style.alignment.SetAllPriority(EAlignPrio::Content);
            } else
                parent_style.Empty();
        } else
            parent_style.Empty();
        current_parent = FormChildName(GetCurrentPrefix(), helper->copy_name);
    }
    return helper;
}

BlockBlock *BlockChart::CreateCopy(gsl::owner<CopyParseHelper*> helper)
{
    if (helper==nullptr)
        return nullptr;
    BlockBlock *ret = nullptr;
    if (helper->block) {
        //We shall copy the alignment of a top level copied object (its descendants yes)
        //But CloneAs will do just that (since it is also used for descendants)
        //So Here we manufacture the right alignment: The running style plus any attribute specified by the user
        //This will be applied after the copied alignment, hopefully overwriting it in both dimensions
        auto use_alignment = std::make_unique<AlignmentAttr>(MyCurrentContext().running_style_blocks.read().alignment);
        if (helper->align) {
            use_alignment->MergeByDimension(*helper->align);
            delete helper->align;
        }
        ret = helper->block->CloneAs(helper->file_pos, helper->copy_name.empty() ? helper->block->name_original : helper->copy_name.view(),
                                     GetCurrentPrefix(), use_alignment.release(),
                                     helper->attributes, helper->modifiers, nullptr);
        //kill parts consumed by Clone()
        helper->align = nullptr;
        helper->attributes = nullptr;
        helper->modifiers = nullptr;
    } //else We got no block. Error already emitted by parser.
    helper->destroy();
    delete helper;
    return ret;
}

void BlockChart::CreateTemplate(gsl::owner<BlockBlock*> block)
{
    if (block==nullptr) return; //silently ignore, we have probably emitted errors.
    if (block->name_original.length()==0) {
        Error.Error(block->file_pos.start, "You need to name a block template. Ignoring this.");
        delete block;
    } else if (Templates[block->name_full] == nullptr) {
        block->MakeTemplate();
        Templates[block->name_full].reset(block);
    } else {
        Error.Error(block->file_pos.start, "A template with this name already exits. Ignoring this one.");
        Error.Error(Templates[block->name_full]->file_pos.start, block->file_pos.start,
                    "Here is the previous definition.");
        delete block;
    }
}

/** Applies additional attributes to an already existing block
 * and/or create the block during parsing.
 * If the block is not found (as looked from the current context),
 * we emit an error. If auto_define is true, we auto-create any
 * non-existent block and return it. Else, we return a null.
 * Note that 'name' may be a full arrow end with '@' symbols.
 * In that case we are either alone on the line, or
 * are followed by attributes (al!=null if there are attrs).
 * Give appropriate errors.
 * (When UpdateBlock(StringWithPosList,...) calls this,
 * it makes sure it only passes block names in 'name').
 * The parameter 'align' may contain an alignment attribute set,
 * which is written before the name, like below a b; We apply it
 * before the attribute list.*/
std::unique_ptr<BlockBlock>
BlockChart::UpdateBlock(std::string_view name, const FileLineColRange & l,
						const AttributeList *al, const AlignmentAttr * align)
{
	if (name.find_first_of("@,()+-")!=std::string_view::npos) {
		//If there is no attribute, assume the user wanted an arrow.
		if (al)
			Error.Error(l.start,
						StrCat("The string '", name,
							   "' is not a block name. I cannot apply attributes to it. Ignoring it."));
		else
			Error.Error(l.end.NextChar(), "Missing arrow symbol here.");
		return {};
	}
	//Look up the block. If we don't have one with the exact same name, we create one.
	auto b = GetBlockByName(std::string(name), MyCurrentContext().parent, false, false);
	if (b[1]) {
		//More than one block, give error.
		GetBlockByNameWithError(std::string(name), l.start, MyCurrentContext().parent, "here");
		return {};
	}
	std::unique_ptr<BlockBlock> ret;
	if (!b[0]) {
		if (!MyCurrentContext().pedantic.value_or(false))
			//OK, define this block
			ret = AutoCreateBlock(name, l, al, align);
		else
			Error.Error(l.start, StrCat("There is no such block named as '", name,
                                        GetCurrentPrefix().length()
                                        ? StrCat("' when looked up from '", GetCurrentPrefix())
                                        : "", "'."),
						"Use chart option 'pedantic=no;' to automatically create blocks when used first.");
	} else {
        //just apply attributes
        if (align)
            const_cast<BlockBlock*>(b[0])->style.alignment.MergeByDimension(*align);
        if (al)
            const_cast<BlockBlock*>(b[0])->ApplyFurtherAttributes(al);

	}
	return ret;
}


std::unique_ptr<BlockInstrList>
BlockChart::UpdateBlock(const StringWithPosList * spl, const AttributeList * al,
                        const AlignmentAttr * align)
{
	if (spl==nullptr)
		return std::unique_ptr<BlockInstrList>();
	auto ret = std::make_unique<BlockInstrList>();
	StringWithPosList problems;
	for (auto &sp : *spl) {
		if (sp.name.find_first_of("@,()+-")==std::string_view::npos)
			ret->Append(UpdateBlock(sp.name, sp.file_pos, al, align));
		else
			problems.push_back(sp);
        align = nullptr; //just apply to the first block in the series.
	}
	if (problems.size()) {
		//just report the first problematic element
		//If there is no attribute, assume the user wanted an arrow.
		if (al)
			Error.Error(problems.front().file_pos.start,
						StrCat("The string '", problems.front().name,
							   "' is not a block name. I cannot apply attributes to it. Ignoring it."));
		else
			Error.Error(problems.back().file_pos.end.NextChar(),
						"Missing arrow symbol here.");
	}
	return ret;
}

/** Search for blocks by a name given by the user or by us. Return multiple if so.
 * You can search by either the original name of the block (as the user specified it) or
 * its unique name as we have mangled it. We perform a longest_prefix match search as below.
 * Note that the name may contain a partial hierarchy, that is c.d will match on a.b.c.d,
 * if invoked within a.b. Note also, that 'c.d' will also match on a.b.c.d if invoked in 'a.b.c'.
 *
 * @param [in] n The name of the block: unique name or as the user specifies it in an attr
 *               or arrow specification. May contain a partial full name (the tail of it).
 * @param [in] used_from The parent block at the definition of the block or arrow. This is used
 *             to disambiguate partial full names.
 * @param [in] liberal If true, we also hit on any block with this original name - if there
 *                     is only one. Else, we force some prefix match.
 * @param [in] only_valid consider valid blocks.
 * @param [in] loc You can supply a location in the file, where 'n' is coming from.
 *                 If there are multiple hits and one of them is at 'loc', we will return
 *                 *only* that.
 * @returns at most three matching names. */
std::array<const BlockBlock *, 3>
BlockChart::GetBlockByName(std::string_view n, const Parent& used_from,
                           bool liberal, bool only_valid, const FileLineCol* loc) const
{
    std::array<const BlockBlock *, 3> ret = {nullptr, nullptr, nullptr};
    if (n.empty()) return ret;
    //first check if this is a unique name. This is what we replace 'prev', etc to.
    const auto unique = BlockNames_Unique.find(n);
    if (unique!=BlockNames_Unique.end()) {
        if (!only_valid || unique->second->valid)
            ret[0] = unique->second;
        return ret;
    }
    //If there is one and only one in original names, use that
    if (liberal) {
        const auto original = BlockNames_Original.equal_range(n);
        if (original.first!=original.second && std::next(original.first)==original.second) {
            if (!only_valid || original.first->second->valid)
                ret[0] = original.first->second;
            return ret;
        }
    }
    //If there is only one <used_from>.<n> name exists, that will be the one.
    //This ensures that we can always refer to our siblings via their original names
    const auto full = BlockNames_Full.equal_range(FormChildName(used_from.GetName(),n));
    if (full.first != full.second) {
        if (std::next(full.first) == full.second) {
            if (!only_valid || full.first->second->valid)
                ret[0] = full.first->second;
            return ret;
        }
        //If there are multiple ones, we search the one whose parent is 'used_from'
        if (used_from.AsBlock())
            for (auto i = full.first; i != full.second; ++i)
                if (i->second->GetParent() == used_from.AsBlock()) {
                    if (!only_valid || i->second->valid)
                        ret[0] = i->second;
                    return ret;
                }
    }
    //OK, no easy find: 'n' probably has a dot. Find all ending in 'n', where the beginning (the part before 'n')
    //equals to (the beginning of) 'used_from'. Rank them based on the length of the name,
    //the longer the better.
    //now cycle through the full names ending in "."+'n'.
    //Note that BlockNames_Full is sorted taking its strings in reverse
    //from is the first that may end in "."+'n', till is the one beyond the last
    //Note in ASCII '.' < alphanumeric chars
    //If we find any block, where 'used_from' is a parent, we will only search for such blocks.
    //This is to disambiguate within a block if a block of the same name exists:
    //E.g.: box A { box X, Y; X->Y; } box A { box X, Y; X->Y; } should have each arrow use the
    //respective X and Y source and dest even if the full name is A.X and A.Y in both cases.
    const std::string dotn = "."+ string(n);
    const auto from = BlockNames_Full.lower_bound(dotn);
    const auto till = BlockNames_Full.lower_bound(StrCat('.', n[0]+1, n.substr(1)));
    const std::string_view used_from_full_name = used_from.GetName();
    bool had_one_in_used_from = false; //did we find any blocks that have 'used_from' as parent?
    for (auto i = from; i!=till; i++) {
        //test that n still matches the end
        //E.g., if we have tested B.A, B.C.A, DDDD.A (in this order) for 'dotn'==".A",
        //we have to stop once we get to X.Y.B (end does not match 'n')
        if (i->first.length()<n.length() ||
            memcmp(i->first.data()+i->first.length()-dotn.length(), dotn.data(), dotn.length()))
            break;
        //The length of used_from + length of n shall not be smaller than the length
        //of 'i'. This would be, e.g. 'used_from'="A.B", n="C.D", but i="A.B.X.Y.C.D" - not a match
        if (i->first.length()>used_from_full_name.length()+dotn.length())
            continue;
        //See if the part before 'n' matches the beginning of 'used_from_full_name'
        //E.g., the full name must start with 'used_from'
        const ptrdiff_t prefix_len = i->first.length() - dotn.length();
        if (memcmp(i->first.data(), used_from_full_name.data(), prefix_len))
            continue;
        const bool i_is_in_used_from = i->second->IsMyParent(used_from.AsBlock());
        if (i_is_in_used_from && !had_one_in_used_from) {
            ret = { nullptr, nullptr, nullptr };
            had_one_in_used_from = true;
        }
        if (only_valid && !i->second->valid)
            continue;
        else if (loc && i->second->file_pos.start==*loc)
            return ret = { i->second, nullptr, nullptr };
        else if (!i_is_in_used_from && had_one_in_used_from)
            continue;
        else if (ret[0]==nullptr)
            ret[0] = i->second;
        else if (ret[1]==nullptr)
            ret[1] = i->second;
        else if (ret[2]==nullptr)
            ret[2] = i->second;
    }
    //Now also add any block that has its name exactly as 'n'
    const auto equal = BlockNames_Full.equal_range(n);
    for (auto i = equal.first; i!=equal.second; i++) {
        if (only_valid && !i->second->valid)
            continue;
        else if (loc && i->second->file_pos.start==*loc)
            return ret = { i->second, nullptr, nullptr };
        else if (ret[0]==nullptr)
            ret[0] = i->second;
        else if (ret[1]==nullptr)
            ret[1] = i->second;
        else if (ret[2]==nullptr)
            ret[2] = i->second;
    }
    return ret;
}

/** Get the block from its name.
 * You can search by either the original name of the block (as the user specified it) or
 * its unique name as we have mangled it.
 * If there are multiple blocks with this (original) name, we emit an error.
 * @param [in] n The name to search for
 * @param [in] l The location of the block name in the file.
 * @param [in] used_from The block from which the name shall be looked up.
 *                       This is kind of the default prefix.
 * @param [in] whattoignore In the text of the error we say we ignore what is in this
 *             string, e.g., "attribute" "arrow", etc.
 * @returns nullptr if no block or multiple blocks by this name.*/
const BlockBlock *
BlockChart::GetBlockByNameWithError(std::string_view n, const FileLineCol &l,
                                    const Parent &used_from, std::string_view whattoignore)
{
    auto v = GetBlockByName(n, used_from, true, true, &l);
    const std::string_view used_from_full_name = used_from.GetName();
    if (v[0]==nullptr) {
        std::string msg = "Could not find a block named '" + string(n);
        if (used_from_full_name.length())
            msg.append(StrCat("' when looked up from prefix '", used_from_full_name));
        Error.Error(l, StrCat(msg , "'. Ignoring this ", whattoignore, '.'));
        return nullptr;
    } else if (v[1]) {
        std::string msg = "You have more than one block with name '" + string(n);
        if (used_from_full_name.length())
            msg.append(StrCat("' when looked up from prefix '", used_from_full_name));
        Error.Error(l, StrCat(msg, "' available and I don't know which one did you mean. Ignoring this ", whattoignore, "."));
        for (unsigned u = 0; u<3 && v[u]; u++)
            Error.Error(v[u]->file_pos.start, l, StrCat("Here is one candidate '", v[u]->name_full, "'."));
        return nullptr;
    }
    return v[0];
}

/** Create a layout for the chart.
 * Parameters 'from' and 'to' can only be 0 or 1 and from<=to must hold.
 * @param canvas The canvas to lay out on.
 * @param [in] from If zero, we lay out the x corrdinates (maybe in addition to
 *                  the y coordinates).
 * @param [in] to If one, we lay out the y corrdinates (maybe in addition to
 *                  the x coordinates).
 * @returns true on error, false on success.*/
bool BlockChart::LayoutWithGLPK(Canvas &canvas, unsigned from, unsigned to)
{
    using namespace std::chrono;
    _ASSERT(from<=to);
    _ASSERT(to<=1);
    const auto started = steady_clock::now();
    const bool ret = Blocks.LayoutWithGLPK(canvas, from, to, block_layout_bulk_per_block*BlockNames_Unique.size());
    Error.TechnicalInfo(FileLineCol{}, layout_stat.Dump(steady_clock::now() - started, BlockNames_Unique.size(), from, to));
    layout_stat.clear();
    return ret;
}

/** After block layout, convert us to an actual coordinate.
 * If the blocks contain a single "parent", we calculate based on 'parent'
 * If the blocks are empty of some blocks are invalid or nonexistent
 * We return empty.*/
OptAttr<double> BlockChart::GetCoord(const AlignTo & a, bool y, const BlockBlock *parent) const
{
    Range r(false);
    if (a.IsParent()) {
        const Block &block = a.edge.margin ? parent->outer_line : parent->inner_line;
        r = y ? block.y : block.x;

    } else for (auto &sp : a.blocks) {
        auto b = GetBlockByName(sp.name, {}, true, true); //no prefix needed, by now these must be resolved unique names
        _ASSERT(b[0] && !b[1]);
        if (b[0] && !b[1]) {
            const Range &o = b.front()->outer_line[y];
            r += o.from - (a.edge.margin ? b.front()->style.min_margin[y].value_or(0) : 0);
            r += o.till + (a.edge.margin ? b.front()->style.max_margin[y].value_or(0) : 0);
        }
    }
    if (r.IsInvalid()) return {};
    return a.edge.pos*r.till + (1-a.edge.pos)*r.from + a.edge.offset;
}


namespace contour {


/** This is how the de-overlapping works:
 * 
 * First we lay out each arrow independently. This is done in Arrow::Layout().
 * Then in BlockChart::CheckArrowOverlap() we check for overlaps between arrows.
 * Between any two arrows, we can have the following situations:
 * 1. No overlap at all.
 * 2. Overlap one end of the arrows (either at the start or at the end, not 
 *    necessarily the same for the two arrows, i.e., they can be in the opposite direction).
 *    This overlap can be just the point, same tangent or same curvature. See Overlap::EType
 * 3. Overlap at both ends of the arrows.
 * 4. Overlap over the full path of the arrows.
 * 
 * Then we "shift" the overlapping arrow ends by moving them perpendicular to the path,
 * this offset is stored in Arrow::Ending::offset_pixel. 
 * To do this, we identify 'groups' of overlapping arrow ending. We also collect the 
 * groups into 'clusters': if an arrow has overlap at its both ends, then the two groups
 * will belong to the same cluster. Essentially clusters can be dealt with independently.
 * 
 * Then we do the following for each cluster:
 * To figure out the order in which we need to shift the arrow ends in a group, we 
 * calculate the relation of the overlapped arrow ends in a group (Overlap::ERelation). 
 * This we do for all groups in the cluster. Then we redo the layout of the arrows in the
 * cluster according to this shift order. This is done in BlockChart::RedoLayout().
 * When doing Arrow::Layout(), we try to avoid blocks and the arrows in the same group as
 * either of our arrow endings (which is only used when re-doing the layout). If that is not 
 * possible, we ignore the other arrows in the group * when finding the layout, but this will
 * result in overlapping arrows (not nice). We strive to create a shift ordering where all 
 * arrows in a group can be laid out avoiding each other.
 * Since the (time) order in which we lay out arrows may also impact if they can be laid out
 * all to avoid each other, in BlockChart::RedoLayout() we try permutating the order of the
 * arrows until they can all avoid each other.
 * So, if after RedoLayout() we find that some of the arrows could not be laid out to 
 * avoid all other arrows, we try permutations of the shift order of the arrows in the group.
 * We do this until we hit a permutation that allows laying out all arrows to avoid each other.
 * If we do not find such a permutation, we take the layout with the best score.
 * 
 * Additionally:
 * Arrow endings in a group are categorized as "shifting" and "non-shifting". Shifting ones
 * are ones that go to the same port/dir, which *must* be shifted to avoid each other.
 * Non-shifting ones are which have no port/dir but accidentally were laid out to this 
 * end, but could also be laid out elsewhere. So we assign no shift to them, when attempting
 * the re-layout. However, When permutating the shift order, we also try combinations when 
 * (some of them) is also shifted to see if that helps.
 * 
 * Additionally2:
 * Arrows that overlap from beginning to the end after the initial layout are grouped into
 * an "overlapping arrow group" and are laid out as a single arrow of increased thickness. 
 * This is to reduce the number of arrows to permute, but also to prevent laying out one 
 * of them too close to some block such that the other does not fit (see test/problematic5g.block).
 * Elements in such an "overlapping arrow groups" are then shifted in the order they appear 
 * in the source file, so that the user has some control over their drawing order. */

/** Collects overlapping segments between two paths.
* The returned overlap segments will contain the zero-based index of the edges.*/
template <typename iter1, typename iter2>
std::vector<std::pair<CPData, CPData>>
Overlaps(iter1 first1, iter1 last1,
         iter2 first2, iter2 last2, bool visible_only)
{
    std::vector<std::pair<CPData, CPData>> ret;
    XY xy[9];
    double my_pos[9], other_pos[9];
    const iter2 first2_save = first2;
    for (unsigned u = 0; first1!=last1; ++u, ++first1)
        if (!visible_only || first1->IsVisible()) {
            const Block T = first1->IsStraight() ? Block(first1->GetStart(), first1->GetEnd()) : first1->GetBezierHullBlock();
            first2 = first2_save;
            for (unsigned v = 0; first2!=last2; ++v, ++first2)
                if (!visible_only || first2->IsVisible()) {
                    const Block O = first2->IsStraight() ? Block(first2->GetStart(), first2->GetEnd()) : first2->GetBezierHullBlock();
                    if (!HullOverlap(T, O, false)) continue;
                    const int num = first1->CrossingWithOverlap(*first2, false, xy, my_pos, other_pos); //num<0 if the two edges overlap
                    if (num>=0) continue;
                    //store so that we increase in me first->second
                    if (my_pos[0]<my_pos[1])
                        ret.push_back({ { xy[0], u, my_pos[0], v, other_pos[0], {} },
                        { xy[1], u, my_pos[1], v, other_pos[1], {} } });
                    else
                        ret.push_back({ { xy[1], u, my_pos[1], v, other_pos[1], {} },
                        { xy[0], u, my_pos[0], v, other_pos[0], {} } });
                }
        }
    //OK, now we have segments of overlap in ret.
    //Combine ones that connect in both paths.
    //Start by sorting by one of the paths.
    std::sort(ret.begin(), ret.end(), [](auto &a, auto &b) {return a.first.me<b.first.me; });
    //Then walk along the series and if we connect in both paths
    for (unsigned u = 1; u<ret.size(); u++)
        if (ret[u-1].second.me.test_equal(ret[u].first.me) &&
            ret[u-1].second.other.test_equal(ret[u].first.other)) {
            //If so, join the two segments
            ret[u-1].second = ret[u].second;
            ret.erase(ret.begin()+u);
            u--;
        }
    return ret;
}

} //namespace contour


/** Checks pairwise if arrows overlap at their ends (or totally).
 * Calculates and adds offset to the arrow ends.
 * Marks each arrow in Arrow::Redo if it needs to be re-calculated.
 * @returns the number of marked arrows.*/
std::vector<ArrowEndGroup> BlockChart::CheckArrowOverlap()
{
    std::vector<ArrowEndGroup> ret;
    //Now check for overlaps between arrows
    //For simplicity we only check arrows starting and/or ending at the
    //same point
    for (auto i = Arrows.begin(); i!=Arrows.end(); i++)
        if (!(*i)->path_visible.IsEmpty() && (*i)->DoIBlockOthers())
            for (auto j = std::next(i); j!=Arrows.end(); j++)
                if (!(*j)->path_visible.IsEmpty() && (*j)->DoIBlockOthers()) {
                    constexpr double comp = 1;
                    bool my_start =
                        (*i)->path_visible.front().GetStart().DistanceSqr((*j)->path_visible.front().GetStart())<comp ||
                        (*i)->path_visible.front().GetStart().DistanceSqr((*j)->path_visible.back().GetEnd())<comp;
                    bool my_end =
                        (*i)->path_visible.back().GetEnd().DistanceSqr((*j)->path_visible.front().GetStart())<comp ||
                        (*i)->path_visible.back().GetEnd().DistanceSqr((*j)->path_visible.back().GetEnd())<comp;
                    //Allow overlaps only if we start from a block
                    my_start &= (*i)->ends[0].deoverlap;
                    my_end   &= (*i)->ends[1].deoverlap;
                    const bool same_ends =
                        (*i)->path_visible.front().GetStart().DistanceSqr((*j)->path_visible.front().GetStart())<comp ||
                        (*i)->path_visible.back().GetEnd().DistanceSqr((*j)->path_visible.back().GetEnd())<comp;
                    //check that the other arrow is also of a block ending
                    my_start &= (*j)->ends[!same_ends].deoverlap;
                    my_end &= (*j)->ends[same_ends].deoverlap;
                    if (!my_start && !my_end) continue;
                    auto overlaps = contour::Overlaps((*i)->path_visible.begin(),
                                                      (*i)->path_visible.end(),
                                                      (*j)->path_visible.begin(),
                                                      (*j)->path_visible.end(), false);
                    //Add point overlaps at start or at end
                    bool add_start = false, add_end = false;
                    if (overlaps.size()==0) {
                        //if we just point touch at start and end, add these touchpoints
                        add_start = my_start;
                        add_end = my_end;
                    } else {
                        //We rely on 'overlaps' being sorted by 'me'
                        add_start = my_start &&
                            (overlaps.front().first.me!=(*j)->path_visible.GetStartPos() ||
                            (same_ends ? overlaps.front().first.other!=(*j)->path_visible.GetStartPos()
                             : overlaps.front().first.other!=(*j)->path_visible.GetEndPos()));
                        add_end = my_end &&
                            (overlaps.back().second.me!=(*j)->path_visible.GetEndPos() ||
                            (same_ends ? overlaps.back().second.other!=(*j)->path_visible.GetEndPos()
                             : overlaps.back().second.other!=(*j)->path_visible.GetStartPos()));
                    }
                    if (add_start) {
                        PathPos other = same_ends ? (*j)->path_visible.GetStartPos() : (*j)->path_visible.GetEndPos();
                        contour::CPData cpd{(*i)->path_visible.front().GetStart(), PathPos{0, 0}, other, {}};
                        overlaps.emplace(overlaps.begin(), cpd, cpd);
                    }
                    if (add_end) {
                        PathPos me = (*i)->path_visible.GetEndPos();
                        PathPos other = same_ends ? (*j)->path_visible.GetEndPos() : (*j)->path_visible.GetStartPos();
                        contour::CPData cpd{(*i)->path_visible.back().GetEnd(), me, other, {}};
                        overlaps.emplace_back(cpd, cpd);
                    }
                    //OK, now the front/back of 'overlaps' is a zero length overlap segment if the arrows
                    //otherwise do not overlap there (just touch as we have tested above).
                    //This is called a 'weak' overlap there. If all overlaps in a group are weak, we don't de-overlap them.
                    //But if some of them is not weak, we will de-overlap those and then we will have to move the weak ones
                    //too.
                    if (my_start && my_end && overlaps.size()==1) {
                        //Here the two arrows fully overlap e2e
                        //Both ends are strong
                        //OK, we are the same second param (pos) will be ignored
                        (*i)->overlaps[0][*j] = (*i)->overlaps[1][*j] =
                        (*j)->overlaps[0][*i] = (*j)->overlaps[1][*i] =
                        { Arrow::Overlap::OVERLAP, { 0,0 }, same_ends, Arrow::Overlap::SAME_TILL_END };
                    } else for (unsigned end = !my_start; end<=(my_end?1U:0U); end++) {
                        //Here we tackle one overlapping end, when the two arrows do not
                        //overlap all the way
                        //Calculate the relation
                        const auto &cp = end ? overlaps.back().first : overlaps.front().second;
                        const auto a_i = (*i)->path_visible.Angle(cp.me, !end, false);
                        const auto a_j = (*j)->path_visible.Angle(cp.other, bool(end) ^ same_ends, false);
                        const Arrow::Overlap::ERelation clockwise = a_i.Smaller(a_j) ?
                            Arrow::Overlap::ERelation::CCLOCKWISE : Arrow::Overlap::ERelation::CLOCKWISE;
                        const Arrow::Overlap::EType type =
                            (end ? !add_end : !add_start) ? Arrow::Overlap::OVERLAP :
                            contour::test_equal(a_i.angle, a_j.angle) ? Arrow::Overlap::SAME_TANGENT :
                            Arrow::Overlap::JUST_ENDPOINT;
                        (*i)->overlaps[end][*j] = { type, overlaps.front().second.me, same_ends, clockwise };
                        if (same_ends)
                            (*j)->overlaps[end][*i] = { type, overlaps.front().second.other, same_ends, Arrow::Overlap::Opposite(clockwise) };
                        else
                            (*j)->overlaps[1-end][*i] = { type, overlaps.back().first.other, same_ends, Arrow::Overlap::Opposite(clockwise) };
                    }
                }

    //OK, now all overlaps are in the respective arrows. Now go through each arrow,
    //identify groups of arrow ends all overlapping each-other
    for (auto pA: Arrows)
        for (bool end : {true, false}) {
            if (pA->overlaps[end].size()==0) continue;
            //OK, we have now a group, where one end overlaps.
            //If any in the group is has already been adjusted, we have already done this
            if (std::ranges::any_of(pA->overlaps[end],
                                    [end](auto &a) {return a.first->adjusted[end ^ !a.second.same_ends]; }))
                continue;
            //If all the touchpoints are just endpoints, don't do a thing.
            if (std::ranges::none_of(pA->overlaps[end],
                                     [](auto &a) {return a.second.type>=Arrow::Overlap::SAME_TANGENT; })) {
                //mark these arrow ends as done
                for (auto &a : pA->overlaps[end])
                    a.first->adjusted[end ^ !a.second.same_ends] = true;
                continue;
            }
            //Create a list of arrow endings overlapping: (arrow, is_its_end)
            ArrowEndGroup group;
            //Add ourselves
            group.Add(pA, end);
            for (auto &ae : pA->overlaps[end])
                group.Add(ae.first, end ^ !ae.second.same_ends);
            //Now sort the group based on how they depart
            std::ranges::sort(group.ends, [this](const auto&a, const auto&b)->bool {
                if (&a==&b) return false; //self comparison
                //sort non-shifting ends last
                if (!b.is_shifting) {
                    if (a.is_shifting) return true;
                    else return std::tie(a.pA->route_order, a.pA->file_pos, a.pA->my_index) < 
                                std::tie(b.pA->route_order, b.pA->file_pos, b.pA->my_index);
                } else if (!a.is_shifting) return false;
                const auto i = a.pA->overlaps[a.is_end].find(b.pA);
                const auto j = b.pA->overlaps[b.is_end].find(a.pA);
                bool reverse = false;
                if (i!=a.pA->overlaps[a.is_end].end() &&
                    j!=b.pA->overlaps[b.is_end].end()) {
                    if (i->second.relation==Arrow::Overlap::SAME_TILL_END) {
                        _ASSERT(j->second.relation==Arrow::Overlap::SAME_TILL_END);
                        //SAME_TILL_END matches are sorted by the order of their definition
                        //at the block defined earlier in the file and opposite in the block
                        //defined later
                        //See which block is defined later
                        const BlockBlock * const b[2] = {
                            GetBlockByName(a.pA->ends[0].pos[0].blocks.front().name, {}, false, false)[0],
                            GetBlockByName(a.pA->ends[1].pos[0].blocks.front().name, {}, false, false)[0]
                        };
                        if (b[0] && b[1]) {
                            //We are at the end of pA signified by 'end' if that block is defined
                            //later than the other end, we need to reverse
                            reverse = b[unsigned(a.is_end)]->file_pos > b[unsigned(!a.is_end)]->file_pos;
                        } else {
                            _ASSERT(0);
                        }
                        //fallthrough to the end
                    } else {
                        _ASSERT(j->second.relation!=Arrow::Overlap::SAME_TILL_END);
                        _ASSERT(j->second.relation!=i->second.relation);
                        //if they split make the cclockwise smaller
                        return i->second.relation==Arrow::Overlap::CCLOCKWISE;
                    }
                } else {
                    _ASSERT(0);
                }
                return a.pA->earlier(*b.pA) ^ reverse;
            });
            _ASSERT(std::ranges::all_of(group.ends, [](const auto& p) { return p.pA->DoIBlockOthers(); }));
            //Mark all ends so that we do not process them again
            for (auto &E : group.ends)
                E.pA->adjusted[E.is_end] = true;

            if (auto i = std::ranges::find(group.ends, true, &ArrowEndGroup::ArrowEnd::is_shifting); i != group.ends.end()) {                
                //Get the leaving direction of the first shifting arrow (if any)
                XY vector = i->is_end
                    ? (i->pA->path_visible.back().PrevTangentPoint(1) -
                       i->pA->path_visible.back().GetEnd())
                    : (i->pA->path_visible.front().NextTangentPoint(0) -
                       i->pA->path_visible.front().GetStart());
                group.shift_vector = vector.Rotate90CCW().Normalize();
            } else
                group.shift_vector = { 0, 0 };
            ret.push_back(std::move(group));
        }
    return ret;
}

/** Checks for arrows that fully overlap and merges them into a single, much thicker arrow. 
 * Removes the merged arrows from 'groups' (except the ones we merged into).
 * Returns the sets of arrows we have merged, indexed by the arrow into which we merged them.
 * The arrow includes the one Arrow we merged them into, in the first place. The arrow also
 * is sorted by source location - so that the user can change the order of the arrows in the group.*/
std::unordered_map<Arrow*, std::vector<Arrow*>> BlockChart::MergeFullOverlapping(std::vector<ArrowEndGroup>& groups) {
    std::vector<std::unordered_set<Arrow*>> overlaps;
    for (auto& group : groups)
        for (auto& E : group.ends)
            if (E.pA->style.routing_allow_joint_layout.value_or(true))
                for (auto& e : E.pA->overlaps[E.is_end])
                    if (e.second.relation == Arrow::Overlap::SAME_TILL_END)
                        if (e.first->style.routing_allow_joint_layout.value_or(true))
                            overlaps.push_back({ E.pA, e.first });
    if (overlaps.empty()) return {};
    //Now merge overlapping sets
    while (true) {
        bool merged = false;
        for (unsigned u = 0; u < overlaps.size(); u++)
            if (overlaps[u].size()) 
                for (unsigned v = u + 1; v < overlaps.size(); v++)
                    if (std::ranges::any_of(overlaps[v], [&overlaps, u](Arrow* pA) { return overlaps[u].contains(pA); })) {
                        overlaps[u].merge(std::move(overlaps[v]));
                        overlaps[v].clear();
                        merged = true;
                    }
        if (!merged) break;
    }
    std::erase_if(overlaps, [](const auto& o) { return o.size() < 2; });
    std::unordered_map<Arrow*, std::vector<Arrow*>> ret;
    for (auto& o : overlaps) {
        //Sort the fully overlapping arrows by source location
        //Make sure we merge them to the first in the vector
        std::vector<Arrow*> v{ o.begin(), o.end() };
        std::ranges::sort(v, {}, [](Arrow* pA) { return std::tie(pA->file_pos, pA->my_index); });
        Arrow* first = v.front();
        ret.try_emplace(first, std::move(v));
    }
    //remove merged: all but the first of the vectors
    for (auto& group : groups)
        std::erase_if(group.ends, [&ret](const ArrowEndGroup::ArrowEnd& E) { 
            return std::ranges::any_of(ret, [&E](auto& arrows) {
                auto i = std::ranges::find(arrows.second, E.pA);
                return i != arrows.second.end() && i != arrows.second.begin();
            });
        });
    //Now pimp remaining arrows of the merged ones
    for (auto [pA, v] : ret) {
        for (auto pB : v)
            pB->repl_lw2 = -1;
        pA->repl_spacing = std::max(v.front()->style.route_arrow_distance.value(),
                                    v.back()->style.route_arrow_distance.value());
        double width = v.front()->style.line.LineWidth();
        for (unsigned u = 1; u < v.size(); u++)
            width += v[u]->style.line.LineWidth()
            + std::max(v[u - 1]->style.route_arrow_distance.value(),
                       v[u]->style.route_arrow_distance.value()) * deoverlap_distance_factor;
        pA->repl_lw2 = width / 2;
    }
    return ret;
}


bool ShiftAtOverlap(const Arrow::Ending& end) {
    return end.dir.number != -1 || end.port.name.size(); //if we have a port or a direction, we shift
}

void ArrowEndGroup::Add(Arrow* pA, bool is_end) {
    const bool shifting = ShiftAtOverlap(pA->ends[is_end]);
    ends.push_back(ArrowEnd{ pA, is_end, shifting, shifting });
}


//We also mark arrows in the group as 'redo'.
void ArrowEndGroup::ApplyShift() {
    //Mark arrows for which we have ends as redo, if not yet marked
    for (auto &p : ends)
        if (!p.pA->redo)
            p.pA->redo = true;

    //Calculate tangent of start and move the ends perpendicularly.
    double total_offset = 0;
    std::vector<double> dist;
    std::vector<size_t> shifting;
    dist.reserve(ends.size());
    shifting.reserve(ends.size());
    for (unsigned u = 0; u < ends.size(); u++) {
        if (ends[u].is_shifting)
            shifting.push_back(u);
        auto& E = ends[u].pA->ends[ends[u].is_end];
        E.offset_pixel = E.offset_pixel_user;
    }
    if (shifting.size() < 2) return; //No two ends shall shift, offset_pixel=zero is OK

    _ASSERT(std::ranges::none_of(shifting, [this](size_t p) {return ends[p].pA->IsMergedToAnother(); }));
    for (unsigned u = 1; u < shifting.size(); u++) {
        const double d = ends[shifting[u-1]].pA->GetHalfLineWidth() + ends[shifting[u]].pA->GetHalfLineWidth()
                        + std::max(ends[shifting[u-1]].pA->GetSpacing(), ends[shifting[u]].pA->GetSpacing()) * deoverlap_distance_factor;
        dist.push_back(d);
        total_offset += d;
    }    
    dist.push_back(0); //dummy to be long enough
    XY current = shift_vector * (total_offset / 2);
    for (unsigned u = 0; u < shifting.size(); current -= shift_vector * dist[u++]) {
        auto& g = ends[shifting[u]];
        auto& E = g.pA->ends[g.is_end];
        E.offset_pixel = E.offset_pixel_user + current;
    }
}

constexpr unsigned factorial(unsigned n) {
    unsigned f = 1;
    for (unsigned i = 1; i <= n; ++i)
        f *= i;
    return f;
}


size_t ArrowEndGroup::number_of_permutations() const {
    const size_t shifting = std::ranges::count_if(ends, [](const ArrowEnd& E) { return E.is_originally_shifting; });
    return factorial(shifting) * (1 << (ends.size() - shifting));
}

void ArrowEndGroup::next_permutation(unsigned permutation_no) {
    const size_t shifting = std::ranges::count_if(ends, [](const ArrowEnd& E) { return E.is_originally_shifting; });
    const size_t mask = (1 << (ends.size() - shifting)) - 1;
    if ((permutation_no & mask)==0)
        std::ranges::next_permutation(ends, {}, [](const ArrowEnd& E) { return std::pair(E.pA->my_index, E.is_end); });
    //apply the bitmask in the lower part of 'permutation_no' to ends originally non-shifting
    //When we start calculations all non-shifting endings were not shifted.
    //The order of trying starts with shifting them all 
    size_t bits = permutation_no & mask;
    for (auto &e : ends)
        if (!e.is_originally_shifting) {
            e.is_shifting = bits & 1;
            bits >>= 1;
        }
}


/** Accumulates the score. In 'index' it counts the number of arrows managing to avoid 
 * other arrows in the group.*/
static PathScore Score(const ArrowToScore& scores) {
    PathScore ret;
    for (auto& [pA, s] : scores) {
        if (s.index == 15) ret.index++;
        ret.length += s.length;
        ret.turn += s.turn;
    }
    return ret;
}

ArrowToScore BlockChart::Redo(Canvas &canvas, const std::vector<Arrow*> &redo) {
    ArrowToScore ret;
    ret.reserve(redo.size());
    for (auto pA : redo)
        pA->ResetPath();
    Progress.RegisterBulk(ESections::LAYOUT_ARROWS, redo.size());
    for (auto pA : redo) {
        const double my_dist = pA->GetMyBlockDistance();
        Contour block;
        for (unsigned end = 0; end < 2; end++)
            for (auto& o : pA->overlaps[end])
                block += o.first->GetMyBlockAreaForDeoverlap(my_dist, o.second.same_ends == bool(end));
        ret.try_emplace(pA, pA->Layout(canvas, std::move(block)));
    }
    //Now check if the arrows overlap.
    //Arrow endings that have no direction will not block other arrows
    //and will not come up as overlapping even if they are.
    for (unsigned u = 0; u < redo.size(); u++)
        if ((ret[redo[u]].index & 2) && redo[u]->DoIBlockOthers() && !redo[u]->path_visible.empty()) {
            const double my_dist = redo[u]->GetMyBlockDistance();
            for (unsigned v = u + 1; v < redo.size(); v++)
                if (redo[v]->DoIBlockOthers() && (ret[redo[v]].index & 2) && !redo[v]->path_visible.empty()) {
                    const Contour area = redo[v]->GetMyBlockAreaForDeoverlap(my_dist, false);
                    if (!area.CrossPoints(redo[u]->path_visible).empty()) {
                        ret[redo[u]].index &= ~2;
                        ret[redo[v]].index &= ~2;
                    }
                }
        }
    return ret;
}

/** Redo the layout of arrows marked as 'redo'
 * @param [in] bail_out_early If set, we stop as soon as we have found a
 * layout where no arrows cross each other or a forbidden block.*/
std::pair<ArrowToScore, std::vector<Arrow*>> BlockChart::RedoLayout(Canvas &canvas, bool bail_out_early) {
    std::pair<ArrowToScore, std::vector<Arrow*>> ret;
    std::vector<Arrow*> redo;
    redo.reserve(Arrows.size());
    std::ranges::copy_if(Arrows, std::back_inserter(redo), [](Arrow* pA) { return pA->redo && !pA->IsMergedToAnother(); });
    constexpr auto sort_by_all = [](const Arrow* pA) { return std::tie(pA->route_order, pA->file_pos, pA->my_index); };
    std::ranges::sort(redo, {}, sort_by_all);
    ret.first.reserve(redo.size());
    PathScore best_score;    
    const PathScoreComp comp(0.5);
    while(true) {        
        auto res = Redo(canvas, redo);
        if (ret.second.empty() || comp(Score(res), best_score)) {
            best_score = Score(res);
            ret = { std::move(res), redo };
            if (bail_out_early && best_score.index == redo.size()) //All managed to avoid each other
                return ret; 
        }
        //Find the next permutation, but skip ones where route order is not correct
        do {
            if (!std::ranges::next_permutation(redo, {}, sort_by_all).found) {
                if (ret.second != redo)
                    ret.first = Redo(canvas, ret.second);
                return ret;
            }
        } while (!std::ranges::is_sorted(redo, {}, [](const Arrow* pA) { return pA->route_order; }));
    }
}

bool have_common_arrow(std::vector<ArrowEndGroup>& g1, ArrowEndGroup& g2) {
    for (auto &g : g1)
        for (auto &E : g.ends)
            if (std::ranges::contains(g2.ends, E.pA, [](auto& a) {return a.pA; }))
                return true;
    return false;
}

/** Cycle the ordering of arrow end groups to find the best order.
 * One plausible ordering is initially passed in 'groups'. 
 * If all overlaps can be laid out with index 15, we do not try.
 * Else we try permutating all the arrow groups until we find the one with the best index.
 * @param [in] canvas The canvas to lay out on.
 * @param [in] groups Contains groups of arrow ends that overlap. The order in the groups is 
 *                    a first attempt at correct ordering. 'Ordering' here means in what order
 *                    do we shift overlapping arrow ends.*/
void BlockChart::HandleArrowOverlap(Canvas& canvas, std::vector<ArrowEndGroup> &&all_groups) {
    if (all_groups.empty()) return;

    //First merge arrows that fully overlap
    auto full_overlap_groups = MergeFullOverlapping(all_groups);

    //Second, collect groups into clusters: if an arrow's two end in two groups, they are in the same cluster.
    //This is to avoid the need to search permutations the entire set of groups, just search in each cluster.
    std::vector<std::vector<ArrowEndGroup>> clusters;
    for (auto& g : all_groups) {
        //group all clusters having a common arrow with 'g' to the end
        auto has_common_arrow_with_g = [&g](auto& c) { return have_common_arrow(c, g); };
        auto c = std::remove_if(clusters.begin(), clusters.end(), has_common_arrow_with_g);
        if (c != clusters.end()) {
            //merge all groups having a common arrow with 'g'
            for (auto i = std::next(c); i != clusters.end(); i++)
                std::move(i->begin(), i->end(), std::back_inserter(*c));
            clusters.erase(std::next(c), clusters.end());
            //add 'g' to the group formed
            clusters.back().push_back(std::move(g));
        } else
            clusters.push_back({ std::move(g) });
    }
    //Now try all permutations in each cluster
    const PathScoreComp comp(0.5);
    for (auto& groups : clusters) {
        //If no arrow in the cluster has 'try harder' routing, we do not search
        //all permutations, but bail out early.
        const bool bail_out_early = std::ranges::none_of(groups, [&full_overlap_groups](const ArrowEndGroup& g) {
            return std::ranges::any_of(g.ends, [&full_overlap_groups](const ArrowEndGroup::ArrowEnd& E) {
                if (E.pA->style.routing_try_harder.value_or(false)) return true;
                auto i = full_overlap_groups.find(E.pA);
                return i!=full_overlap_groups.end() &&
                    std::ranges::any_of(i->second, [](Arrow* pA) { return pA->style.routing_try_harder.value_or(false); });
                    
            }); 
        });
        //Re-layout with the suggested ordering.
        //If no arrows in a group overlap, we take this ordering.
        std::vector<ArrowEndGroup> best = groups;
        for (auto& g : best)
            g.ApplyShift();
        auto [current_scores, best_order] = RedoLayout(canvas, bail_out_early);
        PathScore best_score = Score(current_scores);
        ArrowToScore best_scores = current_scores;
        if (bail_out_early && best_score.index == current_scores.size()) continue;

        //The initial ordering has lead to some arrows not being able to avoid others.
        //Try permutations of the groups in this cluster to find a better ordering.
        //We will try all permutations within each group.
        std::vector<unsigned> permutation;
        permutation.reserve(groups.size());
        for (unsigned u = 0; u < groups.size(); u++)
            permutation.push_back(groups[u].number_of_permutations());
        const std::vector<unsigned> permutation_initial = permutation;
        size_t index_to_permute = permutation.size() - 1;
        while (true) {
            //Do one permutation
            groups[index_to_permute].next_permutation(permutation[index_to_permute]-1);
            groups[index_to_permute].ApplyShift();
            //Calculate the new layout and score it
            auto [new_scores, new_order] = RedoLayout(canvas, bail_out_early);
            new_scores.merge(current_scores); //scores in 'new_scores' take precedence
            current_scores = std::move(new_scores);
            if (PathScore new_score = Score(current_scores); comp(new_score, best_score)) {
                if (bail_out_early && new_score.index == current_scores.size()) goto next_cluster; //All managed to avoid each other. Bail out early
                best = groups;
                best_score = new_score;
                best_order = std::move(new_order);
            }
            //Step to the next permutation
            permutation[index_to_permute]--;
            if (permutation[index_to_permute] == 0) {
                //finished permutating this group, do one permutation in a higher level 
                if (index_to_permute == 0) break;
                //reset the permutations for the subsequent groups
                for (unsigned u = index_to_permute; u < groups.size(); u++) {
                    permutation[u] = permutation_initial[u];
                    groups[u].next_permutation(permutation[u]-1);
                    groups[u].ApplyShift();
                }
                index_to_permute--;
            } else
                index_to_permute = permutation.size() - 1;
        }
        //We have found the best ordering. Apply it.
        for (auto& g : best)
            g.ApplyShift();
        Redo(canvas, best_order);

        //If we have not managed to avoid all arrows, Issue warnings
        for (auto& [pA, score] : best_scores)
            if (score.index < 15)
                Error.Warning(pA->file_pos.start, StrCat("The arrow ", pA->GetDisplayName(), " could not be laid out without overlapping something else."),
                              "Consider adding more space, adding/removing ports to the arrow ends.");

    next_cluster: 
        continue;
    }
    
    //Now copy the path for fully overlapping arrows
    for (auto& [pA, v] : full_overlap_groups) {
        //create the offsets we need to apply
        std::vector<double> off(v.size());
        double width = v.front()->style.line.LineWidth();
        off[0] /= 2;
        for (unsigned u = 1; u < v.size(); u++) {
            width += std::max(v[u - 1]->style.route_arrow_distance.value(),
                              v[u]->style.route_arrow_distance.value()) * deoverlap_distance_factor;
            const double lw = v[u]->style.line.LineWidth();
            off[u] = width + lw / 2;
            width += lw;
        }
        for (double &o : off)
            o -= width / 2;

        //apply the offsets
        Path path = std::move(pA->path);
        Path path_straight = std::move(pA->path_straight);
        Progress.RegisterBulk(ESections::LAYOUT_ARROWS, v.size());
        for (unsigned u = 0; u < v.size(); u++) {
            const EArrowRouting r = v[u]->style.routing.value();
            const bool curvy = r == EArrowRouting::Curvy
                || ((r == EArrowRouting::Manhattan || r == EArrowRouting::Polygon) && v[u]->style.line.radius.value() > 1);
            const auto expand = curvy ? contour::EExpandType::EXPAND_ROUND : contour::EExpandType::EXPAND_MITER_SQUARE;
            v[u]->path = path.CreateSimpleExpand(off[u], false, expand);
            v[u]->path_straight = path_straight.CreateSimpleExpand(off[u], false, expand);
            //Even though the path of the two arrows were the same they may go in opposite directions.
            if (pA != v[u]) {
                auto i = pA->overlaps[0].find(v[u]);
                _ASSERT(i != pA->overlaps[0].end());
                if (i != pA->overlaps[0].end() && !i->second.same_ends) {
                    v[u]->path.Invert();
                    v[u]->path_straight.Invert();
                }
            }
            v[u]->PathFound(canvas);
        }
    }
}

void BlockChart::CompleteParse(bool, bool, XY, bool, bool)
{
    Canvas canvas(Canvas::Empty::Query);
    pageBreakData.clear();
    //Consider the copyright text
    StringFormat sf;
    sf.Default();
    XY crTexSize = Label(copyrightText, canvas, Shapes, sf).getTextWidthHeight().RoundUp();
    copyrightTextHeight = crTexSize.y;

    //Set background
    Blocks.style.f_fill = true; //'invis' style blocks otherwise have no fill
    Blocks.style.fill += Contexts.front().background; //front(): only as set in the outermost context
    Blocks.style.fill.MakeComplete();

	Progress.RegisterBulk(ESections::POSTPROCESS, 3); //One for PostParseProcess, one for ResolveAttrs, one for FinalizeLabels
    //Progress.RegisterBulk(ESections::POSTPOS, 1); //Don't register post pos for MAIN: we wont do such a thing.
	Progress.RegisterBulk(ESections::DRAW, 1);
    Progress.StartSection(ESections::POSTPROCESS);
	Numbering number;
	Blocks.PostParseProcess(canvas, nullptr, number); //BlockNames filled in here
    Blocks.FinalizeLabel(canvas);
    Blocks.ResolveAlignmentAttributes(nullptr, nullptr, nullptr, nullptr);
    Blocks.style.alignment.Empty();
    Progress.CloseSection(true);  //any block that has become invalid will not be processed
    Progress.RegisterBulk(ESections::LAYOUT_BLOCK, 2*block_layout_bulk_per_block*BlockNames_Unique.size()); //2* because we do the X and Y dir independently
    Progress.StartSection(ESections::LAYOUT_BLOCK);
    try {
        const unsigned to = has_xy_correlation ? 1 : 0;
        bool ret = LayoutWithGLPK(canvas, 0, to);
        if (!ret && to==0) ret = LayoutWithGLPK(canvas, 1, 1);
        if (ret)
            Error.FatalError(FileLineCol(), "Internal Error: GLPK could not do a layout.");
    }
    catch (MyGLPKException) {
        Error.FatalError(FileLineCol(), "Internal Error: GLPK terminated.");
    }
    //close the LAYOUT_BLOCK section. It s hard to predict how many steps it would take
    //so we cancel outstanding ones, even at success.
    Progress.CloseSection(true);
    if (Error.hasFatal()) {
        total = {0,0,0,0};
        pageBreakData.clear(); //no pages - nothing to draw
        DrawOrder.clear();
        Progress.StartSection(ESections::LAYOUT_ARROWS);
        Progress.CloseSection(true); //nulls out the registered arrows.
        Progress.StartSection(ESections::POSTPOS);
        Progress.CloseSection(true); //nulls out the registered post_pos stuff
        Progress.StartSection(ESections::DRAW);
        Progress.CloseSection(true); //nulls out the registered draws
    } else {
        //The below 2 steps are not registered in progress
        Blocks.LayoutFinalizeBlocks(canvas, ELayoutContentFinalizePass::BLOCK); //create 'area' for each block
        Blocks.LayoutFinalizeBlocks(canvas, ELayoutContentFinalizePass::JOIN); //create 'area' for each join
        Progress.StartSection(ESections::LAYOUT_ARROWS);
        for (auto p : Arrows)
            p->Layout(canvas, {});
        auto overlap_groups = CheckArrowOverlap();
        HandleArrowOverlap(canvas, std::move(overlap_groups));
        Progress.CloseSection();

        Progress.StartSection(ESections::POSTPOS);
        //Need to do arrows first, so that block 'area' is not yet expanded for tracking.
        for (auto p: DrawOrder)
            p->PostPosProcess(canvas, this);
        Progress.CloseSection(true);
        Blocks.outer_line += total;
        if (Blocks.outer_line.x.till < crTexSize.x+Blocks.outer_line.x.from) {
            Blocks.outer_line.x.till = crTexSize.x+Blocks.outer_line.x.from;
            Blocks.outer_line.x += crTexSize.x+Blocks.outer_line.x.from; //make the bg cover the copyright text, too
        }
        Blocks.outer_line.RoundWider();
        total = Blocks.outer_line;
        total += Block(0, 1, 0, 1); //have at least this size
        if (pageBreakData.size()==0)
            pageBreakData.emplace_back(XY(0, 0), total.Spans()); //we *must* have one page
        bkFill.emplace_back(Blocks.style.fill, total);
        //Prune GUI states: remove GUI commands, referring to no element
        std::erase_if(GUI_State.collapse, [](auto& s) { return !s.second.element_exists; });
    }
    Error.Sort();
}


void BlockChart::CollectIsMapElements(Canvas & canvas)
{
    for (auto p: DrawOrder)
        p->CollectIsMapElements(canvas);
}

void BlockChart::RegisterAllLabels()
{
    for (auto p: DrawOrder)
        p->RegisterLabels();

}

void BlockChart::Draw(Canvas& c, bool pageBreaks, unsigned page)
{
    if (page) {
        Blocks.Draw(c);
        Progress.DoneBulk(ESections::DRAW);
        for (auto p : DrawOrder) {
            p->Draw(c);
            Progress.DoneBulk(ESections::DRAW);
        }
    } else {
        Blocks.Draw(c);
        Progress.DoneBulk(ESections::DRAW);
        if (pageBreaks)
            DrawPageBreaks(c);
        for (auto p : DrawOrder) {
            p->Draw(c);
            Progress.DoneBulk(ESections::DRAW);
        }
    }
}

void BlockChart::DrawComplete(Canvas & c, bool pageBreaks, unsigned page)
{
    if (page > pageBreakData.size() || pageBreakData.size() == 0)
        return;
    Progress.StartSection(ESections::DRAW);
    Draw(c, pageBreaks, page);
    DrawHeaderFooter(c, page);
    Progress.CloseSection(true); //many objects can be invalid...
}


void BlockChart::SetToEmpty()
{
    //Your own code to clear all internal data and create a chart that shows an empty page
    //(same as if you had parsed the empty file)
}
