/*
    This file is part of Msc-generator.
    Copyright (C) 2008-2024 Zoltan Turanyi
    Distributed under GNU Affero General Public License.

    Msc-generator is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    Msc-generator is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with Msc-generator.  If not, see <http://www.gnu.org/licenses/>.
*/
/** @file msc.cpp The definition of the classes MscChart, EntityDistanceMap and DistanceRequirements .
 * @ingroup libmscgen */

#include <cmath>
#include "msc.h"
#include "msccsh.h"
#include "msc_parser_compile.h"

using namespace std;
using namespace msc;

std::string MscChart::GetLanguageDefaultText() const
{
    return "#This is the default signalling chart.\n"
        "#Edit and press F2 to see the result.\n"
        "#You can change the default chart\n"
        "#with the leftmost button on the Preferences pane of the ribbon.\n"
        "\n"
        "a,b,c;\n"
        "b->c:trallala;\n"
        "a->b:new message;\n";
}

std::unique_ptr<Csh> MscChart::CshFactory(Csh::FileListProc proc, const LanguageCollection* languages) const
{
    return std::make_unique<MscCsh>(proc, languages);
}

std::map<std::string, std::string> MscChart::RegisterLibraries() const
{
    return {{std::string("cairo v") + cairo_version_string(), {}}};
}



///////////////////////////////////////////////////////////////////////

/** Inserts a distance requirement into this section.
 * @param [in] e1 The index of the first entity.
 * @param [in] e2 If>=0, the index of the second entity and the distance
 *                requirement is between e1 and e2.
 *                if ==DISTANCE_LEFT or ==DISTANCE_RIGHT, then this is a
 *                side distance on the left or right side of e1, respectively.
 * @param [in] d The width of the distance required in pixels. If a previous
 *               requirement larger than this have been made, the call takes
 *               no effect.*/
void DistanceRequirementsSection::Insert(unsigned e1, int e2, double d)
{
    if (d<=0) return;
    if (e2!=DISTANCE_LEFT && e2!=DISTANCE_RIGHT)
        map_manip::Insert(pairs, IPair(e1, e2), d);
    else
        map_manip::Insert(e2==DISTANCE_LEFT ? left : right, e1, d);
}

/** Increases an existing distance requirement in this section.
 * @param [in] e1 The index of the first entity.
 * @param [in] e2 If>=0, the index of the second entity and the distance
 *                requirement is between e1 and e2.
 *                if ==DISTANCE_LEFT or ==DISTANCE_RIGHT, then this is a
 *                side distance on the left or right side of e1, respectively.
 * @param [in] d The width of the distance increase required in pixels. If no
 *               previous requirement exists, the call takes no effect.*/
bool DistanceRequirementsSection::IncreaseIfNonZero(unsigned e1, int e2, double d)
{
    if (e2!=DISTANCE_LEFT && e2!=DISTANCE_RIGHT)
        return map_manip::IncreaseIfNonZero(pairs, IPair(e1, e2), d);
    else
        return map_manip::IncreaseIfNonZero(e2==DISTANCE_LEFT ? left : right, e1, d);
}

/** Queries a distance requirement in this section.
 * @param [in] e1 The index of the first entity.
 * @param [in] e2 If>=0, the index of the second entity and the distance
 *                requirement is between e1 and e2.
 *                if ==DISTANCE_LEFT or ==DISTANCE_RIGHT, then this is a
 *                side distance on the left or right side of e1, respectively.
 * @returns the distance requirement previously stored, 0 if none.*/
double DistanceRequirementsSection::Query(unsigned e1, int e2) const
{
    if (e2!=DISTANCE_LEFT && e2!=DISTANCE_RIGHT)
        return map_manip::Query(pairs, IPair(e1, e2));
     else
         return map_manip::Query(e2==DISTANCE_LEFT ? left : right, e1);
}

/** Takes the maximum of distances stored for each entity. Ignores marker name.*/
DistanceRequirementsSection &DistanceRequirementsSection::operator += (const DistanceRequirementsSection &d)
{
    map_manip::MergeMax(left, d.left);
    map_manip::MergeMax(right, d.right);
    map_manip::MergeMax(pairs, d.pairs);
    left_entity  = std::min(left_entity,  d.left_entity);
    right_entity = std::max(right_entity, d.right_entity);
    return *this;
}

/** If there is X on the right side of e1 and Y on the e1+1 entity
 * convert these to a distance of max(X,Y)+gap between e1 and e1+1.
 * @param gap [in] We maintain this additional distance when we combine a left
 *                 and a right requirement to a pair.*/
void DistanceRequirementsSection::CombineLeftRightToPair_Add(double gap)
{
    std::map<unsigned, double>::iterator i, j;
    for (i = right.begin(); i!=right.end(); i++) {
        const unsigned index = i->first;
        double next_left = 0;
        j = left.find(index+1);
        if (j != left.end()) {
            next_left = j->second;
            left.erase(j);
        }
        const double requirement = i->second + next_left + gap;
        Insert(index, index+1, requirement);
    }
    right.clear();
    //Add distances for whatever remained in "left"
    for (i = left.begin(); i!=left.end(); i++) {
        const unsigned index = i->first;
        if (index>0)  //ignore distances left of noentity
            Insert(index-1, index, i->second + gap);
    }
    left.clear();
}



/** Inserts a distance requirement into a series of sections.
 * @param [in] e1 The index of the first entity.
 * @param [in] e2 If>=0, the index of the second entity and the distance
 *                requirement is between e1 and e2.
 *                if ==DISTANCE_LEFT or ==DISTANCE_RIGHT, then this is a
 *                side distance on the left or right side of e1, respectively.
 * @param [in] d The width of the distance required in pixels. In each section
 *               we increase the requirement to this if it was smaller than this.
 * @param [in] since The first section.
 * @param [in] to The second section. The operation is performed on all sections
 *                between them. Note that they may come in either order and that
 *                the operation is performed on the earlier section, but on not the later.*/
void DistanceRequirements::Insert(unsigned e1, int e2, double d, iterator since, iterator to)
{
    if (d<=0 || since==to) return;
    //Find first marker
    auto i = elements.begin();
    while (i!=elements.end() && i!=since && i!=to)
        i++;
    if (i==elements.end()) return;
    //insert in all subsequent ranges till we find the other one
    do {
        (i++)->Insert(e1, e2, d);
    } while (i!=elements.end() && i!=since && i!=to);
}

/** Search for the section beginning by a marker by name.
 * If the marker is a centerline marker, linking to an element
 * the parameter 't' specifies if we return the section beginning
 * by the top or the bottom of that element. If the marker is not
 * a centerline marker, we ignore 't'.*/
DistanceRequirements::iterator DistanceRequirements::GetIterator(const string & m, bool t)
{
    auto i = marker_translations.find(m);
    if (i==marker_translations.end())
        return std::find_if(elements.begin(), elements.end(), [&](const DistanceRequirementsSection&e) {return e.marker==m; });
    return GetIterator(i->second, t);
}

/** Search for the section beginning by a marker by name.
 * If the marker is a centerline marker, linking to an element
 * the parameter 't' specifies if we return the section beginning
 * by the top or the bottom of that element. If the marker is not
 * a centerline marker, we ignore 't'.*/
DistanceRequirements::const_iterator DistanceRequirements::GetIterator(const string & m, bool t) const
{
    auto i = marker_translations.find(m);
    if (i==marker_translations.end())
        return std::find_if(elements.begin(), elements.end(), [&](const DistanceRequirementsSection&e) {return e.marker==m; });
    return GetIterator(i->second, t);
}


/** Combined horizontal distance requirements between two sections.
 * Takes the max of distances between the two sections.
 * The two sections are equal, we return zero distances. */
DistanceRequirementsSection DistanceRequirements::Get(const_iterator m1, const_iterator m2) const
{
    DistanceRequirementsSection ret;
    if (m1==m2)
        return ret;
    //Find first marker
    auto i = elements.begin();
    while (i!=elements.end() && i!=m1 && i!=m2)
        i++;
    if (i==elements.end()) return ret;
    do {
        ret += *(i++);
    } while (i!=elements.end() && i!=m1 && i!=m2);
    return ret;
}

/** Queries a distance requirement in this section.
 * @param [in] e1 The index of the first entity.
 * @param [in] e2 If>=0, the index of the second entity and the distance
 *                requirement is between e1 and e2.
 *                if ==DISTANCE_LEFT or ==DISTANCE_RIGHT, then this is a
 *                side distance on the left or right side of e1, respectively.
 * @param [in] from The first section.
 * @param [in] to The second section. The query returns the largest such distance
 *                among all sections between them. Note that they may come in either order and that
 *                the query considers the earlier section, but on not the later.
 * @returns the largest distance requirement previously stored, 0 if none.*/
double DistanceRequirements::Query(unsigned e1, int e2, const_iterator from, const_iterator to) const
{
    double ret = 0;
    //Find first marker
    auto i = elements.begin();
    while (i!=elements.end() && i!=from && i!=to)
        i++;
    if (i==elements.end()) return 0;
    //insert in all subsequent ranges till we find the other one
    do {
        ret = std::max(ret, (i++)->Query(e1, e2));
    } while (i!=elements.end() && i!=from && i!=to);
    return ret;
}

/** Combines the left-right requirements in all sections into pairwise distances,
 * sums all the pairwise distances, adds them to a last section and kills all
 * other sections. This effective reduces the number of sections to 1.
 * If there is X on the right side of e1 and Y on the e1+1 entity
 * convert these to a distance of max(X,Y)+gap between e1 and e1+1.
 * @param gap [in] We maintain this additional distance when we combine a left
 *                 and a right requirement to a pair.*/
void DistanceRequirements::CombineAllLeftRightToPair_Add(double gap)
{
    for (auto &e : elements)
        e.CombineLeftRightToPair_Add(gap);
    DistanceRequirementsSection sum = GetAll();
    elements.clear();
    elements.push_back(std::move(sum));
}

///////////////////////////////////////////////////////////////////////

/** We do some processing only for tracking (which is the ability on the GUI.
    to highlight elements and map them to their line number and vice versa).
    If `prepare_for_tracking` is not set, we omit these steps.
    - Each element computes its `area` and potentially `area_to_draw` members.
    - `AllCovers` AreaList is filled with a copy of the `area` member of each Element (to search in).
    - `AllElements` is a file-position index of all Element objects.
*/
//"Entities" member is responsible to delete ist contents
//"AutoGenEntities" is not, as its contents will be inserted into an
//EntityCommand in MscChart::PostParseProcess()
MscChart::MscChart(FileReadProcedure *p, void *param, const LanguageCollection* languages) :
    ChartBase(p, param, languages),
    drawing(0,0,0,0), headingSize(0), width_attr(-1)
{
    chartTailGap = 3;
    selfArrowYSize = 12;
    headingVGapAbove = 2;
    headingVGapBelow = 2;
    boxVGapOutside = 2;
    boxVGapInside = 2;
    arcVGapAbove = 0;
    arcVGapBelow = 3;
    discoVgap = 3;
    titleVgap = 10;
    subtitleVgap = 5;
    nudgeSize = 4;
    activeEntitySize = 14;
    compressGap = 2;
    hscaleAutoXGap = 5;
    sideNoteGap = 5;
    defWNoteWidth = 0.8;
    entityShapeHeight = 60;

    pedantic=false;
    simple_arc_parallel_layout = false;
    mscgen_compat = EMscgenCompat::AUTODETECT;
    Error.warn_mscgen = true;

    unsigned current_file = Error.AddFile("[config]");

    //Add "plain" and "mscgen" styles

    Designs.emplace("plain", MscContext(true, EContextParse::NORMAL, EContextCreate::PLAIN, FileLineCol()));
    Designs.emplace("mscgen", MscContext(true, EContextParse::NORMAL, EContextCreate::EMPTY, FileLineCol()));
    //Add global context, with "plain" design
    Contexts.emplace_back(true, EContextParse::NORMAL, EContextCreate::PLAIN, FileLineCol());

    //Add virtual entities
    //NoEntity will be the one representing "nullptr"
    //Arrows come between left_side and right_side.
    //Notes on the side come between left_note and left_side; and right_side & right_note
    AllEntities.emplace_back(START_ENT_STR, START_ENT_STR, START_ENT_STR, -1002, -1002,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    StartEntity = &AllEntities.back();
    AllEntities.emplace_back(LNOTE_ENT_STR, LNOTE_ENT_STR, LNOTE_ENT_STR, -1001, -1001,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    LNote = &AllEntities.back();
    AllEntities.emplace_back(LSIDE_ENT_STR, LSIDE_ENT_STR, LSIDE_ENT_STR, -1000, -1000,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    LSide = &AllEntities.back();
    AllEntities.emplace_back(RSIDE_ENT_STR, RSIDE_ENT_STR, RSIDE_ENT_STR, 10000, 10000,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    RSide = &AllEntities.back();
    AllEntities.emplace_back(RNOTE_ENT_STR, RNOTE_ENT_STR, RNOTE_ENT_STR, 10001, 10001,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    RNote = &AllEntities.back();
    AllEntities.emplace_back(END_ENT_STR , END_ENT_STR , END_ENT_STR , 10002, 10002,
                             MyCurrentContext().styles["entity"], FileLineCol(current_file, 0), false, false);
    EndEntity = &AllEntities.back();

    //Add default markers
    Markers[MARKER_BUILTIN_CHART_TOP_STR].line.MakeInvalid();
    Markers[MARKER_BUILTIN_CHART_BOTTOM_STR].line.MakeInvalid();

    //This sets the global context to "plain" (redundant)
    //and adds CommandEntities for lcomment.* and CommandBackground if needed.
    ArcBase *toadd;
    SetDesign(true, "plain", true, &toadd);
    Arcs.Append(toadd);
}

MscChart::~MscChart()
{
    //Proper deletion order
    Arcs.clear();    //This must be before Notes, since Element::~ will use chart->Notes
    //Also Arcs must go before ArrowHeads, as well.
}



bool MscChart::AddCommandLineArg(const std::string& arg)
{
    if (arg == "-Wno-mscgen") {
        Error.warn_mscgen = false;
    } else
        return false;
    return true;
}

void MscChart::AddCommandLineOption(const Attribute &a)
{
    if (a.Is("force-mscgen")) {
        if (a.yes==false)
            Error.Error(a.linenum_attr.start, "I can not unset 'force-mscgen'. Use '--prevent-mscgen' if you want to turn off mscgen backwards compatibility.");
        else
            mscgen_compat = EMscgenCompat::FORCE_MSCGEN;
        return;
    } else if (a.Is("prevent-mscgen")) {
        if (a.yes==false)
            Error.Error(a.linenum_attr.start, "I can not unset 'prevent-mscgen'. Use '--force-mscgen' if you want to force using mscgen backwards compatibility.");
        else
            mscgen_compat = EMscgenCompat::NO_COMPAT;
        return;
    }
    //nullptr return is not appended.
    Arcs.Append(AddAttribute(a));
}

bool MscChart::ApplyForcedDesign(const string& name)
{
    ArcBase *a = nullptr;
    bool ret = 0!=SetDesign(true, name, true, &a);
    if (ret)
        ignore_designs = true;
    Arcs.Append(a);
    return Chart::ApplyForcedDesign(name);
}

bool MscChart::DeserializeGUIState(std::string_view s)
{
    return force_entity_collapse.Deserialize(s) && force_box_collapse.Deserialize(s) && inline_gui_state.Deserialize(s);
}

std::string MscChart::SerializeGUIState() const
{
    std::string ret = force_entity_collapse.Serialize() + force_box_collapse.Serialize() + inline_gui_state.Serialize();
    if (ret=="0\n0\n0\n0\n0\n0\n") ret.clear();
    return ret;
}

bool MscChart::ControlClicked(Element * arc, EGUIControlType t)
{
    EntityApp *ed = dynamic_cast<EntityApp*>(arc);
    if (ed) {
        const bool collapsed = t==EGUIControlType::COLLAPSE;
        auto i = force_entity_collapse.insert({ed->name, collapsed});
        //signal change if we inserted or if the existing element was different
        bool ret = i.second || collapsed!=i.first->second;
        //set actual state in case we did not insert
        i.first->second = collapsed;
        return ret;
    }
    ArcBase *ab = dynamic_cast<ArcBase *>(arc);
    if (ab) {
        const BoxSignature *sig = ab->GetSignature();
        if (!sig)
            return false;
        const EBoxCollapseType type = t==EGUIControlType::COLLAPSE ? EBoxCollapseType::COLLAPSE :
                                      t==EGUIControlType::EXPAND ? EBoxCollapseType::EXPAND :
                                      t==EGUIControlType::ARROW? EBoxCollapseType::BLOCKARROW :
                                      EBoxCollapseType::EXPAND;
        auto i = force_box_collapse.insert({*sig, type});
        bool ret = i.second || type!=i.first->second;
        i.first->second = type;
        return ret;
    }
    return false;
}

/** Applies a design to the chart.
 * @param [in] full True if the design to add is a full design, else it is a partial only
 * @param [in] name The name of the design to apply
 * @param [in] force If true, we apply the design even though MscChart::ignore_designs is true
 * @param [out] ret An arc (a list of arcs) that should be inserted to the chart at this point  (background changes, comment line changes), or nullptr if none
 * @param [in] l The place in the input file, where the design change was proscribed.
 * @returns 0 if design not found; 1 if found and applied or found and ignored (via ignore_designs);
 *          2 if found and applied, but is full, whereas should be partial;
 *          3 if found and applied, but is partial, whereas should be full */
int MscChart::SetDesign(bool full, const string&name, bool force, ArcBase **ret, const FileLineColRange &l)
{
    *ret = nullptr;
    auto i = Designs.find(name);
    if (i==Designs.end())
        return 0;
    if (ignore_designs &&!force)
        return 1;
    MscContext &ctx = i->second;
    MyCurrentContext().ApplyContextContent(ctx);
    std::unique_ptr<ArcList> list = std::make_unique<ArcList>();
    if (!ctx.defBackground.IsEmpty()) {
        std::unique_ptr<ArcBase> arc = std::make_unique<SetBackground>(this, ctx.defBackground);
        arc->AddAttributeList(nullptr);
        list->Append(arc.release());
    }
    if (!ctx.defCommentFill.IsEmpty() || !ctx.defCommentLine.IsEmpty()) {
        MscStyle s; //empty
        s.vline += ctx.defCommentLine;
        s.fill += ctx.defCommentFill;
        list->Append(CEForComments(s, l));
    }
    //ParallelBlocks here becomes one to unroll and one that shall come before the auto generated
    //entities (if at the very beginning of the chart)
    if (list->size()==0)
        *ret = 0;
    else if (list->size()==1)
        *ret = list->front().release();
    else
        *ret = new ParallelBlocks(this, list.release(), nullptr, true, true);
    if (full == ctx.IsFull()) return 1;
    return full ? 3 : 2;
}

/** If our compatibility mode is AUTODETECT, we switch to MSCGEN compatibility mode*/
void MscChart::SwitchToMscgenCompatMode()
{
    if (mscgen_compat != EMscgenCompat::AUTODETECT) return;
    mscgen_compat = EMscgenCompat::FORCE_MSCGEN;
    ArcBase *ab;
    int ret = SetDesign(false, "mscgen", true, &ab);
    _ASSERT(ret==1 && ab==nullptr);
    (void)ret; //to prevent from complaining if _ASSERT is empty.
}


//Helper function. If the pos of *value is smaller (or larger) than i
//if one of the elements is .end() always the other is returned, different
//from operator < above (where .end() is smaller)
EntityRef MscChart::EntityMinMaxByPos(EntityRef i, EntityRef j, bool min) const
{
    if (j==nullptr) return i;
    if (i==nullptr) return j;
    if (min ^ (i->pos < j->pos))
        return j;
    else
        return i;
};

CEntityRef MscChart::EntityMinMaxByPos(CEntityRef i, CEntityRef j, bool min) const
{
    if (j==nullptr) return i;
    if (i==nullptr) return j;
    if (min ^ (i->pos < j->pos))
        return j;
    else
        return i;
};

EntityRef MscChart::FindAllocEntity(std::string_view e, const FileLineColRange &l)
{
    if (e.empty() || e=="|") {
        //During procedure parse this is when a $aaa param is used alone as an entity - it is normal.
        if (SkipContent())
            //but lets prevent this returning a nullptr
            return AllEntities.Find_by_Name(END_ENT_STR);
        //Invalid, empty entities return nullptr
        //So does the '|' symbol
        return nullptr;
    }
    EntityRef ei = AllEntities.Find_by_Name(e);
    if (ei == nullptr) {
        if (SkipContent()) {
            ei = AllEntities.Find_by_Name(END_ENT_STR);
        } else {
            if (pedantic)
                Error.Warning(l.start, "Unknown entity '" + string(e)
                            + "'. Assuming implicit definition.",
                            "This may be a mistyped entity name."
                            " Try turning 'pedantic' off to remove these messages.");
            std::unique_ptr<EntityApp> ed = std::make_unique<EntityApp>(e, this);
            ed->SetLineEnd(l);
            ed->show = true; //start turned on
            std::unique_ptr<EntityAppHelper> eah = ed->AddAttributeList(nullptr, nullptr, l.start); //ownership of 'ed' is transferred to eah->entities
            ed.release();
            AutoGenEntities.Append(std::move(eah->entities));
            ei = AllEntities.Find_by_Name(e);
        }
    }
    return ei;
}

/** Find the leftmost or rightmost descendant of an entity.
 * We search for the ultimate descendant, which is not a grouped entity itself and which
 * may be several nested groups below.
 * @param [in] ei An iterator in AllEntities to search the descendant for.
 *                If  not a group entity itself is returned.
 * @param [in] left Governs if the leftmost or rightmost descendant is searched.
 * @param [in] stop_at_collapsed If true, we do not go down to the descendants of a
 *             collapsed entity, so we may return a group entity, which is collapsed or a
 *             non group entity. If false, we work as if all entities were expanded.
 * @returns An iterator pointing to AllEntities. */
EntityRef MscChart::FindLeftRightDescendant(EntityRef ei, bool left, bool stop_at_collapsed)
{
    while (ei->children_names.size() && !(stop_at_collapsed && ei->collapsed)) {
        if (ei->children_names.size()==0) break;
        EntityRef tmp = AllEntities.Find_by_Name(*ei->children_names.begin());
        for (auto i = ++(ei->children_names.begin()); i!=ei->children_names.end(); i++)
            tmp = EntityMinMaxByPos(tmp, AllEntities.Find_by_Name(*i), left);
        ei = tmp;
    }
    return ei;
}

/** Find the lowest ancestor, which is not hidden due to collapsed group entities.
 * - If `i` is not part (child) of a group entity, we return `i`.
 * - If `i` is part of a group and none of our parents and ancestors are collapsed, we return `i`.
 * - If `i` is part of a group and some parents are collapsed, we return the highest, which is
 *   collapsed (and thus is not itself hidden).
 * In essence we return, which entity is shown on the chart for `i`.
 * (Virtual entities (NoEntity, leftside, leftnote, etc.) and nullptr will return themselves.)*/
EntityRef MscChart::FindActiveParentEntity(EntityRef i)
{
    if (i==nullptr || i->parent_name.length() == 0) return i;
    EntityRef j = AllEntities.Find_by_Name(i->parent_name);
    if (j==nullptr) return i;
    EntityRef k = FindActiveParentEntity(j);
    if (j!=k) return k;  //An ancestor of our parent is collapsed: we show that guy
    if (j->collapsed) return j;  //our parent would show and is collapsed: we show him
    return i; //our parent would show and is not collapsed: we show up
}

/** Return a non-grouped or collapsed entity that shows for `ei`.
 * If `ei` itself is a not hidden not collapsed group entity, we return
 * either the left or rightmost descendant that is not hidden.
 * If `ei` itself does not show, we return an ascendant that does.*/
EntityRef MscChart::FindWhoIsShowingInsteadOf(EntityRef ei, bool left)
{
    if (ei==nullptr) return ei;
    EntityRef sub = FindActiveParentEntity(ei);
    if (sub != ei) return sub; //a parent of ours show instead of us
    return FindLeftRightDescendant(sub, left, true); //a child of us shows
}

/** Return a comma separated list of names of the descendant entities in single quotation marks.
 * If `ei` is not a group entity its name is returned.
 * The order of the descendants is a result of a depth first tree traversal. */
string MscChart::ListGroupedEntityChildren(CEntityRef ei) const
{
    if (ei->children_names.size() == 0) return "'" + ei->name + "'";
    string s;
    for (const auto &i : ei->children_names)
        s.append(ListGroupedEntityChildren(AllEntities.Find_by_Name(i))).append(", ");
    s.erase(s.length()-2);
    return s;
}

/** Emit an error and return true if the entity is grouped. */
bool MscChart::ErrorIfEntityGrouped(CEntityRef ei, FileLineCol l)
{
    if (ei==nullptr || ei->children_names.size()==0) return false;
    Error.Error(l, "Group entity '" + ei->name + "' cannot be used here.",
                "Use one of its members (" + ListGroupedEntityChildren(ei) + ") instead.");
    return true;
}
/** Returns true if `child` is a (not necessarily direct) descendant of `parent`*/
bool MscChart::IsMyParentEntity(const string &child, const string &parent) const
{
    if (child == parent) return false;
    const string myparent = AllEntities.Find_by_Name(child)->parent_name;
    if (myparent.length()==0) return false;
    if (myparent == parent) return true;
    return IsMyParentEntity(myparent, parent);
}

/** Returns true if `child` is a (not necessarily direct) descendant of `parent`*/
bool MscChart::IsMyParentEntity(CEntityRef child, CEntityRef parent) const
{
    if (child == parent) return false;
    CEntityRef myparent = AllEntities.Find_by_Name(child->parent_name);
    if (myparent == nullptr) return false;
    if (myparent == parent) return true;
    return IsMyParentEntity(myparent, parent);
}



/** Get the "pos" of the highest entity (excluding virtual entities)*/
double MscChart::GetEntityMaxPos() const
{
    double ret = -1;  //first entity will be return + 1, which will be zero
    for (auto i=AllEntities.begin(); i!=AllEntities.end(); i++)
        if (!IsVirtualEntity(i) && ret < i->pos)
            ret = i->pos;
    return ret;
}

/** Get the "pos_exp" of the highest entity (excluding virtual entities)*/
double MscChart::GetEntityMaxPosExp() const
{
    double ret = -1;  //first entity will be return + 1, which will be zero
    for (auto i=AllEntities.begin(); i!=AllEntities.end(); i++)
        if (!IsVirtualEntity(i) && ret < i->pos_exp)
            ret = i->pos_exp;
    return ret;
}

/** Return an entity right or left of 'e'.
 * @param [in] e The entity we seek a neighbour for.
 * @param [in] right if true, we seek the right neighbour, else left
 * @param [in] all If true, we return a neighbour from AllEntities
 *                 (a non-grouped entity specified immediately left or right of e).
 *                 If false, we return an iterator from ActiveEntities,
 *                 (the entity that shows immediately left or right from e
 *                  - it may also be a collapsed group entity.)
 * @returns an iterator to the entity, or end() on error.*/
EntityRef MscChart::GetNeighbour(EntityRef e, bool right, bool all)
{
    if (all) {
        auto ei = AllEntities.Find_by_Ptr(e);
        if (ei == AllEntities.end()) return nullptr;
        do {
            right ? ++ei : --ei;
        } while (ei != AllEntities.end() && ei->children_names.size() && (all || !ei->collapsed));
        return ei==AllEntities.end() ? nullptr : &*ei;
    } else {
        auto ei = ActiveEntities.Find_by_Ptr(e);
        if (ei == ActiveEntities.end()) return nullptr;
        do {
            right ? ++ei : --ei;
        } while (ei != ActiveEntities.end() && (*ei)->children_names.size() && (all || !(*ei)->collapsed));
        return ei==ActiveEntities.end() ? nullptr : *ei;
    }
}

/** Create either an SelfArrow or an DirArrow.
 * @param [in] data The type of the arrow segment (dotted, double, etc.)
 *                  and the position of any loss indication (*).
 * @param [in] s The name of the source entity.
 * @param [in] sl The place where `s` is mentioned in the input file.
 * @param [in] d The name of the destination entity.
 * @param [in] fw True if the arrow is defined as 's->d'; false if 'd<-s'
 * @param [in] dl The place where `d` is mentioned in the input file.
 * @param [in] had_param If true one of 's' or 'd' had a parameter definition in it.
 *                       Thus we cannot determine if they are equal. If SkipContent(), we
 *                       always allocate a directional arrow to avoid warnings about e.g.,
 *                       lost messages and so on. The most typical case is when both
 *                       's' and 'd' are empty due to "$a<-$b;".
 * @returns The created object(with the right default style).*/
Arrow *MscChart::CreateArrow(ArrowSegmentData data, std::string_view s, const FileLineColRange &sl,
                             std::string_view d, bool fw, const FileLineColRange &dl, bool had_param)
{
    _ASSERT(IsArcSymbolArrow(data.type));
    if (s==d && (!SkipContent() || !had_param)) {
        if (data.lost != EArrowLost::NOT)
            Error.Error(data.lost_pos.start, "No support for lost arrows pointing to the same entity in this version. Ignoring asterisk.");
        return new SelfArrow(data.type, s, sl, this, selfArrowYSize);
    }
    //'s' and 'd' can be empty even when !SkipContent(), e.g., if the user typed a->"".
    bool s_special = false, d_special = false;
    if (s.empty() && !SkipContent()) {
        s_special = true;
        s = fw ? LSIDE_ENT_STR : RSIDE_ENT_STR;
        Error.Error(sl.start, "Missing entity name.");
    } else if (s==LSIDE_PIPE_ENT_STR) {
        s_special = true;
        s = LSIDE_ENT_STR;
    } else if (s==RSIDE_PIPE_ENT_STR) {
        s_special = true;
        s = RSIDE_ENT_STR;
    }
    if (d.empty() && !SkipContent()) {
        d_special = true;
        d = fw ? RSIDE_ENT_STR : LSIDE_ENT_STR;
        Error.Error(dl.start, "Missing entity name.");
    } else if (d==LSIDE_PIPE_ENT_STR) {
        d_special = true;
        d = LSIDE_ENT_STR;
    } else if (d==RSIDE_PIPE_ENT_STR) {
        d_special = true;
        d = RSIDE_ENT_STR;
    }
    if (s_special && d_special)
        Error.Error(sl.start, "Internal error. We cannot have |->|.");
    DirArrow * ret = new DirArrow(data, s, sl, d, dl, this, fw);
    if (s_special) ret->AddStartPos();
    if (d_special) ret->AddEndPos();
    return ret;
}

BlockArrow *MscChart::CreateBlockArrow(ArcBase *base)
{
    DirArrow *arrow = dynamic_cast<DirArrow *>(base);
    //We can only get SelfArrow or DirArrow here
    if (!arrow) {
        if (base)
            Error.Error(base->file_pos.start, "Big arrows cannot point back to the same entity. Ignoring it.");
        return nullptr;
    }
    return new BlockArrow(std::move(*arrow));
}

/** Creates a EntityCommand object for the two comment lines and areas.
 * @param [in] s The new style of the comment lines and areas - only the vline and fill part will be used.
 * @param [in] l The range of the input file, that results in the change of comment line style.
 * @returns The returned EntityCommand will hold two EntityApp objects: one for the left and
 * one for the right virtual entity representing the line of the comments.*/
EntityCommand *MscChart::CEForComments(const MscStyle &s, const FileLineColRange &l)
{
    std::unique_ptr<EntityApp> led = std::make_unique<EntityApp>(LNOTE_ENT_STR, this);
    led->SetLineEnd(l);
    led->style.write() += s;
    std::unique_ptr<EntityAppHelper> ledh = led->AddAttributeList(nullptr, nullptr, FileLineCol()); //ownership of led is moved to ledh
    led.release();
    std::unique_ptr<EntityApp> red = std::make_unique<EntityApp>(RNOTE_ENT_STR, this);
    red->SetLineEnd(l);
    red->style.write() += s;
    std::unique_ptr<EntityAppHelper> redh = red->AddAttributeList(nullptr, nullptr, FileLineCol()); //ownership of red is moved to redh
    red.release();
    redh->Prepend(ledh.get());
    EntityCommand *ce = new EntityCommand(redh.release(), this, true);
    ce->AddAttributeList(nullptr);
    return ce;
}

/** Applies a chart option
 * @param [in] a The attribute to add
 * @returns An arc to be inserted to the list of arcs at this point.
 * These can be CommandBackground, EntityCommand (to change comment line style), or
 * both of them in an ArcList (for applying design changes).*/
ArcBase *MscChart::AddAttribute(const Attribute &a)
{
    //Chart options cannot be styles
    _ASSERT(a.type != EAttrType::STYLE);

    if (a.Is("msc") || a.Is("msc+")) {
        if (SkipContent()) return nullptr; //we accept everything while in a proc def
        const bool full = a.Is("msc");
        if (!a.CheckType(EAttrType::STRING, Error)) return nullptr;
        const FileLineColRange line(a.linenum_attr.start, a.linenum_value.end);
        ArcBase *ret;
        switch (SetDesign(full, a.value, false, &ret, line)) {
        case 0:
            Error.Error(a, true, "Unknown chart design: '" + a.value +
                "'. Ignoring design selection.",
                "Available designs are: " + GetDesignNamesAsString(full) +".");
            break;
        case 2:
            Error.Warning(a, true, "Use of '+=' to set a full design.", "Use 'msc = " + a.value + "' to suppress this warning.");
            break;
        case 3:
            Error.Warning(a, true, "Use of '=' to apply a partial design.", "Use 'msc += " + a.value + "' to suppress this warning.");
            break;
        case 1:
            break;
        default:
            _ASSERT(0);
        }
        return ret;
    }
    if (a.Is("arcgradient")) {
        Error.WarnMscgenAttr(a, true, "angle");
        if (a.type != EAttrType::NUMBER || a.number<0) {
            Error.Error(a, true, "Expecting a nonnegative number in pixels.");
            return nullptr;
        }
        MyCurrentContext().slant_depth = a.number;
        MyCurrentContext().slant_angle = 0;
        return nullptr;
    }
    if (a.Is("wordwraparcs")) {
        Error.WarnMscgenAttr(a, true, "text.wrap");
        int value = -1;
        if (a.type == EAttrType::NUMBER)
            value = a.number ? 1 : 0;
        else if (a.type == EAttrType::STRING) {
            if (CaseInsensitiveEqual(a.value, "on") || CaseInsensitiveEqual(a.value, "true"))
                value = 1;
            else if (CaseInsensitiveEqual(a.value, "off") || CaseInsensitiveEqual(a.value, "false"))
                value = 0;
        } else if (a.type == EAttrType::BOOL)
            value = a.yes;
        if (value == -1) {
            a.InvalidValueError("yes', 'on', 'true' or '1", Error);
            return nullptr;
        }
        //treat this as text.wrap
        return AddAttribute(Attribute("text.wrap", value==1 ? "yes" : "no"));
    }
    if (a.Is("width")) {
        Error.WarnMscgen(a.linenum_attr.start, "Deprecated mscgen option.","Use the -x command-line option or bitmap export options instead.");
        if (a.type != EAttrType::NUMBER || a.number<10 || a.number>32000) {
            a.InvalidValueError("10..32000", Error);
            return nullptr;
        }
        width_attr = a.number;
        return nullptr;
    }
    if (a.Is("hscale")) {
        if (a.type == EAttrType::STRING && a.value == "auto") {
            MyCurrentContext().hscale = -1;
            return nullptr;
        }
        if (a.type != EAttrType::NUMBER || a.number<0.01 || a.number>100) {
            a.InvalidValueError("0.01..100' or 'auto", Error);
            return nullptr;
        }
        MyCurrentContext().hscale = a.number;
        return nullptr;
    }
    if (a.Is("compress")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        MyCurrentContext().vspacing = a.yes ? DBL_MIN : 0;
        return nullptr;
    }
    if (a.Is("vspacing")) {
        if (a.type == EAttrType::STRING && a.value == "compress") {
            MyCurrentContext().vspacing = DBL_MIN;
        }
        if (!a.CheckType(EAttrType::NUMBER, Error)) return nullptr;
        MyCurrentContext().vspacing = a.number;
        return nullptr;
    }
    if (a.Is("indicator")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        MyCurrentContext().indicator = a.yes;
        return nullptr;
    }
    if (a.Is("auto_heading")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        MyCurrentContext().auto_heading = a.yes;
        return nullptr;
    }
    if (a.StartsWith("text")) {
        MyCurrentContext().text.AddAttribute(a, this, EStyleType::OPTION); //generates error if needed
        return nullptr;
    }
    if (a.Is("numbering")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        MyCurrentContext().numbering = a.yes;
        return nullptr;
    }
    if (a.Is("numbering.pre")) {
        if (MyCurrentContext().SkipContent()) return nullptr;
        MyCurrentContext().numberingStyle.pre = a.value;
        StringFormat::ExpandReferences(MyCurrentContext().numberingStyle.pre.value(), this,
                                       a.linenum_value.start, nullptr,
                                       false, true, StringFormat::LABEL, true);
        return nullptr;
    }
    if (a.Is("numbering.post")) {
        if (MyCurrentContext().SkipContent()) return nullptr;
        MyCurrentContext().numberingStyle.post = a.value;
        StringFormat::ExpandReferences(MyCurrentContext().numberingStyle.post.value(), this,
                                       a.linenum_value.start, nullptr,
                                       false, true, StringFormat::LABEL, true);
        return nullptr;
    }
    if (a.Is("numbering.append")) {
        if (MyCurrentContext().SkipContent()) return nullptr;
        std::vector<NumberingStyleFragment> nsfs;
        if (NumberingStyleFragment::Parse(this, a.linenum_value.start, a.value.c_str(), nsfs))
            MyCurrentContext().numberingStyle.Push(nsfs);
        return nullptr;
    }
    if (a.Is("numbering.format")) {
        if (MyCurrentContext().SkipContent()) return nullptr;
        std::vector<NumberingStyleFragment> nsfs;
        if (NumberingStyleFragment::Parse(this, a.linenum_value.start, a.value.c_str(), nsfs)) {
            int off = MyCurrentContext().numberingStyle.Apply(nsfs);
            if (off > 0) {
                string msg = "Numbering here is ";
                msg << off << " levels deep, and you specified more (" << nsfs.size();
                msg << ") levels of formatting. Ignoring option.";
                Error.Error(a, true, msg);
            }
        }
        return nullptr;
    }
    if (a.Is("numbering.increment")) {
        if (MyCurrentContext().SkipContent()) return nullptr;
        if (!a.CheckType(EAttrType::NUMBER, Error)) return nullptr;
        if (a.value.find('.') != std::string::npos || a.number == 0 || abs(a.number) > 1000000)
            Error.Error(a, true, "The numbering increment must be a nonzero integer, between plus and minus a million.");
        else
            MyCurrentContext().numberingStyle.Last().increment = int(a.number);
        return nullptr;
    }
    if (a.Is("pedantic")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        pedantic = a.yes;
        return nullptr;
    }
    if (a.Is("angle")) {
        if (!a.EnsureNotClear(Error, EStyleType::ELEMENT)) return nullptr;
        if (!a.CheckType(EAttrType::NUMBER, Error)) return nullptr;
        if (a.number<0 || a.number>45) {
            string x;
            if (a.number<0) x = "0";
            if (a.number>45) x = "45";
            Error.Warning(a, true, "Using " + x + " degrees instead of the specified value.",
                "The slant angle must be between 0 and 45 degrees.");
            if (a.number<0) MyCurrentContext().slant_angle = 0;
            else if (a.number>45) MyCurrentContext().slant_angle = 45;
        } else
            MyCurrentContext().slant_angle = a.number;
        MyCurrentContext().slant_depth = 0;
        return nullptr;
    }
    if (a.StartsWith("background")) {
        FillAttr fill;
        fill.Empty();
        if (fill.AddAttribute(a, this, EStyleType::OPTION)) { //generates error if needed
            MyCurrentContext().defBackground += fill;
            ArcBase *arc = new SetBackground(this, fill);
            arc->AddAttributeList(nullptr);
            return arc;
        }
        return nullptr;
    }
    if (a.StartsWith("comment")) {
        if (CaseInsensitiveBeginsWith(a.name.substr(8), "line") ||
            CaseInsensitiveBeginsWith(a.name.substr(8), "fill")) {
            const bool line = CaseInsensitiveBeginsWith(a.name.substr(8), "line");
            MscStyle toadd; //empty
            bool OK;
            if (line) {
                OK = toadd.line.AddAttribute(a, this, EStyleType::OPTION); //generates errors if needed
                if (OK) std::swap(toadd.line, toadd.vline); //option shall be stored in vline
            } else
                OK = toadd.fill.AddAttribute(a, this, EStyleType::OPTION); //generates errors if needed
            if (OK) {
                MyCurrentContext().defCommentFill += toadd.fill;
                MyCurrentContext().defCommentLine += toadd.vline;
                return CEForComments(toadd, FileLineColRange(a.linenum_attr.start, a.linenum_value.end));
            }
            //fallthrough till error if not "OK"
        } else if (CaseInsensitiveBeginsWith(a.name.substr(8), "side") ||
                   CaseInsensitiveBeginsWith(a.name.substr(8), "text")) {
            Attribute new_a(a);
            new_a.name.erase(0,8); //erase "comment."
            MyCurrentContext().styles["comment"].write().AddAttribute(new_a, this);
            return nullptr;
        }
    }
    if (a.Is("classic_parallel_layout")) {
        if (!a.CheckType(EAttrType::BOOL, Error)) return nullptr;
        simple_arc_parallel_layout = a.yes;
        Error.Warning(a, false, "This chart option is deprecated and will be removed.",
            "Use the '[layout=overlap]' attribute before a series of parallel blocks to achieve the same effect.");
        return nullptr;
    }
    if (a.Is("file.url")) {
        if (!a.EnsureNotClear(Error, EStyleType::OPTION)) return nullptr;
        file_url = a.value;
        return nullptr;
    }
    if (a.Is("file.info")) {
        if (!a.EnsureNotClear(Error, EStyleType::OPTION)) return nullptr;
        if (file_info.length())
            file_info.append("\n").append(a.value);
        else
            file_info = a.value;
        return nullptr;
    }

    string ss;
    Error.Error(a, false, "Option '" + a.name + "' not recognized. Ignoring it.");
    return nullptr;
}

/** Add an attribute only if it can be part of a design.
 * Other attributes trigger error. This is called when a design definition is in progress.
 * @param [in] a The attribute to add.
 * @returns True if the attribute is recognized and added, false if not (error is generated)*/
bool MscChart::AddDesignAttribute(const Attribute &a)
{
    if (a.Is("numbering.append"))
        goto error;
    if (a.Is("numbering.format")) {
        std::vector<NumberingStyleFragment> nsfs;
        if (NumberingStyleFragment::Parse(this, a.linenum_value.start, a.value.c_str(), nsfs)) {
            int off = MyCurrentContext().numberingStyle.Apply(nsfs);
            if (off > 0) {
                string msg = "Only the format of the top level number can be set as part of the design definition.";
                msg << " Ignoring option.";
                Error.Error(a, true, msg);
            }
        }
        return true;
    }
    if (a.Is("numbering.increment")) {
        if (!a.CheckType(EAttrType::NUMBER, Error)) return true;
        if (a.value.find('.') != std::string::npos || a.number == 0 || abs(a.number) > 1000000)
            Error.Error(a, true, "The numbering increment must be a nonzero integer, between plus and minus a million.");
        else
            MyCurrentContext().numberingStyle.Last().increment = int(a.number);
        return true;
    }
    if (a.StartsWith("numbering") || a.Is("compress") || a.Is("auto_heading") ||
        a.Is("hscale") || a.Is("msc") || a.Is("msc+") || a.StartsWith("text") ||
        a.StartsWith("comment") || a.StartsWith("background")) {
        //Since we set chart options inside a design definition,
        //we allow "msc = design" commands to take effect.
        bool saved_ignore_designs = ignore_designs;
        ignore_designs = false;
        ArcBase *ret = AddAttribute(a);
        ignore_designs = saved_ignore_designs;
        if (ret)
            delete ret;
        return true;
    }
error:
    Error.Error(a, false, "Cannot set option '" + a.name +
                  "' as part of a design definition. Ignoring it.");
    return false;
}

/** Adds the attribute names as hints to `csh`.
 * If `designOnly` is true, only the attributes applicable for chart designs are added.*/
void MscChart::AttributeNames(Csh &csh, bool designOnly)
{
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "msc",
        "Apply a full or partial chart design using 'msc=<full_design> or msc+=<partial_design>, respectively.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "hscale",
        "Set the horizontal scaling factor (default is 1) and thereby infuence the width of the chart. Use 'auto' for automatic scaling.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "compress",
        "Turn this on to shift all elements upwards until it bumps in to the element above resulting in compressing the chart vertically.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRNAME) + "vspacing",
        "Specify the vertical spacing above each element.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "auto_heading",
        "Turn this on to automatically display entity headings at the top of each page (after 'newpage;' commands).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering",
        "Turning this on will make labelled elements auto-numbered.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.pre",
        "Set the text to prepend to label numbers. E.g., use 'Step ' to achieve 'Step 1', 'Step 2', etc.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.post",
        "Set the text to append to label numbers. E.g., use ':' to achieve '1:', '2:', etc.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.format",
        "Set the format of auto-numbering, including text format, number type (like roman numbers) and the number of numbering levels.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.append",
        "Append a new numbering level, to have, e.g., 2.1, which lasts until the next closing brace ('}').",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.increment",
        "Sets by how much the currently deepest numbering level shall be incremented at each label. Defaults to 1.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "numbering.*",
        "Auto numbering related options, like format or the number of levels.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "angle",
        "Make arrows slanted by this degree.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.color",
        "Set the color of the background from this point downwards.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.color2",
        "Set the second color of the background (useful for two-color gradients).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.gradient",
        "Set the gradient type of the background from this point downwards.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "background.*",
        "Set the background fill from this point downwards.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.color",
        "Set the color of the line separating side comments from the chart (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.type",
        "Set the type of the line separating side comments from the chart (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.width",
        "Set the width of the line separating side comments from the chart (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.*",
        "Set the attributes of the line separating side comments from the chart (if any).",
        EHintType::ATTR_NAME));
    //csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.radius", EHintType::ATTR_NAME));
    //csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.line.corner", EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.fill.color",
        "Set the color of the background of the side comments (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.fill.color2",
        "Set the second color of the background of the side comments (if any), useful for two-color gradients.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.fill.gradient",
        "Set the gradient of the background fill of the side comments (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.fill.*",
        "Set the attributes of the background fill of the side comments (if any).",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.side",
        "Use this option to set the default side for comments.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.*",
        "Options governing the appearance and location of chart comments.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "file.url",
        "Use this option to attach the URL of the file. Useful for design libraries.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "file.info",
        "Use this option to attach a description to the file. Useful for design libraries.",
        EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "file.*",
        "Use this option to attach a meta information to the file. Useful for design libraries.",
        EHintType::ATTR_NAME));

    StringFormat::AttributeNames(csh, "comment.text.");
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "comment.text.*",
        "Set the text attributes of the side comments (if any).",
        EHintType::ATTR_NAME));
    StringFormat::AttributeNames(csh, "text.");
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "text.*",
        "Set the default text attributes.",
        EHintType::ATTR_NAME));
    if (designOnly) return;

    //this is DEPRECATED, do not hint
    //csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "classic_parallel_layout",
    //    "Turn this on to make parallel blocks be laid out allowing overlaps. (Deprecated, will be removed.)",
    //    EHintType::ATTR_NAME));
    csh.AddToHints(CshHint(csh.HintPrefix(COLOR_OPTIONNAME) + "pedantic",
        "Turn this on to generate an error for entities that are used without explicit declaration.",
        EHintType::ATTR_NAME));
}

/** Adds the possible attribute values for a given attribute to csh.
 * Returns true if we have recognize the attribute*/
bool MscChart::AttributeValues(std::string_view attr, Csh &csh)
{
    if (CaseInsensitiveEqual(attr,"msc")) {
        csh.AddDesignsToHints(true);
        return true;
    }
    if (CaseInsensitiveEqual(attr,"msc+")) {
        csh.AddDesignsToHints(false);
        return true;
    }
    if (CaseInsensitiveEqual(attr,"hscale")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<number>",
            "The default scaling is 1 and you can specify a multiplier. Use a larger number to make the space between entities larger.",
            EHintType::ATTR_VALUE, false));
        csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE) + "auto",
            "This automatically sets the spacing between the entities to just as much as needed.",
            EHintType::ATTR_VALUE));
        return true;
    }
    if (CaseInsensitiveEqual(attr,"angle")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<number>",
            "Makes arrows to slant downwards and is measured in degrees.",
            EHintType::ATTR_VALUE, false));
        return true;
    }
    if (CaseInsensitiveEqual(attr,"compress") ||
        CaseInsensitiveEqual(attr,"numbering") ||
        CaseInsensitiveEqual(attr,"auto_heading") ||
        /* DEPRECATED CaseInsensitiveEqual(attr,"classic_parallel_layout") ||*/
        CaseInsensitiveEqual(attr,"pednatic")) {
        csh.AddYesNoToHints();
        return true;
    }
    if (CaseInsensitiveBeginsWith(attr, "text") ||
        CaseInsensitiveBeginsWith(attr, "comment.text") )
        return StringFormat::AttributeValues(attr, csh);
    if (CaseInsensitiveBeginsWith(attr, "comment.line"))
        return LineAttr::AttributeValues(attr, csh);
    if (CaseInsensitiveBeginsWith(attr, "comment.fill"))
        return FillAttr::AttributeValues(attr, csh);
    if (CaseInsensitiveBeginsWith(attr, "comment.side")) {
        const char * const descriptions[] =
        {nullptr, "Place comments on the left side.", "Place comments on the right side.",
        "Place comments to the end of the chart (i.e., convert to endnotes).", ""};

        for (auto s = ESide::LEFT; s<=ESide::END; s = ESide(int(s)+1))
            if (IsValidSideValue(ESideType::ANY, s))
                csh.AddToHints(CshHint(csh.HintPrefix(COLOR_ATTRVALUE)+EnumEncapsulator<ESide>::names[unsigned(s)],
                                       descriptions[unsigned(s)],
                                       EHintType::ATTR_VALUE, true, CshHintGraphicCallbackForSide,
                                       CshHintGraphicParam(s)));
        return true;
    }

    if (CaseInsensitiveBeginsWith(attr,"background")) {
        FillAttr::AttributeValues(attr, csh);
        return true;
    }
    if (CaseInsensitiveEqual(attr,"numbering.pre")||
        CaseInsensitiveEqual(attr,"numbering.post")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"text\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    if (CaseInsensitiveEqual(attr,"numbering.format")||
        CaseInsensitiveEqual(attr,"numbering.append")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"numbering format\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    if (CaseInsensitiveEqual(attr, "file.url")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"quoted URL of file\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    if (CaseInsensitiveEqual(attr, "file.info")) {
        csh.AddToHints(CshHint(csh.HintPrefixNonSelectable() + "<\"quoted description of file\">", nullptr, EHintType::ATTR_VALUE, false));
        return true;
    }
    return false;
}

 /** Search for a marker.
 * This function returns a marker of exactly this name.*/
 MscChart::MarkerData *MscChart::GetMarker(std::string_view name)
 {
     auto i = Markers.find(name);
     if (i!=Markers.end())
         return &i->second;
     return nullptr;
 }


 std::unique_ptr<ArcBase> MscChart::PopContext()
{
     if (Contexts.size()<2) return {};
    const bool full = MyCurrentContext().IsFull();
    const size_t old_size = full ? MyCurrentContext().numberingStyle.Size() : 0;
    ChartBase::PopContext();
    //if numbering continues with the same amount of levels (or if this was a partial
    //context used to specify a design), no action will be needed
    //in PostParseProcess, so we do not generate any Command arcs.
    if (!full || old_size == MyCurrentContext().numberingStyle.Size())
        return{};
    //if number of levels is less in the outer context, we will need to trim the Numbering list
    //during PostParseProcess after processing all arcs in the inner context, so we insert
    //a SetNumbering, which first trims the Numbering list to the new size and then increments
    //the last number, so after 1.2.2 comes 1.3
    if (old_size > MyCurrentContext().numberingStyle.Size())
        return std::make_unique<SetNumbering>(this, MyCurrentContext().numberingStyle.Size(), MyCurrentContext().numberingStyle.Last().increment);
    //We should never get here, but if the length is increasing, we just expand the Numbering list
    return std::make_unique<SetNumbering>(this, MyCurrentContext().numberingStyle.Size(), 0);
}

void MscChart::ParseText(std::string_view input, const FileLineCol& first_char)
{
    Progress.RegisterParse(input.length());
    sv_reader<false> reader(input, first_char);
    procedure_parse_helper proc_helper;
    msc::lexing_state state = mscgen_compat == EMscgenCompat::FORCE_MSCGEN ? msc::lexing_state::mscgen_compat : msc::lexing_state::basic;
    msc_compile_parse(*this, proc_helper, state, reader); //return value ignored
    force_box_collapse = std::move(force_box_collapse_instead);
    force_box_collapse_instead.clear();
    inline_gui_state = std::move(inline_gui_state_instead);
    inline_gui_state_instead.clear();
}

/** Get an (unordered) list of entities that arrows and boxes in this list touches.
 * We do not include virtual entities (LSide, RSide) and entities we skip.
 * We also determine the general direction of the arrows contained within.
 * If there are arrows in both directions (or any bidir arrow) we return
 * MSC_DIR_BIDIR. If there are no directional arrows (only self-arrows or boxes)
 * we return MSC_DIR_INDETERMINATE.
 * @param [in] al The list of arcs to consider.
 * @param el We append the resulting entity list to this.
 * @param [in] dir The direction of the arrows contained on a previous
 *                 call to this function (with which we combine our directions).
 * @return The direction of the arrows in `al` */
EDirType MscChart::GetTouchedEntitiesArcList(const ArcList &al, NEntityList &el,
                                             EDirType dir) const
{
    for (auto &i : al) {
        NEntityList el2;
        EDirType dir2 = i->GetToucedEntities(el2);
        //update combined direction
        switch (dir2) {
        case EDirType::BIDIR:
            dir = EDirType::BIDIR;
            break;
        case EDirType::LEFT:
        case EDirType::RIGHT:
            if (dir == EDirType::INDETERMINATE) dir = dir2;
            else if (dir != dir2) dir = EDirType::BIDIR;
            break;
        case EDirType::INDETERMINATE:
            break;
        }
        //merge the two lists
        for (auto &ei2 : el2)
            if (el.Find_by_Ptr(ei2) == el.end())
                el.Append(ei2);
    }
    return dir;
}

/** Perform post-parse processing on an ArcList
 * See the notes for the libmscgen module for a details of what it includes.
 * @param [in] canvas The canvas that can be used to learn geometry (font sizes, etc.)
 * @param [in] hide True if this list will get hidden due to a box around it being collapsed.
 *                  We still post-parse process them to collect info, reflect chart options, etc.
 * @param arcs The list to process.
 * @param [in] resetiterators If true, we call each arc in the list with a freshly empty left
 *                            and right iterators - thus we do not attempt to collect the left
 *                            and right extents of the arc list. Useful only for the top level
 *                            arc list.
 * @param left If any element in the list touches an entity leftward than this, we update
 *             this parameter with that entity. Used for box auto-sizing.
 * @param right If any element in the right touches an entity leftward than this, we update
 *              this parameter with that entity. Used for box auto-sizing.
 * @param number This is the running value for automatic numbering. It always contain the next
 *               number to assign.
 * @param target Contains the arc to which a subsequent note or comment will be targeted.
 *               Updated on return for a potential note or command after this arc list.
*/
void MscChart::PostParseProcessArcList(Canvas &canvas, bool hide, ArcList &arcs, bool resetiterators,
                                       EntityRef& left, EntityRef& right,
                                       Numbering& number, MscElement** target)
{
    if (arcs.size()==0) return;
    //If a Note is immediately after an EntityCommand or arrow , take it out
    //and temporarily store it in the EntityCommand or arrow (for re-insertion further below)
    for (ArcList::iterator i = ++arcs.begin(); i != arcs.end(); /*none*/) {
        if (Note* cn = dynamic_cast<Note*>(i->get()))
            if ((*std::prev(i))->StoreNoteTemporarily(cn)) {
                i->release();
                i = arcs.erase(i);
                continue;
            }
        i++;
    }

    //Create the final arc list by merging CommandEntities & unrolling stuff
    for (ArcList::iterator i = arcs.begin(); i != arcs.end(); /*none*/) {
        //Merge in ParallelBlocks defined as "unroll"
        //(E.g., we normally insert an internally defined ParallelBlocks at the beginning,
        //which contains entities for the comment lines. These shall be
        //merged with autogen Entities.)
        //Unrolled ArcParallels sure have just one block.
        ParallelBlocks *par = dynamic_cast<ParallelBlocks*>(i->get());
        if (par && par->unroll) {
            arcs.splice(std::next(i), par->blocks.front().arcs);
            arcs.erase(i++);
            continue;
        }
        //Merge subsequent CommandEntities
        EntityCommand * const ce = dynamic_cast<EntityCommand *>(i->get());
        while (ce) {
            ArcList::iterator j = std::next(i);
            if (j==arcs.end()) break;
            EntityCommand * const ce2 = dynamic_cast<EntityCommand *>(j->get());
            if (!ce2 || ce->internally_defined != ce2->internally_defined) break;
            ce->Combine(ce2);
            arcs.erase(j); //deletes ce2
            //i remains at this very same EntityCommand!
        }
        i++;
    }

    //Form JointSeries objects when elements are marked by the 'join' keyword.
    //test if first element is not specified with 'join'.
    if (arcs.front()->join_pos.IsValid()) {
        Error.Error(arcs.front()->join_pos, "There is no previous element in this scope to join to. Ignoring 'join' keyword.");
        arcs.front()->join_pos.MakeInvalid();
    }
    ESide side = ESide::END;
    for (auto i = ++arcs.begin(); i!=arcs.end(); /*nope*/) {
        if ((*i)->join_pos.IsInvalid()) {
            side = ESide::END; //reset. Any new join can be from either side.
            i++;
            continue;
        }
        JoinableResult res = JoinSeries::CanJoinTo(std::prev(i)->get(), i->get(), side, &Error);
        if (res.IsError()) {
            side = ESide::END;
            (*i)->join_pos.MakeInvalid();
            i++;
            continue;
        }
        //Ok, we join
        JoinSeries *js = dynamic_cast<JoinSeries*>(std::prev(i)->get());
        //if the previous element is not a join series, create one
        if (js==nullptr) {
            js = new JoinSeries(this, std::prev(i)->release());
            std::prev(i)->reset(js);
        }
        js->Append(i->release());
        arcs.erase(i++); //i advances
        side = res.left_to_right ? ESide::RIGHT : ESide::LEFT; //side of the next element
    }

    //Main loop, PostParseProcess each arc in the list
    ArcBase *byref_terget = nullptr;
    for (ArcList::iterator i = arcs.begin(); i != arcs.end(); /*none*/) {
        if (resetiterators)
            right = left = nullptr;
        MscElement * const old_target = *target;
        const bool i_can_be_noted = (*i)->CanBeNoted();
        ArcBase *replace = (*i)->PostParseProcess(canvas, hide, left, right, number, target, byref_terget);
        //Beware: 'replace' may be equal to i->get(), in which case 'i' owns it.
        //Or, it can be something different, in which case the object is owned by 'replace'

        //if the new target is somewhere inside "i" (or is exactly == to "i")
        //NOTE: *target is never set to nullptr, only to DELETE_NOTE or to an Arc
        if (*target != old_target && replace != i->get()) {
            //If we remove an arc that could have been noted, we set the target
            //to DELETE_NOTE, as well, to remove the target, too.
            if (replace == nullptr)
                *target = DELETE_NOTE;
            //if we replace to a different Arc, redirect notes to that
            else if (replace->CanBeNoted())
                *target = replace;
            else if (i_can_be_noted)
                *target = DELETE_NOTE; //If the original arc took a note, but the replacement (e.g., an indicator) doesn't we drop the note
            else
                *target = old_target;
        }
        //If we are about to replace to an indicator, remove it,
        //if previous thing was also an Indicator on the same entity
        Indicator * const ai = dynamic_cast<Indicator*>(replace);
        if (ai && i!=arcs.begin()) {
            _ASSERT(replace != i->get()); //this is to make sure, 'replace' owns the object it points to, see above
            //Iterate backwards until we find the element that separates indicators
            for (ArcList::iterator j = std::prev(i); j!=arcs.end(); --j)
                if ((*j)->SeparatesIndicators()) {
                    Indicator* const ai2 = dynamic_cast<Indicator*>(j->get());
                    if (ai2 && ai2->Combine(ai)) {
                        if (*target == replace) //redirect note to previous indicator
                            *target = ai2;
                        delete replace;
                        replace = nullptr;
                    }
                    break;
                }
        }
        //If 'i' is to be dropped and the previous element is an indicator, we expand the line coverage of that
        if (!replace && i!=arcs.begin())
            if (Indicator* const ai2 = dynamic_cast<Indicator*>(std::prev(i)->get()))
                ai2->ExtendFilePos(i->get());

        //Ok, reinsert temporarily stored notes from CommandEntities/arrows (if any)
        if (replace)
            replace->ReinsertTmpStoredNotes(arcs, i);

        //Ok, final actions: replacement plus report progress.
        //We wither remove i from the list or increase the iterator to the next element
		//Below we only do progress report for elements done and not deleted
        if (replace == i->get()) {
            //The arc requested no replacement
			Progress.DoneItem(EArcSection::POST_PARSE, (*i)->myProgressCategory);
            //If this is an endnote, move it to EndNotes            
            if (Note* cn = dynamic_cast<Note*>(i->get()); cn && cn->GetStyle().read().side == ESide::END) {
                EndNotes.Append(cn); //ownership moved to EndNote...
                i->release();        //..so we release it before erasing
                i = arcs.erase(i); 
            } else {
                if ((*i)->CanBeAlignedTo())
                    byref_terget = i->get();
                i++;
            }
		} else {
            //The arc requested a replacement. 'replace' owns its pointed object here.
            //Note that any replacement already has its PostParseProcess() called
            if (replace) {
                Progress.DoneItem(EArcSection::POST_PARSE, replace->myProgressCategory);
                i->reset(replace); //'i' takes ownership, old object pointed by 'i' is deleted
                i++;
                if (replace->CanBeAlignedTo())
                    byref_terget = replace;
            } else
                i = arcs.erase(i); //Nothing to replace to: remove pointer from list (and also delete it)
        }
    }
}

/** Performs the post parse processing for the whole chart
 * See the notes for the libmscgen module for a details of what it includes.
 * (We sort entities, determine which shows, append automatically generated
 * entities, and process MscChart::Arcs.) */
void MscChart::PostParseProcess(Canvas &canvas)
{
	Progress.StartSection(EArcSection::POST_PARSE);
    //remove those entities from "force_entity_collapse" which are not defined as entities
    for (auto i = force_entity_collapse.begin(); i!=force_entity_collapse.end(); /*nope*/)
        if (AllEntities.Find_by_Name(i->first) == nullptr)
            force_entity_collapse.erase(i++);
        else i++;

    //Sort the defined entities as will be displayed from left to right
    AllEntities.sort([](auto &a, auto &b) {return a.pos<b.pos; });

    //Now create a list of active Entities
    //Note that virtual entities will be added
    ActiveEntities.clear();
    for (auto &e : AllEntities) {
        EntityRef j = FindActiveParentEntity(&e);
        if (&e==j && e.children_names.size() && !e.collapsed) continue;
        if (ActiveEntities.size()==0)
            ActiveEntities.Append(j);  //first active entity, likely be NoEntity
        else if (ActiveEntities.Find_by_Name(j->name) == nullptr)
            ActiveEntities.Append(j);  //a new active entity, not yet added
        else if (j != ActiveEntities.back()) {
            //already added but not as last one
            Error.Error(j->file_pos, "Entities grouped into entity '" + j->name +"' are not consecutive.",
                                        "May result in unintended consequences.");
            Error.Error(e.file_pos, j->file_pos, "Entity '" + e.name, "' may be one outlier.");
        }
    }

    //Set index field in Entities
    unsigned index = 0;
    for (auto pEntity : ActiveEntities)
        pEntity->index = index++;

    //Find the first real entity
    NEntityList::iterator iLeftmost = ActiveEntities.begin();
    while (iLeftmost != ActiveEntities.end() && IsVirtualEntity(iLeftmost))
        iLeftmost++;

    //Ensure that leftmost real entity pos == 2*MARGIN
    //Note that the distance between LSide and the first real entity
    //will be auto-scaled.
    double rightmost = 0;
    if (iLeftmost != ActiveEntities.end()) {
        double leftmost = (*iLeftmost)->pos - 2*MARGIN;
        for  (auto &e : AllEntities)
            e.pos -= leftmost;
        //Find rightmost entity's pos
        for  (auto pEntity : ActiveEntities)
            if (!IsVirtualEntity(pEntity) && rightmost < pEntity->pos)
                rightmost = pEntity->pos;
    } else {
        //if there are no real entities, just noentity, lside & rside, set this wide
        //so that copyright banner fits
        rightmost = 3*MARGIN;
    }
    //Set the position of the virtual side entities & resort
    const_cast<double&>(StartEntity->pos) = 0;
    const_cast<double&>(LNote->pos) = 0;
    const_cast<double&>(LSide->pos) = MARGIN;
    const_cast<double&>(RSide->pos) = rightmost + MARGIN;
    const_cast<double&>(RNote->pos) = rightmost + MARGIN + MARGIN;
    const_cast<double&>(EndEntity->pos) = rightmost + MARGIN + MARGIN;
    ActiveEntities.sort([](auto &a, auto &b) {return a->pos<b->pos; });

    if (Arcs.size()==0) return;

    if (AutoGenEntities.size()) {
        //Add the Auto generated entities to the first definition of entities
        //Search for this kind of first definition EntityCommand from the
        //beginning or Arcs, but skip objects of the following type
        //- CommandArcList: these are inserted by chart options only
        //- ArcDividers with no entities: these are likely (sub)title commands
        //If we find a EntityCommand having only the above before it,
        //append the automatically generated entities to that.
        //Otherwise, generate a new entity command at that location
        ArcList::iterator i = Arcs.begin();
        //Skip over the titles and chart options
        while (i!=Arcs.end() && (*i)->BeforeAutoGenEntities())
            i++;
        //Insert a new EntityCommand if we have none
        if (i==Arcs.end() || dynamic_cast<EntityCommand*>(i->get())==nullptr) {
            i = Arcs.insert(i, make_unique<EntityCommand>(new EntityAppHelper, this, false));
            (*i)->AddAttributeList(nullptr);
        }
        dynamic_cast<EntityCommand*>(i->get())->AppendToEntities(std::move(AutoGenEntities));
        AutoGenEntities.clear();
    }
    //Set all entity's shown to false, to avoid accidentally showing them via (heading;) before definition
    for (auto i = ActiveEntities.begin(); i!=ActiveEntities.end(); i++) {
        (*i)->running_shown = EEntityStatus::SHOW_OFF; //not shown not active
        (*i)->running_defined = false;
    }

    //Traverse Arc tree and perform post-parse processing
    Numbering number; //starts at a single level from 1
    EntityRef dummy1 = nullptr, dummy2 = nullptr;
    MscElement *note_target = nullptr;
    PostParseProcessArcList(canvas, false, Arcs, true, dummy1, dummy2, number, &note_target);

    //Reinsert end-notes at the end (they have been post-processed)
    if (EndNotes.size()) {
        //Add separator
        std::unique_ptr<EndNoteSeparator> ens = std::make_unique<EndNoteSeparator>(this);
        ens->AddAttributeList(nullptr);
        Arcs.Append(std::move(ens));
        //Append end-notes
        Arcs.splice(Arcs.end(), EndNotes);
    }
}

void MscChart::FinalizeLabelsArcList(ArcList& arcs, Canvas& canvas) {
    for (auto& i : arcs) {
        i->FinalizeLabels(canvas);
        Progress.DoneItem(EArcSection::FINALIZE_LABELS, i->myProgressCategory);
    }
}


/** Draw the entity lines in a vertical region and between two entities
 * We draw entity lines according to their styles and activation status.
 * We do not draw them where they shall be hidden (by, e.g., a label).
 * @param canvas The canvas onto which w draw.
 * @param [in] y Top of the region to draw in
 * @param [in] height Height of the region to draw in
 * @param [in] from The leftmost entity to draw the line for
 * @param [in] to The entity right of the rightmost entity to draw for.*/
void MscChart::DrawEntityLines(Canvas &canvas, double y, double height,
                               NEntityList::const_iterator from, NEntityList::const_iterator to)
{
    //No checking of iterators!! Call with caution
    //"to" is not included!!
    canvas.ClipInverse(HideELinesHere);
    double act_size = canvas.HasImprecisePositioning() ? floor(activeEntitySize/2) : activeEntitySize/2;
    while(from != to) {
        const EntityStatusMap & status= (*from)->status;
        XY up(XCoord(*from), y);
        XY down(up.x, y);
        const double till = y+height;
        for (/*none*/; up.y < till; up.y = down.y) {
            const double show_till = status.ShowTill(up.y);
            const double style_till = status.StyleTill(up.y);
            const MscStyle &style = dynamic_cast<const MscStyle&>(status.GetStyle(up.y).read());
            down.y = std::min({ show_till, style_till, till });
            //we are turned off below -> do nothing
            if (!status.GetStatus(up.y).IsOn())
                continue;
            if (status.GetStatus(up.y).IsActive()) {
                const double show_from = status.ShowFrom(up.y);
                Block outer_edge;
                //if we may start an active rectangle here...
                if (!status.GetStatus(show_from).IsActive() || !status.GetStatus(show_from).IsOn())
                    outer_edge.y.from = up.y;
                //...or an active rectangle may have started earlier
                else
                    outer_edge.y.from = std::max(show_from, 0.);
                outer_edge.y.till = std::min({ show_till, drawing.y.till, y + height });
                outer_edge.x.from = up.x - act_size;
                outer_edge.x.till = up.x + act_size;
                const bool doClip = outer_edge.y.from < up.y || outer_edge.y.till > down.y;

                if (canvas.does_graphics()) {
                    if (doClip)
                        canvas.Clip(Block(outer_edge.x, Range(up.y, down.y)));
                    outer_edge.Expand(-style.vline.LineWidth()/2);  //From now on this is the midpoint of the line, as it should be
                    canvas.Fill(style.vline.CreateRectangle_ForFill(outer_edge), style.vfill);
                    canvas.Line(style.vline.CreateRectangle_Midline(outer_edge), style.vline);
                    if (doClip)
                        canvas.UnClip();
                } else {
                    DoubleMap<bool> e(false);
                    e.Set(outer_edge.y, true);
                    const Contour w = outer_edge * HideELinesHere;
                    for (const auto& c : w)
                        e.Set(c.GetBoundingBox().y, false);
                    e.Prune();
                    _ASSERT(std::next(e.begin())->second);
                    double from = 0;
                    for (auto i = std::next(e.begin()); i!=e.end(); i++)
                        if (i->second) from = i->first;
                        else {
                            const Block b{XY(outer_edge.x.from, from), XY(outer_edge.x.till, i->first)};
                            const bool top = from == std::next(e.begin())->first;
                            const bool bottom = i == std::prev(e.end());
                            if (top && bottom)
                                canvas.Add(GSBox(b, style.vline, style.vfill));
                            else {
                                canvas.Add(GSBox(b, LineAttr(ELineType::NONE, ColorType()), style.vfill));
                                if (top) {
                                    canvas.Add(GSPath({b.LowerLeft(), b.UpperLeft(), b.UpperRight(), b.LowerRight()}, style.vline));
                                } else if (bottom) {
                                    canvas.Add(GSPath({b.UpperLeft(), b.LowerLeft(), b.LowerRight(), b.UpperRight()}, style.vline));
                                } else {
                                    canvas.Add(GSPath({b.UpperLeft(), b.LowerLeft()}, style.vline));
                                    canvas.Add(GSPath({b.LowerRight(), b.UpperRight()}, style.vline));
                                }
                            }
                        }
                }
            } else {
                const double offset = canvas.HasImprecisePositioning() ? 0 : fmod_negative_safe(style.vline.width.value()/2, 1.);
                if (canvas.NeedsStrokePath_rclBoundsFix()) {
                    /* We draw such a shape for an entity line:
                     *  .....   Where dotted lines are invisible.
                     *  :   :   The visible vertical part is the entity line
                     *  : +--   The visible horizontal parts are serifs to expand the
                     *  : |     bounding box of the stroke.
                     *  : |     This is needed so that the resulting metafile
                     *  --+     retains the entity line even when shrunk.
                     *          Surprisingly the bounds remain large even if we clip
                     *          away the serifs (which we do). */
                    const double gap = 10 + style.vline.LineWidth();
                    const Block clip (up   + XY(-gap+offset,0),
                                      down + XY( gap+offset,0));
                    canvas.Clip(clip);
                    const XY points[] = {down + XY(-2*gap+offset,  +gap),
                                         up   + XY(-2*gap+offset, -gap*2),
                                         up   + XY(+2*gap+offset, -gap*2),
                                         up   + XY(+2*gap+offset, -gap),
                                         up   + XY(     0+offset, -gap),
                                         down + XY(     0+offset, +gap)};
                    Contour line_shape;
                    line_shape.assign_dont_check(points);
                    line_shape.front().Outline()[0].SetVisible(false);
                    line_shape.front().Outline()[1].SetVisible(false);
                    line_shape.front().Outline()[2].SetVisible(false);
                    canvas.Line(line_shape, style.vline);
                    canvas.UnClip();
                } else if (canvas.does_graphics())
                    canvas.Line(up+XY(0,offset), down+XY(0,offset), style.vline);
                else {
                    DoubleMap<bool> e(false);
                    e.Set(Range(up.y+offset, down.y+offset), true);
                    const double lw = style.vline.LineWidth();
                    const Contour w = Block(up+XY(-lw/2, offset), down+XY(lw/2, offset)) * HideELinesHere;
                    for (const auto& c : w)
                        e.Set(c.GetBoundingBox().y, false);
                    e.Prune();
                    _ASSERT(std::next(e.begin())->second);
                    double from = 0;
                    for (auto i = std::next(e.begin()); i!=e.end(); i++)
                        if (i->second) from = i->first;
                        else canvas.Add(GSPath({XY(up.x, from), XY(down.x, i->first)}, style.vline));
                }
            }
        }
        from++;
    }
    canvas.UnClip();
}

/** Calls the Width() function for all arcs in the list and collects distance requirements to `vdist` */
void MscChart::WidthArcList(Canvas &canvas, ArcList &arcs, DistanceRequirements &vdist)
{
    for (auto &pArc : arcs) {
        pArc->Width(canvas, vdist);
        Progress.DoneItem(EArcSection::WIDTH, pArc->myProgressCategory);
        if (pArc->IsByRef())
            ByRefs.emplace(pArc->GetRefTarget(), pArc.get());
	}
}


/** Places a full list of elements starting at y position==0
 * Calls Layout() for each element (recursively) and takes "compress" and
 * "parallel" attributes into account.
 * We always place each element on an integer y coordinate.
 * Automatic pagination is ignored by this function and is applied later instead.
 * Ensures that elements in the list will have non-decreasing yPos order -
 * thus an element later in the list will have same or higher yPos as any previous.
 * @param canvas The canvas to calculate geometry on.
 * @param arcs The list of arcs to place
 * @param cover We add the area covered by each arc to this list.
 * @returns the total height of the list*/
double MscChart::LayoutArcList(Canvas &canvas, ArcList &arcs, AreaList *cover)
{
    std::list<LayoutColumn> y;
    y.emplace_back(&arcs, 0);
    return LayoutParallelArcLists(canvas, y, cover);
}


/** Find how much we need to push arc_cover down (assumed to be laid out at y==0)
 * to be below all children of 'sibling' (but not 'sibling' itself, just its
 * 'completed_children_covers'). We completely ignore mainlines here.
 * Uses offsets meaning upwards movement, as OffsetBelow.*/
double FindOffsetInChildren(const std::list<LayoutColumn> &columns, const LayoutColumn *sibling,
                            const AreaList &arc_cover, double &touchpoint, double offset)
{
    offset = sibling->completed_children_covers.OffsetBelow(arc_cover, touchpoint, offset, false);
    for (auto &col : columns)
        if (col.parent == sibling) {
            offset = col.covers.OffsetBelow(arc_cover, touchpoint, offset, false);
            offset = col.completed_children_covers.OffsetBelow(arc_cover, touchpoint, offset, false);
            if (col.number_of_children)
                offset = FindOffsetInChildren(columns, &col, arc_cover, touchpoint, offset);
        }
    return offset;
}

/** Finds how much 'arc_cover' (laid out at y==0) shall be shifted down not to
 * collide with previously laid out elements of 'columns'.
 * In general we test our own column and sibling colums - but for sibling columns
 * we ignore mainlines, since we can be besides elements of sibling columns, there
 * only overlap needs to be avoided.
 * Since LayoutColumn::covers only contain the cover of the elements in that column,
 * and nothing from the parent column, we also need to test the parents (and the
 * sibling of the parents, without mainlines, too).
 * So the algoritm starts form the current column 'to_test' checks offset of it and
 * its siblings (mainlines ignored), then steps to the parent and repeats until
 * we check the siblings at the top level (where parent==nullptr).
 * 'compress' tells us if the arc in question shall be compressed or not.
 * In the latter case there is no need to check if the element to be placed
 * collides with prior elements in its column or that of parents, since we
 * will attempt to place it strictly below all those. So as a performance
 * optimization we avoid calling OffsetBelow().
 * @param [in] columns The list of columns under processing
 * @param [in] to_test The column we are processing now.
 * @param [in] arc_cover The cover of the element we place, assumed to be laid out at y==0.
 * @param [out] touchpoint The y value at which the prior elements and this element touches.
 * @param [in] compress True if the element will be compressed (or the prior element was
 *                      parallel) and we need to check elements in our column, too.
 * @returns how much we shall shift the element down (from its current position at y==0)*/
double FindOffset(const std::list<LayoutColumn> &columns, const LayoutColumn *to_test,
                  const AreaList &arc_cover, double &touchpoint, bool compress)
{
    //now try pushing upwards against previous elements in this column,
    //against the body (no mainline) of previous elements parallel to us
    //(sibling columns) and all our parents (and their siblings).
    double offset = CONTOUR_INFINITY;
    while (to_test) {
        for (auto &col : columns)
            if (&col == to_test->parent)
                //For our parents, we test against any covers (without mainlines)
                //that are from completed sibling columns of ours.
                offset = col.completed_children_covers.OffsetBelow(arc_cover, touchpoint, offset, false);
            else if (col.parent == to_test->parent && (compress || &col!=to_test)) {
                //For our column we consider the mainlines, but for
                //neighbouring columns (the same parent as our column, but not our column),
                //we do not (we can be besides them, but not overlap)
                offset = col.covers.OffsetBelow(arc_cover, touchpoint, offset, &col==to_test);
                //for our siblings, we traverse their children - we need to avoid those, too.
                if (&col!=to_test && col.number_of_children)
                    offset = FindOffsetInChildren(columns, &col, arc_cover, touchpoint, offset);
            }
        to_test = to_test->parent;
    }
    return -offset;
}

/** Places a set of parallel lists of elements starting at y position==0.
 * Calls Layout() for each element (recursively) and takes "compress" and
 * "parallel" attributes into account.
 * Attempts to avoid collisions and a balanced progress in each list.
 * If we encounter ArcParallels with layout==one_by_one_merge, we do not
 * call their ParallelBlocks::Layout(), but merge them among these lists.
 * This is the main (only) layout function for lists, this one is used
 * even for non-parallel lists.
 * We always place each element on an integer y coordinate.
 * Automatic pagination is ignored by this function and is applied later instead.
 * Ensures that elements in the list will have non-decreasing yPos order -
 * thus an element later in the list will have same or higher yPos as any previous.
 * @param canvas The canvas to calculate geometry on.
 * @param columns The list of arc lists to place
 * @param cover We add the area covered by each arc to this list, if non nullptr.
 * @returns the total combined height of the lists.*/
double MscChart::LayoutParallelArcLists(Canvas &canvas, std::list<LayoutColumn> &columns, AreaList *cover)
{
    //first remove empty columns
    columns.remove_if([](LayoutColumn &c) {return c.list->size()==0; });

    //we count arcs we placed, so that we can see which column got treated last
    unsigned current_action = 1;
    //use this counter to add new columns
    unsigned counter = columns.size();
    //The lowestmost element - to return as height
    double total_bottom = 0;
    //cover of those elements that were in top-level columns (no parent)
    //that have completed. It is important to keep this to avoid
    //overlap with completed columns (since we delete completed columns)
    AreaList top_completed_children_cover;

    //test for fast path
    //if 0) caller requested no cover returned; 1) we do a single column only;
    //2) have no ArcParallels with ONE_BY_ONE_MERGE;
    //3) have no elements marked as compress and 4) have no elements marked as parallel
    //Then we do not need to maintain covers
    //(We could optimize this in the case when we have one or more ArcParallels with
    // ONE_BY_ONE_MERGE, but only a single column without any element with compress.
    // In that case we still don't need covers - but it is such a rarity...)
    const bool need_covers = cover || columns.size() != 1 ||
        columns.front().list->end() != std::find_if(columns.front().list->begin(), columns.front().list->end(),
        [](const std::unique_ptr<ArcBase> &arc) {return arc->IsParallel() || arc->IsCompressed() ||
        (dynamic_cast<ParallelBlocks*>(arc.get()) &&
         dynamic_cast<ParallelBlocks*>(arc.get())->layout==ParallelBlocks::ONE_BY_ONE_MERGE); });

    //Zero-height arcs shall be positioned to the same place
    //as the first non-zero height arc below them (so that
    //if that arc is compressed up, they are not _below_
    //that following arc. So we store them until we know
    //where shall we put them.
    NPtrList<ArcBase> zero_height;

    while (columns.size()) {
        //Select the next column to pick an element from.
        //- lists with no children are sorted earlier (we must have at least
        //  one column with no children)
        //- among those we select the one with lowest y,
        //- if there are several with the same y then we select the one with
        //  the lowest 'last_action' (the one we took an element longest time ago)
        //- finally (and this can happen only after the insertion of new columns from
        //  an ParallelBlocks where last action is all equal) we tie break on the 'column' number
        LayoutColumn &now = *std::min_element(columns.begin(), columns.end(),
            [](const LayoutColumn &a, const LayoutColumn &b)
            {return std::tie(a.number_of_children, a.y, a.last_action, a.column) <
                    std::tie(b.number_of_children, b.y, b.last_action, b.column);});
        _ASSERT(now.number_of_children==0);
        _ASSERT(now.list->end()!=now.arc);

        //Start cycle till the last element (arc) in the list, but we
        //will exit as soon as we added an element of nonzero height.
        for (; now.arc!=now.list->end(); now.arc++) {
            _ASSERT((*now.arc)->IsValid()); //should have been removed by PostParseProcess
            //By ref symbols/verticals are laid out with their target
            if ((*now.arc)->IsByRef()) continue;
            AreaList arc_cover;
            //Check if this is a ParallelBlocks with one_by_one_merge layout.
            //Note: it is important to avoid one_by_one_merge for parallel blocks that have
            //any by-ref symbols/verticals attached to them, if possible. This is because
            //if any of those extend _above_ the top of the ParallelBlocks, we would need to
            //shift the ParallelBlocks down, to make room. We cannot do that if we do 
            //one_by_one_merge, but can do that if we do one_by_one. So we try to do
            //one_by_one instead of one_by_one_merge, if this does not create any visual 
            //change.
            //There are 2 cases, when there is a visual change between one_by_one_merge
            //and one_by_one.
            //1. if the there is any other column, which is not our ancestor. In this case
            //   the elements of the ParallelBlocks have to be mixed with those of the other
            //   columns, which results in a difference compared to when we lay them out as
            //   one block (one_by_one).
            //2. If the various columns of the ParallelBlock can be shifted upwards differently
            //   due to compress.
            // 
            //Thus the algorithm for deciding fallback to one_by_one merge is as follows:
            //1. If the ParallelBlocks has no attached by-ref symbols or verticals, we keep
            //   doing one_by_one_merge.
            //2. If the existing columns are all our parents (or parents' parents, etc.), we
            //   switch to one_by_one, even though issue #2 above may result in a visual
            //   change. 
            if (ParallelBlocks* const pParallelBlocks = dynamic_cast<ParallelBlocks*>(now.arc->get())
                    ; pParallelBlocks && pParallelBlocks->IsValid() && pParallelBlocks->blocks.size()
                    && pParallelBlocks->layout==ParallelBlocks::ONE_BY_ONE_MERGE) {
                //Count how many columns are our parents
                size_t num_our_family = 1; //us
                for (LayoutColumn *c = now.parent; c; c = c->parent)
                    num_our_family++;
                if (not ByRefs.contains(pParallelBlocks) || num_our_family < columns.size()) {
                    //'comp' tells us if the first element can be compressed or not
                    //if not, we set the upper limit to use_y
                    const bool comp = now.previous_was_parallel || pParallelBlocks->IsCompressed();
                    const double upper_limit = comp ? now.y_upper_limit : now.y;
                    //Store value for y_upper_limit to be used after all columns of this
                    //ParallelBlocks has been completed.
                    now.y_upper_limit = now.y;
                    //do not consider vspacing if at the beginning of a scope.
                    if (now.y) now.y += pParallelBlocks->GetVSpacing();
                    pParallelBlocks->SetYPos(now.y);
                    for (auto &col : pParallelBlocks->blocks)
                        columns.emplace_back(&col.arcs, counter++, now.y, upper_limit,
                                             comp, now.previous_x_extent, &now, now.last_action);
                    now.number_of_children = pParallelBlocks->blocks.size();
                    break; // this will jump over element processing
                    //now.arc is left pointing to the ParallelBlocks
                    //now.y_upper_limit points to upper limits used if this ParallelBlocks is 'parallel'
                    //Note: we kept any items potentially in zero_length. They will be
                    //placed at the top of the first non-zero element in one of the columns now inserted
                }
            }

            (*now.arc)->Layout(canvas, need_covers || ByRefs.contains(now.arc->get()) ? &arc_cover : nullptr); //'ByRefs' may be changed by this call (for ParallelBlocks)
            Progress.DoneItem(EArcSection::LAYOUT, (*now.arc)->myProgressCategory);
            //Call Layout for all Symbols that are laid out by reference of this element            
            double min_byref_top = 0;
            const auto [byrefs_from, byrefs_till] = ByRefs.equal_range(now.arc->get());
            if (byrefs_from != byrefs_till) {
                for (auto i = byrefs_from; i != byrefs_till; ++i) {
                    AreaList byref_cover;
                    const bool make_room = i->second->LayoutByRef(canvas, arc_cover, &byref_cover);
                    min_byref_top = std::min(min_byref_top, byref_cover.GetBoundingBox().y.from);
                    if (make_room) { //subsequent symbols/verticals shall be placed considering this larger cover.
                        byref_cover.InvalidateMainLine();                        
                        arc_cover += std::move(byref_cover);
                    }
                }
                if (min_byref_top < 0) {
                    const double shift = -min_byref_top;
                    arc_cover.Shift(XY(0, shift));
                    (*now.arc)->ShiftBy(shift);
                    for (auto i = byrefs_from; i != byrefs_till; ++i)
                        i->second->ShiftBy(shift);
                }
            }
            double h = (*now.arc)->GetFormalHeight();
            //increase h, if arc_cover.Expand() (in "Layout()") pushed outer boundary. This ensures that we
            //maintain at least compressGap/2 amount of space between elements even without compress.
            //Note we fail to do this unless need_cover is true. No problem for now.
            h = std::max(h, arc_cover.GetBoundingBox().y.till);
            //if arc is of zero height, just collect it.
            //Its position may depend on the next arc if that is compressed.
            if (h==0) {
                zero_height.Append(now.arc->get());
                continue;
            }
            now.last_action = ++current_action;
            double touchpoint = now.y;
            double use_y = now.y;
            if ((*now.arc)->IsCompressed() ||
                (now.previous_was_parallel && !now.previous_x_extent.Overlaps(arc_cover.GetBoundingBox().x))) {
                //now try pushing upwards against previous elements in this column,
                //against the body (no mainline) of previous elements parallel to us
                //(sibling columns) and all our parents (and their siblings).
                double offset = FindOffset(columns, &now, arc_cover, touchpoint, true);
                offset = -top_completed_children_cover.OffsetBelow(arc_cover, touchpoint, -offset, false);
                use_y = std::max(now.y_upper_limit, offset);
                if (!(*now.arc)->IsCompressed()) {
                    //we must have previous_was_parallel==true here
                    //if the immediately preceding element (not including zero_height_ones) was
                    //marked with "parallel", we attempt to shift this element up to the top of that one
                    //even if the current element is not marked by "compress".
                    //If we can shift it all the way to the top of the previous element, we place it
                    //there. But if we can shift only halfway, we place it strictly under the previous
                    //element - as we are not compressing.
                    //Note that "y_upper_limit" contains the top of the preceding element (marked with "overlap")
                    if (use_y == now.y_upper_limit)
                        touchpoint = now.y_upper_limit;
                    else
                        //Note: OffsetBelow() may have destroyed touchpoint above
                        touchpoint = use_y; //(which is == now.y).
                }
            } else {
                //This element is not compressed. But as we may lay out parallel blocks
                //it may be that we overlap with an element from another column, so we want to avoid that.
                //But we place zero_height elements to just below the previous one nevertheless
                //We also keep touchpoint==now.y for this very reason.
                double dummy_touchpoint;
                double offset = FindOffset(columns, &now, arc_cover, dummy_touchpoint, false);
                offset = -top_completed_children_cover.OffsetBelow(arc_cover, touchpoint, -offset, false);
                //if we are placing higher than the lowest element so far (because that was 'compressed',
                //for example), we test if we should shift lower due to the elements in our own column.
                if (now.y_bottom_all > use_y)
                    offset = -now.covers.OffsetBelow(arc_cover, touchpoint, -offset, false);
                use_y = std::max(use_y, offset);
            }
            //Now use_y contains where this element shall be placed, and now.y the
            //bottommost in this column (and its parents) (except elements marked with parallel or overlap).
            //Add extra space (even if above was parallel), move touchpoint by half
            //But do not add extra space at the beginning of a scope
            const double extra = now.y ? (*now.arc)->GetVSpacing() : 0;
            touchpoint += extra/2;
            use_y += extra;
            touchpoint = floor(touchpoint+0.5);
            use_y = ceil(use_y);
            //We got a non-zero height or a non-compressed one, flush zero_height ones (if any)
            for (auto pArc : zero_height) {
                pArc->ShiftBy(touchpoint);
                pArc->FinalizeLayout(canvas, nullptr); //arc with zero height has no cover to update
            }
            zero_height.clear();
            //Shift the arc in question to its place
            (*now.arc)->ShiftBy(use_y);
            arc_cover.Shift(XY(0, use_y));
            (*now.arc)->FinalizeLayout(canvas, &arc_cover);
            for (auto i = byrefs_from; i != byrefs_till; ++i) {
                i->second->ShiftBy(use_y);
                //arc_cover contains more than the cover of 'i'
                i->second->FinalizeLayout(canvas, &arc_cover);
            }
            //If the caller wanted to have all covers back, add this one to the list.
            if (cover)
                *cover += arc_cover;
            //update column state
            now.y_upper_limit = use_y; //Do not allow anyone later in the list to be placed above us
            now.previous_was_parallel = (*now.arc)->IsParallel();
            now.previous_x_extent = arc_cover.GetBoundingBox().x;
            now.y_bottom_all = std::max(now.y_bottom_all, use_y+h);
            if ((*now.arc)->IsOverlap()) {
                //Set the normal (non-compressed pos of the next arc)
                //as the top of the current arc
                now.y = use_y;
            } else {
                now.y = std::max(now.y, use_y + h);
                now.y = use_y + h;
                //Update covers
                if ((*now.arc)->IsParallel())
                    arc_cover.InvalidateMainLine(); //kill the mainline if we are parallel
                now.covers += std::move(arc_cover); //arc_cover contains the mainline here (unless parallel)
            }
            //This was a non-zero height element, we break and pick
            //the next arc from the column with the topmost current bottom.
            now.arc++; //if we break for loop increment will not be called, so we increment now.arc here.
            ByRefs.erase(byrefs_from, byrefs_till);
            if (now.arc!=now.list->end() && (*now.arc)->IsStickToPrev()) {
                now.arc--; //so that the for increment does its job
                continue; //We do not break if the next element was 'stick_to_prev'. In that case we continue here
            }
            break;
        } //for cycling the zero height elements
        //We have changed now.y, now.last_action (or have inserted new columns
        //if we process an ParallelBlocks of ONE_BY_ONE_MERGE layout), so
        //the "smallest" element in y has changed.
        //if this column is not done repeat cycle - at the beginning
        //we will select a new column to pick an element from.
        if (now.arc!=now.list->end())
            continue;

        //Ok, we have finished with this column - close it.
        //first, position any remaining zero-height items at the bottom
        for (auto pArc : zero_height) {
            pArc->ShiftBy(now.y);
            pArc->FinalizeLayout(canvas, nullptr);  //arc with zero height has no cover to update
        }
        zero_height.clear();
        LayoutColumn* us = &now;
        while (1) {
            //Next, see if we have a parent.
            LayoutColumn* parent = us->parent;
            if (parent) {
                //If so, check if that has any
                //other children left. If not continue processing that column.
                //If that column is also completed, check if that has any parent too.
                //->this leads to a cycle as long as we have childless parents of
                //completed columns.
                //Start by updating our parent column
                if (!(*parent->arc)->IsOverlap()) {
                    //if we are not part of an ParallelBlocks that has been marked as 'overlap'
                    //move the bottom (where the next uncompressed will be placed)
                    parent->y = std::max(parent->y, us->y);
                }
                //bottom_all needs to be moved always to be able to calculate total height
                parent->y_bottom_all = std::max(parent->y_bottom_all, us->y_bottom_all);
                parent->last_action = std::max(parent->last_action, us->last_action);
                //copy our cover to the cover of the parent column
                parent->completed_children_covers += std::move(us->covers);
            } else
                top_completed_children_cover += std::move(us->covers);
            total_bottom = std::max(total_bottom, us->y_bottom_all);
            //erase 'us' from the list
            columns.erase(std::find_if(columns.begin(), columns.end(),
                [us](const LayoutColumn &a) {return &a==us; }));
            //'us' is now invalid
            if (!parent)
                break; //stop if no parent
            _ASSERT(parent->number_of_children);
            _ASSERT(parent->list->end() != parent->arc);
            //decrement the children list of the parent
            if (0 < --parent->number_of_children)
                break; //if the parent has children, we do not delete the parent, just continue processing columns
            //The parent has no more children, finish processing the ParallelBlocks we have just completed
            //(when we added parallel blocks, we left 'arc' pointing to the ParallelBlocks)
            ParallelBlocks* const pParallelBlocks = dynamic_cast<ParallelBlocks*>(parent->arc->get());
            _ASSERT(pParallelBlocks);
            //Set its height (for verticals if they refer to it <- this may be obsolete)
            pParallelBlocks->SetBottom(parent->y_bottom_all);
            Progress.DoneItem(EArcSection::LAYOUT, pParallelBlocks->myProgressCategory);
            //If this parallel block had by reference verticals/symbols attached, lay them out
            const auto [byrefs_from, byrefs_till] = ByRefs.equal_range(pParallelBlocks);
            for (auto i = byrefs_from; i != byrefs_till; ++i) {
                AreaList byref_cover;
                const bool make_room = i->second->LayoutByRef(canvas, parent->completed_children_covers, &byref_cover);
                //If a by-ref symbol/vertical extends below the actual content of the ParallelBlocks,
                //increase the cursor.
                parent->y = std::max(parent->y, byref_cover.GetBoundingBox().y.till);
                parent->y_bottom_all = std::max(parent->y_bottom_all, byref_cover.GetBoundingBox().y.till);
                if (byref_cover.GetBoundingBox().y.from < pParallelBlocks->yPos) {
                    const auto pSymbol = dynamic_cast<Symbol*>(i->second);
                    const auto pVertical = dynamic_cast<Vertical*>(i->second);
                    _ASSERT(pSymbol || pVertical);
                    if ((pSymbol && pSymbol->layout_warning) || (pVertical && pVertical->layout_warning)) {
                        Error.Warning(i->second->file_pos.start, "This 'by ref' symbol/vertical extends above the top of the parallel block.",
                                      "Parallel blocks with layout=one_by_one_merge, are laid out intermixed with other blocks. Once their ...");
                        Error.Warning(i->second->file_pos.start, i->second->file_pos.start,
                                      "... vertical extent is known, they cannot be shifted down to make room for an 'around' or 'above' symbol.");
                        Error.Warning(i->second->file_pos.start, i->second->file_pos.start,
                                      "Add 'layout_warning=no' to silence this warning.");
                        Error.Warning(pParallelBlocks->file_pos.start, i->second->file_pos.start,
                                      "This is the parallel block series. Consider prefixing it with [layout=one_by_one].");
                    }
                }
                if (make_room) { //subsequent symbols/verticals shall be placed considering this larger cover.
                    byref_cover.InvalidateMainLine();
                    parent->completed_children_covers += std::move(byref_cover);
                }
            }
            for (auto i = byrefs_from; i != byrefs_till; ++i)
                //parent->completed_children_covers contains more than the cover of 'i'
                i->second->FinalizeLayout(canvas, &parent->completed_children_covers);
            ByRefs.erase(byrefs_from, byrefs_till);
            //Since the parent has no more children, continue processing arcs in it.
            //this is testing if the parallel block has 'parallel' keyword in front
            parent->previous_was_parallel = pParallelBlocks->IsParallel();
            parent->previous_x_extent = parent->completed_children_covers.GetBoundingBox().x;
            if (pParallelBlocks->IsOverlap()) {
                //and keep the position of the next element the same
                parent->y = parent->y_upper_limit;
            } else {
                if (parent->previous_was_parallel)
                    //if ParallelBlocks was 'parallel', remove the mainline of all elements inside the parallel block
                    parent->completed_children_covers.InvalidateMainLine();
                //copy children's cover to us (does not happen if ParallelBlocks was 'overlap')
                parent->covers += std::move(parent->completed_children_covers);
                //Note: in the next cycle we will re-use parent->completed_children_covers,
                parent->completed_children_covers.clear();
            }            
            //go to next arc
            parent->arc++;
            if (parent->arc != parent->list->end())
                break; //we have more arcs in the parent column, so we continue processing that
            //OK 'parent' is a completed column - do the cycle again.
            us = parent;             
        } //while(proc->parent)
    }
    return total_bottom;
}


/** Places a list of arcs at below an already laid out part of the chart.
 * The list of arcs is normally placed at y coordinate `start_y` (which is supposed
 * to be well below the already laid out part. The elements in the arc list are
 * placed by calling their Layout() and ShiftBy() functions.
 * Then, if the first element has its 'compress' attribute set or the `forcecompress`
 * parameter is true, it shifts the whole newly laid out list upwards until it bumps
 * into `area_top`, but no higher than `top_y`.
 * The whole reason for this function is to prevent a big empty space
 * between the first and second arrow, when the first arrow can be
 * compressed a lot, but the second not. Here we move all the arrows (arcs)
 * as one block only up till some of them collides with something.
 * No matter what input parameters we get we always place the list at an integer
 * y coordinate.
 * @param canvas The canvas to calculate geometry on.
 * @param arcs The list of arcs to place.
 * @param [in] start_y Place the list here (or above if we compress)
 * @param [in] top_y Never compress the top of the list above this y coordinate.
 * @param [in] area_top When the list is compressed, avoid overlap with these areas
 * @param [in] forceCompress Always attempt to move the list upwards, even if the
 *                           first arc has its vspacing attribute set to non DBL_MIN.
 * @param [out] retCover If not nullptr, we return the resulting cover of the list
 *                       at the final position placed.
 * @returns The bottommost y coordinate touched by any arc on the list after completion. */
double MscChart::PlaceListUnder(Canvas &canvas, ArcList &arcs, double start_y,
                                double top_y, const AreaList &area_top, bool forceCompress,
                                AreaList *retCover)
{
    if (arcs.size()==0) return 0;
    AreaList cover;
    double h = LayoutArcList(canvas, arcs, &cover);
    double touchpoint;
    double new_start_y = std::max(top_y, -area_top.OffsetBelow(cover, touchpoint));
    //if we shifted up, apply shift only if compess is on
    if (new_start_y < start_y) {
        if (forceCompress || (*arcs.begin())->IsCompressed())
            start_y = new_start_y;
    } else //if we shifted down, apply it in any case
        start_y = new_start_y;
    //Add extra space (even if above was parallel)
    start_y = ceil(start_y + (*arcs.begin())->GetVSpacing());
    ShiftByArcList(arcs, start_y);
    if (retCover)
        retCover->swap(cover.Shift(XY(0, start_y)));
    return start_y + h;
}

/** Shifts a whole arc list up or down */
void MscChart::ShiftByArcList(ArcList &arcs, double y)
{
    for (auto &pArc : arcs)
        pArc->ShiftBy(y);
}

/** Inserts an automatic page breaks to a list of arcs
 * @param canvas The canvas to calculate geometry on.
 * @param arcs The list of arc lists to insert the page break into
 * @param [in] i The arcs before which the page break shall be inserted.
 * @param [in] pageBreak The y coordinate of the page break in chart space
 * @param [in] addHeading If true, the page break will also display an automatic heading.*/
void MscChart::InsertAutoPageBreak(Canvas &canvas, ArcList &arcs, ArcList::iterator i,
                              double pageBreak, bool addHeading)
{
    std::unique_ptr<Newpage> cnp = std::make_unique<Newpage>(this, false);
    const Attribute a("auto_heading", addHeading ? "yes" : "no");
    cnp->AddAttribute(a);
    cnp->AddAttributeList(nullptr);
    //We skip FinalizeLabels as they do nothing for Newpage nor EntityCommand
    EntityRef dummy1 = nullptr;
    EntityRef dummy2 = nullptr;
    Numbering dummy3; //will not be used by a EntityCommand
    MscElement *dummy4 = nullptr; //target of any notes, pretend we have none
    cnp->PostParseProcess(canvas, false, dummy1, dummy2, dummy3, &dummy4, nullptr); //if addHeading is set it will generate a EntityCommand and call its PostParseProcess
    DistanceRequirements vd;
    cnp->Width(canvas, vd);
    cnp->Layout(canvas, nullptr);
    cnp->ShiftBy(pageBreak);
    arcs.insert(i, std::move(cnp));
}


/** Helper to automatic pagination.
 * Walk with 'keep_with_next__from' in 'arcs' until 'i' and
 * record the max bottom in lowest.
 * Set 'keep_with_next__from' to arcs.end() in the end */
void clear_keep_with_next__from(ArcList &arcs, ArcList::iterator &keep_with_next__from,
                                ArcList::iterator i, double &lowest)
{
    if (keep_with_next__from != arcs.end()) {
        do {
            lowest = std::max(lowest, (*keep_with_next__from)->GetVisualYExtent(true).till);
        } while (keep_with_next__from++!=i);
        keep_with_next__from = arcs.end();
    } else
        lowest = std::max(lowest, (*i)->GetVisualYExtent(true).till);
}

/** Adds a page break to MscChart::pageBreakData, adjusts the size of the
 * previous page. Assumes total.x is correct.*/
void MscChart::AddPageBreak(double y, bool manual, EntityCommand *autoHeading)
{
    if (pageBreakData.size()) {
        auto &prev = pageBreakData.back();
        prev.wh.x = total.x.Spans();
        prev.wh.y = y - prev.xy.y;
    }
    pageBreakData.push_back(MscPageBreakData(y, 0, manual, autoHeading));
}


/** Walks an ArcList and inserts a page break and optionally a heading.
 *
 * We do not insert a page break, if the indicated position is all above or
 * below the whole list.
 * We also do not insert a page break, if it has already been inserted,
 * in that case we merely shift the elements that fall after the page
 * break down to accommodate a potential heading.
 * Elements that the page break goes through that can split themselves
 * around the page break, will do so. Others will be shifted to the next
 * page.
 * @param canvas Used to determine the size of the heading to insert.
 * @param arcs The list to walk. Can get modified if we insert a page
 *             break, plus some elements may get shifted down.
 * @param [in] netPrevPageSize The size of the page before the pageBreak to insert.
 *             Used to determine if a non-splittable object is larger
 *             than the full page. In this case we break it.
 * @param [in] pageBreak The y coordinate where we shall insert the page break.
 *                       Essentially the bottom of the page.
 *                       Note that due to rounding errors, all elements shifted
 *                       to the next page will see the page to start at
 *                       `pageBreak+1`. Also, we shift or split elements if they
 *                       are not completely above `pageBreak-1`
 * @param addCommandNewpage True if no CommandNewPage is added yet and it needs
 *                    to be added. If we have inserted the CommandNewPage
 *                    object, then false is returned in it.
 * @param [in] addHeading True if we shall add a heading after automatic
 *                        page breaks.
 * @param [in] canChangePBPos If true, we can change the page break
                              position. Due to the fact that we do a single-
                              step process, this shall be set only for
                              the top level MscChart::Arcs list.
   @param [in] dontshiftall If true and all elements of the list need to be
                            shifted to the next page, do nothing. This is used
                            for boxes, when the label of the box wants to be kept
                            on the same page as the first content element.
   @returns How much the list got longer, or -1 if `dontshiftall` was true
            and all elements of the page would need to be shifted.
 */
double MscChart::PageBreakArcList(Canvas &canvas, ArcList &arcs, double netPrevPageSize,
                             double pageBreak, bool &addCommandNewpage, bool addHeading,
                             bool canChangePBPos, bool dontshiftall)
{
    //if no arcs or we are all below the page Break, there is nothing to do
    //Here we test for formal beginning as it may be that it is not the first element
    //that has its visual top the topmost.
    if (arcs.size()==0 || (*arcs.begin())->GetFormalPos() > pageBreak) return -1;
    double shift = 0;
    /** Indicates which element was the first in a series of elements
     * where all of them have `keep_with_next` or `parallel` set.
     * Will be maintained only as long as `shift` is zero and used only
     * when it becomes non-zero, thus we abandon its value after. */
    ArcList::iterator keep_with_next__from = arcs.end();
    double lowest_on_prev_page = pageBreak - netPrevPageSize+1;
    double lowest_on_next_page = pageBreak+1;
    double topmost_in_kwn = 0;
    for (auto i=arcs.begin(); i!=arcs.end(); i++) {
        /* Our rules on how to handle stuff.
         * SplitByPageBreak() must be called for all elements, as this is
         * responsible to maintain the running state of entities.
         * The main tool to handle pagination is that we insert the page
         * break and shift all elements below to get to the next page.
         * We maintain a "shift" value showing how much we need to
         * shift upcoming elements in the list. As a general rule
         * this value can only increase, to preserve layout.
         * However, if at any element this would push the topmost
         * point of that element much below than the bottommost point of
         * prior elements, we reduce shift for these two points to equal.
         * (This is a poor measure to prevent full re-layout.)
         * To this end we also maintain "lowest_on_next_page",
         * in which we maintain the lowest point of elements
         * on the next page processed (after shifted).
         * We also maintain 'topmost_in_kwn' holding the topmost
         * point among the elements between 'lowest_till_kwn_from'
         * and 'i' - undefined if 'lowest_till_kwn_from' is arcs.end().
         * We also maintain "lowest_on_prev_page" to see if at the end
         * we can move the page break upwards (only if canChangePBPos
         * is true).
         * Plus, we pay attention not to insert a page break, where the
         * next upcoming element is a (manual) page break (ignoring
         * zero-height elements in-between).
         * Thus the rule is
         * - If the element is fully above the page break, we leave it.
           - If the element is on the page break,
              + if this is the first such element, we shift it below
                as much as needed to get to the top of the new page.
              + if there have been previous shifts, ideally we should
                reflow the chart. For that we would need a copy of
                the cover for all elements - I deem that too expensive now.
                So for now we merely attempt to shift so that it falls
                below the page break - and increase the shift by that
                much.
           - If the element is fully below the page break, we shift it as
             much as the maximum of the shift yet.
         */
        ArcList res;
        const double s = (*i)->SplitByPageBreak(canvas, netPrevPageSize, pageBreak,
                                                addCommandNewpage, addHeading, res);
        if (s==-2) {
            //ignore (verticals, etc)
            //but break keep_with_next series.
            //Note that the keep_with_next list contains elements only before
            //we have reached pageBreak - after that it is always empty
            if (keep_with_next__from != arcs.end())
                clear_keep_with_next__from(arcs, keep_with_next__from, i, lowest_on_prev_page);
            continue;
        }
        _ASSERT(lowest_on_prev_page<=pageBreak);
        const Range r = (*i)->GetVisualYExtent(true);
        lowest_on_next_page = std::max(lowest_on_next_page, r.till);
        if (s>=0) {
            //The element fell on the page break and split itself.
            ///We do not shift it, just increase shift for subsequent elements.
            if (res.size()) {
                //The element has split itself and some parts of it ended up in
                //new elements. These are returned in "res". We need to insert
                //those after the current element, but add page break before.
                auto j = i;
                j++; //the element following the current element - insert before
                if (addCommandNewpage)
                    InsertAutoPageBreak(canvas, arcs, j, pageBreak, addHeading);
                addCommandNewpage = false;
                i = j; i--; //The page break (or heading) - continue processing just after this
                arcs.splice(j, res, res.begin(), res.end());
            }
            shift = ceil(std::max(shift, s)); //increase shift for later elements
            //we do not shift `i`, but perhaps later elements we got in 'res'
            //we also clear kwn_from so that we do not get shifted by a later element
            keep_with_next__from = arcs.end();
            lowest_on_prev_page = pageBreak-1; //previous page is full.
            dontshiftall = false; //we kept something on previous page, so we will not shift all anyway
            continue;
        }
        _ASSERT(res.size()==0);
        _ASSERT(s==-1);
        //if the element is fully above the page break,
        //we do nothing, just maintain keep_with_next__from
        if (r.till <= pageBreak-1) {
            //if we keep an element of nonzero height on the
            //previous page, clean `dontshiftall`, since we will not shift
            //all by now.
            if (r.from < r.till)
                dontshiftall = false;
            if (!(*i)->IsKeepWithNext() && !(*i)->IsParallel()) {
                clear_keep_with_next__from(arcs, keep_with_next__from, i, lowest_on_prev_page);
                continue;
            }
            if (keep_with_next__from == arcs.end()) {
                keep_with_next__from = i;
                topmost_in_kwn = r.from;
            } else
                topmost_in_kwn = std::min(topmost_in_kwn, r.from);
            continue;
        }
        //if the element falls on the page break or is below and cannot split itself
        //- we check `dontshiftall` - if true we return from this function with -1
        //- we insert PB (if not yet inserted)
        //- adjust the shift, so that we do not over-shift

        if (dontshiftall) return -1; //we may have not shifted anyone yet - the list is intact

        if (addCommandNewpage) {
            //Insert CommandPageBreak if needed. It may modify headingLeftingSize.
            //Insert only if next element is not a page break.
            //Insert before keep_with_next__from or if that is arcs.end() then before 'i'.

            //first find if next element is a page break
            auto iii = keep_with_next__from==arcs.end() ? i : keep_with_next__from;
            while (iii != arcs.end() && (*iii)->GetFormalHeight()==0 &&
                   dynamic_cast<Newpage*>(iii->get())==nullptr)
                iii++;
            if (iii != arcs.end() && dynamic_cast<Newpage*>(iii->get())) {
                //an upcoming page break. Adjust shift so that that element gets to `pageBreak`
                shift = ceil(pageBreak - (*iii)->GetVisualYExtent(true).from);
                keep_with_next__from = arcs.end(); //do not move prior elements
            } else {
                InsertAutoPageBreak(canvas, arcs,
                                    keep_with_next__from == arcs.end() ? i : keep_with_next__from,
                                    pageBreak, addHeading);
                addCommandNewpage = false;
            }
            lowest_on_next_page = std::max(lowest_on_next_page, pageBreak+1); //next page empty
        }

        //ensure we fall on the next page
        if (keep_with_next__from == arcs.end()) {
            topmost_in_kwn = r.from;
            keep_with_next__from = i;
        }
        const double old_shift = shift;
        shift = ceil(std::max(shift, pageBreak+1 - topmost_in_kwn));
        //if we shift a whole page, do not shift at all - this is an element bigger than a page
        if (shift >= netPrevPageSize) {
            shift = old_shift;
            keep_with_next__from = arcs.end();
            lowest_on_prev_page = pageBreak-1; //previous page is full.
            continue;
        }
        //Test if we shift too much
        //shift = ceil(std::min(shift, old_lowest_on_next_page - topmost_in_kwn));

        //Now shift elements between 'keep_with_next__from` till 'i' (inclusive)
        //beware of the ++ in the while()
        do {
            (*keep_with_next__from)->ShiftBy(shift);
            lowest_on_next_page = std::max(lowest_on_next_page,
                (*keep_with_next__from)->GetVisualYExtent(true).till);
        } while (keep_with_next__from++ != i && keep_with_next__from != arcs.end());

        keep_with_next__from  = arcs.end();
    }

    //Now test if we can move the page break up
    lowest_on_prev_page = ceil(lowest_on_prev_page);
    if (!canChangePBPos || lowest_on_prev_page >= pageBreak-1)
        return shift;
    for (auto &pArc : arcs)
        if (pArc->GetVisualYExtent(true).from>=pageBreak)
            pArc->ShiftBy(lowest_on_prev_page - (pageBreak-1)); //negative
    return shift + lowest_on_prev_page - (pageBreak-1);
}

/** Fill in pageBreakData with all data from CommandNewPage:s */
void MscChart::CollectPageBreaks()
{
    pageBreakData.clear(); //this frees all the MscPageBreakData objects
    AddPageBreak(total.y.from, false); //start of first page
    CollectPageBreakArcList(Arcs);
    AddPageBreak(total.y.till, false); //end of last page: not really a page start, but this sets the size of the one before
}


/** Collect the y position of page breaks in the list to pageBreakData. */
void MscChart::CollectPageBreakArcList(ArcList &arcs)
{
    for (auto &pArc : arcs)
        pArc->CollectPageBreak();
}

/** Automatically paginate an already laid out chart.
 * @param canvas The canvas on which to calculate geometries.
 * @param [in] pageSize The height of a page in pixels.
 * @param [in] addHeading If true automatically inserted page breaks will also
 *                        have entity headings. */
void MscChart::AutoPaginate(Canvas &canvas, double pageSize, bool addHeading)
{
    _ASSERT(floor(pageSize)==pageSize);
    //Here `total` is set, `drawing` is set the same
    //All elements (except notes, verticals and floating symbols) have been placed.
    //But page and entity status is not yet collected.
    CollectPageBreaks();
    for (unsigned u=0; u<pageBreakData.size()-1; u++) {
        if (pageBreakData[u+1].xy.y - pageBreakData[u].xy.y < pageSize-2)
            continue;
        //We need to insert a page break, find the effective size of the page
        double netPrevPageSize = pageSize;
        if (pageBreakData[u].headingLeftingSize.y < pageSize)
            netPrevPageSize -= pageBreakData[u].headingLeftingSize.y;
        bool addCommandNewpage = true;
        //Here we re-use Entity::running_* members of AllEntities.
        //(These were used during parsing, not needed any longer.)
        //We re-create the running status so that if we need to insert a heading,
        //we know what to insert.
        if (addHeading)
            for (auto &e : AllEntities)
                e.running_shown = EEntityStatus::SHOW_OFF;
        total.y.till += PageBreakArcList(canvas, Arcs, netPrevPageSize, pageBreakData[u].xy.y + netPrevPageSize+1,
                                         addCommandNewpage, addHeading, true, false);
        //Regenerate page breaks
        CollectPageBreaks();
    }
    pageBreakData.pop_back();
    drawing.y = total.y; //keep this invariant before placing notes
}


/** Helper to hscale=auto positioning.
 * Find the smallest elements between indexes [i, j) and bring them up to the
 * value of the second smallest element, but overall do not add more than
 * max_sum (round up). Return how much of max_sum has remained.
 * E.g, if [i, j) are 10, 13, 15, 10, 10 and max_sum=10
 *              make: 13, 13, 15, 13, 13 and return 10-9 = 1;
 * E.g., max_sum = 5, return 12, 13, 15, 12, 12 and return 0 (round up).*/
double  MscSpreadBetweenMins(vector<double> &v, unsigned i, unsigned j, double max_sum)
{
    double minvalue = v[i];     //the value of the smallest member
    double secondvalue = v[i];  //the value of the second smallest member
    unsigned count = 1;           //count of how many do we have of minvalue
    for (unsigned t=i+1; t<j; t++)
        if (v[t]<minvalue) {
            secondvalue = minvalue;
            minvalue = v[t];
            count = 1;
        } else if (v[t] == minvalue)
            count ++;
        else if (secondvalue > v[t])
            secondvalue = v[t];
    double amount;
    //See if we distribute all of it: if all elements are equal or max_sum is small
    if (secondvalue == minvalue || (secondvalue - minvalue)*count > max_sum) {
        amount = (max_sum+count-1)/count; //round up
        max_sum = 0;
    } else {
        amount = secondvalue-minvalue;
        max_sum -= amount*count;
    }
    //Do the distribution
    for (;i<j; i++)
        if (v[i] == minvalue)
            v[i]+= amount;
   return max_sum;
}

void GeneratePositions(const std::map<IPair, double> &pairs, vector<double> &dist)
{
    //'pairs' starts with requirements between neighbouring entities
    //(an IPair is "smaller" for entities closer to each other)
    //and continues with requirements between second neighbours, ... etc.
    //we process these sequentially
    for (auto &i : pairs) {
        //Get the requirement
        double toadd = i.second;
        //Substract the distance already there
        for (unsigned f = i.first.first; f!=i.first.second; f++)
            toadd -= dist[f];
        //If there is need to increase start with the closer ones
        //and gradually move to larger ones.
        while (toadd>0)
            toadd = MscSpreadBetweenMins(dist, i.first.first,
                i.first.second, toadd);
    }
}


/** Lay out a parsed chart and calculate its extents.
 * This function calls the Width() and Layout() functions of
 * arcs to set x and y coordinates, respectively.
 * It also performs automatic pagination, if needed.
 * As a result, we fill in MscChart::total, MscChart::drawing, MscChart::comments_right_side,
 * and MscChart::copyrightTextHeight. We ensure that MscChart::total is all integer.
 * @param canvas The canvas to calculate geometry on
 * @param [in] autoPaginate If true, we do automatic pagination
 * @param [in] addHeading If true, the automatic page breaks will also add a heading
 * @param [in] pageSize Determines the page size for automatic pagination
 * @param [in] fitWidth If true and automatic pagination is on, we select a zoom value
 *                      to fit the width of the page and normalize the height of the page
 *                      with it.*/
void MscChart::CalculateWidthHeight(Canvas &canvas, bool autoPaginate,
                                    bool addHeading, XY pageSize, bool fitWidth)
{
    pageBreakData.assign(1, MscPageBreakData(0.0, 0.0, false));
    HideELinesHere.clear();
    if (Arcs.size()==0) return;
    if (total.y.Spans() > 0) return; //already done?

    //start with width calculation, that is used by many elements
    Progress.StartSection(EArcSection::WIDTH);
    DistanceRequirements vdist;
    vdist.AddMarker(MARKER_BUILTIN_CHART_TOP_STR);
    ByRefs.clear();
    //Add distance for arcs,
    //needed for hscale=auto, but also for entity and box width and comment size calculation
    WidthArcList(canvas, Arcs, vdist);
    //Merges all required distance requirements to a single unified section in vdist
    vdist.CombineAllLeftRightToPair_Add(hscaleAutoXGap);

    //Turn on entity line for side note lines if there are side notes
    if (vdist.had_l_comment)
        LNote->status.SetStatus(0, EEntityStatus::SHOW_ON);
    if (vdist.had_r_comment)
        RNote->status.SetStatus(0, EEntityStatus::SHOW_ON);

    //Add a gap to side comment lanes (if any)
    //The algorithm to calculate spacing for side comments is as follows.
    //If there are no comments, we use zero.
    //Else if we have hscale=auto, we use the maximum of hscale specified and
    //     label requested spaces. If that is zero, we use XCoord(DEFAULT_COMMENT_SIZE)
    //Else if the user specified something (via hspace), we use that
    //Else we use XCoord(DEFAULT_COMMENT_SIZE)
    //So, here is how we implement it: if no space required (*), but we have comments,
    //we insert to hspace_distances
    //(*) For hscale=auto, we check both normal and hspace distances, for fixed hscale, only
    //hspace distances.
    const bool lhs  = vdist.IncreaseIfNonZeroHSpace(StartEntity->index, LNote->index, sideNoteGap*2);
    const bool ls = lhs || vdist.IncreaseIfNonZero(StartEntity->index, LNote->index, sideNoteGap*2);
    if (vdist.had_l_comment && (GetHScale()>=0 ? !lhs : !ls))
        vdist.InsertHSpace(StartEntity->index, LNote->index, XCoord(DEFAULT_COMMENT_SIZE));
    const bool rhs = vdist.IncreaseIfNonZeroHSpace(RNote->index, EndEntity->index, sideNoteGap*2);
    const bool rs = rhs || vdist.IncreaseIfNonZero(RNote->index, EndEntity->index, sideNoteGap*2);
    if (vdist.had_r_comment && (GetHScale()>=0 ? !rhs : !rs))
        vdist.InsertHSpace(RNote->index, EndEntity->index, XCoord(DEFAULT_COMMENT_SIZE));

    //Add some margins between note line and side line.
    //Add to hspace distance, since these will be needed even for fixed h scale
    vdist.InsertHSpace(LNote->index, LSide->index, XCoord(MARGIN));
    vdist.InsertHSpace(RSide->index, RNote->index, XCoord(MARGIN));

    const double unit = XCoord(1);
    if (GetHScale()>=0) { //Fixed scale, no autoscale
        //Copy the actual pos' of entities to hscape_pairs (where we may have
        //additional requirements from hspace commands).
        for (auto ei = ++ActiveEntities.begin(); ei!=ActiveEntities.end(); ei++)
            vdist.InsertHSpace((*std::prev(ei))->index, (*ei)->index,
                               unit*((*ei)->pos - (*std::prev(ei))->pos));
        //Copy the collected distance between LSide and the leftmost real entity;
        //and between the rightmost real entity and RSide to the hspace
        //distances. We do autoscaling at the left and right side, such that
        //no element of the chart falls off.
        //(If there are no real entities the two below commands work on the
        //same indexes. They are still needed e.g., for charts with a
        //title only.)
        vdist.InsertHSpace(LSide->index, LSide->index+1,
                           vdist.QueryAll(LSide->index, LSide->index+1));
        vdist.InsertHSpace(RSide->index-1, RSide->index,
                           vdist.QueryAll(RSide->index-1, RSide->index));
        //Do the same for the space between LNote<->LSide and RSide<->RNote for
        //verticals operating there.
        vdist.InsertHSpace(LNote->index, LSide->index,
                           vdist.QueryAll(LNote->index, LSide->index));
        vdist.InsertHSpace(RSide->index, RNote->index,
                           vdist.QueryAll(RSide->index, RNote->index));
    } else { //autoscale
        //Merge all distance requirements to that of hspace
        vdist.MergeAllToHSpace();
    }
    //Now the pairwise space requirements between entities are all in
    //vdist.hspace_pairs - but these contain requirements between
    //non-neighbouring entity pairs.
    //Go through all the pairwise requirements and calc actual entity pos.
    //'dist[i]' will hold required distance to the right of entity with index 'i'
    vector<double> dist(ActiveEntities.size(), 0);
    GeneratePositions(vdist.GetHSpacePairs(), dist);
    //GeneratePositions(distances.GetPairs(), dist);

    //Kill comment space if we have no comments on that side (user could still have
    //specified a hspace requirement for comments, but no comments).
    if (!vdist.had_l_comment) dist[0] = 0;
    if (!vdist.had_r_comment) dist[dist.size()-2] = 0;

    //Copy dist[i] values to the pos field of entities
    double curr_pos = 0;
    unsigned index = 0;
    for (auto pEntity : ActiveEntities) {
        pEntity->pos = curr_pos;
        curr_pos += ceil(dist[index++])/unit;    //take integer, so XCoord(pEntity->pos) will return integer
    }

    total.x.from = 0;
    total.x.till = XCoord(EndEntity->pos)+1; //XCoord is always integer

    //Consider the copyright text for width
    StringFormat sf;
    sf.Default();
    XY crTexSize = Label(copyrightText, canvas, Shapes, sf).getTextWidthHeight().RoundUp();
    if (total.x.till < crTexSize.x) total.x.till = crTexSize.x;
    copyrightTextHeight = crTexSize.y;

    //finally record the space of the chart we actually draw element on (but not comments)
    drawing.x.from = XCoord(LNote->pos) + sideNoteGap;
    drawing.x.till = XCoord(RNote->pos) - sideNoteGap;

    //Width calculation ends here. Now lay out the chart and determine its height.

    Progress.StartSection(EArcSection::LAYOUT);
    noLabels = noOverflownLabels = 0;
    total.y.from = 0;
    total.y.till = LayoutArcList(canvas, Arcs, nullptr) + chartTailGap;
    drawing.y = total.y;
    _ASSERT(ByRefs.empty());

    //Record the chart.top and chart.bottom marker positions.
    //Note that notes may extend above and below, resp., but this is OK,
    //we want these to refer to the top and bottom without considering notes.
    Markers[MARKER_BUILTIN_CHART_TOP_STR].y = total.y.from;
    Markers[MARKER_BUILTIN_CHART_BOTTOM_STR].y = total.y.till;

    //If we have copyright text, increase the chart text by one pixel,
    //so that the copyright text (the background of which is drawn one pixel
    //above total.y.till) does not cover the last pixel row of the chart.
    //Required for active entities' closing line to remain shown.
    //We do this only if we have copyright text, so as not to change the 
    //canonical images.
    if (copyrightTextHeight > 0)
        total.y.till += 1; 

    //If at least 10% of the labels are overflown (but at least 3), emit warning
    if (noOverflownLabels>2 && noOverflownLabels > noLabels/10)
        if (GetHScale()>0 && !MyCurrentContext().text.IsWordWrap()) {
            string ss;
            ss << noOverflownLabels;
            Error.Warning(FileLineCol(Error.Files.size()-1, 1, 1),
                 "There are " + ss + " labels wider than the space available.",
                 "Consider using horizontal auto-scaling ('hscale=auto;') or "
                 "word wrapping ('text.wrap=yes;').");
        }

    if (autoPaginate) {
        //Do automatic pagination, if needed...
        if (fitWidth)
            pageSize.y *= total.x.Spans()/pageSize.x;
        pageSize.y = floor(pageSize.y);
        //This scaling may be screwed if notes expand the chart in the Y direction.
        //But what can we do..?
        Progress.StartSection(EBulkSection::AUTOPAGINATE);
        AutoPaginate(canvas, pageSize.y - copyrightTextHeight, addHeading);
        Progress.DoneBulk(EBulkSection::AUTOPAGINATE, 1);
    } else {
        //...or else, just record the locations of the manual page breaks.
        CollectPageBreaks();
        pageBreakData.pop_back(); //remove the empty last page
    }
}

/** Places elements that can be placed only when marker positions are known (verticals, some symbols).
 * Calls ArcBase::PlaceWithMarkers for all elements in the list.*/
void MscChart::PlaceWithMarkersArcList(Canvas &canvas, ArcList &arcs)
{
    for (auto &pArc : arcs) {
        pArc->PlaceWithMarkers(canvas);
        Progress.DoneItem(EArcSection::PLACEWITHMARKERS, pArc->myProgressCategory);
    }
}

/** Cycle through and places all floating notes.
 * This could be improved... */
void MscChart::PlaceFloatingNotes(Canvas &canvas)
{
    Block new_total;
    new_total.MakeInvalid();
    for (auto pNote : Notes) {
        pNote->PlaceFloating(canvas);
        new_total += pNote->GetAreaToDraw().GetBoundingBox();
        Progress.DoneItem(EArcSection::NOTES, pNote->myProgressCategory);
    }
    if (new_total.IsInvalid()) return;
    new_total.Expand(sideNoteGap);
    total += new_total;
}

/** Invalidate all notes that point to this element.
 * Used in the destructor of MscElement - we also hide notes pointing to it.*/
void MscChart::InvalidateNotesToThisTarget(const MscElement *target)
{
    for(auto pNote : Notes)
        if (pNote->GetTarget() == target)
            pNote->Invalidate();
}

/** Remove a note from the list of notes.
 * Used from the destructor of Note */
void MscChart::RemoveFromNotes(const Note *note)
{
    for(auto i=Notes.begin(); i!=Notes.end(); /*nope*/)
        if (*i==note) Notes.erase(i++);
        else i++;
}

/** Calls PostPosProcess for all members of the arc list */
void MscChart::PostPosProcessArcList(Canvas &canvas, ArcList &arcs, Chart *ch)
{
    for (auto &pArc : arcs) {
        pArc->PostPosProcess(canvas, ch);
		Progress.DoneItem(EArcSection::POST_POS, pArc->myProgressCategory);
	}
}

/** Calls RegisterCover for all members of the arc list */
void MscChart::RegisterCoverArcList(const ArcList &arcs, EMscDrawPass pass)
{
    for (auto &pArc : arcs)
        pArc->RegisterCover(pass);
}


/** Prepares a chart for drawing after parse.
 * We perform post-parse processing, lay out the chart and perform
 * post-positioning processing. After this Draw() functions can be called.
 * If autoPaginate is true, pagesize.y is the page height.
 * If fitWidth is true, we adjust page height to get a scaling
 * so that the chart fits the width.
 * @param [in] autoPaginate If true, we do automatic pagination
 * @param [in] addHeading If true, the automatic page breaks will also add a heading
 * @param [in] pageSize Determines the page size for automatic pagination
 * @param [in] fitWidth If true and automatic pagination is on, we select a zoom value
 *                      to fit the width of the page and normalize the height of the page
 *                      with it.
 * @param [in] collectLinkInfo If true, we also fill the ismapData member, collecting
 *                             where are links on the chart.*/
void MscChart::CompleteParse(bool autoPaginate,
                             bool addHeading, XY pageSize,
                             bool fitWidth, bool collectLinkInfo)
{
    if (autoPaginate)
        Progress.RegisterBulk(EBulkSection::AUTOPAGINATE, 1);

    //Allocate (non-sized) output object and assign it to the chart
    //From this point on, the chart sees xy dimensions
    Canvas canvas(Canvas::Empty::Query);

    //Sort Entities, add numbering, fill in auto-calculated values,
    //and throw warnings for badly constructed diagrams.
    headingSize = 0;
    PostParseProcess(canvas);
	Progress.StartSection(EArcSection::FINALIZE_LABELS);
    FinalizeLabelsArcList(Arcs, canvas);

    //Calculate chart size
    CalculateWidthHeight(canvas, autoPaginate, addHeading, pageSize, fitWidth);

    //If the chart ended up empty we emit one pixel
    if (total.y.till <= chartTailGap)
        total.x.till = total.y.till = 1; //"from" members still zero, only notes can take it negative

    Progress.StartSection(EArcSection::PLACEWITHMARKERS);
    PlaceWithMarkersArcList(canvas, Arcs);
    Progress.StartSection(EArcSection::NOTES);
    PlaceFloatingNotes(canvas);

    total.RoundWider();

    //update page sizes & positions
    //adjust page widths in case a note was placed on top
    for (auto &p : pageBreakData) {
        p.xy.x = total.x.from;
        p.wh.x = total.x.Spans();
    }
    //adjust first page in case a note was placed on top
    pageBreakData.front().wh.y += pageBreakData.front().xy.y - total.y.from;
    pageBreakData.front().xy.y = total.y.from;
    //adjust last page size in case a note was placed at the end
    pageBreakData.back().wh.y = total.y.till - pageBreakData.back().xy.y;

    Progress.StartSection(EArcSection::POST_POS);
    //A final step of prcessing, checking for additional drawing warnings
    PostPosProcessArcList(canvas, Arcs, this);

    //Collect the covers in AllCovers
    if (prepare_for_tracking) {
        RegisterCoverArcList(Arcs, EMscDrawPass::BEFORE_BACKGROUND);
        RegisterCoverArcList(Arcs, EMscDrawPass::BEFORE_ENTITY_LINES);
        RegisterCoverArcList(Arcs, EMscDrawPass::AFTER_ENTITY_LINES);
        RegisterCoverArcList(Arcs, EMscDrawPass::DEFAULT);
        RegisterCoverArcList(Arcs, EMscDrawPass::AFTER_DEFAULT);
        RegisterCoverArcList(Arcs, EMscDrawPass::NOTE);
        RegisterCoverArcList(Arcs, EMscDrawPass::AFTER_NOTE);
    }

    //Delete LSide and RSide actions if there are no side comments
    if (LNote->pos == LSide->pos)
        LSide->status.Reset();
    if (RNote->pos == RSide->pos)
        RSide->status.Reset();

    Error.Sort();

    if (collectLinkInfo)
        CollectIsMapElementsArcList(Arcs, canvas);

    //collect background
    if (Background.size()) {
        FillAttr fill_bkg = FillAttr::Solid(ColorType::white());
        fill_bkg += Background.begin()->second;
        double y = std::min(Background.begin()->first, total.y.from);
        for (auto i = ++Background.begin(); i!=Background.end(); i++) {
            const Block area(total.x, {y, i->first});
            bkFill.emplace_back(fill_bkg, area);
            fill_bkg += i->second;
            y = i->first;
        }
        if (y < total.y.till)
            bkFill.emplace_back(fill_bkg, Block(total.x, {y, total.y.till}));
    }
}

/** Draw arcs in an arc list
 * @param canvas The canvas to draw on
 * @param [in] arcs The list of arcs to draw
 * @param [in] yDrawing Draw only arcs in this vertical range
 *                      (speedup for drawing only a single page)
 * @param [in] pass Draw only elements which need to be drawn in this drawing pass*/
void MscChart::DrawArcList(Canvas &canvas, const ArcList &arcs, Range yDrawing, EMscDrawPass pass)
{
    for (auto &pArc : arcs)
        if (pArc->GetVisualYExtent(true).Overlaps(yDrawing)) {
            pArc->Draw(canvas, pass);
            if (pArc->draw_pass == pass)
                Progress.DoneItem(EArcSection::DRAW, pArc->myProgressCategory);
        }
}

/** Draw the chart
 * This function draws entity lines and arcs (all passes)
 * @param canvas The canvas to draw on
 * @param [in] yDrawing Draw only arcs in this vertical range
 *                      (speedup for drawing only a single page)
 * @param [in] pageBreaks If true, we draw dashed lines for page breaks and page numbers. */
void MscChart::DrawChart(Canvas &canvas, Range yDrawing, bool pageBreaks)
{
    if (total.y.Spans() <= 0) return;
    //Draw arcs to be drawn before the background
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::BEFORE_BACKGROUND);
    //draw background
    for (const auto& [fill, area] : bkFill)
        if (!canvas.does_graphics())
            canvas.Add(GSBox(area * Block(total.x, yDrawing), LineAttr::None(), fill));
        else if (canvas.shall_avoid_aligned_bk_clip()
                 && (area.y.from == yDrawing.from || area.y.till == yDrawing.till))
            canvas.Fill(area.CreateExpand2D(XY(0,1)), fill);
        else
            canvas.Fill(area, fill);
    //Add background for side notes (if any)
    for (unsigned u = 0; u<2; u++) {
        const EntityRef note = u==0 ? LNote : RNote;
        const EntityRef side = u==0 ? LSide : RSide;
        if (note->pos == side->pos) continue;
        const Range x = u==0 ? Range(total.x.from, XCoord(LNote->pos)) : Range(XCoord((RNote)->pos), total.x.till);
        double pos = total.y.from, till = total.y.from;
        do {
            till = note->status.StyleTill(till);
            if (till<total.y.till && note->status.GetStyle(pos).read().fill == note->status.GetStyle(till).read().fill)
                continue;  //go until "fill" changes
            if (till>total.y.till) till = total.y.till;
            const Block B(x, Range(pos, till)*yDrawing);  //if pos-till is outside yDrawing, Y is an invalid range
            if (B.y.Spans()>0) {
                const FillAttr& fill = note->status.GetStyle(pos).read().fill;
                if (canvas.does_graphics())
                    canvas.Fill(B, fill);
                else
                    canvas.Add(GSBox(B, LineAttr::None(), fill));
            }
            pos = till;
        } while (pos<total.y.till);
    }
	//Draw page breaks
    if (pageBreaks)
        DrawPageBreaks(canvas);
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::BEFORE_ENTITY_LINES);
	//Draw initial set of entity lines (boxes will cover these and redraw)
    DrawEntityLines(canvas, yDrawing.from, yDrawing.Spans());
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::AFTER_ENTITY_LINES);
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::DEFAULT);
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::AFTER_DEFAULT);
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::NOTE);
    DrawArcList(canvas, Arcs, yDrawing, EMscDrawPass::AFTER_NOTE);
}

void MscChart::DrawHeaderFooter(Canvas &canvas, unsigned page)
{
    ChartBase::DrawHeaderFooter(canvas, page);

     //Draw autoheading, if any
    if (page && pageBreakData[page-1].headingLeftingSize.y) {
        const auto& pb = pageBreakData[page-1];
        const Range X = {pb.xy.x, pb.xy.x+pb.wh.x};
        const Range Y = {pb.xy.y-pb.headingLeftingSize.y, pb.xy.y};
        const Block B(X, Y.CreateExpanded(0.1));
        const FillAttr F = GetBkFill(Y, true);
        if (canvas.does_graphics())
            canvas.Fill(B, F);
        else
            canvas.Add(GSBox(B, LineAttr::None(), F));
        //autoHeading is supposedly shifted to just above the page break
        pageBreakData[page-1].autoHeading->Draw(canvas, pageBreakData[page-1].autoHeading->draw_pass);
    }
}



void MscChart::Draw(Canvas& canvas, bool pageBreaks, unsigned page)
{
    Range yDrawing;
    if (page) {
        yDrawing.from = pageBreakData[page - 1].xy.y;
        yDrawing.till = page < pageBreakData.size() ? pageBreakData[page].xy.y : total.y.till;
    } else
        yDrawing = total.y;
    DrawChart(canvas, yDrawing, page ? false : pageBreaks);
}

void MscChart::DrawComplete(Canvas &canvas, bool pageBreaks, unsigned page)
{
    Progress.StartSection(EArcSection::DRAW);
    if (page>pageBreakData.size() || total.x.Spans()<=0) return;
    Draw(canvas, pageBreaks, page);
    DrawHeaderFooter(canvas, page);
}

void MscChart::SetToEmpty()
{
    pageBreakData.emplace_back(0, 0, false);
    force_entity_collapse.clear();
    force_box_collapse.clear();
    force_box_collapse_instead.clear();
    Markers.clear();
    Background.clear();
    HideELinesHere.clear();
    Notes.clear();
    NoteBlockers.clear();
    Arcs.clear();    //This must be before Notes, since Element::~ will use chart->Notes
    EndNotes.clear();
    AutoGenEntities.clear();
    StartEntity = LNote = LSide = RSide = RNote = EndEntity = nullptr;
    ActiveEntities.clear();
    AllEntities.clear();
    drawing = Block(0, 0, 0, 0);
    headingSize = 0;
    width_attr = -1;
    ChartBase<MscContext, MscProgress, MscRefNameData, MscPageBreakData>::SetToEmpty();
}
