#pragma once
#include "maphoon-lexer/includes.h"
#include "parser_tools.h"
#include "msc_parse_tools.h"
#include "msc.h"
#include "msccsh.h"


namespace msc {

enum class lexing_state : uint8_t {
    no_change = 0,
    basic,
    mscgen_compat,
    mscgen_compat_box
};

constexpr lexing_state start_box_compat(lexing_state s) noexcept {
    return s == lexing_state::mscgen_compat ? lexing_state::mscgen_compat_box : lexing_state::no_change;
}

constexpr lexing_state end_box_compat(lexing_state s) noexcept {
    return s == lexing_state::mscgen_compat_box ? lexing_state::mscgen_compat : lexing_state::no_change;
}

template <typename TOKEN>
struct token_state_symbol {
    TOKEN token;
    lexing_state new_state;
    EArcSymbol symbol;
    constexpr auto operator ==(const token_state_symbol<TOKEN>& o) const noexcept { return token == o.token; } //only token type counts
    constexpr auto operator <(const token_state_symbol<TOKEN>& o) const noexcept { return token < o.token; } //only token type counts
    token_state_symbol(TOKEN token, EArcSymbol symbol = EArcSymbol::INVALID, lexing_state new_state = lexing_state::no_change) noexcept
        : token(token), new_state(new_state), symbol(symbol) {}
    token_state_symbol(TOKEN token, lexing_state new_state) noexcept : token(token), new_state(new_state), symbol(EArcSymbol::INVALID) {}
    friend std::ostream& operator<<(std::ostream& out, const token_state_symbol<TOKEN>& S) {
        out << StrCat('{', std::to_string(int(S.token)), ',', std::to_string(int(S.new_state)), ',', std::to_string(int(S.symbol)), '}');
        return out;
    }
};

struct int_token_state_symbol {
    int16_t token;
    uint8_t new_state;
    uint8_t symbol;
};

/* This class contains the tokenizer grammar.
 * It cannot generate the full symbol, just the token type. */
template <typename TOKEN>
lexing::classifier<char, token_state_symbol<TOKEN>> build_classifier(const lexing_state state) {
    _ASSERT(state != lexing_state::no_change);
    using namespace lexing;
    classifier<char, token_state_symbol<TOKEN>> cl(TOKEN::TOK_UNRECOGNIZED_CHAR);

    /* Newline characters in all forms accepted */
    const auto newline = just('\n') | just('\r') | just('\r') * just('\n');
    cl.insert(newline, TOKEN::TOK_NEWLINE);
    /* # starts a comment last until end of line */
    auto noCRLF = every<char>().without("\n\r");
    cl.insert(just('#') * noCRLF.star(), TOKEN::TOK_COMMENT);
    /* // starts a comment last until end of line */
    cl.insert(just('/') * just('/') * noCRLF.star(), TOKEN::TOK_COMMENT);
    /* / * .. * / block comments*/
    cl.insert(word("/*") *
              (every<char>().without('*') |
               (just('*').plus() * every<char>().without("/*")).star()
               ).star() * just('*').plus() * just('/'), TOKEN::TOK_COMMENT);
      /* / * ... unclosed block comments */
    cl.insert(word("/*"), TOKEN::TOK_COMMENT);
    /* ignore whitespace */
    auto WS = just(' ') | just('\t');
    cl.insert(WS, TOKEN::TOK_WHITESPACE);

    cl.insert(word(INLINE_OPEN_STR), TOKEN::TOK_OINLINE);
    cl.insert(word(INLINE_CLOSE_STR), TOKEN::TOK_CINLINE);

    cl.insert(word("..."), {TOKEN::TOK_SPECIAL_ARC, EArcSymbol::DIV_DISCO});
    cl.insert(word("---"), {TOKEN::TOK_SPECIAL_ARC, EArcSymbol::DIV_DIVIDER});
    cl.insert(word("|||"), {TOKEN::TOK_SPECIAL_ARC, EArcSymbol::DIV_VSPACE});
    cl.insert(word("->"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_SOLID});
    cl.insert(word("<-"), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_SOLID});
    cl.insert(word("<->"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_SOLID_BIDIR});
    cl.insert(word("=>"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DOUBLE});
    cl.insert(word("<="), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DOUBLE});
    cl.insert(word("<=>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DOUBLE_BIDIR});
    cl.insert(word(">>"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DASHED});
    cl.insert(word("<<"), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DASHED});
    cl.insert(word("<<>>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DASHED_BIDIR});
    cl.insert(word(">"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DOTTED});
    cl.insert(word("<"), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DOTTED});
    cl.insert(word("<>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DOTTED_BIDIR});
    cl.insert(word("=>>"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DBLDBL});
    cl.insert(word("<<="), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DBLDBL});
    cl.insert(word("<<=>>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DBLDBL_BIDIR});
    cl.insert(word("==>"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DOUBLE2});
    cl.insert(word("<=="), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DOUBLE2});
    cl.insert(word("<==>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DOUBLE2_BIDIR});

    switch (state) {
    case lexing_state::basic:
         /* The box symbols in Msc-generator*/
        cl.insert(word("--"), {TOKEN::TOK_EMPH, EArcSymbol::BOX_SOLID});
        cl.insert(word("++"), {TOKEN::TOK_EMPH_PLUS_PLUS, EArcSymbol::BOX_DASHED});
        cl.insert(word(".."), {TOKEN::TOK_EMPH, EArcSymbol::BOX_DOTTED});
        cl.insert(word("=="), {TOKEN::TOK_EMPH, EArcSymbol::BOX_DOUBLE});
        break;
    case lexing_state::mscgen_compat_box:
        /* The box symbols in Msc-generator, temporarily used after a BOX keyword
         * we switch back to compat mode after.*/
        cl.insert(word("--"), {TOKEN::TOK_EMPH, EArcSymbol::BOX_SOLID, lexing_state::mscgen_compat});
        cl.insert(word("++"), {TOKEN::TOK_EMPH_PLUS_PLUS, EArcSymbol::BOX_DASHED, lexing_state::mscgen_compat});
        cl.insert(word(".."), {TOKEN::TOK_EMPH, EArcSymbol::BOX_DOTTED, lexing_state::mscgen_compat});
        cl.insert(word("=="), {TOKEN::TOK_EMPH, EArcSymbol::BOX_DOUBLE, lexing_state::mscgen_compat});
        break;
    case lexing_state::mscgen_compat:
        /* The mscgen compatibility mode of interpreting these. */
        cl.insert(word("--"), {TOKEN::TOK_REL_MSCGEN, EArcSymbol::ARC_SOLID_BIDIR});
        cl.insert(word("::"), {TOKEN::TOK_REL_MSCGEN, EArcSymbol::ARC_DOUBLE2_BIDIR});
        cl.insert(word(".."), {TOKEN::TOK_REL_MSCGEN, EArcSymbol::ARC_DOTTED_BIDIR});
        cl.insert(word("=="), {TOKEN::TOK_REL_MSCGEN, EArcSymbol::ARC_DOUBLE_BIDIR});
        cl.insert(word(":>"), {TOKEN::TOK_REL_TO, EArcSymbol::ARC_DOUBLE2});
        cl.insert(word("<:"), {TOKEN::TOK_REL_FROM, EArcSymbol::ARC_DOUBLE2});
        cl.insert(word("<:>"), {TOKEN::TOK_REL_BIDIR, EArcSymbol::ARC_DOUBLE2_BIDIR});
        break;
    case lexing_state::no_change:
    default:
        _ASSERT(0);
        break;
    }

    cl.insert(word("+="), TOKEN::TOK_PLUS_EQUAL);
    cl.insert(just('-'), TOKEN::TOK_DASH);
    cl.insert(just('+'), TOKEN::TOK_PLUS);
    cl.insert(just('='), TOKEN::TOK_EQUAL);
    cl.insert(just('~'), TOKEN::TOK_TILDE);
    /* For these we switch back form box mode to mscgen mode */
    cl.insert(just(','), {TOKEN::TOK_COMMA, end_box_compat(state)});
    cl.insert(just(';'), {TOKEN::TOK_SEMICOLON, end_box_compat(state)});
    cl.insert(just('['), {TOKEN::TOK_OSBRACKET, end_box_compat(state)});
    cl.insert(just(']'), TOKEN::TOK_CSBRACKET);
    cl.insert(just('('), TOKEN::TOK_OPARENTHESIS);
    cl.insert(just(')'), TOKEN::TOK_CPARENTHESIS);
    cl.insert(just('*'), TOKEN::TOK_ASTERISK);
    cl.insert(just('|'), TOKEN::TOK_PIPE_SYMBOL);
    cl.insert(just('{'), TOKEN::TOK_OCBRACKET);
    cl.insert(just('}'), TOKEN::TOK_CCBRACKET);

    /* This is a colon-quoted string, finished by a quotation mark (UTF-8 allowed)
     * : "<string>"
     * ... or a colon-quoted string, finished by a newline or file-end (UTF-8 allowed)
     * : "<string>$
     * <string> can contain escaped quotation marks, hashmarks, but no line breaks
     */
    cl.insert(just(':') * WS.star() * just('"') *
              (noCRLF.without('"') | (word("\\\""))).star() *
              just('"').optional(), TOKEN::TOK_COLON_QUOTED_STRING);
    /* This is a non quoted colon-string
     * : <string>
     * terminated by any of: [ { or ;
     * Honors escaping of the above via a backslash. (UTF-8 allowed)
     * Can contain quotation marks (escaped or unescaped), but can not start with it
     * If it contains a hashmark, unescaped [ { or ; is allowed till the end of the line
     * (representing a commented section inside a label)
     * The mscgen_compat version mandates that no '>' or ':' follows the initial colon
     */
    auto noterm = every<char>().without(";[{#\\");
    if (state==lexing_state::basic)
        cl.insert(just(':') * (WS | newline).star() *
                  (
                      just('#') * noCRLF.star() | //starts with a comment
                      just('\\') * noCRLF |       //or an escape sequence
                      noterm.without("\" \t\n\r") //or some other char, but not a quotation mark or whitespace
                  ) * (
                      just('#') * noCRLF.star() | // continues with a comment
                      just('\\') * noCRLF |       // or an escape sequence
                      noterm                      // or something else
                  ).star(),
                  TOKEN::TOK_COLON_STRING);
    else if (state==lexing_state::mscgen_compat)
        cl.insert(just(':') * (WS | newline).star() *
                  (
                      just('#') * noCRLF.star() |   //starts with a comment
                      just('\\') * noCRLF |         //or an escape sequence
                      noterm.without("\" \t\n\r:>")|//or some other char, but not a quotation mark, '>' or ':' or whitespace
                      (WS|newline)*noterm.without("\" \t\n\r:>") //or '>' and ':' allowed only after a whitespace
                  ) * (
                      just('#') * noCRLF.star() | // continues with a comment
                      just('\\') * noCRLF |       // or an escape sequence
                      noterm                      // or something else
                  ).star(),
                  TOKEN::TOK_COLON_STRING);
    //degenerate colon string: empty or just a solo escape char
    cl.insert(just(':') * WS.star() * just('\\').optional(), TOKEN::TOK_COLON_STRING);

    /* A simple quoted string, that can have escaped quotation marks inside. (UTF-8 allowed)
     * ...or a quoted string missing the terminating quotation mark (at line-end).*/
    cl.insert(just('"') * (noCRLF.without('"') | word("\\\"")).star() * just('"').optional(),
              TOKEN::TOK_QSTRING);

    /* A simple quoted string with line breaks inside (UTF-8 allowed)
     *  Just for msc-gen compatibility */
    if (state == lexing_state::mscgen_compat)
        cl.insert(just('"') * (every<char>().without("\\\"") | (just('\\') * every<char>())).star() * just('"'),
                  TOKEN::TOK_MULTILINE_QSTRING);

    /* Numbers */
    auto digit = range('0', '9');
    auto dot = just('.');
    auto sign = oneof("+-");
    auto num = sign.optional() * digit.plus() * (dot * digit.star()).optional(); //no sign
    cl.insert(num, TOKEN::TOK_NUMBER);

    /* Strings (UTF-8 allowed)
     * Starts with letter or underscore, may also include numbers or dots
     * (but not 2 consecutive dots and dots may not be followed by digits).
     * May end in zero, one or two dots. */
    auto utf8 = range('A', 'Z') | range('a', 'z') | just('_') | range('\x80', '\xff');
    auto str = utf8 * (utf8 | digit | dot * utf8).star() * dot.optional(); //may end in dots
    cl.insert(str * dot.optional(), TOKEN::TOK_STRING);

    /* We need to list only those style names, which are not conforming to
     * TOK_STRING above. */
    cl.insert(word("pipe--"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("pipe++"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("pipe=="), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("pipe.."), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("block->"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("block>"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("block>>"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("block=>"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical--"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical++"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical=="), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical.."), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical->"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical>>"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical>"), TOKEN::TOK_STYLE_NAME);
    cl.insert(word("vertical=>"), TOKEN::TOK_STYLE_NAME);

    /* Color definitions. We allow any non ASCII UTF-8 character in the color name.
     * Also, color name may contain (single) dots and end in a (single) dot. */
    /* string+-number[,number]*/
    auto comma = just(',');
    cl.insert(str * sign * num * (comma * num).optional(), TOKEN::TOK_COLORDEF);
     /* string,number[+-number].*/
    cl.insert(str * comma * num * (sign * num).optional(), TOKEN::TOK_COLORDEF);
     /* number,number[,number[,number]] */
    cl.insert(num * comma * num * (comma * num).optional() * (comma * num).optional(), TOKEN::TOK_COLORDEF);

   /* Parameter names: $ followed by a string not ending with a dot.
    * A single '$" also matches.*/
    cl.insert(just('$') * utf8 * (utf8 | digit | dot * utf8).star(), TOKEN::TOK_PARAM_NAME);
    cl.insert(just('$'), TOKEN::TOK_PARAM_NAME);
    /* Parameter names: $$ representing a value unique to the actual invocation.*/
    cl.insert(just('$') * just('$'), TOKEN::TOK_PARAM_NAME);

    //Have concrete keywords (also matching strings) late as for
    //equal length matches, the later will be selected.

    /* These shape definition keywords are case sensitive */
    cl.insert(oneof("MLCESTHP"), TOKEN::TOK_SHAPE_COMMAND);

    if (state == lexing_state::mscgen_compat) {
        cl.insert(oneof("xX"), TOKEN::TOK_REL_X);
        cl.insert(just('-') * oneof("xX") * oneof(" \t\0xa\0xd[,;"), TOKEN::TOK_REL_DASH_X);
    }

    /* These keywords are case insensitive */
    cl.insert(iword("msc"), TOKEN::TOK_MSC);
    cl.insert(iword("heading"), TOKEN::TOK_COMMAND_HEADING);
    cl.insert(iword("nudge"), TOKEN::TOK_COMMAND_NUDGE);
    cl.insert(iword("defshape"), TOKEN::TOK_COMMAND_DEFSHAPE);
    cl.insert(iword("defstyle"), TOKEN::TOK_COMMAND_DEFSTYLE);
    cl.insert(iword("defcolor"), TOKEN::TOK_COMMAND_DEFCOLOR);
    cl.insert(iword("defdesign"), TOKEN::TOK_COMMAND_DEFDESIGN);
    cl.insert(iword("defproc"), TOKEN::TOK_COMMAND_DEFPROC);
    cl.insert(iword("replay"), TOKEN::TOK_COMMAND_REPLAY);
    cl.insert(iword("set"), TOKEN::TOK_COMMAND_SET);
    cl.insert(iword("include"), TOKEN::TOK_COMMAND_INCLUDE);
    cl.insert(iword("newpage"), TOKEN::TOK_COMMAND_NEWPAGE);
    cl.insert(iword("block"), TOKEN::TOK_COMMAND_BIG);
    cl.insert(iword("box"), {TOKEN::TOK_COMMAND_BOX, start_box_compat(state)});
    cl.insert(iword("pipe"), TOKEN::TOK_COMMAND_PIPE);
    cl.insert(iword("mark"), TOKEN::TOK_COMMAND_MARK);
    cl.insert(iword("parallel"), TOKEN::TOK_COMMAND_PARALLEL);
    cl.insert(iword("overlap"), TOKEN::TOK_COMMAND_OVERLAP);
    cl.insert(iword("vertical"), TOKEN::TOK_VERTICAL);
    cl.insert(iword("start"), TOKEN::TOK_START);
    cl.insert(iword("before"), TOKEN::TOK_BEFORE);
    cl.insert(iword("end"), TOKEN::TOK_END);
    cl.insert(iword("after"), TOKEN::TOK_AFTER);
    cl.insert(iword("at"), TOKEN::TOK_AT);
    cl.insert(iword("lost"), TOKEN::TOK_LOST);
    cl.insert(iword("left"), TOKEN::TOK_AT_POS);
    cl.insert(iword("center"), TOKEN::TOK_AT_POS);
    cl.insert(iword("right"), TOKEN::TOK_AT_POS);
    cl.insert(iword("show"), TOKEN::TOK_SHOW);
    cl.insert(iword("hide"), TOKEN::TOK_HIDE);
    cl.insert(iword("activate"), TOKEN::TOK_ACTIVATE);
    cl.insert(iword("deactivate"), TOKEN::TOK_DEACTIVATE);
    cl.insert(iword("bye"), TOKEN::TOK_BYE);
    cl.insert(iword("vspace"), TOKEN::TOK_COMMAND_VSPACE);
    cl.insert(iword("hspace"), TOKEN::TOK_COMMAND_HSPACE);
    cl.insert(iword("symbol"), TOKEN::TOK_COMMAND_SYMBOL);
    cl.insert(iword("note"), TOKEN::TOK_COMMAND_NOTE);
    cl.insert(iword("comment"), TOKEN::TOK_COMMAND_COMMENT);
    cl.insert(iword("endnote"), TOKEN::TOK_COMMAND_ENDNOTE);
    cl.insert(iword("footnote"), TOKEN::TOK_COMMAND_FOOTNOTE);
    cl.insert(iword("title"), TOKEN::TOK_COMMAND_TITLE);
    cl.insert(iword("subtitle"), TOKEN::TOK_COMMAND_SUBTITLE);
    cl.insert(iword("text"), TOKEN::TOK_COMMAND_TEXT);
    cl.insert(iword("brace"), TOKEN::TOK_VERTICAL_SHAPE);
    cl.insert(iword("bracket"), TOKEN::TOK_VERTICAL_SHAPE);
    cl.insert(iword("range"), TOKEN::TOK_VERTICAL_SHAPE);
    cl.insert(iword("pointer"), TOKEN::TOK_VERTICAL_SHAPE);
    cl.insert(iword("rbox"), TOKEN::TOK_MSCGEN_RBOX);
    cl.insert(iword("abox"), TOKEN::TOK_MSCGEN_ABOX);
    cl.insert(iword("join"), TOKEN::TOK_JOIN);
    cl.insert(iword("if"), TOKEN::TOK_IF);
    cl.insert(iword("then"), TOKEN::TOK_THEN);
    cl.insert(iword("else"), TOKEN::TOK_ELSE);
    
    cl.insert(iword("around"), TOKEN::TOK_SYMBOLPOS_AROUND);
    cl.insert(iword("leftof"), TOKEN::TOK_SYMBOLPOS_LEFTOF);
    cl.insert(iword("rightof"), TOKEN::TOK_SYMBOLPOS_RIGHTOF);
    cl.insert(iword("above"), TOKEN::TOK_SYMBOLPOS_ABOVE);
    cl.insert(iword("below"), TOKEN::TOK_SYMBOLPOS_BELOW);

    cl = make_deterministic(cl);
    //cl = minimize(cl); //TODO: uncomment, when maphoon works.
    return cl;
}

//When we do Color Syntax Highlight parsing the chart object is Csh instead.
template <bool CSH>
using ChartOrCsh = std::conditional<CSH, MscCsh, MscChart>::type;

/* This class tokenizes the input and generates a full symbol. */
template <typename TOKEN, bool CSH, typename READER_FN, typename CSH_STYPE, typename CSH_LTYPE> requires std::is_enum_v<TOKEN>
token_state_symbol<TOKEN> get_token(sv_reader<CSH>& inp, ChartOrCsh<CSH>& C, READER_FN readandclassify, CSH_STYPE& val, CSH_LTYPE& loc) {
    while (true) {
        inp.try_pop();
        if (!inp.has(1)) {
            loc = inp.template commit<false, false>(0);
            return TOKEN::TOK_EOF;
        }

        auto [int_token_state_symbol, len] = readandclassify(0, inp);
    re_scanned:
        TOKEN token = TOKEN(int_token_state_symbol.token);
        lexing_state new_state = lexing_state(int_token_state_symbol.new_state);
        if (len == 0) {
            loc = inp.template commit<false, false>(1);
            return {TOKEN::TOK_UNRECOGNIZED_CHAR, new_state};
        }
        switch (token) {
        case TOKEN::TOK_WHITESPACE:
            inp.template commit<false, false>(len);
            //whitespace does not change the state, keep using current one
            continue;
        case TOKEN::TOK_NEWLINE:
            inp.template commit<true, false>(len);
            //whitespace does not change the state, keep using current one
            continue;
        case TOKEN::TOK_COMMENT: {
            //if equals to "/*" then this is a comment, that is not closed(file contains no "*/")
            const bool open_only = inp.view(len) == "/*";
            auto loc = inp.template commit<true, false>(len);
            if constexpr (CSH)
                C.AddCSH(loc, COLOR_COMMENT);
            if (open_only) {
                if constexpr (CSH)
                    C.AddCSH_Error(loc, "Unpaired beginning of block comment '/" "*'.");
                else
                    C.Error.Error(loc.start(), "Unpaired beginning of block comment '/" "*'.");
            }
            //comments does not change the state, keep using current one
            continue;
        }
        case TOKEN::TOK_OINLINE: {
            const auto loc_oinline = inp.template commit<false, false>(len);
            loc = inp.template commit<false, false>(0);
            const char *start_text_ptr = inp.view(0).data(), *end_text_ptr = start_text_ptr;
            unsigned nesting_level = 0;
            while (inp.has(1)) {
                auto [int_token_state_symbol, len] = readandclassify(0, inp);
                switch (TOKEN(int_token_state_symbol.token)) {
                case TOKEN::TOK_OINLINE:
                    nesting_level++;
                    FALLTHROUGH;
                default:
                    if (!len) len = 1;
                    end_text_ptr = inp.view(len).data() + len;
                    loc.merge(inp.template commit<true, false>(len));
                    break;
                case TOKEN::TOK_CINLINE:
                    if (nesting_level) {
                        nesting_level--;
                        end_text_ptr = inp.view(len).data() + len;
                        loc.merge(inp.template commit<true, false>(len));
                        break;
                    }
                    inp.template commit<true, false>(len);
                    val.str = std::string_view(start_text_ptr, end_text_ptr - start_text_ptr);
                    return TOKEN::TOK_INLINE;
                }
            }
            //Unpaired OINLINE
            if constexpr (CSH) {
                C.AddCSH_Error(loc_oinline, "Unpaired beginning of inlined chart '" INLINE_OPEN_STR "'.");
                C.AddCSH(loc, COLOR_COMMENT); //we have consumed the whole file, color as comment
            } else
                C.Error.Error(loc_oinline.start(), "Unpaired beginning of inlined chart '" INLINE_OPEN_STR "'.");
            loc = loc_oinline;
            return TOKEN::TOK_UNMATCHED_INLINE;
        }
        case TOKEN::TOK_SPECIAL_ARC:
        case TOKEN::TOK_REL_TO:
        case TOKEN::TOK_REL_FROM:
        case TOKEN::TOK_REL_BIDIR:
        case TOKEN::TOK_EMPH:
        case TOKEN::TOK_EMPH_PLUS_PLUS:
        case TOKEN::TOK_REL_MSCGEN:
            loc = inp.template commit<false, false>(len);
            val.arcsymbol = EArcSymbol(int_token_state_symbol.symbol);
            return token;

        case TOKEN::TOK_PLUS_EQUAL:
        case TOKEN::TOK_DASH:
        case TOKEN::TOK_PLUS:
        case TOKEN::TOK_EQUAL:
        case TOKEN::TOK_TILDE:
        case TOKEN::TOK_COMMA:
        case TOKEN::TOK_SEMICOLON:
        case TOKEN::TOK_OSBRACKET:
        case TOKEN::TOK_CSBRACKET:
        case TOKEN::TOK_OPARENTHESIS:
        case TOKEN::TOK_CPARENTHESIS:
        case TOKEN::TOK_ASTERISK:
        case TOKEN::TOK_PIPE_SYMBOL:
        case TOKEN::TOK_CINLINE:
        case TOKEN::TOK_UNRECOGNIZED_CHAR:
        case TOKEN::TOK_EOF:
            loc = inp.template commit<false, false>(len);
            return {token, new_state}; //comma, semicolon and osbracket may change the state

        case TOKEN::TOK_REL_DASH_X:
            _ASSERT(len == 3);
            loc = inp.template commit<false, false>(len-1);  //here tokenizer includes the post-condition, too
            return token;

        case TOKEN::TOK_OCBRACKET:
        case TOKEN::TOK_CCBRACKET:
            val.input_text_ptr = inp.view(1).data();
            loc = inp.template commit<false, false>(len);
            return token;

        case TOKEN::TOK_SHAPE_COMMAND: {
            _ASSERT(len == 1);
            const char cmd = inp.peek(0);
            loc = inp.template commit<false, false>(len);
            switch (cmd) {
            case 'M': val.shapecommand = ShapeElement::MOVE_TO; return TOKEN::TOK_SHAPE_COMMAND;
            case 'L': val.shapecommand = ShapeElement::LINE_TO; return TOKEN::TOK_SHAPE_COMMAND;
            case 'C': val.shapecommand = ShapeElement::CURVE_TO; return TOKEN::TOK_SHAPE_COMMAND;
            case 'E': val.shapecommand = ShapeElement::CLOSE_PATH; return TOKEN::TOK_SHAPE_COMMAND;
            case 'S': val.shapecommand = ShapeElement::SECTION_BG; return TOKEN::TOK_SHAPE_COMMAND;
            case 'T': val.shapecommand = ShapeElement::TEXT_AREA; return TOKEN::TOK_SHAPE_COMMAND;
            case 'H': val.shapecommand = ShapeElement::HINT_AREA; return TOKEN::TOK_SHAPE_COMMAND;
            case 'P': val.shapecommand = ShapeElement::PORT; return TOKEN::TOK_SHAPE_COMMAND;
            default: _ASSERT(0); return TOKEN::TOK_UNRECOGNIZED_CHAR;
            }
        }
        //keywords
        case TOKEN::TOK_MSC:
        case TOKEN::TOK_COMMAND_HEADING:
        case TOKEN::TOK_COMMAND_NUDGE:
        case TOKEN::TOK_COMMAND_DEFSHAPE:
        case TOKEN::TOK_COMMAND_DEFSTYLE:
        case TOKEN::TOK_COMMAND_DEFCOLOR:
        case TOKEN::TOK_COMMAND_DEFDESIGN:
        case TOKEN::TOK_COMMAND_DEFPROC:
        case TOKEN::TOK_COMMAND_REPLAY:
        case TOKEN::TOK_COMMAND_SET:
        case TOKEN::TOK_COMMAND_INCLUDE:
        case TOKEN::TOK_COMMAND_NEWPAGE:
        case TOKEN::TOK_COMMAND_BIG:
        case TOKEN::TOK_COMMAND_PIPE:
        case TOKEN::TOK_COMMAND_MARK:
        case TOKEN::TOK_COMMAND_PARALLEL:
        case TOKEN::TOK_COMMAND_OVERLAP:
        case TOKEN::TOK_VERTICAL:
        case TOKEN::TOK_START:
        case TOKEN::TOK_BEFORE:
        case TOKEN::TOK_END:
        case TOKEN::TOK_AFTER:
        case TOKEN::TOK_AT:
        case TOKEN::TOK_LOST:
        case TOKEN::TOK_AT_POS:
        case TOKEN::TOK_SHOW:
        case TOKEN::TOK_HIDE:
        case TOKEN::TOK_ACTIVATE:
        case TOKEN::TOK_DEACTIVATE:
        case TOKEN::TOK_BYE:
        case TOKEN::TOK_COMMAND_VSPACE:
        case TOKEN::TOK_COMMAND_HSPACE:
        case TOKEN::TOK_COMMAND_SYMBOL:
        case TOKEN::TOK_COMMAND_NOTE:
        case TOKEN::TOK_COMMAND_COMMENT:
        case TOKEN::TOK_COMMAND_ENDNOTE:
        case TOKEN::TOK_COMMAND_FOOTNOTE:
        case TOKEN::TOK_COMMAND_TITLE:
        case TOKEN::TOK_COMMAND_SUBTITLE:
        case TOKEN::TOK_COMMAND_TEXT:
        case TOKEN::TOK_VERTICAL_SHAPE:
        case TOKEN::TOK_MSCGEN_RBOX:
        case TOKEN::TOK_MSCGEN_ABOX:
        case TOKEN::TOK_JOIN:
        case TOKEN::TOK_IF:
        case TOKEN::TOK_THEN:
        case TOKEN::TOK_ELSE:
        case TOKEN::TOK_SYMBOLPOS_AROUND:
        case TOKEN::TOK_SYMBOLPOS_ABOVE:
        case TOKEN::TOK_SYMBOLPOS_BELOW:
        case TOKEN::TOK_SYMBOLPOS_LEFTOF:
        case TOKEN::TOK_SYMBOLPOS_RIGHTOF:

        //special tokens, whose value is their text
        case TOKEN::TOK_COLORDEF:
        case TOKEN::TOK_NUMBER:
        case TOKEN::TOK_PARAM_NAME:
        case TOKEN::TOK_STYLE_NAME:
        case TOKEN::TOK_REL_X:
            val.str = inp.view(len);
            loc = inp.template commit<false, false>(len);
            return token;

        case TOKEN::TOK_COMMAND_BOX:
            val.str = inp.view(len);
            loc = inp.template commit<false, false>(len);
            return {token, new_state}; //box may change the state

        case TOKEN::TOK_COLON_STRING: {
            const std::string_view text = inp.view(len);
            _ASSERT(text.starts_with(':'));
            loc = inp.template commit<true, false>(len);
            if constexpr (CSH) {
                val.multi_str.init();
                C.AddCSH_ColonString_CheckAndAddEscapeHint(loc, text, true);
                C.AddColonLabel(loc, text);
            } else
                val.multi_str.set_owning(process_colon_string(text, loc.start()));
            return TOKEN::TOK_COLON_STRING;
        }
        case TOKEN::TOK_COLON_QUOTED_STRING: { //This is transmuted to a TOK_COLON_STRING
            //if last char is not '"' the quotation is not closed
            std::string_view text = inp.view(len);
            _ASSERT(text.starts_with(':'));
            loc = inp.template commit<false, false>(len);
            if constexpr (CSH) {
                if (!text.ends_with('"'))
                    C.AddCSH_ErrorAfter(loc, "Missing closing quotation mark.");
                C.AddCSH_ColonString_CheckAndAddEscapeHint(loc, text, false);
                C.AddColonLabel(loc, text);
                val.multi_str.init();
            } else {
                const char* const colon_pos = text.data();
                const bool closed = text.ends_with('"');
                if (closed)
                    text.remove_suffix(1);
                text.remove_prefix(1); //the colon
                remove_head_tail_whitespace(text);
                FileLineCol pos = loc.start();
                pos.AdvanceCol(text.data() - colon_pos);
                if (!closed)
                    C.Error.Error(loc.start(),
                                  "This opening quotation mark misses its closing pair. "
                                  "Assuming string termination at line-end.",
                                  "Quoted strings cannot have line breaks. Use \'\\n\' to insert a line break.");
                text.remove_prefix(1); //leading quotation mark
                pos.AdvanceCol(1);
                val.multi_str.CombineThemToMe(pos.Print(), text);
            }
            return TOKEN::TOK_COLON_STRING;
        }
        case TOKEN::TOK_STRING: {
            std::string_view text = inp.view(len);
            //TOK_STRING: may end in two dots. In that case we should
            //re-scan without the two dots. (E.g., in "box..", we want to
            //scan "box" as TOK_COMMAND_BOX and not as TOK_STRING
            if (text.ends_with("..")) {
                text.remove_suffix(2);
                sv_reader<CSH> input(text, {});
                std::tie(int_token_state_symbol, len) = readandclassify(0, input);
                goto re_scanned;
            }
            loc = inp.template commit<false, false>(len);
            val.str.set(text);
            return TOKEN::TOK_STRING;
        }
        case TOKEN::TOK_QSTRING: {
            std::string_view text = inp.view(len);
            loc = inp.template commit<true, false>(len); //mscgen_compat mode has multiline quoted strings
            text.remove_prefix(1); //opening quotation mark
            //TOK_QSTRING: if last char is not '"' the quotation is not closed
            if (text.ends_with('"'))
                text.remove_suffix(1); //closing quotation mark
            else if constexpr (CSH)
                C.AddCSH_ErrorAfter(loc, "Missing closing quotation mark.");
            else
                C.Error.Error(loc.start(),
                              "This opening quotation mark misses its closing pair. "
                              "Assuming string termination at line-end.",
                              "Quoted strings cannot have line breaks. Use \'\\n\' to insert a line break.");
            val.str.set(text);
            return TOKEN::TOK_QSTRING;
        }
        case TOKEN::TOK_MULTILINE_QSTRING: {
            //mscgen_compat-only quoted-string: ends in a quotation mark and may be multiline
            std::string_view text = inp.view(len);
            loc = inp.template commit<true, false>(len);
            if constexpr (CSH) {
                val.str.set(text.substr(1, text.size()-2));
                return TOKEN::TOK_QSTRING;
            } else {
                val.multi_str.set_owning(process_multiline_qstring(text, loc.start()));
                return TOKEN::TOK_MULTILINE_QSTRING;
            }
        }
        default:
            _ASSERT(0);
            loc = inp.template commit<false, false>(len);
            return token;
        } //switch(token)
    }
}
}
