r"""
Models of the django app *mini_buildd*

.. attention:: **Daemon**: When changing ftp network settings, then PCA: Still not active, needs explicit restart?
.. attention:: **Distribution**: When adding Architecture: No indication that we need to re-index (run check on repository)
.. attention:: **Distribution** (``workaround in __str__``): Not unique for base source (historic fluke)
.. attention:: **AptKeys**: Duplicates possible (will break API call ``setup``)
.. attention:: **Chroot**: LVM chroots fail running lvcreate with 'not found: device not cleared' (:debbug:`705238`)

  Unclear, ancient. See also: http://lists.debian.org/debian-user/2012/12/msg00407.html.

  "--noudevsync" workaround makes lvcreate work again, but the
  chroot will not work later anyway later.

.. note:: **Daemon**: Instance does not touch the GPG setup once it's created -- *unless you do an explicit remove* on the instance.
.. note:: **Uploader**: Don't add or delete instances manually!

  These are bound to users, and come automatically when new users are created. The administrator only changes these
  instances to grant rights.

.. note:: **Chroot** (``mbd_workaround_debmirror_script``): Be able to strap 'newer Ubuntus' with Debian's ``debootstrap``

  Installed ``debootstrap`` might be outdated and not be able to strap newer codenames. Generally, the solution
  is to simply install an updated ``debootstrap`` that can handle this.

  However, newer Ubuntu releases are usually not supported in a timely fashion in Debian's ``debootstrap``.
  Furthermore, Ubuntu's variant no longer provides scripts (or symlinks): See
  http://changelogs.ubuntu.com/changelogs/pool/main/d/debootstrap/debootstrap_1.0.132ubuntu1/changelog

  This workaround basically does the same as the hardcoded change in Ubuntu's ``debootstrap``: Just use 'gutsy'
  as default. In case that change is incorporated into Debian's ``debootstrap``, this workaround is obsolete again.

.. note:: **Chroot** (setup workaround in ``dist.py``): debootstrap fails for <=lenny chroots on >=jessie host kernel (uname) (:debbug:`642031`)

  This should ideally be worked around in debootstrap itself eventually.

  mini-buildd comes with a workaround wrapper :mbdcommand:`mini-buildd-debootstrap-uname-2.6`. Just use
  ``/usr/sbin/mini-buildd-debootstrap-uname-2.6`` as ``debootstrap_command`` in resp. chroots.

  Fwiw, this is due to older libc6 packaging's preinst, which will
  meekly fail if ``uname -r`` starts with a two-digit version;
  i.e.::

    FINE : 3.2.0-4-amd64      Standard wheezy kernel
    FAILS: 3.10-2-amd64       Standard jessie/sid kernel
    FAILS: 3.9-0.bpo.1-amd64  Wheezy backport of the jessie/sid kernel

.. tip:: **Chroot**: Install :debpkg:`qemu-user-static` for seamless access to foreign archs (albeit with a speed penalty)

Naming conventions
==================

Model class and field names
---------------------------
All model class names and all field names must be **human
readable with no abbreviations** (as django, per default,
displays the internal names intelligently to the end user).

*Model class names* must be in **CamelCase**.

*Field names* must be all **lowercase** and **separated by underscores**.

For example, **don't** try to do sort of "grouping" using names like::

  email_notify
  email_allow_regex

This should rather read::

  notify
  allow_emails_to

To group fields together for the end user, use AdminModel's *fieldset* option.

Methods
-------
Any methods that represent mini-buildd logic should go into the models
directly, but must be prefixed with "mbd\_". This avoids conflicts
with method names form the django model's class, but still keeps the
logic where it belongs.

"""

import collections
import contextlib
import datetime
import functools
import glob
import inspect
import logging
import os
import re
import shlex
import shutil
import tempfile
import time
import urllib.error
import urllib.request
from contextlib import closing

import debian.deb822
import debian.debian_support
import django.contrib.admin
import django.contrib.auth.models
import django.contrib.messages
import django.core.exceptions
import django.core.mail
import django.core.validators
import django.db
import django.db.models
import django.template.response
import django.utils.deconstruct
import django.utils.html
import django.utils.timezone

from mini_buildd import api, call, client, config, daemon, dist, events, files, ftpd, gnupg, net, reprepro, sbuild, schroot, util

LOG = logging.getLogger(__name__)


def help_html(help_plain):
    """Prepare plain help for HTML"""
    return f"<pre>{django.utils.html.escape(help_plain)}</pre>"


def mbd_msg(request, msg_lines, level=django.contrib.messages.INFO, level_followups=django.contrib.messages.INFO):
    """Convey message to user via django messaging, with extra multiline ('followups') formatting support"""
    if request is not None:
        prefix = ""
        for s in msg_lines.split("\n"):
            msg = f"{prefix}{s}"
            django.contrib.messages.add_message(request, level, django.utils.html.format_html(msg))
            level = level_followups
            prefix = "↳ "


@django.utils.deconstruct.deconstructible
class RegexValidator(django.core.validators.RegexValidator):
    def __init__(self, regex, message=None):
        # Note: 'code' argument seems to be needed: https://code.djangoproject.com/ticket/17051
        super().__init__(regex=regex,
                         message=f"Value must match regex '{regex}'" if message is None else message,
                         code="mbd_regex_invalid")


@django.utils.deconstruct.deconstructible
class ForbiddenKeysValidator:
    def __init__(self, forbidden_keys):
        self.forbidden_keys = forbidden_keys

    def __call__(self, value):
        for key in value:
            if key in self.forbidden_keys:
                raise django.core.exceptions.ValidationError(f"Don't use key '{key}' ({','.join(self.forbidden_keys)} are forbidden).")


@django.utils.deconstruct.deconstructible
class ServerEndpointValidator:
    def __call__(self, value):
        try:
            net.ServerEndpoint(value, protocol="ftp")  # protocol needed but does not matter for validation
        except BaseException as e:
            raise django.core.exceptions.ValidationError(f"Can't parse: {util.e2http(e)}")


class UnixTextField(django.db.models.TextField):
    r"""Normalizes ``\r\n`` newlines (usually send by browsers) to Unix newlines ``\n``"""

    def get_prep_value(self, value):
        return value.replace("\r\n", "\n")


class Model(django.db.models.Model):
    """
    Abstract father model for all mini-buildd models

    This just makes sure no config is changed or deleted while
    the daemon is running.
    """

    @classmethod
    def mbd_get_or_none(cls, **kwargs):
        """Get unambiguous instance or ``None`` if no such instance (``objects.get()`` w/o exception)"""
        try:
            return cls.objects.get(**kwargs)
        except cls.DoesNotExist:
            return None

    @classmethod
    def mbd_get_fields(cls, exclude=None):
        return [f for f in cls._meta.get_fields() if f.name not in ([] if exclude is None else exclude)]

    class Meta():
        abstract = True
        app_label = "mini_buildd"

    class Admin(django.contrib.admin.ModelAdmin):
        save_on_top = True
        save_as = True
        save_as_continue = False

        @classmethod
        def _mbd_on_change(cls, obj):
            """Global actions to take when an object changes"""
            for o in obj.mbd_get_reverse_dependencies():
                if o.LETHAL_DEPENDENCIES:
                    o.mbd_set_changed()
                    o.save()

        @classmethod
        def _mbd_on_activation(cls, obj):
            """Global actions to take when an object becomes active"""

        @classmethod
        def _mbd_on_deactivation(cls, obj):
            """Global actions to take when an object becomes inactive"""

        def save_model(self, request, obj, form, change):
            if change:
                self._mbd_on_change(obj)
            super().save_model(request, obj, form, change)

        def delete_model(self, request, obj):
            self._mbd_on_change(obj)

            is_prepared_func = getattr(obj, "mbd_is_prepared", None)
            if is_prepared_func and is_prepared_func():
                self.mbd_remove(obj)

            super().delete_model(request, obj)

    def mbd_class_name(self):
        """Allow to get class name in templates"""
        return self.__class__.__name__

    def mbd_get_dependencies(self):
        return []

    def mbd_get_reverse_dependencies(self):
        return []

    @classmethod
    def mbd_get_default(cls, field_name):
        return cls._meta.get_field(field_name).get_default()

    def mbd_validate_field_unchanged(self, name):
        current_obj = type(self).mbd_get_or_none(pk=self.pk)
        if current_obj is not None:
            if getattr(self, name) != getattr(current_obj, name):
                raise django.core.exceptions.ValidationError({name: "You can't modify this field on already existing instance (use 'save as new' to save a new instance)"})


class StatusModel(Model):
    """Abstract model class for all models that carry a status"""

    # The main statuses: removed, prepared, active
    STATUS_REMOVED = 0
    STATUS_PREPARED = 1
    STATUS_ACTIVE = 2
    STATUS_CHOICES = [
        (STATUS_REMOVED, "Removed"),
        (STATUS_PREPARED, "Prepared"),
        (STATUS_ACTIVE, "Active")]
    status = django.db.models.IntegerField(choices=STATUS_CHOICES, default=STATUS_REMOVED, editable=False)

    # Statuses of the prepared data, relevant for status "Prepared" only.
    # For "Removed" it's always NONE, for "Active" it's always the stamp of the last check.
    CHECK_NONE = datetime.datetime(datetime.MINYEAR, 1, 1, tzinfo=datetime.timezone.utc)
    CHECK_CHANGED = datetime.datetime(datetime.MINYEAR, 1, 2, tzinfo=datetime.timezone.utc)
    CHECK_FAILED = datetime.datetime(datetime.MINYEAR, 1, 3, tzinfo=datetime.timezone.utc)
    CHECK_REACTIVATE = datetime.datetime(datetime.MINYEAR, 1, 4, tzinfo=datetime.timezone.utc)
    _CHECK_MAX = CHECK_REACTIVATE
    CHECK_STRINGS = {
        CHECK_NONE: {"char": "-", "string": "Unchecked -- please run check"},
        CHECK_CHANGED: {"char": "*", "string": "Changed -- please prepare again"},
        CHECK_FAILED: {"char": "x", "string": "Failed -- please fix and check again"},
        CHECK_REACTIVATE: {"char": "A", "string": "Failed in active state -- will auto-activate when check succeeds again"}}
    last_checked = django.db.models.DateTimeField(default=CHECK_NONE, editable=False)

    LETHAL_DEPENDENCIES = True

    class Meta(Model.Meta):
        abstract = True

    class Admin(Model.Admin):
        def save_model(self, request, obj, form, change):
            if change:
                obj.mbd_set_changed()
            super().save_model(request, obj, form, change)

        @classmethod
        def _mbd_run_dependencies(cls, obj, func, **kwargs):
            """
            Run action for all dependencies

            But don't fail and run all checks for models with
            LETHAL_DEPENDENCIES set to False. Practical use case is
            the Daemon model only, where we want to run all checks on
            all dependencies, but not fail ourselves.
            """
            for o in obj.mbd_get_dependencies():
                try:
                    func(o, **kwargs)
                except BaseException as e:
                    if obj.LETHAL_DEPENDENCIES:
                        raise
                    util.log_exception(LOG, f"Check on '{o}' failed", e)

        @classmethod
        def mbd_prepare(cls, obj):
            if not obj.mbd_is_prepared():
                # Fresh prepare
                cls._mbd_run_dependencies(obj, cls.mbd_prepare)
                obj.mbd_prepare()
                obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_NONE
                obj.save()
            elif obj.mbd_is_changed():
                # Update data on change
                cls._mbd_run_dependencies(obj, cls.mbd_prepare)
                obj.mbd_sync()
                obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_NONE
                obj.save()

        @classmethod
        def mbd_check(cls, obj, force=False, needs_activation=False):
            if not obj.mbd_is_prepared() or obj.mbd_is_changed():
                raise util.HTTPBadRequest(f"Can't check removed or changed object (run 'prepare' first): {obj}")

            try:
                # Also run for all status dependencies
                cls._mbd_run_dependencies(obj, cls.mbd_check,
                                          force=force,
                                          needs_activation=obj.mbd_is_active() or obj.last_checked == obj.CHECK_REACTIVATE)

                if force or not obj.mbd_is_checked():
                    obj.mbd_check()

                    # Handle special flags
                    reactivated = False
                    if obj.last_checked == obj.CHECK_REACTIVATE:
                        obj.status = StatusModel.STATUS_ACTIVE
                        reactivated = True

                    # Finish up
                    obj.last_checked = django.utils.timezone.now()
                    obj.save()

                    # Run activation hook if reactivated
                    if reactivated:
                        cls._mbd_on_activation(obj)

                if needs_activation and not obj.mbd_is_active():
                    raise util.HTTPBadRequest(f"Not active, but a (tobe-)active item depends on it. Activate this first: {obj}")
            except BaseException:
                # Check failed, auto-deactivate and re-raise exception
                obj.last_checked = max(obj.last_checked, obj.CHECK_FAILED)
                if obj.mbd_is_active():
                    obj.status, obj.last_checked = obj.STATUS_PREPARED, obj.CHECK_REACTIVATE
                obj.save()
                raise

        @classmethod
        def mbd_pc(cls, obj, force=False, needs_activation=False):
            cls.mbd_prepare(obj)
            cls.mbd_check(obj, force=force, needs_activation=needs_activation)

        @classmethod
        def mbd_activate(cls, obj):
            if not obj.mbd_is_active():
                if obj.mbd_is_prepared() and obj.mbd_is_checked():
                    cls._mbd_run_dependencies(obj, cls.mbd_activate)
                    obj.status = obj.STATUS_ACTIVE
                    obj.save()
                elif obj.mbd_is_prepared() and obj.last_checked in [obj.CHECK_FAILED, obj.CHECK_NONE]:
                    obj.last_checked = obj.CHECK_REACTIVATE
                    obj.save()
                else:
                    raise util.HTTPBadRequest(f"Prepare and check first: {obj}")

            if obj.mbd_is_active():
                cls._mbd_on_activation(obj)

        @classmethod
        def mbd_pca(cls, obj, force=False, needs_activation=False):
            cls.mbd_prepare(obj)
            cls.mbd_check(obj, force=force, needs_activation=needs_activation)
            cls.mbd_activate(obj)

        @classmethod
        def mbd_deactivate(cls, obj):
            cls._mbd_on_deactivation(obj)
            obj.status = min(obj.STATUS_PREPARED, obj.status)
            if obj.last_checked == obj.CHECK_REACTIVATE:
                obj.last_checked = obj.CHECK_FAILED
            obj.save()

        @classmethod
        def mbd_remove(cls, obj):
            cls._mbd_on_deactivation(obj)
            if obj.mbd_is_prepared():
                obj.mbd_remove()
                obj.status, obj.last_checked = obj.STATUS_REMOVED, obj.CHECK_NONE
                obj.save()

        @classmethod
        def mbd_action(cls, request, queryset, action, **kwargs):
            """
            Try to run action on each object in queryset

            Emit error message on failure, but don't fail ourself.
            """
            name = action.__name__[4:]
            for o in queryset:
                try:
                    action(o, **kwargs)

                    msg = f"{name} succeeded: {o}"
                    mbd_msg(request, msg)
                    LOG.info(msg)
                except BaseException as e:
                    msg = f"{name} failed: {o}"
                    mbd_msg(request, f"{msg}: {util.e2http(e)}", level=django.contrib.messages.ERROR, level_followups=django.contrib.messages.WARNING)
                    util.log_exception(LOG, msg, e)

        # Admin actions: pylint: disable=no-self-argument
        #
        # Unlike ``mbd_<action>`` class methods, these act on querysets && will not fail but use the django message system to deliver errors
        #
        @django.contrib.admin.action(description="Prepare")
        def mbd_action_prepare(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_prepare)

        @django.contrib.admin.action(description="Check")
        def mbd_action_check(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_check, force=True)

        @django.contrib.admin.action(description="PC")
        def mbd_action_pc(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_pc)

        @django.contrib.admin.action(description="Activate")
        def mbd_action_activate(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_activate)

        @django.contrib.admin.action(description="PCA")
        def mbd_action_pca(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_pca)

        @django.contrib.admin.action(description="Deactivate")
        def mbd_action_deactivate(modeladmin, request, queryset):
            modeladmin.mbd_action(request, queryset, modeladmin.mbd_deactivate)

        @django.contrib.admin.action(description="Remove")
        def mbd_action_remove(modeladmin, request, queryset):
            if request.POST.get("confirm"):
                modeladmin.mbd_action(request, queryset, modeladmin.mbd_remove)
                return None
            return django.template.response.TemplateResponse(
                request,
                "admin/mini_buildd/mbd_action_remove_confirm.html",
                {
                    "title": ("Are you sure?"),
                    "queryset": queryset,
                    "action": "mbd_action_remove",
                    "desc": (
                        "Unpreparing means all the data associated by preparation will be\n"
                        "removed from the system. Especially for repositories,\n"
                        "this would mean losing all packages!\n"
                    ),
                    "action_checkbox_name": django.contrib.admin.helpers.ACTION_CHECKBOX_NAME})

        @classmethod
        def _status(cls, obj):
            return django.utils.html.format_html(f'<span class="mbd-model-status-{obj.get_status_display()}" title="{obj.mbd_get_status_display(typ="string")}">{obj.mbd_get_status_display(typ="char")}</span>')

        actions = (mbd_action_prepare, mbd_action_check, mbd_action_pc, mbd_action_activate, mbd_action_pca, mbd_action_deactivate, mbd_action_remove)
        list_display = ("_status", "__str__")
        list_display_links = ("__str__",)

    def mbd_set_changed(self):
        if self.mbd_is_active():
            self.status = self.STATUS_PREPARED
            LOG.warning("Deactivated due to changes: %s", self)
        self.last_checked = self.CHECK_CHANGED
        LOG.warning("Marked as changed: %s", self)

    #
    # Action hooks helpers
    #
    def _mbd_remove_and_prepare(self):
        StatusModel.Admin.mbd_remove(self)
        StatusModel.Admin.mbd_prepare(self)

    #
    # Status abstractions and helpers
    #
    def mbd_is_prepared(self):
        return self.status >= self.STATUS_PREPARED

    def mbd_is_active(self):
        return self.status >= self.STATUS_ACTIVE

    def mbd_is_checked(self):
        return self.last_checked > self._CHECK_MAX

    def mbd_is_changed(self):
        return self.last_checked == self.CHECK_CHANGED

    def mbd_is_reactivate(self):
        return self.last_checked == self.CHECK_REACTIVATE

    @classmethod
    def mbd_get_active(cls):
        return cls.objects.filter(status__gte=cls.STATUS_ACTIVE)

    @classmethod
    def mbd_get_active_or_auto_reactivate(cls):
        return cls.objects.filter(django.db.models.Q(status__gte=cls.STATUS_ACTIVE) | django.db.models.Q(last_checked=cls.CHECK_REACTIVATE))

    @classmethod
    def mbd_get_prepared(cls):
        return cls.objects.filter(status__gte=cls.STATUS_PREPARED)

    def mbd_get_check_display(self, typ="string"):
        return {"string": self.last_checked.strftime("Checked on %Y-%m-%d %H:%M"), "char": "C"}[typ] if self.mbd_is_checked() else self.CHECK_STRINGS[self.last_checked][typ]

    def mbd_get_status_display(self, typ="string"):
        return f"{self.get_status_display()} ({self.mbd_get_check_display(typ)})"


class GnuPGPublicKey(StatusModel):
    key_id = django.db.models.CharField(
        max_length=100, blank=True, default="",
        validators=[django.core.validators.MinLengthValidator(8, message="The key id, if given, must be at least 8 bytes long")],
        help_text=help_html("Key id to retrieve the actual key per key server (leave ``key`` below blank)")
    )
    key = UnixTextField(
        blank=True, default="",
        help_text=help_html("Actual (ASCII-armored) GnuPG public key (leave ``key_id`` above blank)")
    )

    key_long_id = django.db.models.CharField(max_length=254, blank=True, editable=False, default="")
    key_created = django.db.models.CharField(max_length=254, blank=True, editable=False, default="")
    key_expires = django.db.models.CharField(max_length=254, blank=True, editable=False, default="")
    key_name = django.db.models.CharField(max_length=254, blank=True, editable=False, default="")
    key_fingerprint = django.db.models.CharField(max_length=254, blank=True, editable=False, default="")

    class Meta(StatusModel.Meta):
        abstract = True
        app_label = "mini_buildd"

    class Admin(StatusModel.Admin):
        list_display = StatusModel.Admin.list_display + ("key_created", "key_expires", "key_fingerprint")
        search_fields = ("key_id", "key_long_id", "key_name", "key_fingerprint")

    def __str__(self):
        return f"GnuPGPublicKey '{self.key_long_id if self.key_long_id else self.key_id}' ({self.key_name})"

    def mbd_key_id(self):
        """
        Get valid key id while preferring long key id

        Traditionally, also short ids were used for ``key_id``. This assures a valid key is returned regardless of the
        state (if unprepared, ``key_long_id`` would be empty) while also preferring the long key if it exists.
        """
        return self.key_long_id if self.key_long_id else self.key_id

    @classmethod
    def mbd_filter_key(cls, key_id):
        regex = rf"{key_id[-8:]}$"
        return cls.objects.filter(django.db.models.Q(key_long_id__iregex=regex) | django.db.models.Q(key_id__iregex=regex))

    def mbd_prepare(self):
        key, info = gnupg.PublicKeyCache().keyinfo(self.key_id, self.key)
        self.key, self.key_long_id, self.key_name, self.key_fingerprint, self.key_created, self.key_expires = key, info["key"], info["user"], info["fingerprints"][0], info["created"], info["expires"]
        self.key_id = self.key_long_id

    def mbd_remove(self):
        self.key_long_id = ""
        self.key_created = ""
        self.key_expires = ""
        self.key_name = ""
        self.key_fingerprint = ""

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_check(self):
        """Check that we actually have the key and long_id. This should always be true after ``prepare``"""
        if not self.mbd_key_id():
            raise util.HTTPBadRequest("GnuPG key with inconsistent state -- try remove,prepare to fix")

    def mbd_verify(self, signed_message):
        with closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-pubkey-"})) as _gnupg:
            _gnupg.add_pub_key(self.key)
            data, _ = _gnupg.gpgme_verify(signed_message)
            return data


class AptKey(GnuPGPublicKey):
    def clean(self):
        super().clean()

        if self.key_id:
            matching_key = self.mbd_filter_key(self.key_id)
            if matching_key.count() > 0 and self.pk != matching_key.get().pk:
                raise django.core.exceptions.ValidationError({"key_id": f"Another such key id already exists: {matching_key[0]}"})


class Uploader(GnuPGPublicKey):
    user = django.db.models.OneToOneField(django.contrib.auth.models.User, editable=False, on_delete=django.db.models.CASCADE)
    may_upload_to = django.db.models.ManyToManyField("Repository", blank=True)

    class Admin(GnuPGPublicKey.Admin):
        search_fields = GnuPGPublicKey.Admin.search_fields + ("user__username",)
        readonly_fields = GnuPGPublicKey.Admin.readonly_fields + ("user",)
        filter_horizontal = ("may_upload_to",)

    def __str__(self):
        return f"Uploader '{self.user}' may upload to '{','.join([r.identity for r in self.may_upload_to.all()])}' with key '{super().__str__()}'"


def cb_create_user_profile(sender, instance, created, **kwargs):
    """Automatically create a user profile with every user that is created"""
    if created:
        Uploader.objects.create(user=instance)


django.db.models.signals.post_save.connect(cb_create_user_profile, sender=django.contrib.auth.models.User)


class Remote(GnuPGPublicKey):
    http = django.db.models.CharField(
        primary_key=True, max_length=255, default="tcp:host=YOUR_REMOTE_HOST:port=8066",
        help_text=help_html(f"HTTP Remote Endpoint: {inspect.getdoc(net.ClientEndpoint)}")
    )

    wake_command = django.db.models.CharField(
        max_length=255, default="", blank=True,
        help_text=help_html("Command to wake this remote (for example ``/usr/bin/wakeonlan -i DEST_IP MAC``)")
    )

    class Admin(GnuPGPublicKey.Admin):
        search_fields = GnuPGPublicKey.Admin.search_fields + ("http",)
        readonly_fields = GnuPGPublicKey.Admin.readonly_fields + ("key_id", "key")

    def mbd_url(self):
        return net.ClientEndpoint(self.http, protocol="http").geturl()

    def mbd_api(self, command, timeout=None):
        return client.Client(self.mbd_url()).api(command, timeout=timeout)

    def __str__(self):
        return f"Remote '{self.mbd_url()}' with {super().__str__()}"

    def mbd_get_status(self, timeout=2, wake=False, wake_sleep=5, wake_attempts=2):
        attempt = 0
        while True:
            try:
                return self.mbd_api(api.Status(), timeout=timeout)
            except util.HTTPUnavailable as e:
                if not wake or attempt >= wake_attempts:
                    raise
                if not self.wake_command:
                    raise util.HTTPUnavailable(f"{e.rfc7807.detail} (no wake command configured for remote {self})")

                LOG.info("Trying to wake '%s' via '%s' (#%s/%s)...", self.mbd_url(), self.wake_command, attempt, wake_attempts)
                call.Call(shlex.split(self.wake_command)).check()
                attempt += 1
                time.sleep(wake_sleep)

    def mbd_prepare(self):
        # Be sure to wake if needed
        self.mbd_get_status(wake=True)

        # We prepare the GPG data from downloaded key data, so key_id _must_ be empty (see super(mbd_prepare))
        self.key_id = ""
        self.key = self.mbd_api(api.PubKey())

        if self.key:
            LOG.warning("Downloaded remote key integrated: Please check key manually before activation!")
        else:
            raise util.HTTPBadRequest(f"Empty remote key from '{self.mbd_url()}' -- maybe the remote is not prepared yet?")
        super().mbd_prepare()

    def mbd_check(self):
        """Check whether the remote mini-buildd is up, running and serving for us"""
        super().mbd_check()

        # Get status
        status = self.mbd_get_status(wake=True)

        # GnuPG handshake
        signed_message = self.mbd_api(api.Handshake(signed_message=daemon.get().handshake_message()))
        self.mbd_verify(signed_message)

        # Check if our instance is activated on remote
        if util.http_endpoint().geturl() not in status["remotes"]:
            raise util.HTTPBadRequest("Our instance is not activated on remote")

        # Check if our instance is started
        if status["load"] < 0.0:
            raise util.HTTPBadRequest("Remote is stopped")


class Builders(list):
    """
    All active builders (ourselves as Meta-Remote plus all actual Remotes) with status

    Is used to choose builder to upload to, and as data for the Builder's View.
    """

    Builder = collections.namedtuple("Builder", "remote status exception ourselves")

    def __init__(self, wake=False, check=False):
        super().__init__()

        # Always add ourselves
        ourselves = Remote(http=util.http_endpoint().geturl(), status=Remote.STATUS_ACTIVE, last_checked=django.utils.timezone.now())
        self.append(Builders.Builder(ourselves, ourselves.mbd_get_status(), None, True))

        for r in Remote.mbd_get_active_or_auto_reactivate():
            status, exception = None, None
            try:
                status = r.mbd_get_status(wake=wake)
                if check:
                    Remote.Admin.mbd_check(r, force=True)
            except BaseException as e:
                LOG.warning("%s: %s", r, e)
                exception = util.e2http(e)
            finally:
                self.append(Builders.Builder(r, status, exception, False))

    def get(self, codename, arch):
        """Get a dict {<load>: <status>} of active builders for <codename>/<arch>"""
        return {builder.status["load"]: builder.status for builder in self if builder.remote.mbd_is_active() and builder.status and builder.status["load"] >= 0.0 and f"{codename}:{arch}" in builder.status["chroots"]}


class Archive(Model):
    URLOPEN_TIMEOUT = 15  # seconds

    url = django.db.models.URLField(
        primary_key=True, max_length=512,
        default="http://ftp.debian.org/debian/",
        validators=[RegexValidator(r"^.*[^/]+/$", message="Must have exactly one trailing slash (like 'http://example.org/path/')")],
        help_text=help_html(
            "The URL of an APT archive (there must be a 'dists/' infrastructure below)\n"
            "\n"
            "Use the 'directory' notation with exactly one trailing slash (like 'http://example.org/path/')."
        ))
    ping = django.db.models.FloatField(default=-1.0, editable=False)

    class Meta(Model.Meta):
        ordering = ["url"]

    class Admin(Model.Admin):
        search_fields = ("url",)

    def __str__(self):
        return f"Archive '{self.url}' (ping {round(self.ping, 2)} ms)"

    class ReleaseFile():
        @classmethod
        def _download(cls, file_, url):
            with urllib.request.urlopen(url, timeout=Archive.URLOPEN_TIMEOUT) as response:
                file_.write(response.read())
                file_.flush()
                file_.seek(0)

        def __init__(self, url_base):
            self.url_base = url_base

            # pylint: disable=consider-using-with  # See close(), use this class with contextlib.closing()
            self.release_file = tempfile.NamedTemporaryFile()
            self.signature_file = tempfile.NamedTemporaryFile()
            self.release_url, self.signature_url = None, None

            try:
                self.release_url = os.path.join(self.url_base, "InRelease")
                self._download(self.release_file, self.release_url)
            except urllib.error.HTTPError:
                try:
                    self.release_url = os.path.join(self.url_base, "Release")
                    self._download(self.release_file, self.release_url)
                    self.signature_url = self.release_url + ".gpg"
                    self._download(self.signature_file, self.signature_url)
                except urllib.error.HTTPError as e:
                    raise util.HTTPNotFound(f"{self.url_base}: Neither 'InRelease' nor 'Release[.gpg]' found") from e

            self.release = debian.deb822.Release(self.release_file)

        def close(self):
            for t in [self.release_file, self.signature_file]:
                if t is not None:
                    t.close()

        def match(self, source):
            LOG.debug("match(): %s matches %s?", self.release_url, source)

            # Check release file fields
            for key, value in source.imbd_release_file_values():
                if value != self.release.get(key):
                    raise util.HTTPNotFound(f"'{self.release_url}' does not match (has '{key}: {self.release.get(key)}', we need '{value}')")

            # Pre-Check 'Valid-Until'
            #
            # Some Release files contain an expire date via the
            # 'Valid-Until' tag. If such an expired archive is used,
            # builds will fail. Furthermore, it could be only the
            # selected archive not updating, while the source may be
            # perfectly fine from another archive.
            #
            # This pre-check avoids such a situation, or at least it
            # can be fixed by re-checking the source.
            try:
                valid_until = util.Datetime.parse(self.release["Valid-Until"])
                if valid_until < util.Datetime.now():
                    LOG.warning("%s expired, maybe the archive has problems? (Valid-Until='%s').", self.release_url, valid_until)
                    if source.check_valid_until:
                        raise util.HTTPUnavailable(f"{self.release_url} expired (Valid-Until='{valid_until}')")
            except KeyError:
                pass  # We can assume Release file has no "Valid-Until", and be quiet
            except BaseException as e:
                util.log_exception(LOG, f"Ignoring error checking 'Valid-Until' on {self.release_url}", e)

        def verify(self, _gnupg):
            return _gnupg.gpgme_verify_release(self.release_file, self.signature_file if self.signature_url is not None else None, accept_expired=True, name=self.url_base)

    def mbd_ping(self):
        """Ping and update the ping value"""
        try:
            t0 = django.utils.timezone.now()
            # Append dists to URL for ping check: Archive may be
            # just fine, but not allow to access to base URL
            # (like ourselves ;). Any archive _must_ have dists/ anyway.
            try:
                with urllib.request.urlopen(f"{self.url}/dists", timeout=self.URLOPEN_TIMEOUT) as _:
                    pass
            except urllib.error.HTTPError as e:
                # Allow HTTP 4xx client errors through; these might be valid use cases like:
                # 404 Usage Information: apt-cacher-ng
                if not 400 <= e.code <= 499:
                    raise

            delta = django.utils.timezone.now() - t0
            self.ping = delta.total_seconds() * (10 ** 3)
            self.save()
        except Exception as e:
            self.ping = -1.0
            self.save()
            raise util.HTTPUnavailable(f"'{self}' does not ping: {e}") from e

    def mbd_get_reverse_dependencies(self):
        """Return all sources (and their deps) that use us"""
        result = list(self.source_set.all())
        for s in self.source_set.all():
            result += s.mbd_get_reverse_dependencies()
        return result


class Architecture(Model):
    name = django.db.models.CharField(primary_key=True, max_length=50)

    def __str__(self):
        return f"Architecture '{self.name}'"


@functools.total_ordering
class Component(Model):
    """A Debian component (like 'main', 'contrib', 'non-free')"""

    #: Convenience component sorting to get some well known components sorted as we are used to
    _SORT_KEYS = {
        # Debian
        "main": "0",
        "contrib": "1",
        "non-free": "2",
        "non-free-firmware": "3",
        # Ubuntu (also uses 'main')
        "universe": "4",
        "restricted": "5",
        "multiverse": "6",
    }

    name = django.db.models.CharField(primary_key=True, max_length=50)

    def __str__(self):
        return f"Component '{self.name}'"

    __hash__ = Model.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.name == other.name
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return self._SORT_KEYS.get(self.name, 1000) < self._SORT_KEYS.get(other.name, 1000)
        return NotImplemented


@functools.total_ordering
class Source(StatusModel):
    origin = django.db.models.CharField(
        max_length=60,
        default="Debian",
        help_text=help_html("The exact string of the 'Origin' field of the resp. Release file")
    )
    codename = django.db.models.CharField(
        max_length=60,
        default="sid",
        help_text=help_html(
            "The name of the directory below ``dists/`` in archives this source refers to (this is also the 3rd part of an APT line)\n"
            "\n"
            "Also the exact string of the ``Codename`` field of the resp, Release file (unless overwritten in ``extra_identifiers``)."
        ))

    extra_identifiers = django.db.models.JSONField(
        blank=True,
        default=dict,
        validators=[ForbiddenKeysValidator(["Origin"])],
        help_text=help_html(
            "Extra release file identifiers (JSON dict)\n"
            "\n"
            "Some sources need additional fields (same as in a Release file) to further specify this source. These are also later used to pin the source via APT.\n"
            "\n"
            "Most likely candidates are ``Codename``, ``Suite``, ``Archive``, ``Version`` or ``Label``.\n"
            "\n"
            "For some sources, ``Codename`` (as we use it above) does not match resp. value as given in the Release file.\n"
            "When ``Codename`` is overridden in this manner, be sure to also add one further flag to identify the source -- else apt pinning later would likely not be unambiguous.\n"
            "Real world examples that need this extra handling are ``Debian Security`` (``buster`` or older), ``Ubuntu Security and Backports``:\n"
            "\n"
            "{'Codename': 'stretch', 'Label': 'Debian-Security'}\n"
            "{'Codename': 'bionic', 'Suite': 'bionic-backports'}"
        ))

    apt_keys = django.db.models.ManyToManyField(
        AptKey, blank=True,
        help_text=help_html(
            "At least one APT key this source's 'Release' file is signed with\n"
            "\n"
            "Hint: If you leave this empty and run 'check' on it, you will get a message listing all signer keys."
        ))

    codeversion_override = django.db.models.CharField(
        max_length=50, blank=True,
        default="",
        help_text=help_html(
            "Permanently override with a static ``codeversion`` value\n"
            "\n"
            "Leave blank to make ``codeversion`` be guessed every time the source is checked.\n"
            "\n"
            "The ``codeversion`` (like '12' for ``Debian bookworm``) is relevant for base sources only, to be used for version restrictions."
        ))
    codeversion_guessed = django.db.models.CharField(max_length=50, editable=False, blank=True, default="")
    codeversion = django.db.models.CharField(max_length=50, editable=False, blank=True, default="")

    check_valid_until = django.db.models.BooleanField(
        default=True,
        help_text=help_html(
            "Fail if outdated (via ``Valid-Until`` field)\n"
            "\n"
            "Some sources have a ``Valid-Until`` field that is no longer updated. Disable this if you really still want to use it anyway.\n"
            "\n"
            "When disabled, this will 1st ignore mini-buildd's own 'Valid-Until check' and 2nd, create APT lines for this source with the ``[check-valid-until=no]`` option.\n"
            "\n"
            "Note: For ``jessie`` or older (where this APT line option does not work), ``apt-no-check-valid-until`` sbuild setup block is still needed."
        ))

    remove_from_component = django.db.models.CharField(
        max_length=100, blank=True,
        default="",
        help_text=help_html(
            "Prefix to be removed from component\n"
            "\n"
            "Some (actually, we only know of ``Debian Security``) sources have weird ``Components`` that need to be fixed to work with mini-buildd.\n"
            "\n"
            "For example, ``Debian Security`` needs ``updates/``."
        ))

    # Extra
    description = django.db.models.CharField(max_length=100, editable=False, blank=True, default="")
    archives = django.db.models.ManyToManyField(Archive, editable=False, blank=True)
    components = django.db.models.ManyToManyField(Component, editable=False, blank=True)
    architectures = django.db.models.ManyToManyField(Architecture, editable=False, blank=True)

    class Meta(StatusModel.Meta):
        unique_together = ["origin", "codename"]

    class Admin(StatusModel.Admin):
        def _meta_field_apt_line(self, obj):
            try:
                return obj.mbd_get_apt_line().get()
            except BaseException as e:
                return str(util.e2http(e))

        list_display = StatusModel.Admin.list_display + ("codeversion", "_meta_field_apt_line")
        search_fields = ("origin", "codename")

        readonly_fields = ("codeversion_guessed", "description", "archives", "components", "architectures")
        filter_horizontal = ("apt_keys",)

        @classmethod
        def mbd_filter_active_base_sources(cls):
            """Filter active base sources; needed in chroot and distribution wizards"""
            return Source.objects.filter(status__gte=Source.STATUS_ACTIVE,
                                         origin__in=dist.SETUP["origin"].keys(),
                                         codename__regex=r"^[a-z]+$").exclude(codename="experimental")

    def __str__(self):
        return f"Source '{self.origin} {self.codename}' ({self.codeversion})"

    def imbd_release_file_values(self):
        """Yield key/values a matching release file must match"""
        # Origin and Codename (may be overwritten) from fields
        yield "Origin", self.origin
        if "Codename" not in self.extra_identifiers:   # pylint: disable=unsupported-membership-test
            yield "Codename", self.codename

        yield from self.extra_identifiers.items()

    def mbd_get_archive(self):
        """Get fastest archive"""
        oa_list = self.archives.all().filter(ping__gte=0.0).order_by("ping")
        if oa_list:
            return oa_list[0]
        raise util.HTTPBadRequest(f"{self}: No archive found (network down? ``Archive`` missing? ``Source`` with incorrect keys?)")

    def mbd_get_apt_line(self, limit_components=None):
        return files.AptLine(self.mbd_get_archive().url,
                             "",
                             self.codename,
                             [c.name for c in sorted(self.components.all()) if limit_components is None or c in limit_components],
                             options=f"check-valid-until={'yes' if self.check_valid_until else 'no'}",
                             comment=f"{self}: {self.mbd_get_apt_pin()}")

    def mbd_get_apt_pin(self):
        """Apt 'pin line' (for use in a apt 'preference' file)"""
        # See man apt_preferences for the field/pin mapping
        supported_fields = {"Origin": "o", "Codename": "n", "Suite": "a", "Archive": "a", "Version": "v", "Label": "l"}
        pins = []
        for key, value in self.imbd_release_file_values():
            k = supported_fields.get(key)
            if k:
                pins.append(f"{k}={value}")
        return "release " + ", ".join(pins)

    def mbd_prepare(self):
        self.full_clean()

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_remove(self):
        self.archives.set([])
        self.components.set([])
        self.architectures.set([])
        self.description = ""

    def mbd_check(self):
        """Rescan all archives, and check that there is at least one working"""
        self.archives.set([])

        # Gather failed information to debug if no archive was found
        failed = {}

        with contextlib.closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-sourcecheck-"})) as gpg:
            for k in self.apt_keys.all():
                gpg.add_pub_key(k.key)

            for archive in Archive.objects.all():
                try:
                    archive.mbd_ping()

                    with closing(Archive.ReleaseFile(os.path.join(archive.url, "dists", self.codename))) as release_file:
                        release_file.match(self)
                        release_file.verify(gpg)

                        # Implicitly save ping value for this archive
                        self.archives.add(archive)
                        self.description = release_file.release.get("Description", "")

                        # Set codeversion
                        self.codeversion_guessed = dist.guess_codeversion(release_file.release.get("Origin"), release_file.release.get("Codename"), release_file.release.get("Version"))
                        self.codeversion = self.codeversion_override if self.codeversion_override else self.codeversion_guessed

                        # Set architectures and components (may be auto-added)
                        if release_file.release.get("Architectures"):
                            for a in release_file.release["Architectures"].split(" "):
                                new_arch, _created = Architecture.objects.get_or_create(name=a)
                                self.architectures.add(new_arch)
                        if release_file.release.get("Components"):
                            for c in release_file.release["Components"].split(" "):
                                new_component, _created = Component.objects.get_or_create(name=c.replace(self.remove_from_component, ""))
                                self.components.add(new_component)
                except util.HTTPError as e:
                    util.log_exception(LOG, f"{self}: Skipping {archive}", e, level=logging.DEBUG)
                    failed[archive] = e

        # Check that at least one archive can be found
        try:
            fastest_archive = self.mbd_get_archive()
            LOG.debug("%s: Fastest archive: %s", self, fastest_archive)
        except util.HTTPError as e:
            errs = '\n'.join([f'{archive}: {error}' for archive, error in failed.items()])
            raise util.HTTPNotFound(f"{e.detail}:\n{errs}") from e

    def mbd_get_dependencies(self):
        return self.apt_keys.all()

    def mbd_get_reverse_dependencies(self):
        """Return all chroots and repositories that use us"""
        result = list(self.chroot_set.all())
        for d in self.distribution_set.all():
            result += d.mbd_get_reverse_dependencies()
        return result

    def clean(self):
        """Forbid to change identity on existing source (usually a bad idea; repos/chroots that refer to us may break)"""
        super().clean()
        self.mbd_validate_field_unchanged("origin")
        self.mbd_validate_field_unchanged("codename")

    __hash__ = StatusModel.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return dist.Codename(self.codename, origin=self.origin) == dist.Codename(other.codename, origin=other.origin)
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return dist.Codename(self.codename, origin=self.origin) < dist.Codename(other.codename, origin=other.origin)
        return NotImplemented


class PrioritySource(Model):
    source = django.db.models.ForeignKey(Source, on_delete=django.db.models.CASCADE)
    priority = django.db.models.IntegerField(
        default=1,
        help_text=help_html("A APT pin priority value (see 'man apt_preferences'; examples: 1=not automatic, 1001=downgrade)")
    )

    class Meta(Model.Meta):
        unique_together = ["source", "priority"]

    class Admin(Model.Admin):
        search_fields = ("source__origin", "source__codename")

    def __str__(self):
        return f"{self.source} with priority {self.priority}"

    def mbd_get_apt_preferences(self):
        return f"Package: *\nPin: {self.source.mbd_get_apt_pin()}\nPin-Priority: {self.priority}\n"


class Suite(Model):
    name = django.db.models.CharField(
        max_length=100,
        validators=[RegexValidator(regex=r"^[a-z]+$")],
        help_text=help_html("A suite name (usually s.th. like 'experimental', 'unstable', 'testing' or 'stable')")
    )

    def __str__(self):
        return f"Suite '{self.name}'"


@functools.total_ordering
class SuiteOption(Model):
    layout = django.db.models.ForeignKey("Layout", on_delete=django.db.models.CASCADE)
    suite = django.db.models.ForeignKey("Suite", on_delete=django.db.models.CASCADE)

    uploadable = django.db.models.BooleanField(
        default=True,
        help_text=help_html("Whether this suite should accept user uploads")
    )

    experimental = django.db.models.BooleanField(
        default=False,
        help_text=help_html(
            "Whether this suite is flagged experimental\n"
            "\n"
            "Experimental suites must be uploadable and must not migrate.\n"
            "Also, the packager treats failing extra QA checks (like lintian) as non-lethal, and will install anyway."
        ))

    migrates_to = django.db.models.ForeignKey(
        "self", blank=True, null=True, on_delete=django.db.models.CASCADE,
        help_text=help_html("Give another suite where packages may migrate to (you may need to save this 'blank' first before you see choices here)")
    )

    rollback = django.db.models.IntegerField(
        default=0,
        help_text=help_html("Keep this number of old versions in this suite")
    )

    build_keyring_package = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Build keyring package for this suite (i.e., when the resp. Repository action is called)")
    )

    not_automatic = django.db.models.BooleanField(
        default=True,
        help_text=help_html("Include 'NotAutomatic' in the Release file")
    )

    but_automatic_upgrades = django.db.models.BooleanField(
        default=True,
        help_text=help_html("Include 'ButAutomaticUpgrades' in the Release file")
    )

    class Meta(Model.Meta):
        unique_together = ["suite", "layout"]

    def __str__(self):
        return f"{self.suite} for {self.layout}"

    def clean(self):
        super().clean()
        if self.build_keyring_package and not self.uploadable:
            raise django.core.exceptions.ValidationError("You can only build keyring packages on uploadable suites")
        if self.experimental and self.migrates_to:
            raise django.core.exceptions.ValidationError("Experimental suites may not migrate")
        if self.experimental and not self.uploadable:
            raise django.core.exceptions.ValidationError("Experimental suites must be uploadable")
        if self.migrates_to and self.migrates_to.layout != self.layout:
            raise django.core.exceptions.ValidationError("Migrating suites must be in the same layout (you need to save once to make new suites visible)")
        if self.migrates_to and self.migrates_to.uploadable:
            raise django.core.exceptions.ValidationError("You may not migrate to an uploadable suite")

    __hash__ = Model.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.suite.name == other.suite.name
        return NotImplemented

    def mbd_migrates_to(self, other):
        """Check if we, directly or transitively, migrate to other suite"""
        migrates_to = self.migrates_to
        while migrates_to:
            if migrates_to == other:
                return True
            migrates_to = migrates_to.migrates_to  # pylint: disable=no-member
        return False

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            if self.mbd_migrates_to(other):
                return False
            if self.experimental and not other.experimental:
                return False
            if self.experimental and other.experimental and self.suite.name == "experimental":
                return False
            return True
        return NotImplemented


class SuiteOptionInline(django.contrib.admin.TabularInline):
    model = SuiteOption
    extra = 1


class Layout(Model):
    name = django.db.models.CharField(primary_key=True, max_length=100)

    suites = django.db.models.ManyToManyField(Suite, through=SuiteOption)

    # Version magic
    default_version = django.db.models.CharField(
        max_length=100,
        default="~%IDENTITY%%CODEVERSION%+1",
        help_text=help_html(
            "Version string to append to the original version for automated ports\n"
            "\n"
            "You may use these placeholders:\n"
            "* %IDENTITY%: Repository identity (see 'Repository')\n"
            "* %CODEVERSION%: Numerical base distribution version (see 'Source')"
        ))

    mandatory_version_regex = django.db.models.CharField(
        max_length=100,
        default=r"~%IDENTITY%%CODEVERSION%\+[1-9]",
        help_text=help_html("Mandatory version regex (you may use the same placeholders as for 'default version')")
    )

    # Version magic (experimental)
    experimental_default_version = django.db.models.CharField(
        max_length=30,
        default="~%IDENTITY%%CODEVERSION%+0",
        help_text=help_html("Like 'default version', but for suites flagged 'experimental'")
    )

    experimental_mandatory_version_regex = django.db.models.CharField(
        max_length=100,
        default=r"~%IDENTITY%%CODEVERSION%\+0",
        help_text=help_html("Like 'mandatory version', but for suites flagged 'experimental'")
    )

    meta_distributions = django.db.models.JSONField(
        blank=True,
        default=dict,
        help_text=help_html(
            "Support METAs alone as distribution identifier\n"
            "\n"
            "Meta distribution identifiers should be unique across all repositories; usually, a layout with meta distributions should only be used by at most one repository.\n"
            "\n"
            "Example:\n"
            "\n"
            "{'unstable': 'sid-unstable', 'experimental'='sid-experimental'}\n"
            "\n"
            "(See standard layout 'Debian Developer'): allows upload/testing of packages (to unstable,experimental,..) aimed for Debian."
        ))

    class Admin(Model.Admin):
        inlines = (SuiteOptionInline,)

    def __str__(self):
        return f"Layout '{self.name}'"

    def mbd_get_reverse_dependencies(self):
        """When the layout changes, all repos that use that layout also change"""
        return list(self.repository_set.all())


class ArchitectureOption(Model):
    architecture = django.db.models.ForeignKey("Architecture", on_delete=django.db.models.CASCADE)
    distribution = django.db.models.ForeignKey("Distribution", on_delete=django.db.models.CASCADE)

    optional = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Don't care if the architecture can't be build")
    )
    build_architecture_all = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Use to build arch-all packages")
    )

    class Meta(Model.Meta):
        unique_together = ["architecture", "distribution"]

    def __str__(self):
        return f"{self.architecture} for {self.distribution}"

    def clean(self):
        super().clean()
        if self.build_architecture_all and self.optional:
            raise django.core.exceptions.ValidationError("Optional architectures must not be architecture all")


class ArchitectureOptionInline(django.contrib.admin.TabularInline):
    model = ArchitectureOption
    extra = 1


@django.utils.deconstruct.deconstructible
class ShebangValidator:
    def __call__(self, value):
        try:
            files.File.shebang_from_data(value)
        except BaseException as e:
            raise django.core.exceptions.ValidationError("Missing shebang") from e


@django.utils.deconstruct.deconstructible
class SbuildConfigBlocksValidator:
    def __call__(self, value):
        try:
            sbuild.CONFIG_BLOCKS.validate(value)
        except util.HTTPError as e:
            raise django.core.exceptions.ValidationError(str(e))


@django.utils.deconstruct.deconstructible
class SbuildSetupBlocksValidator:
    def __call__(self, value):
        try:
            sbuild.SETUP_BLOCKS.validate(value)
        except util.HTTPError as e:
            raise django.core.exceptions.ValidationError(str(e))


@functools.total_ordering
class Distribution(Model):
    base_source = django.db.models.ForeignKey(Source, on_delete=django.db.models.CASCADE)
    extra_sources = django.db.models.ManyToManyField(PrioritySource, blank=True)
    components = django.db.models.ManyToManyField(Component)

    architectures = django.db.models.ManyToManyField(Architecture, through=ArchitectureOption)

    RESOLVER_APT = 0
    RESOLVER_APTITUDE = 1
    RESOLVER_INTERNAL = 2
    RESOLVER_CHOICES = (
        (RESOLVER_APT, "apt"),
        (RESOLVER_APTITUDE, "aptitude"),
        (RESOLVER_INTERNAL, "internal"))
    build_dep_resolver = django.db.models.IntegerField(choices=RESOLVER_CHOICES, default=RESOLVER_APTITUDE)

    sbuild_config_blocks = django.db.models.CharField(
        max_length=255, blank=True,
        default=dist.SETUP["defaults"]["distribution"]["sbuild_config_blocks"],
        validators=[SbuildConfigBlocksValidator()],
        help_text=help_html(sbuild.CONFIG_BLOCKS.usage()))

    sbuild_setup_blocks = django.db.models.CharField(
        max_length=255, blank=True,
        default=dist.SETUP["defaults"]["distribution"]["sbuild_setup_blocks"],
        validators=[SbuildSetupBlocksValidator()],
        help_text=help_html(sbuild.SETUP_BLOCKS.usage()))

    # sbuild checks (lintian, piuparts, autopkgtest)
    lintian_mode = django.db.models.IntegerField(
        choices=dist.SbuildCheck.CHOICES,
        default=dist.SbuildCheck.Mode.ERRFAIL.value,
        help_text=help_html(
            f"{dist.SbuildCheck.desc()}\n"
            "\n"
            f"{dist.SbuildCheck.usage()}"
        ))

    lintian_extra_options = django.db.models.CharField(
        max_length=512,
        default=dist.SETUP["defaults"]["distribution"]["lintian_extra_options"],
        help_text=help_html(
            "Lintian options to always use\n"
            "\n"
            "Useful examples (with hints in what lintian versions they are applicable):\n"
            "\n"
            " * '--info' (all): Show versatile explanations in buildlogs.\n"
            " * '--suppress-tags <tag>' (>= 2.3.0, >= Debian 6 squeeze): Suppress false-positive/too noisy tags.\n"
            "   * 'bad-distribution-in-changes-file' (all): Lintian does not know mini-buildd-style distributions.\n"
            "   * 'bogus-mail-host[-in-debian-changelog]' (>= 2.104.0, >= Debian 11 bullseye, >= Ubuntu 21.10 impish): Not really mandatory for mini-buildd -- but lintian deems them errors, making lintian always fail.\n"
        ))

    lintian_warnfail_options = django.db.models.CharField(
        max_length=512,
        default=dist.SETUP["defaults"]["distribution"]["lintian_warnfail_options"],
        help_text=help_html(
            "Lintian options to use for 'WARNFAIL' mode\n"
            "\n"
            "Humbly up-to-date variants of lintian all work with the default ('--fail-on error,warning').\n"
            "\n"
            "You may set exception here for some older releases if need be.\n"
            "\n"
            "FYI, lintian 'warnfail' option history:\n"
            " * Debian (stretch)  2.5.50: Deprecates '--fail-on-warnings'.\n"
            " * Debian (bullseye) 2.16.0: Drops support for '--fail-on-warnings' (NO WAY TO DO 'WARNFAIL').\n"
            " * Debian (bullseye) 2.77.0: Introduces '--fail-on'.\n"
            " * Ubuntu (focal)    2.62.0: Release that does not have either (NO WAY TO DO 'WARNFAIL').\n"
            " * Ubuntu (groovy)   2.89.0: 1st release >= 2.77.\n"
            "Conclusion:\n"
            " * Debian >=bullseye needs '--fail-on error,warning', below needs '--fail-on-warnings'.\n"
            " * Ubuntu >=groovy needs '--fail-on error,warning', below needs '--fail-on-warnings'.\n"
            " * Ubuntu focal needs '' (just can't do 'WARNFAIL').\n"
        ))

    piuparts_mode = django.db.models.IntegerField(
        choices=dist.SbuildCheck.CHOICES,
        default=dist.SbuildCheck.Mode.DISABLED.value,
        help_text=help_html(
            f"EXPERIMENTAL: {dist.SbuildCheck.desc()}\n"
            "\n"
            f"{dist.SbuildCheck.usage()}\n"
            "\n"
            "EXPERIMENTAL: All modes but 'Run check and ignore errors' are currently ignored, and even that might not be called correctly.\n"
        ))
    piuparts_extra_options = django.db.models.CharField(max_length=200, default="--info")
    piuparts_root_arg = django.db.models.CharField(max_length=200, default="sudo")

    autopkgtest_mode = django.db.models.IntegerField(
        choices=dist.SbuildCheck.CHOICES,
        default=dist.SbuildCheck.Mode.DISABLED.value,
        help_text=help_html(
            f"{dist.SbuildCheck.desc()}\n"
            "\n"
            f"{dist.SbuildCheck.usage()}"
        ))

    internal_apt_priority = django.db.models.IntegerField(
        default=1,
        help_text=help_html(
            "Priority for internal APT sources in builds\n"
            "\n"
            "The default is 1, which means you will only build against newer packages in our own repositories in case it's really needed by the package's build dependencies."
            "This is the recommended behaviour, producing sparse dependencies.\n"
            "\n"
            "However, some packages with incorrect build dependencies might break anyway, while they would work fine when just build against the newest version available.\n"
            "\n"
            "You may still solve this on a per-package basis, using the resp. upload option via changelog. "
            "However, in case you don't care about sparse dependencies in this distribution in general, you can pimp the internal priority up here.\n"
            "\n"
            "Examples: 1=sparse dependencies, 500=always build against newer internal packages"
        ))

    deb_build_options = django.db.models.CharField(
        max_length=255, blank=True,
        help_text=help_html(
            "Set extra build options\n"
            "\n"
            "Values to add to environment variable ``DEB_BUILD_OPTIONS`` used when building packages. See\n"
            "\n"
            "* ``https://www.debian.org/doc/debian-policy/ch-source.html#debian-rules-and-deb-build-options`` and\n"
            "* ``https://lists.debian.org/debian-devel/2015/12/msg00262.html'>Debian dbgsym announcement``\n"
            "\n"
            "for valid options. Value may be overridden by the resp. user upload option.\n"
            "\n"
            "Option ``noddebs`` is useful for Ubuntu distributions that for\n"
            "some reason create their automated debug packages with file appendix\n"
            "'ddeb', as (see ``https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=730572``) reprepo fails on them.\n"
            "\n"
            "Example: Never build automatic debug packages (Ubuntu bionic, cosmic):\n"
            "\n"
            "noddebs"
        ))

    deb_build_profiles = django.db.models.CharField(
        max_length=255, blank=True,
        help_text=help_html(
            "Set extra build profiles\n"
            "\n"
            "Contents of DEB_BUILD_PROFILES environment for building (see ``https://wiki.debian.org/BuildProfileSpec``).\n"
            "Value may be overridden by the resp. user upload option."
        ))

    add_depends = django.db.models.CharField(
        max_length=255, blank=True,
        help_text=help_html(
            "Add extra (package) dependencies"
        ))

    chroot_setup_script = UnixTextField(
        blank=True,
        default="",
        validators=[ShebangValidator()],
        help_text=help_html(
            "Custom script that will be run via sbuild's ``--chroot-setup-command``\n"
            "\n"
            "Be sure to provide a proper shebang like ``#!/bin/sh -e``.\n"
            "\n"
            "Note that some predefined script blocks to choose from are available (see 'sbuild_setup_blocks'), which might suite your needs already.\n"
            "\n"
            "Example:\n"
            "---\n"
            "#!/bin/sh -e\n"
            "\n"
            "echo \"I: Foo bar...\"\n"
            "---\n"
        ))
    sbuildrc_snippet = UnixTextField(
        blank=True,
        default="",
        help_text=help_html(
            "Custom perl snippet to be added to ``.sbuildrc`` for each build\n"
            "\n"
            "Note that some predefined config blocks to choose from are available (see 'sbuild_config_blocks'), which might suite your needs already.\n"
            "\n"
            "You may use these placeholders:\n"
            "\n"
            "* ``%LIBDIR%``: Per-chroot persistent dir; may be used for data that should persist between builds (caches, ...).\n"
            "\n"
            "Example:\n"
            "---\n"
            "$build_environment = { 'MY_FOO_BAR' => '%LIBDIR%/my-foo-bar' };\n"
            "---\n"
        ))

    class Meta(StatusModel.Meta):
        ordering = ["base_source"]

    class Admin(Model.Admin):
        inlines = (ArchitectureOptionInline,)
        filter_horizontal = ("extra_sources", "components",)

    def mbd_is_active(self):
        for s in [self.base_source] + [s.source for s in self.extra_sources.all()]:
            if not s.mbd_is_active():
                return False
        return True

    def __str__(self):
        """
        Note: Appends 'id' only to make multiple Distributions for same base source distinguishable for the user
        """
        return f"Distribution for {self.base_source} [pk={self.pk}]"

    def mbd_setup(self):
        return f"{self.base_source.codename}:{'+'.join((a.name for a in self.architectures.all()))}"

    def mbd_get_components(self):
        return [c.name for c in sorted(self.components.all())]

    def mbd_get_archall_architectures(self):
        return [a.architecture.name for a in self.architectureoption_set.all() if a.build_architecture_all]

    def mbd_get_mandatory_architectures(self):
        return [a.architecture.name for a in self.architectureoption_set.all() if not a.optional]

    def mbd_get_reverse_dependencies(self):
        """When the distribution changes, all repos that use that distribution also change"""
        return list(self.repository_set.all())

    __hash__ = Model.__hash__

    def __eq__(self, other):
        if isinstance(other, self.__class__):
            return self.base_source == other.base_source
        return NotImplemented

    def __lt__(self, other):
        if isinstance(other, self.__class__):
            return self.base_source < other.base_source
        return NotImplemented


class EmailAddress(Model):
    address = django.db.models.EmailField(primary_key=True, max_length=255)
    name = django.db.models.CharField(blank=True, max_length=255)

    class Meta(Model.Meta):
        verbose_name_plural = "Email addresses"

    def __str__(self):
        return f"EmailAddress '{self.name} <{self.address}>'"


class Repository(StatusModel):
    identity = django.db.models.CharField(
        primary_key=True,
        max_length=50,
        default="test",
        validators=[RegexValidator(regex=r"^[a-z0-9]+$")],
        help_text=help_html(
            "The id of the reprepro repository, placed in 'repositories/<ID>'\n"
            "\n"
            "It can also be used in 'version enforcement string' (true for the default layout) -- in this context, it "
            "plays the same role as the well-known 'bpo' version string from Debian backports."
        ))

    layout = django.db.models.ForeignKey(Layout, on_delete=django.db.models.CASCADE)
    distributions = django.db.models.ManyToManyField(Distribution)

    allow_unauthenticated_uploads = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Allow unauthenticated user uploads")
    )

    extra_uploader_keyrings = UnixTextField(
        blank=True,
        help_text=help_html(
            "Extra keyrings, line by line, to be allowed as uploaders (in addition to configured django users)\n"
            "\n"
            "Example:\n"
            "---\n"
            "# Allow Debian maintainers (must install the 'debian-keyring' package)\n"
            "/usr/share/keyrings/debian-keyring.gpg\n"
            "# Allow from some local keyring file\n"
            "/etc/my-schlingels.gpg\n"
            "---"
        ))

    notify = django.db.models.ManyToManyField(
        EmailAddress, blank=True,
        help_text=help_html("Addresses that get all notification emails unconditionally")
    )
    notify_changed_by = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Notify the address in the 'Changed-By' field of the uploaded changes file")
    )
    notify_maintainer = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Notify the address in the 'Maintainer' field of the uploaded changes file")
    )

    reprepro_morguedir = django.db.models.BooleanField(
        default=False,
        help_text=help_html("Move files deleted from repo pool to 'morguedir' (see reprepro)")
    )

    external_home_url = django.db.models.URLField(blank=True)

    LETHAL_DEPENDENCIES = False

    class Meta(StatusModel.Meta):
        verbose_name_plural = "Repositories"

    class Admin(StatusModel.Admin):
        filter_horizontal = ("distributions", "notify",)

    def __str__(self):
        return f"Repository '{self.identity}' with distributions for {', '.join([d.base_source.codename + ('' if d.mbd_is_active() else '*') for d in sorted(self.distributions.all())])}"

    def mbd_setup(self, with_dists=True):
        return f"{self.identity}:{self.layout.name}" + (f":{'+'.join((d.base_source.codename for d in self.distributions.all()))}" if with_dists else "")

    def mbd_json(self):
        return {
            "codenames": [d.base_source.codename for d in self.distributions.all()],
            "layout": {
                s.suite.name: {
                    "uploadable": s.uploadable,
                    "experimental": s.experimental,
                    "migrates_to": s.migrates_to.suite.name if s.migrates_to else None,
                } for s in sorted(self.layout.suiteoption_set.all())},
        }

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.mbd_path = config.ROUTES["repositories"].path.join(self.identity)
        self.mbd_reprepro = reprepro.Reprepro(basedir=self.mbd_path)

    def mbd_get_diststr(self, distribution, suite, rollback=None):
        diststr = f"{distribution.base_source.codename}-{self.identity}-{suite.suite.name}"

        if rollback is not None:
            if rollback not in list(range(suite.rollback)):
                raise util.HTTPBadRequest(f"{diststr}: Rollback number out of range: {rollback} ({list(range(suite.rollback))})")
            diststr += f"-rollback{rollback}"

        return diststr

    def mbd_get_diststrs(self, frollbacks=None, distributions_filter=None, suiteoption_filter=None):
        """List of all distribution strings (except rollbacks) in this repo, optionally matching a suite options filter (unstable, experimental,...)"""
        diststrs = []
        for d in self.distributions.all().filter(**({} if distributions_filter is None else distributions_filter)):
            for s in sorted(self.layout.suiteoption_set.filter(**({} if suiteoption_filter is None else suiteoption_filter))):
                diststrs.append(self.mbd_get_diststr(d, s))
                if frollbacks is not None:
                    for r in frollbacks(s):
                        diststrs.append(self.mbd_get_diststr(d, s, rollback=r))
        return diststrs

    def mbd_get_meta_distributions(self, distribution, suite_option):
        try:
            result = []
            for meta, d in self.layout.meta_distributions.items():
                diststr = f"{d.split('-')[0]}-{self.identity}-{d.split('-')[1]}"
                if diststr == self.mbd_get_diststr(distribution, suite_option):
                    result.append(meta)
            return result
        except BaseException as e:
            raise util.HTTPBadRequest(f"Please fix syntax error in Meta Distributions' in layout '{self.layout}'") from e

    def mbd_get_apt_pin(self, distribution, suite):
        return f"release n={self.mbd_get_diststr(distribution, suite)}, o={daemon.get_model().mbd_get_archive_origin()}"

    def mbd_get_apt_preferences(self, distribution, suite, prio=1):
        return f"Package: *\nPin: {self.mbd_get_apt_pin(distribution, suite)}\nPin-Priority: {prio}\n"

    @classmethod
    def mbd_get_apt_keys(cls, distribution):
        result = daemon.get().gnupg.pub_key
        for e in distribution.extra_sources.all():
            for k in e.source.apt_keys.all():
                result += k.key
        return result

    def mbd_get_internal_suite_dependencies(self, suite_option):
        result = []

        # Add ourselves
        result.append(suite_option)

        if suite_option.experimental:
            # Add all non-experimental suites
            for s in self.layout.suiteoption_set.all().filter(experimental=False):
                result.append(s)
        else:
            # Add all suites that we migrate to
            s = suite_option.migrates_to
            while s:
                result.append(s)
                s = s.migrates_to

        return result

    @classmethod
    def __mbd_subst_placeholders(cls, value, repository, distribution):
        return util.subst_placeholders(
            value,
            {"IDENTITY": repository.identity,
             "CODEVERSION": distribution.base_source.codeversion})

    def mbd_get_mandatory_version_regex(self, distribution, suite_option):
        return self.__mbd_subst_placeholders(
            self.layout.experimental_mandatory_version_regex if suite_option.experimental else self.layout.mandatory_version_regex,
            self, distribution)

    def mbd_get_default_version(self, distribution, suite_option):
        return self.__mbd_subst_placeholders(
            self.layout.experimental_default_version if suite_option.experimental else self.layout.default_version,
            self, distribution)

    def mbd_get_apt_line(self, distribution, suite_option, rollback=None, snapshot=None):
        diststr = self.mbd_get_diststr(distribution, suite_option, rollback=rollback)
        if snapshot:
            snapshots = self.mbd_reprepro.get_snapshots(diststr)
            if snapshot not in snapshots:
                raise util.HTTPBadRequest(f"No such snapshot for {diststr}: '{snapshot}' (available: {','.join(snapshots)})")
            diststr += os.path.join("/", "snapshots", snapshot)

        return files.AptLine(config.URIS["repositories"]["static"].url_join(),
                             self.identity,
                             diststr,
                             [c.name for c in sorted(distribution.components.all())],
                             comment=f"{daemon.get_model().mbd_get_archive_origin()} '{diststr}': {self.mbd_get_apt_pin(distribution, suite_option)}")

    def mbd_get_apt_build_sources_list(self, distribution, suite_option):
        sources_list = files.SourcesList()

        sources_list.append(distribution.base_source.mbd_get_apt_line(limit_components=distribution.components.all()))
        for e in distribution.extra_sources.all():
            sources_list.append(e.source.mbd_get_apt_line(limit_components=distribution.components.all()))
        for s in self.mbd_get_internal_suite_dependencies(suite_option):
            sources_list.append(self.mbd_get_apt_line(distribution, s))

        return sources_list

    def mbd_get_apt_build_preferences(self, distribution, suite_option, internal_apt_priority_override=None):
        result = ""

        # Get preferences for all extra (prioritized) sources
        for e in distribution.extra_sources.all():
            result += e.mbd_get_apt_preferences() + "\n"

        # Get preferences for all internal sources
        internal_prio = internal_apt_priority_override if internal_apt_priority_override else distribution.internal_apt_priority
        for s in self.mbd_get_internal_suite_dependencies(suite_option):
            result += self.mbd_get_apt_preferences(distribution, s, prio=internal_prio) + "\n"

        return result

    def _mbd_reprepro_conf_options(self):
        return files.File("options",
                          snippet=(f"gnupghome {config.ROUTES['home'].path.join('.gnupg')}\n"
                                   f"{'morguedir +b/morguedir' if self.reprepro_morguedir else ''}\n"))

    def _mbd_reprepro_conf_distributions(self):
        dist_template = (
            "Codename: {distribution}\n"
            "Suite: {distribution}\n"
            "Label: {distribution}\n"
            "AlsoAcceptFor: {meta_distributions}\n"
            "Origin: {origin}\n"
            "Components: {components}\n"
            "UDebComponents: {components}\n"
            "Architectures: source {architectures}\n"
            "Description: {desc}\n"
            "SignWith: default\n"
            "NotAutomatic: {na}\n"
            "ButAutomaticUpgrades: {bau}\n"
            "DebIndices: Packages Release . .gz .xz\n"
            "DscIndices: Sources Release . .gz .xz\n"
            "Contents: .gz\n\n"
        )

        result = ""
        for d in self.distributions.all():
            for s in sorted(self.layout.suiteoption_set.all()):
                result += dist_template.format(
                    distribution=self.mbd_get_diststr(d, s),
                    meta_distributions=" ".join(self.mbd_get_meta_distributions(d, s)),
                    origin=daemon.get_model().mbd_get_archive_origin(),
                    components=" ".join(d.mbd_get_components()),
                    architectures=" ".join([x.name for x in d.architectures.all()]),
                    desc=self.mbd_get_diststr(d, s),
                    na="yes" if s.not_automatic else "no",
                    bau="yes" if s.but_automatic_upgrades else "no")

                for r in range(s.rollback):
                    result += dist_template.format(
                        distribution=self.mbd_get_diststr(d, s, r),
                        meta_distributions="",
                        origin=daemon.get_model().mbd_get_archive_origin(),
                        components=" ".join(d.mbd_get_components()),
                        architectures=" ".join([x.name for x in d.architectures.all()]),
                        desc=self.mbd_get_diststr(d, s, r),
                        na="yes",
                        bau="no")

        return files.File("distributions", snippet=result)

    def mbd_reprepro_update_config(self, force_reindex=False):
        if files.Dir(os.path.join(self.mbd_path, "conf")).add(self._mbd_reprepro_conf_options()) \
                                                         .add(self._mbd_reprepro_conf_distributions()).update() or force_reindex:
            self.mbd_reprepro.reindex(self.mbd_get_diststrs())

    def _mbd_package_shift_rollbacks(self, distribution, suite_option, package_name):
        diststr = self.mbd_get_diststr(distribution, suite_option)
        ls = self.mbd_reprepro.ls(package_name)
        if ls.filter_rollbacks(diststr, version=ls[diststr]["version"]):
            LOG.info("%s_%s: Already in rollbacks, skipping rollback", package_name, ls[diststr]["version"])
        else:
            for r in range(suite_option.rollback - 1, -1, -1):
                src = self.mbd_get_diststr(distribution, suite_option, None if r == 0 else r - 1)
                dst = self.mbd_get_diststr(distribution, suite_option, r)
                LOG.debug("Rollback: Moving %s: %s to %s", package_name, src, dst)
                try:
                    self.mbd_reprepro.migrate(package_name, src, dst)
                except BaseException as e:
                    util.log_exception(LOG, "Rollback failed (ignoring)", e)

    def _mbd_package_migrate(self, package, distribution, suite, rollback=None, version=None, log_event=True):
        src_diststr = self.mbd_get_diststr(distribution, suite)
        dst_diststr = None
        pkg_ls = self.mbd_reprepro.ls(package)

        if rollback is not None:
            dst_diststr = src_diststr
            LOG.info("Rollback restore of '%s' from rollback %s to '%s'", package, rollback, dst_diststr)

            pkg_ls.filter(diststr=dst_diststr, raise_if_found=f"Package '{package}' exists in '{dst_diststr}': Remove first to restore rollback")
            rob_diststr = self.mbd_get_diststr(distribution, suite, rollback=rollback)
            pkg_ls.filter(ls=pkg_ls.rollbacks(), diststr=rob_diststr, version=version, raise_if_not_found="No such rollback source")

            # Actually migrate package in reprepro
            self.mbd_reprepro.migrate(package, rob_diststr, dst_diststr, version)
        else:
            # Get src and dst dist strings, and check we are configured to migrate
            if not suite.migrates_to:
                raise util.HTTPBadRequest(f"You can't migrate from '{src_diststr}'")
            dst_diststr = self.mbd_get_diststr(distribution, suite.migrates_to)

            # Check if package is in src_dst (&& update 'version' if None)
            version = pkg_ls.filter(diststr=src_diststr, version=version, raise_if_not_found="No such source")[src_diststr]["version"]

            # Check that version is not already migrated
            if pkg_ls.filter(diststr=dst_diststr, version=version):
                raise util.HTTPBadRequest(f"Version '{version}' already migrated to '{dst_diststr}'")

            # Shift rollbacks in the destination distributions
            if pkg_ls.filter(diststr=dst_diststr):
                self._mbd_package_shift_rollbacks(distribution, suite.migrates_to, package)

            # Actually migrate package in reprepro
            self.mbd_reprepro.migrate(package, src_diststr, dst_diststr, version)

        if log_event:
            pkg_ls = self.mbd_reprepro.ls(package)
            daemon.get().events.log(events.Type.MIGRATED, pkg_ls.changes(dst_diststr, None))

    def mbd_package_migrate(self, package, distribution, suite, full=False, rollback=None, version=None):
        if full:
            while suite.migrates_to is not None:
                self._mbd_package_migrate(package, distribution, suite, rollback=rollback, version=version)
                suite = suite.migrates_to
        else:
            self._mbd_package_migrate(package, distribution, suite, rollback=rollback, version=version)

    def mbd_package_remove(self, package, distribution, suite, rollback=None, version=None, without_rollback=False):
        diststr = self.mbd_get_diststr(distribution, suite, rollback)
        src_pkg = self.mbd_reprepro.ls(package)

        if rollback is None:
            # Preliminary check
            src_pkg.filter(diststr=diststr, version=version, raise_if_not_found="No such source")

            # Rollbacks
            if not without_rollback:
                self._mbd_package_shift_rollbacks(distribution, suite, package)

            # Remove package
            self.mbd_reprepro.remove(package, diststr, version)

            # Notify
            daemon.get().events.log(events.Type.REMOVED, src_pkg.changes(diststr, None))
        else:
            # Preliminary check
            src_pkg.filter(ls=src_pkg.rollbacks(), diststr=diststr, version=version, raise_if_not_found="No such source")

            # Rollback removal
            self.mbd_reprepro.remove(package, diststr, version)

            # Fix up empty rollback dist
            for r in range(rollback, suite.rollback - 1):
                src_diststr = self.mbd_get_diststr(distribution, suite, r + 1)
                dst_diststr = self.mbd_get_diststr(distribution, suite, r)
                try:
                    self.mbd_reprepro.migrate(package, src_diststr, dst_diststr)
                    self.mbd_reprepro.remove(package, src_diststr)
                except BaseException as e:
                    util.log_exception(LOG, f"Rollback: Moving '{package}' from '{src_diststr}' to '{dst_diststr}' FAILED (ignoring)", e, logging.WARN)

            # Notify
            daemon.get().events.log(events.Type.REMOVED, src_pkg.changes(diststr, None, ls=src_pkg.rollbacks()))

    def mbd_package_precheck(self, distribution, suite_option, package, version):
        # 1st, check that the given version matches the distribution's version restrictions
        mandatory_regex = self.mbd_get_mandatory_version_regex(distribution, suite_option)
        if not re.compile(mandatory_regex).search(version):
            raise util.HTTPBadRequest(f"Version restrictions failed for suite '{suite_option.suite.name}': '{mandatory_regex}' not in '{version}'")

        pkg_ls = self.mbd_reprepro.ls(package)
        diststr = self.mbd_get_diststr(distribution, suite_option)

        # 2nd: Check whether the very same version is already in any distribution
        pkg_ls.filter(version=version, raise_if_found="Source already installed")

        # 3rd: Check that distribution's current version is smaller than the to-be installed version
        pkg_in_dist = pkg_ls.filter(diststr=diststr)
        if pkg_in_dist:
            pkg_in_dist_version = pkg_in_dist[diststr]["version"]
            if debian.debian_support.Version(version) < debian.debian_support.Version(pkg_in_dist_version):
                raise util.HTTPBadRequest(f"Package '{package}' has greater version '{pkg_in_dist_version}' installed in '{diststr}'")

    def _mbd_buildresult_install(self, buildresult, diststr):
        # Don't try install if skipped
        if buildresult.cget("Sbuild-Status") == "skipped":
            LOG.debug("Skipped: %s", buildresult)
        else:
            with util.tmp_dir(prefix="buildresult-untar-") as tmpdir:
                buildresult.untar(tmpdir)
                self.mbd_reprepro.install(" ".join(glob.glob(os.path.join(tmpdir, "*.changes"))), diststr)
                LOG.info("Installed: %s", buildresult)

    def mbd_package_install(self, distribution, suite_option, changes, buildresults):
        """Install a dict arch:buildresult of successful build results"""
        # Get the full distribution str
        diststr = self.mbd_get_diststr(distribution, suite_option)

        # Check that all mandatory archs are present
        missing_mandatory_archs = [arch for arch in distribution.mbd_get_mandatory_architectures() if arch not in buildresults]
        if missing_mandatory_archs:
            raise util.HTTPBadRequest(f"{len(missing_mandatory_archs)} mandatory architecture(s) missing: {' '.join(missing_mandatory_archs)}")

        # Get the (source) package name
        package = changes["Source"]
        LOG.debug("Package install: Package=%s", package)

        # Shift current package up in the rollback distributions (unless this is the initial install)
        is_installed = self.mbd_reprepro.ls(package).filter(diststr=diststr)
        if is_installed:
            self._mbd_package_shift_rollbacks(distribution, suite_option, package)

        # First, install the dsc
        self.mbd_reprepro.install_dsc(changes.dsc_file_path(), diststr)

        # Second, install all build results; if any buildresult fails to install, rollback changes to repository
        try:
            for buildresult in list(buildresults.values()):
                self._mbd_buildresult_install(buildresult, diststr)
        except Exception as e:
            self.mbd_reprepro.remove(package, diststr)
            if is_installed:
                self._mbd_package_migrate(package, distribution, suite_option, rollback=0, log_event=False)
            util.log_exception(LOG, "Binary install failed", e)
            raise util.HTTPInternal(f"Binary install failed for {package}: {util.e2http(e)}")

    def mbd_icodenames(self):
        for d in sorted(self.distributions.all()):
            yield d.base_source.codename

    def mbd_prepare(self):
        """Idempotent repository preparation. This may be used as-is as mbd_sync"""
        # Architecture sanity checks
        for d in sorted(self.distributions.all()):
            if not d.architectureoption_set.all().filter(optional=False):
                raise util.HTTPBadRequest(f"{d}: There must be at least one mandatory architecture")
            if len(d.architectureoption_set.all().filter(optional=False, build_architecture_all=True)) != 1:
                raise util.HTTPBadRequest(f"{d}: There must be exactly one one arch-all architecture")

        # Check that the codenames of the distribution are unique
        codenames = []
        for d in self.distributions.all():
            if d.base_source.codename in codenames:
                raise util.HTTPBadRequest(f"Multiple distribution codename in: {d}")
            codenames.append(d.base_source.codename)

            # Check for mandatory component "main"
            if not d.components.all().filter(name="main"):
                raise util.HTTPBadRequest(f"Mandatory component 'main' missing in: {d}")

        self.mbd_reprepro_update_config()

    def mbd_sync(self):
        self.mbd_prepare()

    def mbd_remove(self):
        if os.path.exists(self.mbd_path):
            shutil.rmtree(self.mbd_path)

    def mbd_check(self):
        self.mbd_reprepro_update_config()
        self.mbd_reprepro.check()

        # Check for ambiguity with other repos in meta distribution maps
        get_meta_distribution_map()

    def mbd_get_dependencies(self):
        result = []
        for d in self.distributions.all():
            result.append(d.base_source)
            result += [e.source for e in d.extra_sources.all()]
        return result


def get_meta_distribution_map():
    """Get a dict of the meta distributions: meta -> actual"""
    result = {}
    for r in Repository.objects.all():
        for d in r.distributions.all():
            for s in r.layout.suiteoption_set.all():
                for m in r.mbd_get_meta_distributions(d, s):
                    diststr = r.mbd_get_diststr(d, s)
                    if m in result:
                        raise util.HTTPInternal(f"Ambiguous Meta-Distributions ({m}={diststr} or {result[m]}). "
                                                f"Please check Repositories and Layouts (see Layouts/Meta-Distributions in Administrators Manual)")
                    result[m] = diststr

    LOG.debug("Got meta distribution map: %s", result)
    return result


def map_distribution(diststr):
    """Map incoming distribution to internal"""
    return get_meta_distribution_map().get(diststr, diststr)


def get_repository(identity):
    """Get repository object with user error handling"""
    repository = Repository.objects.filter(pk=identity).first()
    if repository is None:
        raise util.HTTPBadRequest(f"No such repository: {identity}")
    return repository


def parse_dist(_dist, check_uploadable=False):
    repository = get_repository(_dist.repository)

    distribution = repository.distributions.all().filter(base_source__codename__exact=_dist.codename).first()
    if distribution is None:
        raise util.HTTPBadRequest(f"No distribution for codename '{_dist.codename}' in repository '{repository.identity}'")

    suite = repository.layout.suiteoption_set.filter(suite__name=_dist.suite).first()
    if suite is None:
        raise util.HTTPBadRequest(f"No distribution for suite '{_dist.suite}' in repository '{repository.identity}'")

    if check_uploadable:
        if _dist.rollback_no is not None or not suite.uploadable:
            raise util.HTTPBadRequest(f"Distribution '{_dist.get()}' not uploadable")

        if not distribution.mbd_is_active():
            raise util.HTTPUnavailable(f"Distribution '{_dist.get()}' not active")

        if not repository.mbd_is_active():
            raise util.HTTPUnavailable(f"Repository '{repository.identity}' not active")

    return repository, distribution, suite


def parse_diststr(diststr, check_uploadable=False):
    return parse_dist(dist.Dist(diststr), check_uploadable=check_uploadable)


class DaemonEmailField(django.db.models.EmailField):
    default_validators = [django.core.validators.EmailValidator(allowlist=["localhost", config.HOSTNAME_FQDN])]


@django.utils.deconstruct.deconstructible
class FtpdOptionsValidator:
    def __call__(self, value):
        try:
            ftpd.FtpD.parse_handler_options(value)
        except BaseException as e:
            raise django.core.exceptions.ValidationError(f"Can't parse: {util.e2http(e)}")


class Daemon(StatusModel):
    MBD_HELP_NEEDS_RESTART = "[Daemon must be restarted to activate this setting.]\n\n"

    # Basics
    identity = django.db.models.CharField(
        max_length=50,
        default=config.default_identity,
        validators=[RegexValidator(regex=r"^[a-zA-Z0-9\-]+$")],
        help_text=help_html(
            "mini-buildd's instance identity\n"
            "\n"
            "Will be used to identify this mini-buildd instance in various places:\n"
            "\n"
            "* 'Name-Real' part of the GnuPG key.\n"
            "* name of the automated keyring packages.\n"
            "* the 'Origin' tag of repository indices (but this may also be overridden via 'custom_archive_origin' option below).\n"
            "* to identify remotes.\n"
        ))

    custom_archive_origin = django.db.models.CharField(
        max_length=255, blank=True,
        default="",
        help_text=help_html("Custom APT 'Origin' field for repository indices")
    )

    email_address = DaemonEmailField(
        max_length=255,
        default=config.default_email_address,
        help_text=help_html(
            "mini-buildd instance's email address\n"
            "\n"
            "Will be used in:\n"
            "\n"
            "* 'Name-Email' part of the GnuPG key.\n"
            "* Sender for notification email.\n"
        ))
    # GnuPG options
    gnupg_template = UnixTextField(
        default=(
            "Key-Type: RSA\n"
            "Key-Length: 4096\n"
            "Expire-Date: 0"
        ),
        help_text=help_html(
            "Template as accepted by 'gpg --batch --gen-key' (see 'man gpg')\n"
            "\n"
            "You should not give 'Name-Real' and 'Name-Email', as these are automatically added.\n"
            "\n"
            "Prepare/Remove actions will generate/remove the GnuPG key."
        ))

    gnupg_keyserver = django.db.models.CharField(
        max_length=200,
        default="keyserver.ubuntu.com",
        help_text=help_html("GnuPG keyserver to use (as fill-in helper)")
    )

    ftpd_bind = django.db.models.CharField(
        max_length=200,
        default="tcp:port=8067",
        validators=[ServerEndpointValidator()],
        help_text=help_html(MBD_HELP_NEEDS_RESTART + f"FTP Server Endpoint: {inspect.getdoc(net.ServerEndpoint)}")
    )

    ftpd_options = django.db.models.CharField(
        max_length=255, blank=True,
        default="",
        validators=[FtpdOptionsValidator()],
        help_text=help_html(MBD_HELP_NEEDS_RESTART + ftpd.FtpD.HANDLER_OPTIONS_USAGE)
    )

    # Load options
    max_parallel_builds = django.db.models.IntegerField(
        default=util.get_cpus,
        help_text=help_html(MBD_HELP_NEEDS_RESTART + "Maximum number of parallel builds")
    )

    event_queue_size = django.db.models.IntegerField(
        default=500,
        help_text=help_html(MBD_HELP_NEEDS_RESTART + "Max number of items that can be stored in the events queue")
    )

    _DURATION_FIELD_FORMAT_HELP = "\n\nFormat hint: <DAYS> <HOURS>:<MINUTES>:<SECONDS>"

    expire_events_after = django.db.models.DurationField(
        default=datetime.timedelta(weeks=1 * 12 * 4),
        help_text=help_html("How long to keep events and associated files (like buildlogs)" + _DURATION_FIELD_FORMAT_HELP)
    )

    expire_builds_after = django.db.models.DurationField(
        default=datetime.timedelta(days=5),
        help_text=help_html("How long to keep builds and associated files" + _DURATION_FIELD_FORMAT_HELP)
    )

    notify = django.db.models.ManyToManyField(
        EmailAddress, blank=True,
        help_text=help_html("Addresses that get all notification emails unconditionally")
    )
    allow_emails_to = django.db.models.CharField(
        max_length=254,
        default=config.default_allow_emails_to,
        help_text=help_html(
            "Regex to allow sending emails to automatically computed addresses\n"
            "\n"
            "Currently the 'Changed-By' or 'Maintainer' addresses from the changelog when this feature is enabled for the resp. repository.\n"
            "\n"
            "Use '.*' to allow all -- it's however recommended to put this to s.th. like '.*@myemail.domain', to avoid original package maintainers to be accidentally spammed.\n"
        ))

    LETHAL_DEPENDENCIES = False

    class Meta(StatusModel.Meta):
        verbose_name_plural = "Daemon"

    class Admin(StatusModel.Admin):
        filter_horizontal = ("notify",)

        @classmethod
        def _mbd_on_activation(cls, obj):
            daemon.start()

        @classmethod
        def _mbd_on_deactivation(cls, obj):
            daemon.stop()

    def __str__(self):
        return (f"Daemon '{self.identity}' serving {len(Repository.mbd_get_active())} repositories, "
                f"entertaining {len(Chroot.mbd_get_active())} chroots, "
                f"using {len(Remote.mbd_get_active())} remotes")

    def mbd_get_ftp_endpoint(self):
        return net.ServerEndpoint(self.ftpd_bind, protocol="ftp")

    def mbd_fullname(self):
        return f"mini-buildd archive {self.identity}"

    def mbd_gnupg(self):
        return gnupg.GnuPG(self.gnupg_template, self.mbd_fullname(), self.email_address)

    def clean(self):
        super().clean()

        if Daemon.objects.count() > 0 and self.pk != Daemon.objects.get().pk:
            raise django.core.exceptions.ValidationError("You can only create one Daemon instance")

    def mbd_prepare(self):
        _gnupg = self.mbd_gnupg()
        _gnupg.prepare()
        daemon.get().gnupg = _gnupg
        LOG.debug("Daemon GnuPG prepared.")

    @classmethod
    def mbd_sync(cls):
        LOG.warning("The GnuPG key will never be updated automatically. Explicitly run remove+prepare to achieve this.")

    def mbd_remove(self):
        self.mbd_gnupg().remove()
        LOG.debug("Daemon GnuPG removed.")

    def mbd_get_dependencies(self):
        """All active or to-be active repositories, remotes and chroots"""
        result = []
        for model in [Repository, Chroot, Remote]:
            for o in model.mbd_get_active_or_auto_reactivate():
                result.append(o)
        return result

    @classmethod
    def mbd_check(cls):
        """Just warn in case there are no repos and no chroots"""
        if not Repository.mbd_get_active() and not Chroot.mbd_get_active():
            LOG.warning("No active chroot or repository.")

    def mbd_get_archive_origin(self):
        return self.custom_archive_origin if self.custom_archive_origin else f"Mini-Buildd archive {self.identity} on {config.HOSTNAME_FQDN}"

    def mbd_dput_values(self):
        ftp = self.mbd_get_ftp_endpoint()
        return {
            "method": "ftps" if ftp.is_ssl() else "ftp",
            "fqdn": ftp.hopo(),
            "login": "anonymous",
            "incoming": "/incoming",
        }

    def mbd_dput_name(self):
        return f"mini-buildd-{self.identity}"

    def mbd_get_dput_conf(self):
        """Get traditional dput.cf snippet"""
        return f"[{self.mbd_dput_name()}]\n" + "\n".join(f"{key:8} = {value}" for key, value in self.mbd_dput_values().items()) + "\n"

    def mbd_get_dputng_profile(self):
        """Get dput-ng profile"""
        return util.json_pretty(self.mbd_dput_values())

    def _mbd_notify_signature(self, typ):
        reason = {"daemon": "Your address is configured to get any notifications (contact administrators if you don't want this).",
                  "repository": "Your address is configured to get any notifications for this repository (contact administrators if you don't want this).",
                  "changed-by": "Your address is the uploader of the package ('Changed-By' in changes).",
                  "maintainer": "Your address is the maintainer of the package ('Maintainer' in changes).",
                  "subscriber": "Your user account has a matching subscription.",
                  }.get(typ, "Unknown")

        return (f"--\N{SPACE}\n"
                f"mini-buildd instance '{self.identity}' at {config.HOSTNAME} <{self.email_address}>\n"
                f"Reason for this mail: {reason}\n"
                f"Visit mini-buildd   : {util.http_endpoint().geturl()}\n")

    def mbd_notify_event(self, event):
        repository, distribution = None, None
        try:
            repository, distribution, _suite = parse_diststr(event.distribution)
        except BaseException:
            pass

        m_automatic_to_allow = re.compile(self.allow_emails_to)
        m_sent = []

        def send_mail(to, typ):
            info = f"EMail notify for '{event}' to '{to}' ({typ})"
            if to in m_sent:
                LOG.info("%s: already sent to this address", info)
            else:
                mail = django.core.mail.EmailMessage(
                    subject=f"[mini-buildd-{self.identity}] {event}",
                    body=f"{event.summary()}\n{self._mbd_notify_signature(typ)}",
                    from_email=self.email_address,
                    to=[to])
                mail.attach(event.json_file_name(), util.json_pretty(event.to_json()), "application/json")
                ok = mail.send(fail_silently=True)
                if ok == 1:
                    LOG.info("%s: sent successfully", info)
                    m_sent.append(to)
                else:
                    LOG.warning("%s: FAILED", info)

        # Hardcoded addresses from daemon
        for m in self.notify.all():
            send_mail(to=m.address, typ="daemon")

        # Hardcoded addresses from repository
        if repository:
            for m in repository.notify.all():
                send_mail(to=m.address, typ="repository")

        # Package uploader (Changed-By): Add even if we do not have a repo, so uploader is informed on such error cases too.
        changed_by = event.extra.get("changed_by")
        if changed_by and (repository is None or repository.notify_changed_by) and m_automatic_to_allow.search(changed_by):
            send_mail(to=changed_by, typ="changed-by")

        # Package maintainer: Add only when we have a repo, and it's configured to do so
        maintainer = event.extra.get("maintainer")
        if maintainer and repository and repository.notify_maintainer and m_automatic_to_allow.search(maintainer):
            send_mail(to=maintainer, typ="maintainer")

        # User subscriptions
        def get_subscriptions():
            source = "" if event.source is None else event.source
            real_distribution = distribution
            if real_distribution is None:
                # If distribution was not given explicitly, try from changes, resolving meta dists if needed.
                changes_dist = "" if event.distribution is None else event.distribution
                real_distribution = map_distribution(changes_dist)
            return Subscription.objects.filter(package__in=[source, ""], distribution__in=[real_distribution, ""])

        for s in get_subscriptions():
            address = s.subscriber.email
            if s.subscriber.is_active:
                send_mail(to=address, typ="subscriber")
            else:
                LOG.debug("Notify: Skipping subscription address: %s: Account disabled", address)


class Chroot(StatusModel):
    PERSONALITIES = {'i386': 'linux32'}

    source = django.db.models.ForeignKey(
        Source, on_delete=django.db.models.CASCADE,
        help_text=help_html(
            "Base source to create the chroot from; its codename must be 'debootstrapable'\n"
            "\n"
            "Examples: Debian 'squeeze', Debian 'wheezy', Ubuntu 'hardy'."
        ))
    architecture = django.db.models.ForeignKey(
        Architecture, on_delete=django.db.models.CASCADE,
        help_text=help_html(
            "Chroot's architecture\n"
            "\n"
            "Using the same arch as the host system will always work, other architectures may work if supported. An 'amd64' host, for example, will also allow for architecture 'i386'.\n"
        ))
    debootstrap_command = django.db.models.CharField(
        max_length=255,
        default="/usr/sbin/debootstrap",
        help_text=help_html(
            "bootstrap command\n"
            "\n"
            "Change only if you need an alternate debootstrap command.\n"
            "\n"
            "For example, ``/usr/sbin/mini-buildd-debootstrap-uname-2.6`` may be used as workaround for urold (``<= lenny``) distributions."
        ))
    debootstrap_extra_options = django.db.models.CharField(
        max_length=255, blank=True,
        default="",
        help_text=help_html(
            "Additional debootstrap options\n"
            "\n"
            "For example, ``--include=foo,bar`` may be used to have these packages pre-installed in build chroots.\n"
        ))

    personality = django.db.models.CharField(
        max_length=50, editable=False, blank=True,
        default="",
        help_text=help_html(
            "Schroot 'personality' value (see 'schroot')\n"
            "\n"
            "For 32bit chroots running on a 64bit host, this must be 'linux32'."
        ))
    personality_override = django.db.models.CharField(
        max_length=50, blank=True,
        default="",
        help_text=help_html(
            "Leave empty unless you want to override the automated way (via\n"
            "an internal mapping). Please report manual overrides so it can\n"
            "go to the default mapping.\n"
        ))

    class Meta(StatusModel.Meta):
        unique_together = ["source", "architecture"]
        ordering = ["source", "architecture"]

    class Admin(StatusModel.Admin):
        search_fields = ("source__codename", "architecture__name")
        readonly_fields = ("personality",)

    def clean(self):
        """Forbid change source/arch on existing chroot (we would loose the path to the associated data)"""
        super().clean()
        self.mbd_validate_field_unchanged("source")
        self.mbd_validate_field_unchanged("architecture")

    def mbd_key(self):
        return f"{self.source.codename}:{self.architecture.name}"

    def __str__(self):
        return f"{self.__class__.__name__} for {self.source} with {self.architecture}"

    @property
    def mbd_path(self):
        # Note: Property as this code must not be in __init__(): self.source, self.architecture may not be initialized (error on "add" in django admin).
        return config.ROUTES["chroots"].path.join(self.source.codename, self.architecture.name)

    def mbd_get_backend(self):
        """Get actual class of (saved) instance (even if operating on base class ``Chroot`` only)"""
        for cls in ["filechroot", "dirchroot", "looplvmchroot", "lvmchroot", "btrfssnapshotchroot"]:
            if hasattr(self, cls):
                return getattr(self, cls)
        raise util.HTTPBadRequest("No chroot backend found")

    def mbd_schroot_name(self):
        """
        Schroot name w/o namespace (see ``schroot --list``)

        Must produce same as :func:`~changes.Buildrequest.schroot_name`.
        """
        return f"mini-buildd-{self.source.codename}-{self.architecture.name}"

    def mbd_get_tmp_dir(self):
        return os.path.join(self.mbd_path, "tmp")

    def mbd_get_schroot_conf_file(self):
        return os.path.join(self.mbd_path, "schroot.conf")

    def mbd_get_keyring_file(self):
        """Get keyring file path. Holds all keys from the source to verify the release via debootstrap's --keyring option"""
        return os.path.join(self.mbd_path, "keyring.gpg")

    def mbd_get_system_schroot_conf_file(self):
        return os.path.join("/etc/schroot/chroot.d", self.mbd_schroot_name() + ".conf")

    def mbd_get_pre_sequence(self):
        """Get preliminary sequence. Subclasses may implement this to do define an extra preliminary sequence"""
        LOG.debug("%s: No pre-sequence defined.", self)
        return []

    def mbd_workaround_debmirror_script(self):
        if self.source.origin == "Ubuntu" and not os.path.exists(os.path.join("/usr/share/debootstrap/scripts", self.source.codename)):
            return ["gutsy"]
        return []

    def mbd_get_sequence(self):
        # In case a source was removed, we might not be able to get an archive URL for debootstrap.
        # We should be able to assemble the sequence anyway to be able to remove that chroot.
        try:
            debootstrap_url = self.source.mbd_get_archive().url
        except BaseException as e:
            LOG.warning("%s: Can't get archive URL from source (source removed?): %s", self, e)
            debootstrap_url = "no_archive_url_found_maybe_source_is_removed"

        debootstrap_extra_options = shlex.split(self.debootstrap_extra_options)

        return [
            (["/bin/mkdir", "--verbose", self.mbd_get_tmp_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_tmp_dir()])] + self.mbd_get_backend().mbd_get_pre_sequence() + [
                 ([self.debootstrap_command,
                   "--variant", "buildd",
                   "--keyring", self.mbd_get_keyring_file(),
                   "--arch", self.architecture.name,
                   *debootstrap_extra_options,
                   self.source.codename,
                   self.mbd_get_tmp_dir(),
                   debootstrap_url] + self.mbd_workaround_debmirror_script(),
                  ["/bin/umount", "-v", os.path.join(self.mbd_get_tmp_dir(), "proc"), os.path.join(self.mbd_get_tmp_dir(), "sys")])] + self.mbd_get_backend().mbd_get_post_sequence() + [
                      (["/bin/cp", "--verbose", self.mbd_get_schroot_conf_file(), self.mbd_get_system_schroot_conf_file()],
                       ["/bin/rm", "--verbose", self.mbd_get_system_schroot_conf_file()])]

    def mbd_prepare(self):
        os.makedirs(self.mbd_path, exist_ok=True)

        # Set personality
        self.personality = self.personality_override if self.personality_override else self.PERSONALITIES.get(self.architecture.name, "linux")

        files.File("schroot.conf",
                   snippet=(f"[{self.mbd_schroot_name()}]\n"
                            f"description=Mini-Buildd chroot {self.mbd_schroot_name()}\n"
                            f"setup.fstab=mini-buildd/fstab\n"
                            f"groups=sbuild\n"
                            f"users=mini-buildd\n"
                            f"root-groups=sbuild\n"
                            f"root-users=mini-buildd\n"
                            f"source-root-users=mini-buildd\n"
                            f"personality={self.personality}\n"
                            f"\n"
                            f"# Backend specific config\n"
                            f"{self.mbd_get_backend().mbd_get_schroot_conf()}\n")).save_as(self.mbd_get_schroot_conf_file())

        # Gen keyring file to use with debootstrap
        with contextlib.closing(gnupg.TmpGnuPG(tmpdir_options={"prefix": "gnupg-debootstrap-"})) as gpg:
            for k in self.source.apt_keys.all():
                gpg.add_pub_key(k.key)
            gpg.export(self.mbd_get_keyring_file())

        call.call_sequence(self.mbd_get_sequence(), run_as_root=True)

    def mbd_remove(self):
        call.call_sequence(self.mbd_get_sequence(), rollback_only=True, run_as_root=True)
        util.rmdirs(self.mbd_path)

    def mbd_sync(self):
        self._mbd_remove_and_prepare()

    def mbd_backend_check(self):
        """Run backend check. Subclasses may implement this to do extra backend-specific checks"""
        LOG.debug("%s: No backend check implemented.", self)

    def mbd_apt_line(self):
        return self.source.mbd_get_apt_line(limit_components=[Component(name="main")]).get()

    def mbd_check(self):
        with closing(schroot.Session(self.mbd_schroot_name(), "source")) as session:
            # Basic function checks
            session.info()

            # Globally configure debconf frontend to noninteractive.
            # This should avoid possible stalls due to debconf interaction on apt commands on the chroot.
            # (While rare, this definitely occasionally happened with rolling distributions)
            # There is no good way via schroot to convey environment (like DEBIAN_FRONTEND) for specific commands.
            # Also, a generic approach via schroot.conf seems unwise (would need --preserve-environment, and then to manually maintain 'environment-filter' regex).
            session.set_debconf("debconf/frontend", "noninteractive")

            # Backend checks
            self.mbd_get_backend().mbd_backend_check()

            # Update base apt line (source might have switched to another archive).
            session.update_file("/etc/apt/sources.list", self.mbd_apt_line())

            # "apt update/upgrade" check
            for args, fatal in [(["update"], True),
                                (["--ignore-missing", "dist-upgrade"], True),
                                (["--purge", "autoremove"], False),
                                (["clean"], True)]:
                try:
                    session.run(
                        [
                            "--directory", "/",
                            "--",
                        ] + shlex.split(sbuild.APT_GET) + args)
                except BaseException as e:
                    LOG.warning("'apt-get %s' not supported in this chroot: %s", ' '.join(args), e)
                    if fatal:
                        raise

    def mbd_get_dependencies(self):
        return [self.source]


class DirChroot(Chroot):
    """Directory chroot backend"""

    UNION_AUFS = 0
    UNION_OVERLAYFS = 1
    UNION_UNIONFS = 2
    UNION_OVERLAY = 3
    UNION_CHOICES = (
        (UNION_AUFS, "aufs"),
        (UNION_OVERLAYFS, "overlayfs"),
        (UNION_UNIONFS, "unionfs"),
        (UNION_OVERLAY, "overlay"))
    union_type = django.db.models.IntegerField(
        choices=UNION_CHOICES,
        default=util.guess_default_dirchroot_backend(overlay=UNION_OVERLAY, aufs=UNION_AUFS),
        help_text=help_html(
            "See 'man 5 schroot.conf'\n"
        ))

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    def mbd_backend_flavor(self):
        return self.get_union_type_display()

    def mbd_get_chroot_dir(self):
        return os.path.join(self.mbd_path, "source")

    def mbd_get_schroot_conf(self):
        return (f"type=directory\n"
                f"directory={self.mbd_get_chroot_dir()}\n"
                f"union-type={self.get_union_type_display()}\n")

    def mbd_get_post_sequence(self):
        return [
            (["/bin/mv",
              "--verbose",
              self.mbd_get_tmp_dir(),
              self.mbd_get_chroot_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_chroot_dir()])]


class FileChroot(Chroot):
    """File chroot backend"""

    COMPRESSION_NONE = 0
    COMPRESSION_GZIP = 1
    COMPRESSION_BZIP2 = 2
    COMPRESSION_XZ = 3
    COMPRESSION_CHOICES = (
        (COMPRESSION_NONE, "no compression"),
        (COMPRESSION_GZIP, "gzip"),
        (COMPRESSION_BZIP2, "bzip2"),
        (COMPRESSION_XZ, "xz"))

    compression = django.db.models.IntegerField(choices=COMPRESSION_CHOICES, default=COMPRESSION_NONE)

    TAR_ARGS = {
        COMPRESSION_NONE: [],
        COMPRESSION_GZIP: ["--gzip"],
        COMPRESSION_BZIP2: ["--bzip2"],
        COMPRESSION_XZ: ["--xz"]}
    TAR_SUFFIX = {
        COMPRESSION_NONE: "tar",
        COMPRESSION_GZIP: "tar.gz",
        COMPRESSION_BZIP2: "tar.bz2",
        COMPRESSION_XZ: "tar.xz"}

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    def mbd_backend_flavor(self):
        return self.TAR_SUFFIX[self.compression]

    def mbd_get_tar_file(self):
        return os.path.join(self.mbd_path, "source." + self.TAR_SUFFIX[self.compression])

    def mbd_get_schroot_conf(self):
        return (f"type=file\n"
                f"file={self.mbd_get_tar_file()}\n")

    def mbd_get_post_sequence(self):
        return [
            (["/bin/tar",
              "--create",
              "--directory", self.mbd_get_tmp_dir(),
              "--file", self.mbd_get_tar_file()] + self.TAR_ARGS[self.compression] + ["."],
             []),
            (["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_tmp_dir()],
             [])]


class LVMChroot(Chroot):
    """LVM chroot backend"""

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    volume_group = django.db.models.CharField(
        max_length=80,
        default="auto",
        help_text=help_html("Give a pre-existing LVM volume group name. Just leave it on 'auto' for loop lvm chroots.")
    )
    filesystem = django.db.models.CharField(max_length=10, default="ext2")
    snapshot_size = django.db.models.IntegerField(
        default=4,
        help_text=help_html("Snapshot device file size in GB")
    )

    def mbd_backend_flavor(self):
        return f"lvm={self.volume_group}/{self.filesystem}/{self.snapshot_size}G"

    def mbd_get_volume_group(self):
        try:
            return self.looplvmchroot.mbd_get_volume_group()
        except BaseException:
            return self.volume_group

    def mbd_get_lvm_device(self):
        return f"/dev/{self.mbd_get_volume_group()}/{self.mbd_schroot_name()}"

    def mbd_get_schroot_conf(self):
        return (f"type=lvm-snapshot\n"
                f"device={self.mbd_get_lvm_device()}\n"
                f"mount-options=-t {self.filesystem} -o noatime,user_xattr\n"
                f"lvm-snapshot-options=--size {self.snapshot_size}G\n")

    def mbd_get_pre_sequence(self):
        if not os.path.exists("/sbin/lvcreate"):
            raise util.HTTPUnavailable("LVM not installed")

        return [
            (["/sbin/lvcreate", "--size", f"{self.snapshot_size}G", "--name", self.mbd_schroot_name(), self.mbd_get_volume_group()],
             ["/sbin/lvremove", "--verbose", "--force", self.mbd_get_lvm_device()]),

            (["/sbin/mkfs", "-t", self.filesystem, self.mbd_get_lvm_device()],
             []),

            (["/bin/mount", "-v", "-t", self.filesystem, self.mbd_get_lvm_device(), self.mbd_get_tmp_dir()],
             ["/bin/umount", "-v", self.mbd_get_tmp_dir()])]

    def mbd_get_post_sequence(self):
        return [(["/bin/umount", "-v", self.mbd_get_tmp_dir()], [])]

    def mbd_backend_check(self):
        call.Call(["/sbin/fsck", "-a", "-t", self.filesystem, self.mbd_get_lvm_device()],
                  run_as_root=True).check()


class LoopLVMChroot(LVMChroot):
    """Loop LVM chroot backend"""

    class Meta(LVMChroot.Meta):
        pass

    class Admin(LVMChroot.Admin):
        pass

    loop_size = django.db.models.IntegerField(
        default=100,
        help_text=help_html("Loop device file size in GB")
    )

    def mbd_backend_flavor(self):
        return f"{self.loop_size}G loop: {super().mbd_backend_flavor()}"

    def mbd_get_volume_group(self):
        return f"mini-buildd-loop-{self.source.codename}-{self.architecture.name}"

    def mbd_get_backing_file(self):
        return os.path.join(self.mbd_path, "lvmloop.image")

    def mbd_get_loop_device(self):
        for f in glob.glob("/sys/block/loop[0-9]*/loop/backing_file"):
            with util.fopen(f) as ld:
                if os.path.realpath(ld.read().strip()) == os.path.realpath(self.mbd_get_backing_file()):
                    return "/dev/" + f.split("/")[3]
        LOG.debug("No existing loop device for %s, searching for free device", self.mbd_get_backing_file())
        return call.Call(["/sbin/losetup", "--find"], run_as_root=True).check().stdout.rstrip()

    def mbd_get_pre_sequence(self):
        loop_device = self.mbd_get_loop_device()
        LOG.debug("Acting on loop device: %s", loop_device)
        return [
            (["/bin/dd",
              "if=/dev/zero", f"of={self.mbd_get_backing_file()}",
              f"bs={self.loop_size}M",
              "seek=1024", "count=0"],
             ["/bin/rm", "--verbose", self.mbd_get_backing_file()]),

            (["/sbin/losetup", "--verbose", loop_device, self.mbd_get_backing_file()],
             ["/sbin/losetup", "--verbose", "--detach", loop_device]),

            (["/sbin/pvcreate", "--verbose", loop_device],
             ["/sbin/pvremove", "--verbose", loop_device]),

            (["/sbin/vgcreate", "--verbose", self.mbd_get_volume_group(), loop_device],
             ["/sbin/vgremove", "--verbose", "--force", self.mbd_get_volume_group()])] + super().mbd_get_pre_sequence()


class BtrfsSnapshotChroot(Chroot):
    """Btrfs Snapshot chroot backend"""

    class Meta(Chroot.Meta):
        pass

    class Admin(Chroot.Admin):
        pass

    @classmethod
    def mbd_backend_flavor(cls):
        return "btrfs_snapshot"

    def mbd_get_chroot_dir(self):
        return os.path.join(self.mbd_path, "source")

    def mbd_get_snapshot_dir(self):
        return os.path.join(self.mbd_path, "snapshot")

    def mbd_get_schroot_conf(self):
        return (f"type=btrfs-snapshot\n"
                f"btrfs-source-subvolume={self.mbd_get_chroot_dir()}\n"
                f"btrfs-snapshot-directory={self.mbd_get_snapshot_dir()}\n")

    def mbd_get_pre_sequence(self):
        return [
            (["/bin/btrfs", "subvolume", "create", self.mbd_get_chroot_dir()],
             ["/bin/btrfs", "subvolume", "delete", self.mbd_get_chroot_dir()]),

            (["/bin/mount", "-v", "-o", "bind", self.mbd_get_chroot_dir(), self.mbd_get_tmp_dir()],
             ["/bin/umount", "-v", self.mbd_get_tmp_dir()]),

            (["/bin/mkdir", "--verbose", self.mbd_get_snapshot_dir()],
             ["/bin/rm", "--recursive", "--one-file-system", "--force", self.mbd_get_snapshot_dir()])]

    def mbd_get_post_sequence(self):
        return [(["/bin/umount", "-v", self.mbd_get_tmp_dir()], [])]


class Subscription(Model):
    subscriber = django.db.models.ForeignKey(django.contrib.auth.models.User, on_delete=django.db.models.CASCADE)
    package = django.db.models.CharField(max_length=100, blank=True)
    distribution = django.db.models.CharField(max_length=100, blank=True)

    def __str__(self):
        return f"Subscription for '{self.subscriber}' (subscribes to '{self.package if self.package else 'any package'}' in '{self.distribution if self.distribution else 'any distribution'}')"


def mbd_icodenames():
    for r in Repository.objects.all():
        yield from r.mbd_icodenames()
