"""
Backfill_toofull
"""
import logging
import time
from tasks import ceph_manager
from tasks.util.rados import rados
from teuthology import misc as teuthology

log = logging.getLogger(__name__)

def wait_for_pg_state(manager, pgid, state, to_osd):
    log.debug("waiting for pg %s state is %s" % (pgid, state))
    for i in range(300):
        time.sleep(5)
        manager.flush_pg_stats([0, 1, 2, 3])
        pgs = manager.get_pg_stats()
        pg = next((pg for pg in pgs if pg['pgid'] == pgid), None)
        log.info('pg=%s' % pg);
        assert pg
        status = pg['state'].split('+')
        if 'active' not in status:
            log.debug('not active')
            continue
        if state not in status:
            log.debug('not %s' % state)
            continue
        assert to_osd in pg['up']
        return
    assert False, '%s not in %s' % (pgid, state)


def task(ctx, config):
    """
    Test backfill reservation calculates "toofull" condition correctly.

    A pretty rigid cluster is brought up and tested by this task
    """
    if config is None:
        config = {}
    assert isinstance(config, dict), \
        'backfill_toofull task only accepts a dict for configuration'
    first_mon = teuthology.get_first_mon(ctx, config)
    (mon,) = ctx.cluster.only(first_mon).remotes.keys()

    manager = ceph_manager.CephManager(
        mon,
        ctx=ctx,
        logger=log.getChild('ceph_manager'),
        )

    profile = config.get('erasure_code_profile', {
        'k': '2',
        'm': '1',
        'crush-failure-domain': 'osd'
    })
    profile_name = profile.get('name', 'backfill_toofull')
    manager.create_erasure_code_profile(profile_name, profile)
    pool = manager.create_pool_with_unique_name(
        pg_num=1,
        erasure_code_profile_name=profile_name,
        min_size=2)
    manager.raw_cluster_cmd('osd', 'pool', 'set', pool,
                            'pg_autoscale_mode', 'off')

    manager.flush_pg_stats([0, 1, 2, 3])
    manager.wait_for_clean()

    pool_id = manager.get_pool_num(pool)
    pgid = '%d.0' % pool_id
    pgs = manager.get_pg_stats()
    acting = next((pg['acting'] for pg in pgs if pg['pgid'] == pgid), None)
    log.debug("acting=%s" % acting)
    assert acting
    primary = acting[0]
    target = acting[1]

    log.debug("write some data")
    # perform rados bench writes of 4MiB objects. The ec2+1 pool has 150%
    # overhead and based on the total available capacity, calculate the max
    # number of objects necessary to fill only 50% of the cluster.
    cluster_stats = manager.get_cluster_df_stats()
    total_avail_bytes = cluster_stats['total_avail_bytes']
    log.debug("total available bytes: %s" % total_avail_bytes)
    max_objects = int(total_avail_bytes / (4*1024*1024) / 1.5 / 2)
    log.debug("max_objects=%s" % max_objects)
    # Perform writes
    rados(ctx, mon, ['-p', pool, 'bench', '120', 'write', '--max-objects', \
          str(max_objects), '--no-cleanup'])
    df = manager.get_osd_df(target)
    log.debug("target osd df: %s" % df)

    total_kb = df['kb']
    used_kb = df['kb_used']

    log.debug("pause recovery")
    manager.raw_cluster_cmd('osd', 'set', 'noout')
    manager.raw_cluster_cmd('osd', 'set', 'nobackfill')
    manager.raw_cluster_cmd('osd', 'set', 'norecover')

    log.debug("stop target osd %s" % target)
    manager.kill_osd(target)
    manager.wait_till_active()

    pgs = manager.get_pg_stats()
    pg = next((pg for pg in pgs if pg['pgid'] == pgid), None)
    log.debug('pg=%s' % pg)
    assert pg

    log.debug("re-write data")
    rados(ctx, mon, ['-p', pool, 'cleanup'])
    time.sleep(10)
    cluster_stats = manager.get_cluster_df_stats()
    total_avail_bytes = cluster_stats['total_avail_bytes']
    log.debug("total available bytes(after stopping osd.%s): %s" %
              (target, total_avail_bytes))
    # Write nominal data (20% of total_avail_bytes) to test backfillfull.
    # This later helps keep the backfillfull ratio to a level that allows
    # backfilling of new data to the target.
    max_objects = int(total_avail_bytes / (4*1024*1024) / 1.5 / 5)
    log.debug("max_objects=%s" % max_objects)

    rados(ctx, mon, ['-p', pool, 'bench', '60', 'write', '--max-objects', \
          str(max_objects), '--no-cleanup'])

    df = manager.get_osd_df(primary)
    log.debug("primary osd df: %s" % df)

    primary_used_kb = df['kb_used']

    log.info("test backfill reservation rejected with toofull")

    # We set backfillfull ratio less than new data size and expect the pg
    # entering backfill_toofull state.
    #
    # We also need to update nearfull ratio to prevent "full ratio(s) out of order".

    backfillfull = 0.9 * primary_used_kb / total_kb
    nearfull = backfillfull * 0.9

    log.debug("update nearfull ratio to %s and backfillfull ratio to %s" %
              (nearfull, backfillfull))
    manager.raw_cluster_cmd('osd', 'set-nearfull-ratio',
                            '{:.3f}'.format(nearfull + 0.001))
    manager.raw_cluster_cmd('osd', 'set-backfillfull-ratio',
                            '{:.3f}'.format(backfillfull + 0.001))

    log.debug("start target osd %s" % target)

    manager.revive_osd(target)
    manager.wait_for_active()
    manager.wait_till_osd_is_up(target)

    wait_for_pg_state(manager, pgid, 'backfill_toofull', target)

    log.info("test pg not enter backfill_toofull after restarting backfill")

    # We want to set backfillfull ratio to be big enough for the target to
    # successfully backfill new data but smaller than the sum of old and new
    # data, so if the osd backfill reservation incorrectly calculates "toofull"
    # the test will detect this (fail).
    #
    # Note, we need to operate with "uncompressed" bytes because currently
    # osd backfill reservation does not take compression into account.
    #
    # We also need to update nearfull ratio to prevent "full ratio(s) out of order".

    pdf = manager.get_pool_df(pool)
    log.debug("pool %s df: %s" % (pool, pdf))
    assert pdf
    compress_ratio = 1.0 * pdf['compress_under_bytes'] / pdf['compress_bytes_used'] \
        if pdf['compress_bytes_used'] > 0 else 1.0
    log.debug("compress_ratio: %s" % compress_ratio)

    backfillfull = (used_kb + primary_used_kb) * compress_ratio / total_kb
    assert backfillfull < 0.9 * compress_ratio
    nearfull_min = max(used_kb, primary_used_kb) * compress_ratio / total_kb
    assert nearfull_min < backfillfull
    delta = backfillfull - nearfull_min
    nearfull = nearfull_min + delta * 0.1
    backfillfull = nearfull_min + delta * 0.2

    log.debug("update nearfull ratio to %s and backfillfull ratio to %s" %
              (nearfull, backfillfull))
    # Set backfillfull-ratio first to avoid a temporary "full ratio(s) out of
    # order" condition and the resulting cluster error log.
    manager.raw_cluster_cmd('osd', 'set-backfillfull-ratio',
                            '{:.3f}'.format(backfillfull + 0.001))
    manager.raw_cluster_cmd('osd', 'set-nearfull-ratio',
                            '{:.3f}'.format(nearfull + 0.001))

    wait_for_pg_state(manager, pgid, 'backfilling', target)

    pgs = manager.get_pg_stats()
    pg = next((pg for pg in pgs if pg['pgid'] == pgid), None)
    log.debug('pg=%s' % pg)
    assert pg

    log.debug("interrupt %s backfill" % target)
    manager.mark_down_osd(target)
    # after marking the target osd down it will automatically be
    # up soon again

    log.debug("resume recovery")
    manager.raw_cluster_cmd('osd', 'unset', 'noout')
    manager.raw_cluster_cmd('osd', 'unset', 'nobackfill')
    manager.raw_cluster_cmd('osd', 'unset', 'norecover')

    # wait for everything to peer, backfill and recover
    manager.wait_for_clean()

    pgs = manager.get_pg_stats()
    pg = next((pg for pg in pgs if pg['pgid'] == pgid), None)
    log.info('pg=%s' % pg)
    assert pg
    assert 'clean' in pg['state'].split('+')
