# Install and configure a CEPH MON node for OCI
#
# == Parameters
#
# [*block_devices*]
# List of block devices the volume node can use as LVM backend.
class oci::volume(
  $region_name              = 'RegionOne',
  $openstack_release        = undef,
  $machine_hostname         = undef,
  $machine_ip               = undef,
  $first_master             = undef,
  $first_master_ip          = undef,
  $all_masters              = undef,
  $all_masters_ip           = undef,
  $vip_hostname             = undef,
  $vip_ipaddr               = undef,
  $vip_netmask              = undef,
  $self_signed_api_cert     = true,
  $sql_vip_ip               = undef,
  $network_ipaddr           = undef,
  $network_cidr             = undef,
  $use_ssl                  = true,

  $messaging_nodes_for_notifs = false,
  $all_rabbits                = [],
  $all_rabbits_ips            = [],

  $block_devices            = undef,
  $vgname                   = undef,
  $backup_backend           = undef,

  $pass_cinder_db           = undef,
  $pass_cinder_authtoken    = undef,
  $pass_cinder_messaging    = undef,

  # Needed for notifications:
  $pass_nova_authtoken      = undef,
  $pass_keystone_adminuser  = undef,

  $cluster_has_osds         = false,
  $ceph_fsid                = undef,
  $ceph_admin_key           = undef,
  $ceph_openstack_key       = undef,
  $ceph_mon_initial_members = undef,
  $ceph_mon_host            = undef,
  $ceph_libvirtuuid         = undef,

  $extswift_use_external        = false,
  $extswift_auth_url            = undef,
  $extswift_region              = 'RegionOne',
  $extswift_proxy_url           = undef,
  $extswift_project_name        = undef,
  $extswift_project_domain_name = undef,
  $extswift_user_name           = undef,
  $extswift_user_domain_name    = undef,
  $extswift_password            = undef,

  $lvm_backend_name                   = 'LVM_1',
  $availability_zone                  = 'nova',
  $cinder_reserved_percentage         = 10,
  $cinder_max_over_subscription_ratio = 4.0,
  $volume_copy_bps_limit              = 1073741824,

  $disable_notifications        = false,

  $cinder_enabled_backends       = 'no-override',
  $enabled_backend_list          = undef,
  $cinder_separate_volume_groups = 'no',

  $haproxy_use_sni = false,
  $glance_sni_name     = "image",
  $keystone_sni_name   = "identity",

  # From variables.json
  $kernel_from_backports    = false,

  $external_keystone_activate              = false,
  $external_keystone_url                   = 'https://api.example.com/identity',
  $external_keystone_admin_password        = undef,
  $external_keystone_region_prefixed_users = false,

  $configure_dkms = false,

  $rabbitmq_enable_quorum_queues = false,
  $rabbitmq_message_timeout      = 120,

  $use_json_log = false,

  $cinder_debug_log = false,
  $logrotate_hourly = true,
){

  if $logrotate_hourly {
    $logrotate_period = 'hour'
  }else{
    $logrotate_period = 'day'
  }

  $logging_context_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s'
  $logging_default_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s'

  $rabbitmq_transient_queue_expire = $rabbitmq_message_timeout + 60

  if $rabbitmq_enable_quorum_queues {
    $rabbit_ha_queues              = false
    $rabbit_quorum_queue           = true
    $rabbit_transient_quorum_queue = true
    $amqp_durable_queues           = true
    $use_queue_manager             = true
    $rabbit_stream_fanout          = true
    $rabbit_qos_prefetch_count     = 20
  }else{
    $rabbit_ha_queues              = true
    $rabbit_quorum_queue           = false
    $rabbit_transient_quorum_queue = false
    $amqp_durable_queues           = false
    $use_queue_manager             = false
    $rabbit_stream_fanout          = false
    $rabbit_qos_prefetch_count     = $facts['os_service_default']
  }

  if $configure_dkms { class { '::oci::dkms': } }

  # Generic performances tweak (will default to throughput-performance)
  class { '::oci::tuned': }

  if $external_keystone_region_prefixed_users {
    $cinder_user           = "cinder-${region_name}"
  }else{
    $cinder_user           = 'cinder'
  }

  if $messaging_nodes_for_notifs {
    $notif_rabbit_servers = $all_rabbits
  }else{
    $notif_rabbit_servers = $all_masters
  }

  if $use_ssl {
    $proto = 'https'
    $messaging_default_port = '5671'
    $messaging_notify_port = '5671'
    $api_port = 443
  } else {
    $proto = 'http'
    $messaging_default_port = '5672'
    $messaging_notify_port = '5672'
    $api_port = 80
  }
  $messaging_default_proto = 'rabbit'
  $messaging_notify_proto = 'rabbit'

  $base_url = "${vip_hostname}"

  $base_url_glance     = $haproxy_use_sni ? { true => "${glance_sni_name}.${vip_hostname}", false => "${base_url}/image"}
  $base_url_keystone   = $haproxy_use_sni ? { true => "${keystone_sni_name}.${vip_hostname}", false => "${base_url}/identity"}

  $keystone_auth_uri   = $external_keystone_activate ? { true => $external_keystone_url, false => "${proto}://${base_url_keystone}"}
  $keystone_admin_uri  = $external_keystone_activate ? { true => $external_keystone_url, false => "${proto}://${base_url_keystone}"}
  $keystone_admin_pass = $external_keystone_activate ? { true => $external_keystone_admin_password, false => $pass_keystone_adminuser}

  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    ensure_resource('file', '/etc/keystone', {
      'ensure'  => 'directory',
      'mode'    => '0600',
    })
    ensure_resource('file', '/etc/keystone/puppet.conf', {
      'ensure'  => 'present',
      'content' => "
[keystone_authtoken]
region_name=${region_name}
auth_url=${keystone_auth_uri}
username=admin
password=${keystone_admin_pass}
project_name=admin
interface=public
",
      'mode'    => '0600',
    })
  }else{
    ensure_resource('file', '/etc/openstack', {
      'ensure' => 'directory',
      'mode'   => '0755',
      'owner'  => 'root',
      'group'  => 'root',
    })

    ensure_resource('file', '/etc/openstack/puppet', {
      'ensure' => 'directory',
      'mode'   => '0755',
      'owner'  => 'root',
      'group'  => 'root',
    })

    openstacklib::clouds { '/etc/openstack/puppet/admin-clouds.yaml':
      username     => 'admin',
      password     => $keystone_admin_pass,
      auth_url     => $keystone_auth_uri,
      project_name => 'admin',
      system_scope => 'all',
      region_name  => $region_name,
      interface    => 'public',
    }
  }

  if $self_signed_api_cert {
    $api_endpoint_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  if $disable_notifications {
    $notification_driver = 'noop'
  }else{
    $notification_driver = 'messagingv2'
  }

  if $disable_notifications {
    $cinder_notif_transport_url = undef
  } else {
    $cinder_notif_transport_url = os_transport_url({
                                    'transport' => $messaging_notify_proto,
                                    'hosts'     => fqdn_rotate($notif_rabbit_servers),
                                    'port'      => $messaging_notify_port,
                                    'username'  => 'cinder',
                                    'password'  => $pass_cinder_messaging,
                                  })
  }

  # This is needed if we're running backups over Ceph.
  if $cluster_has_osds {
    class { 'ceph':
      fsid                => $ceph_fsid,
      ensure              => 'present',
      authentication_type => 'cephx',
      mon_initial_members => $ceph_mon_initial_members,
      mon_host            => $ceph_mon_host,
    }->
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
    }->
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=cinder, profile rbd pool=nova, profile rbd pool=glance, profile rbd pool=gnocchi, profile rbd pool=cinderback',
    }
  }

  include ::cinder::client
  # Cinder main class (ie: cinder-common config)
  if $openstack_release == 'rocky'{
    class { '::cinder':
      default_transport_url              => os_transport_url({
        'transport' => $messaging_default_proto,
        'hosts'     => fqdn_rotate($all_masters),
        'port'      => $messaging_default_port,
        'username'  => 'cinder',
        'password'  => $pass_cinder_messaging,
      }),
      database_connection                => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
      rabbit_use_ssl                     => $use_ssl,
      rpc_response_timeout               => $rabbitmq_message_timeout,
      amqp_durable_queues                => false,
      amqp_auto_delete                   => false,
      rabbit_ha_queues                   => true,
      kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
      debug                              => true,
      storage_availability_zone          => $availability_zone,
      rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
      rabbit_heartbeat_timeout_threshold => 10,
      rabbit_heartbeat_rate              => 2,
      kombu_failover_strategy            => 'round-robin',
    }
  }else{
    if($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri'){
      class { '::cinder':
        default_transport_url              => os_transport_url({
          'transport' => $messaging_default_proto,
          'hosts'     => fqdn_rotate($all_masters),
          'port'      => $messaging_default_port,
          'username'  => 'cinder',
          'password'  => $pass_cinder_messaging,
        }),
        database_connection                => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        rabbit_use_ssl                     => $use_ssl,
        rpc_response_timeout               => $rabbitmq_message_timeout,
        amqp_durable_queues                => false,
        amqp_auto_delete                   => false,
        rabbit_ha_queues                   => true,
        kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
        storage_availability_zone          => $availability_zone,
        rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
        rabbit_heartbeat_timeout_threshold => 10,
        rabbit_heartbeat_rate              => 2,
        kombu_failover_strategy            => 'round-robin',
      }
      class { '::cinder::logging':
        debug                         => $cinder_debug_log,
        use_json                      => $use_json_log,
        logging_context_format_string => $logging_context_format_string,
        logging_default_format_string => $logging_default_format_string,
      }
      class { 'cinder::ceilometer':
        notification_transport_url => $cinder_notif_transport_url,
        notification_driver        => $notification_driver,
      }
      if($openstack_release == 'train' or $openstack_release == 'ussuri'){
        cinder_config {
          'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
        }
      }
    }else{
      if($openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'){
        class { '::cinder':
          default_transport_url              => os_transport_url({
            'transport' => $messaging_default_proto,
            'hosts'     => fqdn_rotate($all_masters),
            'port'      => $messaging_default_port,
            'username'  => 'cinder',
            'password'  => $pass_cinder_messaging,
          }),
          database_connection                => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
          rabbit_use_ssl                     => $use_ssl,
          rpc_response_timeout               => $rabbitmq_message_timeout,
          amqp_durable_queues                => false,
          amqp_auto_delete                   => false,
          rabbit_ha_queues                   => true,
          kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
          storage_availability_zone          => $availability_zone,
          notification_transport_url         => $cinder_notif_transport_url,
          notification_driver                => $notification_driver,
          rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
          rabbit_heartbeat_timeout_threshold => 10,
          rabbit_heartbeat_rate              => 2,
          kombu_failover_strategy            => 'round-robin',
        }
        cinder_config {
          'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
        }
      # Yoga and up
      }else{
        class { '::cinder::db':
          database_connection   => "mysql+pymysql://cinder:${pass_cinder_db}@${sql_vip_ip}/cinderdb?charset=utf8",
        }
        if($openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'){
          class { '::cinder':
            default_transport_url              => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => false,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => true,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            storage_availability_zone          => $availability_zone,
            notification_transport_url         => $cinder_notif_transport_url,
            notification_driver                => $notification_driver,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
          }
          cinder_config {
            'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
          }
        }elsif $openstack_release == 'bobcat'{
          class { '::cinder':
            default_transport_url => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => false,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => $rabbit_ha_queues,
            rabbit_quorum_queue                => $rabbit_quorum_queue,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            storage_availability_zone          => $availability_zone,
            notification_transport_url         => $cinder_notif_transport_url,
            notification_driver                => $notification_driver,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
          }
          cinder_config {
            'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
          }
        # Caracal and up.
        }else{
          class { '::cinder':
            default_transport_url         => os_transport_url({
              'transport' => $messaging_default_proto,
              'hosts'     => fqdn_rotate($all_masters),
              'port'      => $messaging_default_port,
              'username'  => 'cinder',
              'password'  => $pass_cinder_messaging,
            }),
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => false,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => $rabbit_ha_queues,
            rabbit_quorum_queue                => $rabbit_quorum_queue,
            rabbit_transient_quorum_queue      => $rabbit_transient_quorum_queue,
            rabbit_use_queue_manager           => $use_queue_manager,
            rabbit_stream_fanout               => $rabbit_stream_fanout,
            rabbit_qos_prefetch_count          => $rabbit_qos_prefetch_count,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            storage_availability_zone          => $availability_zone,
            notification_transport_url         => $cinder_notif_transport_url,
            notification_driver                => $notification_driver,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
            rabbit_enable_cancel_on_failover   => true,
          }
        }
      }
      class { '::cinder::logging':
        debug                         => $cinder_debug_log,
        use_json                      => $use_json_log,
        logging_context_format_string => $logging_context_format_string,
        logging_default_format_string => $logging_default_format_string,
      }
    }
  }


  if $extswift_use_external {
    $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
    $provision_cinder_backup = true
  }else{
    case $backup_backend {
      'ceph': {
        $backup_backend_real = 'cinder.backup.drivers.ceph.CephBackupDriver'
        $provision_cinder_backup = false
      }
      'swift': {
        $backup_backend_real = 'cinder.backup.drivers.swift.SwiftBackupDriver'
        $provision_cinder_backup = true
      }
      'none': {
        $backup_backend_real = ''
        $provision_cinder_backup = false
      }
    }
  }

  cinder_config {
    'DEFAULT/snapshot_clone_size':       value => '200';
    'ssl/ca_file':                       value => $api_endpoint_ca_file;
  }
  cinder_config {
    "${lvm_backend_name}/goodness_function": value => '"(capabilities.total_volumes < 200) ? 100 : 50"'
  }
  if $extswift_use_external == false {
    cinder_config {
      'DEFAULT/backup_driver':             value => $backup_backend_real;
      'DEFAULT/backup_swift_auth_url':     value => $keystone_auth_uri;
      'DEFAULT/backup_swift_ca_cert_file': value => $api_endpoint_ca_file;
    }
  }else{
    cinder_config {
      'DEFAULT/backup_ceph_user':          value => 'openstack';
      'DEFAULT/backup_ceph_pool':          value => 'cinderback';
    }
  }
  # Configure the authtoken
  class { '::cinder::keystone::authtoken':
    username             => $cinder_user,
    password             => $pass_cinder_authtoken,
    auth_url             => $keystone_admin_uri,
    www_authenticate_uri => $keystone_auth_uri,
    cafile               => $api_endpoint_ca_file,
    region_name          => $region_name,
  }

  # This is mandatory starting with bobcat, and this is supported in puppet-cinder only in train and up.
  # This is related to CVE-2023-2088.
  # Configure the [service_user].
  if($openstack_release == 'rocky' or $openstack_release == 'stein'){
    cinder_config {
      'service_user/cafile': value => $api_endpoint_ca_file;
    }
    notify { 'No service_user config for this version of OpenStack': }
  }else{
    class { '::cinder::keystone::service_user':
      username                => $cinder_user,
      password                => $pass_cinder_authtoken,
      auth_url                => $keystone_auth_uri,
      project_name            => 'services',
      region_name             => $region_name,
      send_service_user_token => true,
      cafile                  => $api_endpoint_ca_file,
    }
  }

  # Configure the [nova] section.
  if($openstack_release == 'rocky'){
    cinder_config {
      'nova/cafile': value => $api_endpoint_ca_file;
    }
    notify { 'No nova section config for this version of OpenStack': }
  }else{
    class { '::cinder::nova':
      auth_url     => $keystone_auth_uri,
      username     => $cinder_user,
      password     => $pass_cinder_authtoken,
      project_name => 'services',
      cafile       => $api_endpoint_ca_file,
      region_name  => $region_name,
      auth_type    => 'password',
    }
  }

  # Clear volumes on delete (for data security)
  class { '::cinder::volume':
    volume_clear        => 'zero',
    volume_clear_ionice => '-c3',
  }

  # A cinder-backup service on each volume nodes
  if $provision_cinder_backup {
    class { '::cinder::backup': }
    if $extswift_use_external {
      class { '::cinder::backup::swift':
        backup_swift_url             => $extswift_proxy_url,
        backup_swift_auth_url        => "${extswift_auth_url}",
        backup_swift_container       => 'volumebackups',
        backup_swift_object_size     => 1048576000,
#        backup_swift_retry_attempts  => $facts['os_service_default'],
#        backup_swift_retry_backoff   => $facts['os_service_default'],
        backup_swift_user_domain     => $extswift_user_domain_name,
        backup_swift_project_domain  => $extswift_project_domain_name,
        backup_swift_project         => $extswift_project_name,
        backup_compression_algorithm => 'zlib',
      }
      cinder_config {
        'DEFAULT/backup_swift_user':         value => $extswift_user_name;
        'DEFAULT/backup_swift_key':          value => $extswift_password;
        'DEFAULT/backup_swift_auth':         value => 'single_user';
        'DEFAULT/backup_swift_auth_version': value => '3';
      }
    }else{
      cinder_config {
        'DEFAULT/backup_swift_auth': value => 'per_user';
      }
    }
  }

  # Avoids Cinder to lookup for the catalogue
  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal' or $openstack_release == 'dalmatian' or $openstack_release == 'epoxy'){
    class { '::cinder::glance':
      glance_api_servers => "${proto}://${base_url_glance}",
    }
  }else{
    class { '::cinder::glance':
      username           => $cinder_user,
      password           => $pass_cinder_authtoken,
      cafile             => $api_endpoint_ca_file,
      glance_api_servers => "${proto}://${base_url_glance}",
      auth_type          => 'password',
      auth_url           => $keystone_admin_uri,
    }
  }

  # Configure the LVM backend
  if $openstack_release == 'rocky' or $openstack_release == 'stein'{
    if $cinder_enabled_backends == "no-override" {
      cinder::backend::iscsi { $lvm_backend_name:
        iscsi_ip_address   => $machine_ip,
        volume_group       => $vgname,
        iscsi_protocol     => 'iscsi',
        extra_options      => {
          "${lvm_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
          "${lvm_backend_name}/volume_clear_size"             => { 'value' => '0' },
          "${lvm_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
          "${lvm_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
        },
        manage_volume_type => true,
      }
    }else{
      $enabled_backend_list.each |Integer $index_backend, String $custom_backend_name| {
        $this_vg_name = "${vgname}${index_backend}"
        cinder::backend::iscsi { $custom_backend_name:
          iscsi_ip_address   => $machine_ip,
          volume_group       => $this_vg_name,
          iscsi_protocol     => 'iscsi',
          extra_options      => {
            "${custom_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
            "${custom_backend_name}/volume_clear_size"             => { 'value' => '0' },
            "${custom_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
            "${custom_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
          },
          manage_volume_type => true,
        }
      }
    }
  }else{
    if $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat'{
      if $cinder_enabled_backends == "no-override" {
        cinder::backend::iscsi { $lvm_backend_name:
          target_ip_address  => $machine_ip,
          volume_group       => $vgname,
          target_protocol    => 'iscsi',
          extra_options      => {
            "${lvm_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
            "${lvm_backend_name}/volume_clear_size"             => { 'value' => '0' },
            "${lvm_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
            "${lvm_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
          },
          manage_volume_type => true,
        }
      }else{
        $enabled_backend_list.each |Integer $index_backend, String $custom_backend_name| {
          $this_vg_name = "${vgname}${index_backend}"
          cinder::backend::iscsi { $custom_backend_name:
            target_ip_address  => $machine_ip,
            volume_group       => $this_vg_name,
            target_protocol    => 'iscsi',
            extra_options      => {
              "${custom_backend_name}/reserved_percentage"           => { 'value' => $cinder_reserved_percentage },
              "${custom_backend_name}/volume_clear_size"             => { 'value' => '0' },
              "${custom_backend_name}/max_over_subscription_ratio"   => { 'value' => $cinder_max_over_subscription_ratio },
              "${custom_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
            },
            manage_volume_type => true,
          }
        }
      }
    }else{
      if $cinder_enabled_backends == "no-override" {
        cinder::backend::iscsi { $lvm_backend_name:
          target_ip_address           => $machine_ip,
          volume_group                => $vgname,
          target_protocol             => 'iscsi',
          reserved_percentage         => $cinder_reserved_percentage,
          max_over_subscription_ratio => $cinder_max_over_subscription_ratio,
          extra_options               => {
            "${lvm_backend_name}/volume_clear_size"             => { 'value' => '0' },
            "${lvm_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
          },
          manage_volume_type => true,
        }
      }else{
        $enabled_backend_list.each |Integer $index_backend, String $custom_backend_name| {
          $this_vg_name = "${vgname}${index_backend}"
          cinder::backend::iscsi { $custom_backend_name:
            target_ip_address           => $machine_ip,
            volume_group                => $this_vg_name,
            target_protocol             => 'iscsi',
            reserved_percentage         => $cinder_reserved_percentage,
            max_over_subscription_ratio => $cinder_max_over_subscription_ratio,
            extra_options      => {
              "${custom_backend_name}/volume_clear_size"             => { 'value' => '0' },
              "${custom_backend_name}/volume_copy_bps_limit"         => { 'value' => $volume_copy_bps_limit }
            },
            manage_volume_type => true,
          }
        }
      }
    }
  }

  # Configure Ceph backend, so that "openstack volume retype" can work
  if $cluster_has_osds {
    cinder::backend::rbd { 'CEPH_1':
      rbd_user           => 'openstack',
      rbd_pool           => 'cinder',
      rbd_secret_uuid    => $ceph_libvirtuuid,
      manage_volume_type => true,
      backend_host       => $machine_hostname,
    }
  }

  if $cinder_enabled_backends == "no-override" {
    if $cluster_has_osds {
      $lvm_backend_name_real = concat([ $lvm_backend_name] , [ 'CEPH_1' ] )
    }else{
      $lvm_backend_name_real = [ $lvm_backend_name ]
    }
    class { '::cinder::backends':
      enabled_backends => $lvm_backend_name_real,
    }
  }else{
    class { '::cinder::backends':
      enabled_backends => $enabled_backend_list,
    }
  }

  if Integer.new($facts['os']['release']['major']) >= 13 {
    $ioname = 'io'
  }else{
    $ioname = 'blkio'
  }

  # Set cgroupv1 config for volume_copy_bps_limit
  exec { 'create cinder-volume-copy cgroupv1':
    path => [ '/usr/bin', '/bin', '/usr/sbin', '/sbin' ],
    command => "cgcreate -g ${ioname}:cinder-volume-copy",
    unless  => "cgget -g ${ioname}:cinder-volume-copy 2> /dev/null",
  }
  logrotate::rule { 'cinder-common':
    path          => '/var/log/cinder/*.log',
    rotate_every  => $logrotate_period,
    missingok     => true,
    compress      => true,
    delaycompress => true,
    ifempty       => false,
    copytruncate  => true,
    rotate        => '4',
  }
}
