# a messaging node (for notifications)
#
class oci::messaging(
  $region_name               = 'RegionOne',
  $openstack_release         = undef,
  $cluster_name              = undef,
  $machine_hostname          = undef,
  $machine_ip                = undef,
  $machine_iface             = undef,

  $all_masters               = undef,
  $all_masters_ip            = undef,
  $first_master              = undef,
  $first_master_ip           = undef,

  $vip_hostname              = undef,
  $vip_ipaddr                = undef,
  $vip_netmask               = undef,

  $sql_vip_ip                = undef,
  $sql_vip_netmask           = undef,
  $sql_vip_iface             = undef,

  $is_first_rabbit           = false,
  $first_rabbit              = undef,
  $all_rabbits               = [],
  $all_rabbits_short_hostnames = [],
  $all_rabbits_ips           = [],
  $all_rabbits_ids           = [],

  $initial_cluster_setup     = false,

  $messaging_vip_ipaddr      = undef,
  $messaging_vip_network     = undef,
  $messaging_vip_cidr        = undef,

  # IP address where to find MySQL for messaging.
  # Set to ip of messaging VIP if there's no sqlmsg
  # set to VIP of sqlmsg otherwise.
  $vip_sqlmsg_ip             = undef,

  $messaging_other_nodes     = undef,
  $messaging_other_nodes_ips = undef,

  $has_subrole_db            = true,
  $has_subrole_heat          = true,
  $has_subrole_glance        = false,
  $has_subrole_nova          = false,
  $has_subrole_neutron       = false,
  $has_subrole_swift         = true,
  $has_subrole_horizon       = true,
  $has_subrole_barbican      = true,
  $has_subrole_mistral       = true,
  $has_subrole_cinder        = false,
  $has_subrole_gnocchi       = false,
  $has_subrole_ceilometer    = false,
  $has_subrole_panko         = false,
  $has_subrole_cloudkitty    = false,
  $has_subrole_aodh          = false,
  $has_subrole_octavia       = false,
  $has_subrole_magnum        = false,
  $has_subrole_manila        = false,
  $has_subrole_designate     = false,

  $use_ssl                   = true,
  $self_signed_api_cert      = true,
  $rabbit_env                = {},
  $pass_haproxy_stats        = undef,
  $pass_mysql_rootuser       = undef,
  $pass_mysql_backup         = undef,
  $pass_rabbitmq_cookie      = undef,
  $pass_rabbitmq_monitoring  = undef,
  $pass_keystone_messaging   = undef,
  $pass_nova_messaging       = undef,
  $pass_glance_messaging     = undef,
  $pass_cinder_messaging     = undef,
  $pass_neutron_messaging    = undef,
  $pass_heat_messaging       = undef,
  $pass_barbican_messaging   = undef,
  $pass_mistral_messaging    = undef,
  $pass_vmms_messaging       = undef,
  $pass_gnocchi_messaging    = undef,
  $pass_panko_messaging      = undef,
  $pass_ceilometer_messaging = undef,
  $pass_cloudkitty_messaging = undef,
  $pass_aodh_messaging       = undef,
  $pass_octavia_messaging    = undef,
  $pass_magnum_messaging     = undef,
  $pass_manila_messaging     = undef,
  $pass_designate_messaging  = undef,
  $pass_swift_messaging      = undef,
  $pass_zabbix_authtoken     = undef,

  $pass_gnocchi_db           = undef,
  $pass_gnocchi_authtoken    = undef,
  $pass_gnocchi_rscuuid      = undef,
  $pass_keystone_adminuser   = undef,

  $cloudkitty_metrics_yml    = undef,
  $cloudkitty_http_pool_maxsize = 10,

  $pass_cloudkitty_db        = undef,
  $pass_cloudkitty_authtoken = undef,

  $pass_elastic_keystore     = undef,
  $pass_cloudkitty_elastic   = undef,

  $ceph_fsid                 = undef,
  $ceph_bootstrap_osd_key    = undef,
  $ceph_admin_key            = undef,
  $ceph_openstack_key        = undef,
  $ceph_mon_key              = undef,
  $ceph_mon_initial_members  = undef,
  $ceph_mon_host             = undef,

  # Zookeeper ensemble
  $zookeeper_ensemble_string = join($all_masters_ip,':2181,'),

  # Variables from variables.json
  $messaging_node_memcache_memory_limit    = 20,
  $messaging_node_memcache_max_connections = 65536,

  $kernel_from_backports    = false,

  $haproxy_timeout_http_request = '10s',
  $haproxy_timeout_queue        = '1m',
  $haproxy_timeout_connect      = '10s',
  $haproxy_timeout_client       = '1m',
  $haproxy_timeout_server       = '1m',
  $haproxy_timeout_check        = '10s',

  $external_keystone_activate              = false,
  $external_keystone_url                   = 'https://api.example.com/identity',
  $external_keystone_admin_password        = undef,
  $external_keystone_region_prefixed_users = false,

  $ceph_use_cephadm_to_deploy = false,

  $cloudkitty_use_storage_v2 = false,
  $cloudkitty_storage_v2_backend = 'elasticsearch',
  $cloudkitty_storage_v2_host = 'https://user:pass@elastic.example.com',
  $cloudkitty_storage_v2_index_name = 'cloudkitty',

  $gnocchi_incoming_sacks_use_redis = false,
  $gnocchi_metricd_workers          = 0,
  $gnocchi_metricd_processing_delay = 10,
  $gnocchi_metricd_reporting_delay  = 120,
  $gnocchi_metricd_cleanup_delay    = 20,
  $gnocchi_metricd_greedy           = false,

  $configure_dkms = false,

  $ceph_configure_osbpo = false,

  $haproxy_use_sni = false,

  $keystone_sni_name   = "identity",

  $rabbitmq_enable_quorum_queues = false,
  $rabbitmq_message_timeout      = 120,

  $use_json_log = false,

  $gnocchi_debug_log    = false,
  $cloudkitty_debug_log = false,
  $logrotate_hourly     = true,

  $cloudkitty_processor_workers = 12,
  $gnocchi_api_workers          = 24,

  $mysql_max_connections        = 10000,
  $mysql_max_user_connections   = 5000,
){

  if $logrotate_hourly {
    $logrotate_period = 'hour'
  }else{
    $logrotate_period = 'day'
  }

  $logging_context_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(user_identity)s] %(instance)s%(message)s'
  $logging_default_format_string = '%(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s'

  if $facts['networking']['fqdn'] {
    $fqdn = $facts['networking']['fqdn']
  }else{
    $fqdn = $::fqdn
  }

  $rabbitmq_transient_queue_expire = $rabbitmq_message_timeout + 60

  if $rabbitmq_enable_quorum_queues {
    $rabbit_ha_queues              = false
    $rabbit_quorum_queue           = true
    $rabbit_transient_quorum_queue = true
    $amqp_durable_queues           = true
    $use_queue_manager             = true
    $rabbit_stream_fanout          = true
    $rabbit_qos_prefetch_count     = 20
  }else{
    $rabbit_ha_queues              = true
    $rabbit_quorum_queue           = false
    $rabbit_transient_quorum_queue = false
    $amqp_durable_queues           = false
    $use_queue_manager             = false
    $rabbit_stream_fanout          = false
    $rabbit_qos_prefetch_count     = $facts['os_service_default']
  }

  if $configure_dkms { class { '::oci::dkms': } }

  # Configure ceph BPO
  class { 'oci::configure_backports':
    ceph_configure_osbpo       => $ceph_configure_osbpo,
  }

  # Generic performances tweak (will default to throughput-performance)
  class { '::oci::tuned': }

  $proto = 'https'
  $messaging_default_port = '5671'
  $messaging_notify_port = '5671'
  $api_port = 443


  # $base_url = "${proto}://${vip_hostname}"
  $base_url = "${vip_hostname}"

  $base_url_keystone   = $haproxy_use_sni ? { true => "${keystone_sni_name}.${vip_hostname}", false => "${base_url}/identity"}

  $keystone_auth_uri   = $external_keystone_activate ? { true => "${external_keystone_url}", false => "${proto}://${base_url_keystone}"}
  $keystone_admin_uri  = $external_keystone_activate ? { true => "${external_keystone_url}", false => "${proto}://${base_url_keystone}"}
  $keystone_admin_pass = $external_keystone_activate ? { true => "${external_keystone_admin_password}", false => "${pass_keystone_adminuser}"}

  $region_suffix = $external_keystone_region_prefixed_users ? {
    true  => "-${region_name}",
    false => ''
  }

  $gnocchi_user          = "gnocchi${region_suffix}"
  $cloudkitty_user       = "cloudkitty${region_suffix}"

  $messaging_all_nodes_ips = concat([$machine_ip], $messaging_other_nodes_ips)
  $memcached_string = join([join($messaging_all_nodes_ips,':11211,'), ':11211'],'')
  $memcached_servers  = ["${memcached_string}"]

  if $self_signed_api_cert {
    $api_endpoint_ca_file = "/etc/ssl/certs/oci-pki-oci-ca-chain.pem"
  }else{
    $api_endpoint_ca_file = ''
  }
  $oci_pki_root_ca_file = '/etc/ssl/certs/oci-pki-oci-ca-chain.pem'

  class { '::oci::puppet_oci_ca_cert':
    self_signed_api_cert => $self_signed_api_cert,
  }

  $sql_host = $vip_sqlmsg_ip

  if $facts['os']['lsb'] != undef{
    $mycodename = $facts['os']['lsb']['distcodename']
  }else{
    $mycodename = $facts['os']['distro']['codename']
  }

  if $mycodename == 'stretch' or $mycodename == 'buster' or $mycodename == 'bullseye'{
    $sql_default_charset = 'utf8'
    $sql_default_collate = 'utf8_general_ci'
  }else{
    $sql_default_charset = 'utf8mb3'
    $sql_default_collate = 'utf8mb3_general_ci'
  }

  ensure_resource('group', 'zabbix', {
    'ensure' => 'present',
    'gid'    => '966',
  })

  ensure_resource('file', '/etc/oci/monitoring_openstack_api_password', {
    'ensure'  => 'present',
    'content' => "${pass_zabbix_authtoken}",
    'group'   => 'zabbix',
    'mode'    => '0640',
  })

  ensure_resource('file', '/etc/oci/monitoring_openstack_api_url', {
    'ensure'  => 'present',
    'content' => "${$keystone_auth_uri}/v3",
    'group'   => 'zabbix',
    'mode'    => '0640',
  })

  ######################
  ### Setup RabbitMQ ###
  ######################
  if $use_ssl {
    file { "/etc/rabbitmq/ssl/private":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      require                 => File['/etc/rabbitmq/ssl'],
      selinux_ignore_defaults => true,
    }->
    file { "/etc/rabbitmq/ssl/public":
      ensure                  => directory,
      owner                   => 'root',
      mode                    => '0755',
      selinux_ignore_defaults => true,
    }->
    file { "/etc/rabbitmq/ssl/private/${fqdn}.key":
      ensure                  => present,
      owner                   => "rabbitmq",
      source                  => "/etc/ssl/private/ssl-cert-snakeoil.key",
      selinux_ignore_defaults => true,
      mode                    => '0600',
    }->
    file { "/etc/rabbitmq/ssl/public/${fqdn}.crt":
      ensure                  => present,
      owner                   => "rabbitmq",
      source                  => '/etc/ssl/certs/ssl-cert-snakeoil.pem',
      selinux_ignore_defaults => true,
      mode                    => '0644',
      notify        => Service['rabbitmq-server'],
    }
    $rabbit_ssl_cert = "/etc/rabbitmq/ssl/public/${fqdn}.crt"
    $rabbit_ssl_key  = "/etc/rabbitmq/ssl/private/${fqdn}.key"
    $rabbit_ssl_ca   = $oci_pki_root_ca_file
  } else {
    $rabbit_ssl_cert = UNSET
    $rabbit_ssl_key  = UNSET
    $rabbit_ssl_ca   = UNSET
  }

  class { '::rabbitmq':
    delete_guest_user           => true,
    node_ip_address             => $machine_ip,
    ssl_interface               => $machine_ip,
    ssl                         => $use_ssl,
    ssl_only                    => false,
    ssl_cacert                  => $rabbit_ssl_ca,
    ssl_cert                    => $rabbit_ssl_cert,
    ssl_key                     => $rabbit_ssl_key,
    environment_variables       => $rabbit_env,
    repos_ensure                => false,
    # Clustering options...
    config_cluster              => true,
    cluster_nodes               => $all_rabbits_short_hostnames,
    cluster_node_type           => 'ram',
    erlang_cookie               => $pass_rabbitmq_cookie,
    wipe_db_on_cookie_change    => true,
    collect_statistics_interval => 60000,
    cluster_partition_handling  => 'autoheal',
    config_variables            => {
        'vm_memory_high_watermark'   => '0.4',
      },
  }->
  rabbitmq_vhost { '/':
    provider => 'rabbitmqctl',
    require  => Class['::rabbitmq'],
  }
  -> file { "/root/reset-rabbitmq-credentials":
    ensure                  => present,
    owner                   => 'root',
    content                 => "#!/bin/sh

set -e

rabbitmqctl add_vhost / || true

check_rb_user () {
	if ! rabbitmqctl list_users 2>&1 | awk '{print \$1}' | grep -q '^'\${1}'$' ; then
		rabbitmqctl add_user \${1} \${2}
	fi
	rabbitmqctl set_user_tags \${1} administrator
	rabbitmqctl set_permissions -p / \${1} '.*' '.*' '.*'
}

check_rb_user keystone ${pass_keystone_messaging}
check_rb_user monitoring ${pass_rabbitmq_monitoring}
check_rb_user glance ${pass_glance_messaging}
check_rb_user heat ${pass_heat_messaging}
check_rb_user barbican ${pass_barbican_messaging}
check_rb_user mistral ${pass_mistral_messaging}
check_rb_user nova ${pass_nova_messaging}
check_rb_user neutron ${pass_neutron_messaging}
check_rb_user cinder ${pass_cinder_messaging}
check_rb_user gnocchi ${pass_gnocchi_messaging}
check_rb_user panko ${pass_panko_messaging}
check_rb_user ceilometer ${pass_ceilometer_messaging}
check_rb_user swift ${pass_swift_messaging}
check_rb_user cloudkitty ${pass_cloudkitty_messaging}
check_rb_user aodh ${pass_aodh_messaging}
check_rb_user octavia ${pass_octavia_messaging}
check_rb_user magnum ${pass_magnum_messaging}
check_rb_user designate ${pass_designate_messaging}
check_rb_user manila ${pass_manila_messaging}

exit 0
",
    selinux_ignore_defaults => true,
    mode                    => '0740',
  }

  if $initial_cluster_setup {
    if $rabbitmq_enable_quorum_queues {
      $auto_join_rabbitmq_cmd = "/usr/bin/oci-auto-join-rabbitmq-cluster ${first_rabbit} --no-ha-queues"
    }else{
      $auto_join_rabbitmq_cmd = "/usr/bin/oci-auto-join-rabbitmq-cluster ${first_rabbit}"
    }
    exec { 'auto-join-rabbit-cluster':
      command => $auto_join_rabbitmq_cmd,
      unless  => "/bin/false",
      require => Class['::rabbitmq'],
    }
    if $is_first_rabbit {
      rabbitmq_user { 'keystone':
        admin    => true,
        password => $pass_keystone_messaging,
        require  => Class['::rabbitmq'],
      }->
      rabbitmq_user_permissions { "keystone@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
        require              => Class['::rabbitmq'],
      }

      rabbitmq_user { 'monitoring':
        admin    => true,
        password => $pass_rabbitmq_monitoring,
        require  => Class['::rabbitmq'],
      }->
      rabbitmq_user_permissions { "monitoring@/":
        configure_permission => '.*',
        write_permission     => '.*',
        read_permission      => '.*',
        provider             => 'rabbitmqctl',
      }


      if $has_subrole_glance {
        rabbitmq_user { 'glance':
          admin    => true,
          password => $pass_glance_messaging,
        }->
        rabbitmq_user_permissions { "glance@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_heat {
        rabbitmq_user { 'heat':
          admin    => true,
          password => $pass_heat_messaging,
        }->
        rabbitmq_user_permissions { "heat@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_barbican {
        rabbitmq_user { 'barbican':
          admin    => true,
          password => $pass_barbican_messaging,
        }->
        rabbitmq_user_permissions { "barbican@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_mistral {
        rabbitmq_user { 'mistral':
          admin    => true,
          password => $pass_mistral_messaging,
        }->
        rabbitmq_user_permissions { "mistral@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }

      if $has_subrole_nova {
        rabbitmq_user { 'nova':
          admin    => true,
          password => $pass_nova_messaging,
        }->
        rabbitmq_user_permissions { "nova@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_neutron {
        rabbitmq_user { 'neutron':
          admin    => true,
          password => $pass_neutron_messaging,
        }->
        rabbitmq_user_permissions { "neutron@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_cinder {
        rabbitmq_user { 'cinder':
          admin    => true,
          password => $pass_cinder_messaging,
        }->
        rabbitmq_user_permissions { "cinder@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_gnocchi {
        rabbitmq_user { 'gnocchi':
          admin    => true,
          password => $pass_gnocchi_messaging,
        }->
        rabbitmq_user_permissions { "gnocchi@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_panko {
        rabbitmq_user { 'panko':
          admin    => true,
          password => $pass_panko_messaging,
        }->
        rabbitmq_user_permissions { "panko@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_ceilometer {
        rabbitmq_user { 'ceilometer':
          admin    => true,
          password => $pass_ceilometer_messaging,
        }->
        rabbitmq_user_permissions { "ceilometer@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_swift {
        rabbitmq_user { 'swift':
          admin    => true,
          password => $pass_swift_messaging,
        }->
        rabbitmq_user_permissions { "swift@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_cloudkitty {
        rabbitmq_user { 'cloudkitty':
          admin    => true,
          password => $pass_cloudkitty_messaging,
        }->
        rabbitmq_user_permissions { "cloudkitty@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_aodh {
        rabbitmq_user { 'aodh':
          admin    => true,
          password => $pass_aodh_messaging,
        }->
        rabbitmq_user_permissions { "aodh@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_octavia {
        rabbitmq_user { 'octavia':
          admin    => true,
          password => $pass_octavia_messaging,
        }->
        rabbitmq_user_permissions { "octavia@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_magnum {
        rabbitmq_user { 'magnum':
          admin    => true,
          password => $pass_magnum_messaging,
        }->
        rabbitmq_user_permissions { "magnum@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_manila {
        rabbitmq_user { 'manila':
          admin    => true,
          password => $pass_manila_messaging,
        }->
        rabbitmq_user_permissions { "manila@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
      if $has_subrole_designate {
        rabbitmq_user { 'designate':
          admin    => true,
          password => $pass_designate_messaging,
        }->
        rabbitmq_user_permissions { "designate@/":
          configure_permission => '.*',
          write_permission     => '.*',
          read_permission      => '.*',
          provider             => 'rabbitmqctl',
          require              => Class['::rabbitmq'],
        }
      }
    }
  }

  ################################
  ### Add a virtual IP address ###
  ################################
  # Setup corosync
  if ($openstack_release == 'rocky' or $openstack_release == 'stein') {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      unicast_addresses        => $all_rabbits_ips,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_rabbits_ips,
      quorum_members_ids       => $all_rabbits_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  } else {
    class { 'corosync':
      authkey                  => '/var/lib/puppet/ssl/certs/ca.pem',
      bind_address             => $machine_ip,
      cluster_name             => 'mycluster',
      enable_secauth           => true,
      set_votequorum           => true,
      quorum_members           => $all_rabbits_ips,
      quorum_members_ids       => $all_rabbits_ids,
      log_stderr               => false,
      log_function_name        => true,
      syslog_priority          => 'debug',
      debug                    => true,
      enable_totem_interface   => false,
    }
    corosync::service { 'pacemaker':
      version => '0',
    }->
    cs_property { 'stonith-enabled':
      value   => 'false',
    }->
    cs_property { 'no-quorum-policy':
      value   => 'stop',
    }
  }
  if $messaging_vip_cidr == 32 {
    $vip_messaging_iface = 'lo'
  } else {
    $vip_messaging_iface = $sql_vip_iface
  }
  if $initial_cluster_setup {
    cs_primitive { 'openstack-messaging-vip':
      primitive_class => 'ocf',
      primitive_type  => 'IPaddr2',
      provided_by     => 'heartbeat',
      parameters      => { 'ip' => $messaging_vip_ipaddr, 'cidr_netmask' => "${messaging_vip_cidr}", 'nic' => $vip_messaging_iface },
      operations      => { 'monitor' => { 'interval' => '10s' } },
      require         => Cs_property['no-quorum-policy'],
    }
  }

  #####################
  ### Setup haproxy ###
  #####################
  # Here, we assume that stretch & buster don't have
  # a version of haproxy >= 2.1.
  if $mycodename == 'stretch' or $mycodename == 'buster' {
    # This used to work with haproxy 1.8,
    # let's see if we can get rid of it
    $haproxy_replace_path = 'reqrep'
    $haproxy_replace_head = 'rspirep'
    $use_haproxy_21_format = false
  }else{
    # This works with haproxy >= 2.1
    $haproxy_replace_path  = 'http-request replace-path'
    $haproxy_replace_head  = 'http-response replace-header'
    $use_haproxy_21_format = true
  }

  $haproxy_schedule = ['gnocchi::db::begin', 'cloudkitty::db::begin'];

  package { 'haproxy-cmd':
    ensure => present,
  }

  class { 'haproxy':
    restart_command => 'systemctl reload haproxy',
    global_options   => {
      'log'     => '/dev/log local0',
      'maxconn' => '40960',
      'user'    => 'haproxy',
      'group'   => 'haproxy',
      'stats'   => [
        'socket /var/lib/haproxy/stats',
        'socket /var/lib/haproxy/admin.sock mode 600 level admin',
        'timeout 30s'
      ],
      'daemon'   => '',
      'nbthread' => '4',
      'ssl-default-bind-ciphers'   => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-bind-options'   => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
      'ssl-default-server-ciphers' => 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256',
      'ssl-default-server-options' => 'no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets',
    },
    defaults_options => {
      'mode'      => 'http',
      'option'    => [
          'httplog',
        ],
      'timeout' => [
          "http-request ${haproxy_timeout_http_request}",
          "queue ${haproxy_timeout_queue}",
          "connect ${haproxy_timeout_connect}",
          "client ${haproxy_timeout_client}",
          "server ${haproxy_timeout_server}",
          "check ${haproxy_timeout_check}",
        ],
      'stats'     => 'enable',
    },
    merge_options => true,
    before        => Anchor[$haproxy_schedule],
    require       => Sysctl::Value['net.ipv4.ip_nonlocal_bind'],
  }->
  haproxy::listen { 'hapstats':
    section_name => 'statshttp',
    bind         => { "${machine_ip}:8088" => 'user root'},
    mode         => 'http',
    options      => {
        'stats' => [ 'uri /', "auth admin:${pass_haproxy_stats}", 'refresh 15', 'admin if TRUE', 'realm Haproxy Statistics', ],
      },
  }->
  haproxy::listen { 'haphealthcheck':
    section_name => 'healthcheck',
    bind         => { "${machine_ip}:8081" => 'user root'},
    mode         => 'http',
    options => [
      { 'option' => 'httpchk' },
    ]
  }

  # Define galera frontend and backend
  haproxy::frontend { 'galerafe':
    mode      => 'tcp',
    bind      => { "${messaging_vip_ipaddr}:3306" => [] },
    options   => [
      { 'timeout'         => 'client 3600s'},
      { 'default_backend' => 'galerabe'},
    ],
    before => Anchor[$haproxy_schedule],
  }
  haproxy::backend { 'galerabe':
    options => [
      { 'mode'    => 'tcp' },
      { 'balance' => 'roundrobin' },
      { 'timeout' => 'check 5000' },
      { 'timeout' => 'server 3600s' },
      { 'option'  => 'httpchk'},
    ],
    before => Anchor[$haproxy_schedule],
  }
  haproxy::balancermember { 'galerabm':
    listening_service => 'galerabe',
    ipaddresses       => $machine_ip,
    server_names      => $machine_hostname,
    options           => 'check inter 4000 port 9200 fall 3 rise 5',
    before => Anchor[$haproxy_schedule],
  }
  haproxy::balancermember { 'galerabmback':
    listening_service => 'galerabe',
    ipaddresses       => $messaging_other_nodes_ips,
    server_names      => $messaging_other_nodes,
    options           => 'check inter 4000 port 9200 fall 3 rise 5 backup',
    before => Anchor[$haproxy_schedule],
  }
  if $gnocchi_incoming_sacks_use_redis{
    haproxy::frontend { 'redisfe':
      mode      => 'tcp',
      bind      => { "${messaging_vip_ipaddr}:6380" => [] },
      options   => [
        { 'timeout'         => 'client 3600s'},
        { 'default_backend' => 'redisbe'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::backend { 'redisbe':
      options => [
        { 'mode'    => 'tcp' },
        { 'balance' => 'roundrobin' },
        { 'timeout' => 'check 5000' },
        { 'timeout' => 'server 3600s' },
        { 'option'  => 'redis-check'},
      ],
      before => Anchor[$haproxy_schedule],
    }
    haproxy::balancermember { 'redisbm':
      listening_service => 'redisbe',
      ipaddresses       => $all_rabbits_ips,
      server_names      => $all_rabbits,
      options           => 'check check-ssl ssl verify required ca-file /etc/ssl/certs/oci-pki-oci-ca-chain.pem inter 4000 port 6380 fall 3 rise 5',
      before            => Anchor[$haproxy_schedule],
    }
  }

  #######################
  ### Setup memcached ###
  #######################
  class { '::memcached':
    listen_ip       => $machine_ip,
    udp_port        => 0,
    max_memory      => "${messaging_node_memcache_memory_limit}%",
    max_connections => $messaging_node_memcache_max_connections,
    pidfile         => '/var/run/memcached/memcached.pid',
    user            => 'memcache',
  }

  #######################
  ### Setup databases ###
  #######################
  if $initial_cluster_setup and $has_subrole_db {
    if $mycodename != 'stretch'{
      package { 'mariadb-backup':
        ensure => present,
        before => Class['galera'],
      }
      $wsrep_sst_method = 'mariabackup'
    }else{
      $wsrep_sst_method = 'mariabackup'
    }

    if $is_first_rabbit {
      # If this is the first node, it must publish itself as working
      # before waiting for the other nodes to join.
      Class['galera::status'] -> Exec['galera-size-is-correct']
      Service['xinetd'] -> Exec['galera-size-is-correct']
    }else{
      exec {'galera-first-master-is-up':
      command => "/usr/bin/oci-wait-for-first-galera-node ${$first_rabbit} 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        timeout => 5000,
      }
      Exec['galera-first-master-is-up'] -> Class['galera']
    }

    if $mycodename == 'stretch' or $mycodename == 'buster' {
      $galera_package_name = 'galera-3'
    }else{
      $galera_package_name = 'galera-4'
    }

    $ram_total_in_mb = Integer($facts['memorysize_mb'])
    $ram_total_in_gb = $ram_total_in_mb / 1024
    $ram_total_in_gb_div_4 = $ram_total_in_gb / 4

    if $ram_total_in_gb_div_4 > 64 {
      $innodb_buffer_pool_size = 64
    }elsif $ram_total_in_gb_div_4 > 32{
      $innodb_buffer_pool_size = 32
    }elsif $ram_total_in_gb_div_4 > 16{
      $innodb_buffer_pool_size = 16
    }elsif $ram_total_in_gb_div_4 > 8{
      $innodb_buffer_pool_size = 8
    }elsif $ram_total_in_gb_div_4 > 4{
      $innodb_buffer_pool_size = 4
    }else{
      $innodb_buffer_pool_size = 2
    }

    $pre_trixie_options = {
      'bind_address'                    => $machine_ip,
      'wait_timeout'                    => '28800',
      'interactive_timeout'             => '30',
      'connect_timeout'                 => '30',
      'character_set_server'            => 'utf8',
      'collation_server'                => 'utf8_general_ci',
      'innodb_buffer_pool_size'         => "${innodb_buffer_pool_size}G",
      'innodb_flush_log_at_trx_commit'  => '2',
      'max_connections'                 => "${mysql_max_connections}",
      'max_user_connections'            => "${mysql_max_user_connections}",
      'binlog_cache_size'               => '1M',
      'log-bin'                         => 'mysql-bin',
      'binlog_format'                   => 'ROW',
      'performance_schema'              => '1',
      'log_warnings'                    => '2',
      'wsrep_sst_auth'                  => "backup:${pass_mysql_backup}",
      'wsrep_sst_method'                => $wsrep_sst_method,
      'wsrep_cluster_name'              => $cluster_name,
      'wsrep_node_name'                 => $machine_hostname,
    }
    if $mycodename == 'stretch' or $mycodename == 'buster' or $mycodename == 'bullseye' or $mycodename == 'bookworm'{
      $mysqld_options = $pre_trixie_options
    }else{
      $mysqld_options = deep_merge($pre_trixie_options, { 'innodb_snapshot_isolation' => 'OFF' })
    }

    class { 'galera':
      galera_servers      => $all_rabbits_ips,
      galera_master       => $first_rabbit,
      local_ip            => $machine_ip,
      mysql_package_name  => 'mariadb-server',
      client_package_name => 'default-mysql-client',
      vendor_type         => 'mariadb',
      root_password       => $pass_mysql_rootuser,
      status_password     => $pass_mysql_rootuser,
      deb_sysmaint_password => $pass_mysql_rootuser,
      configure_repo      => false,
      configure_firewall  => false,
      galera_package_name => $galera_package_name,
      override_options => {
        'mysqld' => $mysqld_options
      },
      require             => Cs_primitive['openstack-messaging-vip'],
    }->
    mysql_user { 'backup@%':
      ensure        => present,
      password_hash => mysql_password($pass_mysql_backup),
    }->
    mysql_grant{'backup@%/*.*':
      ensure     => 'present',
      options    => ['GRANT'],
      privileges => ['SELECT', 'PROCESS', 'CREATE TEMPORARY TABLES', 'LOCK TABLES', 'SHOW VIEW', 'RELOAD', 'TRIGGER', 'REPLICATION CLIENT'],
      table      => '*.*',
      user       => 'backup@%',
    }->

    # Wait until SHOW STATUS LIKE 'wsrep_cluster_status' shows Primary
    exec {'galera-is-up':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_status'\" Primary mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_connected' shows ON
    exec {'galera-wsrep-connected-on':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_connected'\" ON mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-up'],
      timeout => 5000,
    }

    # Wait until SHOW STATUS LIKE 'wsrep_local_state_comment' shows Synced
    exec {'galera-is-synced':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_local_state_comment'\" Synced mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-wsrep-connected-on'],
      timeout => 5000,
    }

    # Wait until all nodes are connected to the cluster
    $galera_cluster_num_of_nodes = sprintf('%i', $all_masters.size)
    exec {'galera-size-is-correct':
      command => "/usr/bin/oci-wait-for-sql \"SHOW STATUS LIKE 'wsrep_cluster_size'\" ${galera_cluster_num_of_nodes} mysql 4800",
      unless  => '/bin/false # comment to satisfy puppet syntax requirements',
      require => Exec['galera-is-synced'],
      timeout => 5000,
    }

    ######################################
    ### Create or wait for db creation ###
    ######################################
    # Gnocchi DB
    if $is_first_rabbit {
      class { '::gnocchi::db::mysql':
        dbname        => 'gnocchidb',
        password      => $pass_gnocchi_db,
        allowed_hosts => '%',
        charset       => $sql_default_charset,
        collate       => $sql_default_collate,
        require       => Exec['galera-size-is-correct'],
        before        => Anchor['gnocchi::service::begin'],
      }
    } else {
      exec { 'gnocchi-db-user':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='gnocchi'\" gnocchi mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['gnocchi::service::begin'],
        timeout => 5000,
      }
    }

    # Cloudkitty DB
    if $is_first_rabbit {
      class { '::cloudkitty::db::mysql':
        dbname        => 'cloudkittydb',
        password      => $pass_cloudkitty_db,
        allowed_hosts => '%',
        charset       => $sql_default_charset,
        collate       => $sql_default_collate,
        require       => Exec['galera-size-is-correct'],
        before        => Anchor['cloudkitty::service::begin'],
      }
    } else {
      exec { 'cloudkitty-db-user':
        command => "/usr/bin/oci-wait-for-sql \"SELECT User FROM user WHERE User='cloudkitty'\" cloudkitty mysql 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Exec['galera-size-is-correct'],
        before  => Anchor['cloudkitty::service::begin'],
        timeout => 5000,
      }
    }
  }
  if $initial_cluster_setup == true and $has_subrole_db == false {
    package {'default-mysql-client':
      ensure => present,
    }
    if $is_first_rabbit {
      exec {'wait-for-cloudkitty-db':
        command => "/usr/bin/oci-wait-for-remote-sql --sql-query \"SHOW TABLES\" --server-address ${sql_host} --db-name cloudkittydb --username cloudkitty --password ${pass_cloudkitty_db} --timeout 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Package['default-mysql-client'],
        before  => Anchor['cloudkitty::service::begin'],
        timeout => 5000,
      }
    }else{
      exec {'wait-for-cloudkitty-db':
        command => "/usr/bin/oci-wait-for-remote-sql --sql-query \"DESCRIBE hashmap_fields\" --server-address ${sql_host} --db-name cloudkittydb --username cloudkitty --password ${pass_cloudkitty_db} --timeout 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => [Package['default-mysql-client'], Service['rabbitmq-server']],
        before  => Anchor['cloudkitty::service::begin'],
        timeout => 5000,
      }
    }
    if $is_first_rabbit {
      exec {'wait-for-gnocchi-db':
        command => "/usr/bin/oci-wait-for-remote-sql --sql-query \"SHOW TABLES\" --server-address ${sql_host} --db-name gnocchidb --username gnocchi --password ${pass_gnocchi_db} --timeout 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => Package['default-mysql-client'],
        before  => Anchor['gnocchi::service::begin'],
        timeout => 5000,
      }
    }else{
      exec {'wait-for-gnocchi-db':
        command => "/usr/bin/oci-wait-for-remote-sql --sql-query \"DESCRIBE metric\" --server-address ${sql_host} --db-name gnocchidb --username gnocchi --password ${pass_gnocchi_db} --timeout 4800",
        unless  => '/bin/false # comment to satisfy puppet syntax requirements',
        require => [Service['rabbitmq-server'], Package['default-mysql-client']],
        before  => Anchor['gnocchi::service::begin'],
        timeout => 5000,
      }
    }
  }
  ##################
  ### Setup Ceph ###
  ##################
  class { 'ceph':
    fsid                => $ceph_fsid,
    ensure              => 'present',
    authentication_type => 'cephx',
    mon_initial_members => $ceph_mon_initial_members,
    mon_host            => $ceph_mon_host,
  }

  if $ceph_use_cephadm_to_deploy == false {
    ceph::key { 'client.admin':
      secret  => $ceph_admin_key,
      cap_mon => 'allow *',
      cap_osd => 'allow *',
      cap_mds => 'allow',
      require => Class['ceph'],
    }
    -> ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=gnocchi',
    }
    $ceph_pools = ['gnocchi']
    ceph::pool { $ceph_pools: }
    class { '::ceph::profile::params':
      fsid                => $ceph_fsid,
      manage_repo         => false,
      release             => 'nautilus',
      authentication_type => 'cephx',
      mon_host            => $ceph_mon_host,
      mon_initial_members => $ceph_mon_initial_members,
      client_keys         => {
        'client.admin' => {
          secret  => $ceph_admin_key,
          cap_mon => 'allow *',
          cap_osd => 'allow *',
          cap_mds => 'allow',
        },
        'client.openstack' => {
          secret  => $ceph_openstack_key,
          mode    => '0644',
          cap_mon => 'profile rbd',
          cap_osd => 'profile rbd pool=gnocchi',
        }
      }
    }
  }else{
    ceph::key { 'client.openstack':
      secret  => $ceph_openstack_key,
      mode    => '0644',
      cap_mon => 'profile rbd',
      cap_osd => 'profile rbd pool=gnocchi',
      require => Class['ceph'],
    }
  }

  #####################
  ### Setup Gnocchi ###
  #####################
  oci::sslkeypair {'gnocchi':
    notify_service_name => 'gnocchi-api',
  }
  $gnocchi_key_file = "/etc/gnocchi/ssl/private/${fqdn}.pem"
  $gnocchi_crt_file = "/etc/gnocchi/ssl/public/${fqdn}.crt"
  if $has_subrole_db {
    $gnocchi_sql_host = $machine_ip
  }else{
    $gnocchi_sql_host = $sql_host
  }
  if $openstack_release == 'rocky'{
    class { '::gnocchi':
      debug                 => true,
      database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${gnocchi_sql_host}/gnocchidb?charset=utf8",
    }
  }else{
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::gnocchi':
        database_connection   => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${gnocchi_sql_host}/gnocchidb?charset=utf8",
      }
    } else {
      class { '::gnocchi':
        coordination_url          => "zookeeper://${zookeeper_ensemble_string}:2181/",
        coordination_url_metricd  => $facts['os_service_default'],
        use_multiple_config_files => $gnocchi_incoming_sacks_use_redis,
      }
      class { 'gnocchi::db':
        database_connection => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${gnocchi_sql_host}/gnocchidb?charset=utf8",
      }
    }
    class { '::gnocchi::logging':
      debug    => $gnocchi_debug_log,
      use_json => $use_json_log,
    }
  }

  class { '::gnocchi::keystone::authtoken':
    username                   => $gnocchi_user,
    password                   => $pass_gnocchi_authtoken,
    auth_url                   => $keystone_admin_uri,
    www_authenticate_uri       => $keystone_auth_uri,
    memcached_servers          => $memcached_servers,
    memcache_use_advanced_pool => true,
    token_cache_time           => 3600,
    cafile                     => $api_endpoint_ca_file,
    region_name                => $region_name,
  }

  if $gnocchi_incoming_sacks_use_redis {
    $sync_extra_opts = '--config-file /etc/gnocchi/gnocchi.conf --config-file /etc/gnocchi/gnocchi-metricd.conf'
  } else {
    $sync_extra_opts = '--config-file /etc/gnocchi/gnocchi.conf'
  }

  # db::sync is included by api, call it first
  class { '::gnocchi::db::sync':
    extra_opts => $sync_extra_opts,
  }
  class { '::gnocchi::api':
    enabled                      => true,
    service_name                 => 'gnocchi-api',
    sync_db                      => false,
    enable_proxy_headers_parsing => true,
  }

  class { '::gnocchi::client': }
  # Fix the number of uwsgi processes
  class { '::gnocchi::wsgi::uwsgi':
    processes => $gnocchi_api_workers,
  }
  gnocchi_api_uwsgi_config {
    'uwsgi/disable-logging': value => false;
    'uwsgi/close-on-exec':   value => true;
    'uwsgi/close-on-exec2':  value => true;
  }
  if $gnocchi_metricd_workers == 0 {
    $gnocchi_metricd_workers_real = $facts['os_workers']
  }else{
    $gnocchi_metricd_workers_real = $gnocchi_metricd_workers
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::gnocchi::metricd':
      workers       => $gnocchi_metricd_workers_real,
      cleanup_delay => 20,
    }
  }elsif $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    class { '::gnocchi::metricd':
      workers                 => $gnocchi_metricd_workers_real,
      metric_processing_delay => $gnocchi_metricd_processing_delay,
      cleanup_delay           => $gnocchi_metricd_cleanup_delay,
    }
    gnocchi_config {
      'metricd/metric_reporting_delay':  value => $gnocchi_metricd_reporting_delay;
      'metricd/greedy':                  value => $gnocchi_metricd_greedy;
    }
  }else{
    class { '::gnocchi::metricd':
      workers                 => $gnocchi_metricd_workers_real,
      metric_cleanup_delay    => $gnocchi_metricd_cleanup_delay,
      metric_processing_delay => $gnocchi_metricd_processing_delay,
      metric_reporting_delay  => $gnocchi_metricd_reporting_delay,
      greedy                  => $gnocchi_metricd_greedy,
    }
  }

  systemd::dropin_file { 'runtime_maxsec.conf':
    unit    => 'gnocchi-metricd.service',
    content => "[Service]
RuntimeMaxSec=604800
",
  }

  # The limit to $facts['processorcount'] is mostly for the virtualized PoC.
  $parallel_operations = min($facts['os_workers'] * 4, $facts['processors']['count'] * 0.5)
  if $parallel_operations < 2 {
    $parallel_operations_real = 2
  }else{
    $parallel_operations_real = Integer($parallel_operations)
  }
  gnocchi_config {
    'DEFAULT/parallel_operations': value => $parallel_operations_real;
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    class { '::gnocchi::storage':
      metric_processing_delay => $gnocchi_metric_processing_delay,
      coordination_url        => "zookeeper://${zookeeper_ensemble_string}:2181/",
    }
  }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    class { '::gnocchi::storage::ceph':
      ceph_username => 'openstack',
      ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
      manage_cradox => false,
      manage_rados  => true,
    }
  }else{
    class { '::gnocchi::storage::ceph':
      ceph_username => 'openstack',
      ceph_keyring  => '/etc/ceph/ceph.client.openstack.keyring',
      manage_rados  => true,
    }
  }

  if $gnocchi_incoming_sacks_use_redis{
    class { '::oci::redis':
      bind_host        => $machine_ip,
      machine_hostname => $machine_hostname,
    }
    class { '::gnocchi::storage::incoming::redis':
      use_multiple_config_files => true,
      redis_url                 => "redis://@${messaging_vip_ipaddr}:6380?db=0",
      redis_url_metricd         => "redis://@${machine_ip}:6379?db=0",
    }
    gnocchi_metricd_config {
      'indexer/url': value => $facts['os_service_default'];
    }
    gnocchi_api_config {
      'indexer/url': value => $facts['os_service_default'];
    }
  }else{
    package { 'redis-server':
      ensure => absent,
    }
  }

  if($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena' or $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope' or $openstack_release == 'bobcat' or $openstack_release == 'caracal' or $openstack_release == 'dalmatian' or $openstack_release == 'epoxy'){
    gnocchi_config {
      'database/connection': value => "mysql+pymysql://gnocchi:${pass_gnocchi_db}@${sql_host}/gnocchidb?charset=utf8", secret => true;
    }
  }

  class { '::gnocchi::statsd':
    archive_policy_name => 'ik-medium-rate',
    flush_delay         => '100',
    # random datas:
    resource_id         => $pass_gnocchi_rscuuid,
  }
  logrotate::rule { 'gnocchi-common':
    path          => '/var/log/gnocchi/*.log',
    rotate_every  => $logrotate_period,
    missingok     => true,
    compress      => true,
    delaycompress => true,
    ifempty       => false,
    copytruncate  => true,
    rotate        => '4',
  }

  ########################
  ### Setup CloudKitty ###
  ########################
  if $initial_cluster_setup and $has_subrole_db {
    Class['galera'] -> Anchor['cloudkitty::install::begin']
  }
  oci::sslkeypair {'cloudkitty':
    notify_service_name => 'cloudkitty-api',
  }
  $cloudkitty_key_file = "/etc/cloudkitty/ssl/private/${fqdn}.pem"
  $cloudkitty_crt_file = "/etc/cloudkitty/ssl/public/${fqdn}.crt"
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train'{
    class { '::cloudkitty::db':
      database_connection   => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
      database_idle_timeout => 1800,
    }
  }else{
    class { '::cloudkitty::db':
      database_connection              => "mysql+pymysql://cloudkitty:${pass_cloudkitty_db}@${sql_host}/cloudkittydb?charset=utf8",
      database_connection_recycle_time => 1800,
    }
  }
  $cloudkitty_notif_transport_url = os_transport_url({
                                      'transport' => 'rabbit',
                                      'hosts'     => fqdn_rotate($all_rabbits),
                                      'port'      => '5671',
                                      'username'  => 'cloudkitty',
                                      'password'  => $pass_cloudkitty_messaging,
                                    })

  if $cloudkitty_use_storage_v2 {
    $storage_backend = $cloudkitty_storage_v2_backend
    $storage_version = 2
  }else{
    $storage_backend = 'sqlalchemy'
    $storage_version = 1
  }

  if $cloudkitty_use_storage_v2 {
    # To be called before ::cloudkitty because it's included there.
    class { '::cloudkitty::storage': }
  }

  if $openstack_release == 'rocky'{
    class { '::cloudkitty':
      default_transport_url              => os_transport_url({
                                                      'transport' => 'rabbit',
                                                      'hosts'     => fqdn_rotate($all_masters),
                                                      'port'      => '5671',
                                                      'username'  => 'cloudkitty',
                                                      'password'  => $pass_cloudkitty_messaging,
                                                    }),
      notification_transport_url         => $cloudkitty_notif_transport_url,
      rabbit_use_ssl                     => $use_ssl,
      rpc_response_timeout               => $rabbitmq_message_timeout,
      amqp_durable_queues                => false,
      amqp_auto_delete                   => false,
      rabbit_ha_queues                   => true,
      kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
      tenant_fetcher_backend             => 'keystone',
      auth_section                       => 'keystone_authtoken',
      host                               => $machine_hostname,
      storage_backend                    => $storage_backend,
      storage_version                    => $storage_version,
      rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
      rabbit_heartbeat_timeout_threshold => 10,
      rabbit_heartbeat_rate              => 2,
      kombu_failover_strategy            => 'round-robin',
    }
  }else{
    if ($openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
      class { '::cloudkitty':
        default_transport_url              => os_transport_url({
                                                        'transport' => 'rabbit',
                                                        'hosts'     => fqdn_rotate($all_masters),
                                                        'port'      => '5671',
                                                        'username'  => 'cloudkitty',
                                                        'password'  => $pass_cloudkitty_messaging,
                                                      }),
        notification_transport_url         => $cloudkitty_notif_transport_url,
        rabbit_use_ssl                     => true,
        rpc_response_timeout               => $rabbitmq_message_timeout,
        amqp_durable_queues                => false,
        amqp_auto_delete                   => false,
        rabbit_ha_queues                   => true,
        kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
        tenant_fetcher_backend             => 'keystone',
        auth_section                       => 'keystone_authtoken',
        host                               => $machine_hostname,
        storage_backend                    => $storage_backend,
        storage_version                    => $storage_version,
        rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
        rabbit_heartbeat_timeout_threshold => 10,
        rabbit_heartbeat_rate              => 2,
        kombu_failover_strategy            => 'round-robin',
      }
      if ($openstack_release == 'train' or $openstack_release == 'ussuri') {
        cloudkitty_config {
          'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
        }
      }
    }else{
      if $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
        class { '::cloudkitty':
          default_transport_url              => os_transport_url({
                                                          'transport' => 'rabbit',
                                                          'hosts'     => fqdn_rotate($all_masters),
                                                          'port'      => '5671',
                                                          'username'  => 'cloudkitty',
                                                          'password'  => $pass_cloudkitty_messaging,
                                                        }),
          notification_transport_url         => $cloudkitty_notif_transport_url,
          rabbit_use_ssl                     => $use_ssl,
          rpc_response_timeout               => $rabbitmq_message_timeout,
          amqp_durable_queues                => false,
          amqp_auto_delete                   => false,
          rabbit_ha_queues                   => true,
          kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
          fetcher_backend                    => 'keystone',
          auth_section                       => 'ociadmin_authtoken',
          host                               => $machine_hostname,
          storage_backend                    => $storage_backend,
          storage_version                    => $storage_version,
          rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
          rabbit_heartbeat_timeout_threshold => 10,
          rabbit_heartbeat_rate              => 2,
          kombu_failover_strategy            => 'round-robin',
        }
        cloudkitty_config {
          'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
        }
      # Yoga and up
      }else{
        class { '::cloudkitty::fetcher::keystone':
          auth_section            => 'ociadmin_authtoken',
          ignore_rating_role      => true,
          ignore_disabled_tenants => true,
        }
        if $openstack_release == 'yoga' or $openstack_release == 'zed' or $openstack_release == 'antelope'{
          class { '::cloudkitty':
            default_transport_url              => os_transport_url({
                                                            'transport' => 'rabbit',
                                                            'hosts'     => fqdn_rotate($all_masters),
                                                            'port'      => '5671',
                                                            'username'  => 'cloudkitty',
                                                            'password'  => $pass_cloudkitty_messaging,
                                                          }),
            notification_transport_url         => $cloudkitty_notif_transport_url,
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => false,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => true,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            fetcher_backend                    => 'keystone',
            host                               => $machine_hostname,
            storage_backend                    => $storage_backend,
            storage_version                    => $storage_version,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
          }
          cloudkitty_config {
            'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
          }
        }elsif $openstack_release == 'bobcat'{
          class { '::cloudkitty':
            default_transport_url              => os_transport_url({
                                                            'transport' => 'rabbit',
                                                            'hosts'     => fqdn_rotate($all_masters),
                                                            'port'      => '5671',
                                                            'username'  => 'cloudkitty',
                                                            'password'  => $pass_cloudkitty_messaging,
                                                          }),
            notification_transport_url         => $cloudkitty_notif_transport_url,
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => $amqp_durable_queues,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => $rabbit_ha_queues,
            rabbit_quorum_queue                => $rabbit_quorum_queue,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            fetcher_backend                    => 'keystone',
            host                               => $machine_hostname,
            storage_backend                    => $storage_backend,
            storage_version                    => $storage_version,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
          }
          cloudkitty_config {
            'oslo_messaging_rabbit/enable_cancel_on_failover': value => true;
          }
        # Caracal and up.
        }else{
          class { '::cloudkitty':
            default_transport_url              => os_transport_url({
                                                            'transport' => 'rabbit',
                                                            'hosts'     => fqdn_rotate($all_masters),
                                                            'port'      => '5671',
                                                            'username'  => 'cloudkitty',
                                                            'password'  => $pass_cloudkitty_messaging,
                                                          }),
            notification_transport_url         => $cloudkitty_notif_transport_url,
            rabbit_use_ssl                     => $use_ssl,
            rpc_response_timeout               => $rabbitmq_message_timeout,
            amqp_durable_queues                => $amqp_durable_queues,
            amqp_auto_delete                   => false,
            rabbit_ha_queues                   => $rabbit_ha_queues,
            rabbit_quorum_queue                => $rabbit_quorum_queue,
            rabbit_transient_quorum_queue      => $rabbit_transient_quorum_queue,
            rabbit_use_queue_manager           => $use_queue_manager,
            rabbit_stream_fanout               => $rabbit_stream_fanout,
            rabbit_qos_prefetch_count          => $rabbit_qos_prefetch_count,
            kombu_ssl_ca_certs                 => $oci_pki_root_ca_file,
            fetcher_backend                    => 'keystone',
            host                               => $machine_hostname,
            storage_backend                    => $storage_backend,
            storage_version                    => $storage_version,
            rabbit_transient_queues_ttl        => $rabbitmq_transient_queue_expire,
            rabbit_heartbeat_timeout_threshold => 10,
            rabbit_heartbeat_rate              => 2,
            kombu_failover_strategy            => 'round-robin',
            rabbit_enable_cancel_on_failover   => true,
          }
        }
      }
      class { '::cloudkitty::logging':
        debug                         => $cloudkitty_debug_log,
        use_json                      => $use_json_log,
	logging_context_format_string => $logging_context_format_string,
	logging_default_format_string => $logging_default_format_string,
      }
    }
  }

  class { '::cloudkitty::keystone::authtoken':
    username                   => $cloudkitty_user,
    password                   => $pass_cloudkitty_authtoken,
    auth_url                   => $keystone_admin_uri,
    www_authenticate_uri       => $keystone_auth_uri,
    memcached_servers          => $memcached_servers,
    memcache_use_advanced_pool => true,
    token_cache_time           => 3600,
    cafile                     => $api_endpoint_ca_file,
    region_name                => $region_name,
  }
  class { '::cloudkitty::client': }
  class { '::cloudkitty::api':
    sync_db      => true,
  }
  cloudkitty_api_uwsgi_config {
    'uwsgi/disable-logging': value => false;
    'uwsgi/close-on-exec':   value => true;
    'uwsgi/close-on-exec2':  value => true;
  }
  # Fix the number of uwsgi processes
  class { '::cloudkitty::wsgi::uwsgi': }
  if $openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby' or $openstack_release == 'xena'{
    cloudkitty_config {
      'fetcher_keystone/ignore_disabled_tenants':  value => 'True';
      'fetcher_keystone/ignore_rating_role':       value => 'True';
    }
  }
  cloudkitty_config {
    'fetcher_keystone/cafile':             value => $api_endpoint_ca_file;
    'collector_gnocchi/cafile':            value => $api_endpoint_ca_file;
    'coordination/backend_url':            value => "zookeeper://${zookeeper_ensemble_string}:2181/";
    'fetcher_gnocchi/cafile':              value => $api_endpoint_ca_file;
    'fetcher_gnocchi/auth_section':        value => 'keystone_authtoken';
    'fetcher_gnocchi/http_pool_maxsize':   value => $cloudkitty_http_pool_maxsize;
    'collector_gnocchi/http_pool_maxsize': value => $cloudkitty_http_pool_maxsize;
  }

  # This is needed for the Keystone fetcher to list users
  cloudkitty_config {
    'ociadmin_authtoken/auth_type':            value => 'password';
    'ociadmin_authtoken/auth_url':             value => $keystone_admin_uri;
    'ociadmin_authtoken/project_name':         value => 'admin';
    'ociadmin_authtoken/project_domain_name':  value => 'Default';
    'ociadmin_authtoken/username':             value => 'admin';
    'ociadmin_authtoken/user_domain_name':     value => 'Default';
    'ociadmin_authtoken/password':             value => $keystone_admin_pass;
    'ociadmin_authtoken/www_authenticate_uri': value => $keystone_auth_uri;
    'ociadmin_authtoken/cafile':               value => $api_endpoint_ca_file;
    'ociadmin_authtoken/memcached_servers':    value => $memcached_string;
    'ociadmin_authtoken/region_name':          value => $region_name;
  }
  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri') {
    cloudkitty_config {
      'fetcher/backend': value => 'keystone';
    }
  }

  if $cloudkitty_metrics_yml {
    file { "/etc/cloudkitty/metrics.yml":
      ensure                  => present,
      owner                   => root,
      group                   => root,
      content                 => base64('decode', $cloudkitty_metrics_yml),
      selinux_ignore_defaults => true,
      mode                    => '0644',
      require                 => Anchor['cloudkitty::install::end'],
      notify                  => Service['cloudkitty-processor'],
    }
  }

  $my_orchestrator_max_threads = 30

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby') {
    cloudkitty_config {
      'orchestrator/coordination_url': value => "zookeeper://${zookeeper_ensemble_string}:2181/";
      'orchestrator/max_workers':      value => $cloudkitty_processor_workers;
      'orchestrator/max_threads':      value => $my_orchestrator_max_threads;
    }
  }elsif ($openstack_release == 'xena'){
    cloudkitty_config {
      'orchestrator/coordination_url': value => "zookeeper://${zookeeper_ensemble_string}:2181/";
    }
  }else{
    class { '::cloudkitty::orchestrator':
      coordination_url => "zookeeper://${zookeeper_ensemble_string}:2181/",
      max_workers      => $cloudkitty_processor_workers,
      max_threads      => $my_orchestrator_max_threads,
    }
  }

  if ($openstack_release == 'rocky' or $openstack_release == 'stein' or $openstack_release == 'train' or $openstack_release == 'ussuri' or $openstack_release == 'victoria' or $openstack_release == 'wallaby') {
    class { '::cloudkitty::processor':
      auth_section => 'ociadmin_authtoken',
    }
    cloudkitty_config {
      'collector_gnocchi/region_name': value => $region_name;
      'collector_gnocchi/interface':   value => 'publicURL';
    }
  }else {
    if $openstack_release == 'xena'{
      class { '::cloudkitty::processor':
        auth_section => 'ociadmin_authtoken',
        max_workers  => $cloudkitty_processor_workers,
        region_name  => $region_name,
        interface    => "publicURL",
      }
    }else{
      class { '::cloudkitty::processor':
        auth_section => 'ociadmin_authtoken',
        region_name  => $region_name,
        interface    => "publicURL",
      }
    }
    if ($openstack_release == 'xena'){
      cloudkitty_config {
        'orchestrator/max_threads': value => $my_orchestrator_max_threads;
      }
    }
  }
  if $cloudkitty_use_storage_v2 {
    class { '::cloudkitty::storage::elasticsearch':
      host       => $cloudkitty_storage_v2_host,
      index_name => $cloudkitty_storage_v2_index_name,
    }
    cloudkitty_config {
      'storage_elasticsearch/use_datastream': value => 'true';
    }
  }
  logrotate::rule { 'cloudkitty-common':
    path          => '/var/log/cloudkitty/*.log',
    rotate_every  => $logrotate_period,
    missingok     => true,
    compress      => true,
    delaycompress => true,
    ifempty       => false,
    copytruncate  => true,
    rotate        => '4',
  }
}
