diff --git a/lib/charm/openstack/adapters.py b/lib/charm/openstack/adapters.py index bcf0f7a..41d17f8 100644 --- a/lib/charm/openstack/adapters.py +++ b/lib/charm/openstack/adapters.py @@ -1,6 +1,12 @@ """Adapter classes and utilities for use with Reactive interfaces""" -from charmhelpers.core import hookenv +import charms.reactive.bus as reactive_bus +import charmhelpers.contrib.hahelpers.cluster as ch_cluster +import charmhelpers.contrib.network.ip as ch_ip +import charmhelpers.contrib.openstack.utils as ch_utils +import charmhelpers.core.hookenv as hookenv + +ADDRESS_TYPES = ['admin', 'internal', 'public'] class OpenStackRelationAdapter(object): @@ -82,6 +88,49 @@ def hosts(self): return None +class PeerHARelationAdapter(OpenStackRelationAdapter): + """ + Adapter for cluster relation of nodes of the same service + """ + + interface_type = "cluster" + + def __init__(self, relation): + super(PeerHARelationAdapter, self).__init__(relation) + self.config = hookenv.config() + self.local_address = APIConfigurationAdapter().local_address + self.local_unit_name = APIConfigurationAdapter().local_unit_name + self.cluster_hosts = {} + self.add_network_split_addresses() + self.add_default_addresses() + + def add_network_split_addresses(self): + """Populate cluster_hosts with addresses of peers on a given network if + this node is also on that network""" + for addr_type in ADDRESS_TYPES: + cfg_opt = 'os-{}-network'.format(addr_type) + laddr = ch_ip.get_address_in_network(self.config.get(cfg_opt)) + if laddr: + netmask = ch_ip.get_netmask_for_address(laddr) + self.cluster_hosts[laddr] = { + 'network': "{}/{}".format(laddr, netmask), + 'backends': {self.local_unit_name: laddr}} + key = '{}-address'.format(addr_type) + for _unit, _laddr in self.relation.ip_map(address_key=key): + self.cluster_hosts[laddr]['backends'][_unit] = _laddr + + def add_default_addresses(self): + """Populate cluster_hosts with addresses supplied by private-address + """ + self.cluster_hosts[self.local_address] = {} + netmask = ch_ip.get_netmask_for_address(self.local_address) + self.cluster_hosts[self.local_address] = { + 'network': "{}/{}".format(self.local_address, netmask), + 'backends': {self.local_unit_name: self.local_address}} + for _unit, _laddr in self.relation.ip_map(): + self.cluster_hosts[self.local_address]['backends'][_unit] = _laddr + + class DatabaseRelationAdapter(OpenStackRelationAdapter): """ Adapter for the Database relation interface. @@ -148,6 +197,153 @@ def __init__(self): setattr(self, k, v) +class APIConfigurationAdapter(ConfigurationAdapter): + """This configuration adapter extends the base class and adds properties + common accross most OpenstackAPI services""" + + def __init__(self, port_map=None): + super(APIConfigurationAdapter, self).__init__() + self.port_map = port_map + self.config = hookenv.config() + + @property + def ipv6_mode(self): + """Return if charm should enable IPv6 + + @return True if user has requested ipv6 support otherwise False + """ + return self.config.get('prefer-ipv6', False) + + @property + def local_address(self): + """Return remotely accessible address of charm (not localhost) + + @return True if user has requested ipv6 support otherwise False + """ + if self.ipv6_mode: + addr = ch_ip.get_ipv6_addr(exc_list=[self.config('vip')])[0] + else: + addr = ch_utils.get_host_ip(hookenv.unit_get('private-address')) + return addr + + @property + def local_unit_name(self): + """ + @return local unit name + """ + return hookenv.local_unit().replace('/', '-') + + @property + def local_host(self): + """Return localhost address depending on whether IPv6 is enabled + + @return localhost ip address + """ + return 'ip6-localhost' if self.ipv6_mode else '127.0.0.1' + + @property + def haproxy_host(self): + """Return haproxy bind address depending on whether IPv6 is enabled + + @return address + """ + return '::' if self.ipv6_mode else '0.0.0.0' + + @property + def haproxy_stat_port(self): + """Port to listen on to access haproxy statistics + + @return port + """ + return '8888' + + @property + def haproxy_stat_password(self): + """Password for accessing haproxy statistics + + @return password + """ + return reactive_bus.get_state('haproxy.stat.password') + + @property + def service_ports(self): + """Dict of service names and the ports they listen on + + @return {'svc1': 'portA', 'svc2': 'portB', ...} + """ + service_ports = {} + if self.port_map: + for service in self.port_map.keys(): + service_ports[service] = [ + self.port_map[service]['admin'], + ch_cluster.determine_apache_port( + self.port_map[service]['admin'], + singlenode_mode=True)] + return service_ports + + @property + def service_listen_info(self): + """Dict of service names and attributes for backend to listen on + + @return { + 'svc1': { + 'proto': 'http', + 'ip': '10.0.0.10', + 'port': '8080', + 'url': 'http://10.0.0.10:8080}, + 'svc2': { + 'proto': 'https', + 'ip': '10.0.0.20', + 'port': '8443', + 'url': 'https://10.0.0.20:8443}, + ... + + """ + info = {} + if self.port_map: + for service in self.port_map.keys(): + key = service.replace('-', '_') + info[key] = { + 'proto': 'http', + 'ip': self.local_address, + 'port': ch_cluster.determine_apache_port( + self.port_map[service]['admin'], + singlenode_mode=True)} + info[key]['url'] = '{proto}://{ip}:{port}'.format(**info[key]) + return info + + @property + def external_endpoints(self): + """Dict of service names and attributes that clients use to connect + + @return { + 'svc1': { + 'proto': 'http', + 'ip': '10.0.0.10', + 'port': '8080', + 'url': 'http://10.0.0.10:8080}, + 'svc2': { + 'proto': 'https', + 'ip': '10.0.0.20', + 'port': '8443', + 'url': 'https://10.0.0.20:8443}, + ... + + """ + info = {} + info = {} + ip = self.config.get('vip', self.local_address) + if self.port_map: + for service in self.port_map.keys(): + key = service.replace('-', '_') + info[key] = { + 'proto': 'http', + 'ip': ip, + 'port': self.port_map[service]['admin']} + info[key]['url'] = '{proto}://{ip}:{port}'.format(**info[key]) + return info + + class OpenStackRelationAdapters(object): """ Base adapters class for OpenStack Charms, used to aggregate @@ -171,13 +367,17 @@ class OpenStackRelationAdapters(object): _adapters = { 'amqp': RabbitMQRelationAdapter, 'shared_db': DatabaseRelationAdapter, + 'cluster': PeerHARelationAdapter, } """ Default adapter mappings; may be overridden by relation adapters in subclasses. + + Additional kwargs can be passed to the configuration adapterwhich has been + specified via the options parameter """ - def __init__(self, relations, options=ConfigurationAdapter): + def __init__(self, relations, options=ConfigurationAdapter, **kwargs): self._adapters.update(self.relation_adapters) self._relations = [] for relation in relations: @@ -188,7 +388,7 @@ def __init__(self, relations, options=ConfigurationAdapter): relation_value = OpenStackRelationAdapter(relation) setattr(self, relation_name, relation_value) self._relations.append(relation_name) - self.options = options() + self.options = options(**kwargs) self._relations.append('options') def __iter__(self): diff --git a/lib/charm/openstack/charm.py b/lib/charm/openstack/charm.py index af2ab19..43ab284 100644 --- a/lib/charm/openstack/charm.py +++ b/lib/charm/openstack/charm.py @@ -10,8 +10,8 @@ from charmhelpers.contrib.openstack.utils import ( configure_installation_source, ) -from charmhelpers.core.host import path_hash, service_restart -from charmhelpers.core.hookenv import config, status_set +from charmhelpers.core.host import path_hash, service_restart, pwgen +from charmhelpers.core.hookenv import config, status_set, relation_ids from charmhelpers.fetch import ( apt_install, apt_update, @@ -20,9 +20,17 @@ from charmhelpers.contrib.openstack.templating import get_loader from charmhelpers.core.templating import render from charmhelpers.core.hookenv import leader_get, leader_set -from charms.reactive.bus import set_state, remove_state +from charms.reactive.bus import set_state, remove_state, get_state from charm.openstack.ip import PUBLIC, INTERNAL, ADMIN, canonical_url +import charmhelpers.contrib.network.ip as ip +import charm.openstack.ha as ha +from relations.hacluster.common import CRM +import relations.openstack_ha.peers as ha_peers + +VIP_KEY = "vip" +CIDR_KEY = "vip_cidr" +IFACE_KEY = "vip_iface" class OpenStackCharm(object): @@ -33,8 +41,8 @@ class OpenStackCharm(object): name = 'charmname' - packages = [] - """Packages to install""" + base_packages = [] + """Packages to install unconditionally""" api_ports = {} """ @@ -48,10 +56,15 @@ class OpenStackCharm(object): default_service = None """Default service for the charm""" - restart_map = {} + base_restart_map = {} + """Map of services which must always be restarted when corresponding + configuration file changes + """ sync_cmd = [] services = [] + ha_resources = [] adapters_class = None + HAPROXY_CONF = '/etc/haproxy/haproxy.cfg' def __init__(self, interfaces=None): self.config = config() @@ -59,6 +72,40 @@ def __init__(self, interfaces=None): self.release = 'liberty' if interfaces and self.adapters_class: self.adapter_instance = self.adapters_class(interfaces) + self.set_haproxy_stat_password() + + def enable_haproxy(self): + """Determine if haproxy is fronting the services + + @return True if haproxy is fronting the service""" + return 'haproxy' in self.ha_resources + + @property + def packages(self): + """List of packages to be installed + + @return ['pkg1', 'pkg2', ...] + """ + _packages = [] + _packages.extend(self.base_packages) + if self.enable_haproxy(): + _packages.append('haproxy') + return _packages + + @property + def restart_map(self): + """Map of services to be restarted if a file changes + + @return { + 'file1': ['svc1', 'svc3'], + 'file2': ['svc2', 'svc3'], + ... + } + """ + _restart_map = self.base_restart_map.copy() + if self.enable_haproxy(): + _restart_map[self.HAPROXY_CONF] = ['haproxy'] + return _restart_map def install(self): """ @@ -150,6 +197,54 @@ def db_sync(self): # render_domain_config needs a working system self.restart_all() + def configure_ha_resources(self, hacluster): + """Inform the ha subordinate about each service it should manage. The + child class specifies the services via self.ha_resources + + @param hacluster interface + """ + RESOURCE_TYPES = { + 'vips': self._add_ha_vips_config, + 'haproxy': self._add_ha_haproxy_config, + } + self.resources = CRM() + if not self.ha_resources: + return + for res_type in self.ha_resources: + RESOURCE_TYPES[res_type]() + # TODO Remove hardcoded multicast port + hacluster.bind_on(iface=self.config[IFACE_KEY], mcastport=4440) + hacluster.manage_resources(self.resources) + + def _add_ha_vips_config(self): + """Add a VirtualIP object for each user specified vip to self.resources + """ + for vip in self.config.get(VIP_KEY, []).split(): + iface = (ip.get_iface_for_address(vip) or + self.config(IFACE_KEY)) + netmask = (ip.get_netmask_for_address(vip) or + self.config(CIDR_KEY)) + if iface is not None: + self.resources.add( + ha.VirtualIP( + self.name, + vip, + nic=iface, + cidr=netmask,)) + + def _add_ha_haproxy_config(self): + """Add a InitService object for haproxy to self.resources + """ + self.resources.add( + ha.InitService( + self.name, + 'haproxy',)) + + def set_haproxy_stat_password(self): + """Set a stats password for accessing haproxy statistics""" + if not get_state('haproxy.stat.password'): + set_state('haproxy.stat.password', pwgen(32)) + class OpenStackCharmFactory(object): @@ -167,9 +262,19 @@ class OpenStackCharmFactory(object): @classmethod def charm(cls, release=None, interfaces=None): """ - Get an instance of the right charm for the - configured OpenStack series + Get an instance of the right charm for the configured OpenStack series + + If the cluster relation exists add the cluster interface. It is + forecfully added here as the interface is needed even if there is only + one unit in the service. If only one unit exists the cluster hooks + never fire. """ + if relation_ids('cluster'): + cluster_interface = ha_peers.OpenstackHAPeers('cluster') + if interfaces: + interfaces.append(cluster_interface) + else: + interfaces = [cluster_interface] if release and release in cls.releases: return cls.releases[release](interfaces=interfaces) else: diff --git a/lib/charm/openstack/ha.py b/lib/charm/openstack/ha.py new file mode 100644 index 0000000..2aee2db --- /dev/null +++ b/lib/charm/openstack/ha.py @@ -0,0 +1,54 @@ +"""Classes for enabling ha in Openstack charms with the reactive framework""" + +import ipaddress +import relations.hacluster.common + + +"""Configure ha resources with: +@when('ha.connected') +def cluster_connected(hacluster): + charm = DesignateCharmFactory.charm() + charm.configure_ha_resources(hacluster) + +TODO Proper docs to follow +""" + + +class InitService(relations.hacluster.common.ResourceDescriptor): + def __init__(self, service_name, init_service_name): + self.service_name = service_name + self.init_service_name = init_service_name + + def configure_resource(self, crm): + res_key = 'res_{}_{}'.format( + self.service_name.replace('-', '_'), + self.init_service_name.replace('-', '_')) + clone_key = 'cl_{}'.format(res_key) + res_type = 'lsb:{}'.format(self.init_service_name) + crm.primitive(res_key, res_type, params='op monitor interval="5s"') + crm.init_services(self.init_service_name) + crm.clone(clone_key, res_key) + + +class VirtualIP(relations.hacluster.common.ResourceDescriptor): + def __init__(self, service_name, vip, nic=None, cidr=None): + self.service_name = service_name + self.vip = vip + self.nic = nic + self.cidr = cidr + + def configure_resource(self, crm): + vip_key = 'res_{}_{}_vip'.format(self.service_name, self.nic) + ipaddr = ipaddress.ip_address(self.vip) + if isinstance(ipaddr, ipaddress.IPv4Address): + res_type = 'ocf:heartbeat:IPaddr2' + res_params = 'ip="{}"'.format(self.vip) + else: + res_type = 'ocf:heartbeat:IPv6addr' + res_params = 'ipv6addr="{}"'.format(self.vip) + + if self.nic: + res_params = '{} nic="{}"'.format(res_params, self.nic) + if self.cidr: + res_params = '{} cidr_netmask="{}"'.format(res_params, self.cidr) + crm.primitive(vip_key, res_type, params=res_params) diff --git a/templates/haproxy.cfg b/templates/haproxy.cfg index 8721d8a..c2f1caa 100644 --- a/templates/haproxy.cfg +++ b/templates/haproxy.cfg @@ -1,6 +1,6 @@ global - log {{ local_host }} local0 - log {{ local_host }} local1 notice + log {{ options.local_host }} local0 + log {{ options.local_host }} local1 notice maxconn 20000 user haproxy group haproxy @@ -12,52 +12,52 @@ defaults option tcplog option dontlognull retries 3 -{%- if haproxy_queue_timeout %} - timeout queue {{ haproxy_queue_timeout }} +{%- if options.haproxy_queue_timeout %} + timeout queue {{ options.haproxy_queue_timeout }} {%- else %} timeout queue 5000 {%- endif %} -{%- if haproxy_connect_timeout %} - timeout connect {{ haproxy_connect_timeout }} +{%- if options.haproxy_connect_timeout %} + timeout connect {{ options.haproxy_connect_timeout }} {%- else %} timeout connect 5000 {%- endif %} -{%- if haproxy_client_timeout %} - timeout client {{ haproxy_client_timeout }} +{%- if options.haproxy_client_timeout %} + timeout client {{ options.haproxy_client_timeout }} {%- else %} timeout client 30000 {%- endif %} -{%- if haproxy_server_timeout %} - timeout server {{ haproxy_server_timeout }} +{%- if options.haproxy_server_timeout %} + timeout server {{ options.haproxy_server_timeout }} {%- else %} timeout server 30000 {%- endif %} -listen stats {{ stat_port }} +listen stats {{ options.stat_port }} mode http stats enable stats hide-version stats realm Haproxy\ Statistics stats uri / - stats auth admin:password + stats auth admin:{{ options.haproxy_stat_password }} -{% if frontends -%} -{% for service, ports in service_ports.items() -%} +{% if cluster.cluster_hosts -%} +{% for service, ports in options.service_ports.items() -%} frontend tcp-in_{{ service }} bind *:{{ ports[0] }} {% if ipv6 -%} bind :::{{ ports[0] }} {% endif -%} - {% for frontend in frontends -%} - acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }} + {% for frontend in cluster.cluster_hosts -%} + acl net_{{ frontend }} dst {{ cluster.cluster_hosts[frontend]['network'] }} use_backend {{ service }}_{{ frontend }} if net_{{ frontend }} {% endfor -%} - default_backend {{ service }}_{{ default_backend }} + default_backend {{ service }}_{{ cluster.local_address }} -{% for frontend in frontends -%} +{% for frontend in cluster.cluster_hosts -%} backend {{ service }}_{{ frontend }} balance leastconn - {% for unit, address in frontends[frontend]['backends'].items() -%} + {% for unit, address in cluster.cluster_hosts[frontend]['backends'].items() -%} server {{ unit }} {{ address }}:{{ ports[1] }} check {% endfor %} {% endfor -%}