From 215d4736416f1329dd96da255b2b973b4bc025b3 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 4 Jan 2017 10:23:13 +0100 Subject: [PATCH 01/52] Removed monasca-api role. --- chef/data_bags/crowbar/template-barbican.schema | 2 +- crowbar_framework/app/models/monasca_service.rb | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/chef/data_bags/crowbar/template-barbican.schema b/chef/data_bags/crowbar/template-barbican.schema index ed6729dad7..5757615242 100644 --- a/chef/data_bags/crowbar/template-barbican.schema +++ b/chef/data_bags/crowbar/template-barbican.schema @@ -2,7 +2,7 @@ "required": true, "type": "map", "mapping": { - "id": { "type": "str", "required": true, "pattern": "/^barbican-|^template-barbican$/" }, + "id": { "type": "str", "required": true, "pattern": "/^mybarclamp-|^template-mybarclamp$/" }, "description": { "required": true, "type": "str" }, "attributes": { "required": true, diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index c8ced2b6a5..97c4cc0da4 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -88,11 +88,9 @@ def create_proposal agent_nodes = nodes base["deployment"][@bc_name]["elements"]["monasca-agent"] = agent_nodes - unless server_nodes.nil? - base["deployment"][@bc_name]["elements"] = { - "monasca-server" => [server_nodes.first.name] - } - end + base["deployment"][@bc_name]["elements"] = { + "monasca-server" => [server_nodes.first.name] + } unless server_nodes.nil? base["attributes"][@bc_name]["database_instance"] = find_dep_proposal("database") From aee8cc8ede824a88f908a76cde55038dff8039ec Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 4 Jan 2017 13:13:09 +0100 Subject: [PATCH 02/52] Added up-to-date WSGI plumbing. --- chef/cookbooks/monasca/recipes/api.rb | 29 ++++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/api.rb b/chef/cookbooks/monasca/recipes/api.rb index 361163b4c9..c0d9bb3313 100644 --- a/chef/cookbooks/monasca/recipes/api.rb +++ b/chef/cookbooks/monasca/recipes/api.rb @@ -18,6 +18,9 @@ include_recipe "apache2::mod_wsgi" include_recipe "apache2::mod_rewrite" +application_path = "/srv/www/monasca-api" +application_exec_path = "#{application_path}/app.wsgi" + apache_module "deflate" do conf false enable true @@ -30,8 +33,10 @@ ### TODO: uncomment this once there is a package. # package "openstack-monasca-api" + ### FIXME: remove this once there is a package creating this directory + directory "/etc/monasca/" do owner "root" group "root" @@ -131,18 +136,18 @@ ### FIXME: Uncomment once we actually have a runnable WSGI app from a ### monasca-api package -# crowbar_openstack_wsgi "WSGI entry for monasca-api" do -# bind_host bind_host -# bind_port bind_port -# daemon_process "monasca-api" -# user node[:monasca][:user] -# group node[:monasca][:group] -# processes node[:monasca][:api][:processes] -# threads node[:monasca][:api][:threads] -# end +#crowbar_openstack_wsgi "WSGI entry for monasca-api" do +# bind_host bind_host +# bind_port bind_port +# daemon_process "monasca-api" +# user node[:monasca][:user] +# group node[:monasca][:group] +# processes node[:monasca][:api][:processes] +# threads node[:monasca][:api][:threads] +#end # -# apache_site "monasca-api.conf" do -# enable true -# end +#apache_site "monasca-api.conf" do +# enable true +#end node.save From 0a93eaf2aebbb5d2ecb571296af1fce6ea1b9d27 Mon Sep 17 00:00:00 2001 From: Johannes Grassler Date: Wed, 4 Jan 2017 14:32:22 +0100 Subject: [PATCH 03/52] Addressed Hound comments. --- chef/cookbooks/monasca/recipes/api.rb | 29 ++++++++----------- .../app/models/monasca_service.rb | 8 +++-- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/api.rb b/chef/cookbooks/monasca/recipes/api.rb index c0d9bb3313..361163b4c9 100644 --- a/chef/cookbooks/monasca/recipes/api.rb +++ b/chef/cookbooks/monasca/recipes/api.rb @@ -18,9 +18,6 @@ include_recipe "apache2::mod_wsgi" include_recipe "apache2::mod_rewrite" -application_path = "/srv/www/monasca-api" -application_exec_path = "#{application_path}/app.wsgi" - apache_module "deflate" do conf false enable true @@ -33,10 +30,8 @@ ### TODO: uncomment this once there is a package. # package "openstack-monasca-api" - ### FIXME: remove this once there is a package creating this directory - directory "/etc/monasca/" do owner "root" group "root" @@ -136,18 +131,18 @@ ### FIXME: Uncomment once we actually have a runnable WSGI app from a ### monasca-api package -#crowbar_openstack_wsgi "WSGI entry for monasca-api" do -# bind_host bind_host -# bind_port bind_port -# daemon_process "monasca-api" -# user node[:monasca][:user] -# group node[:monasca][:group] -# processes node[:monasca][:api][:processes] -# threads node[:monasca][:api][:threads] -#end +# crowbar_openstack_wsgi "WSGI entry for monasca-api" do +# bind_host bind_host +# bind_port bind_port +# daemon_process "monasca-api" +# user node[:monasca][:user] +# group node[:monasca][:group] +# processes node[:monasca][:api][:processes] +# threads node[:monasca][:api][:threads] +# end # -#apache_site "monasca-api.conf" do -# enable true -#end +# apache_site "monasca-api.conf" do +# enable true +# end node.save diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 97c4cc0da4..c8ced2b6a5 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -88,9 +88,11 @@ def create_proposal agent_nodes = nodes base["deployment"][@bc_name]["elements"]["monasca-agent"] = agent_nodes - base["deployment"][@bc_name]["elements"] = { - "monasca-server" => [server_nodes.first.name] - } unless server_nodes.nil? + unless server_nodes.nil? + base["deployment"][@bc_name]["elements"] = { + "monasca-server" => [server_nodes.first.name] + } + end base["attributes"][@bc_name]["database_instance"] = find_dep_proposal("database") From 82592465bd53caaffb58068c49578957f4f11e7a Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Wed, 11 Jan 2017 08:29:34 +0100 Subject: [PATCH 04/52] Fix barbican schema template --- chef/data_bags/crowbar/template-barbican.schema | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/data_bags/crowbar/template-barbican.schema b/chef/data_bags/crowbar/template-barbican.schema index 5757615242..ed6729dad7 100644 --- a/chef/data_bags/crowbar/template-barbican.schema +++ b/chef/data_bags/crowbar/template-barbican.schema @@ -2,7 +2,7 @@ "required": true, "type": "map", "mapping": { - "id": { "type": "str", "required": true, "pattern": "/^mybarclamp-|^template-mybarclamp$/" }, + "id": { "type": "str", "required": true, "pattern": "/^barbican-|^template-barbican$/" }, "description": { "required": true, "type": "str" }, "attributes": { "required": true, From 5cc52d8e94215523237ca14bc5ae64e62fc28787 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Tue, 10 Jan 2017 14:43:33 +0100 Subject: [PATCH 05/52] Install monasca-ui horizon plugin --- chef/cookbooks/horizon/recipes/server.rb | 18 ++++++++++++++++++ .../monasca/recipes/role_monasca_server.rb | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/horizon/recipes/server.rb b/chef/cookbooks/horizon/recipes/server.rb index 8955bb9bf3..e1cb4e90bd 100644 --- a/chef/cookbooks/horizon/recipes/server.rb +++ b/chef/cookbooks/horizon/recipes/server.rb @@ -154,6 +154,24 @@ end end +monasca_ui_pkgname = + case node[:platform_family] + when "suse" + "openstack-horizon-plugin-monasca-ui" + when "rhel" + "openstack-monasca-ui" + end + +unless monasca_ui_pkgname.nil? + monasca_servers = search(:node, "roles:monasca-server") || [] + unless monasca_servers.empty? + package monasca_ui_pkgname do + action :install + notifies :reload, resources(service: "apache2") + end + end +end + if node[:platform_family] == "suse" # Get rid of unwanted vhost config files: ["#{node[:apache][:dir]}/vhosts.d/default-redirect.conf", diff --git a/chef/cookbooks/monasca/recipes/role_monasca_server.rb b/chef/cookbooks/monasca/recipes/role_monasca_server.rb index bc300659b7..b8d0a49328 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_server.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_server.rb @@ -16,5 +16,5 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-server") include_recipe "monasca::server" - include_recipe "monasca::api" +# include_recipe "monasca::api" end From 64aa8db3018b56887203d611c8c19aaf64a541fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Tue, 17 Jan 2017 15:02:49 +0100 Subject: [PATCH 06/52] Initial commit for the log-agent Following is the first one that will add installing monasca-log-agent capabilities --- chef/cookbooks/monasca/attributes/default.rb | 19 ++- chef/cookbooks/monasca/recipes/log_agent.rb | 113 ++++++++++++++++++ .../monasca/recipes/role_monasca_log_agent.rb | 18 +++ .../templates/default/log-agent.conf.erb | 38 ++++++ chef/data_bags/crowbar/template-monasca.json | 12 +- .../data_bags/crowbar/template-monasca.schema | 9 +- chef/roles/monasca-log-agent.rb | 7 ++ .../app/models/monasca_service.rb | 11 ++ 8 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 chef/cookbooks/monasca/recipes/log_agent.rb create mode 100644 chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb create mode 100644 chef/cookbooks/monasca/templates/default/log-agent.conf.erb create mode 100644 chef/roles/monasca-log-agent.rb diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index 41af1cfe8b..ed99a689cc 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -13,6 +13,16 @@ # See the License for the specific language governing permissions and # limitation. +monasca_debug = false +monasca_ha_enabled = false + +monasca_log_agent_service_name = "monasca-log-agent" + +default[:monasca][:platform] = { + packages: [], + services: [monasca_log_agent_service_name] +} + default[:monasca][:db][:database] = "monasca" default[:monasca][:db][:user] = "monasca" default[:monasca][:db][:password] = nil # must be set by wrapper @@ -20,7 +30,12 @@ override[:monasca][:group] = "monasca" override[:monasca][:user] = "monasca" -default[:monasca][:debug] = false -default[:monasca][:ha_enabled] = false +default[:monasca][:debug] = monasca_debug +default[:monasca][:ha_enabled] = monasca_ha_enabled default[:monasca][:api][:bind_host] = "*" + +default[:monasca][:log_agent][:service_name] = monasca_log_agent_service_name +default[:monasca][:log_agent][:user] = "monasca-log-agent" +default[:monasca][:log_agent][:group] = "logstash" +default[:monasca][:log_agent][:debug] = monasca_debug diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb new file mode 100644 index 0000000000..69c9f14531 --- /dev/null +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -0,0 +1,113 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +package "monasca-log-agent" + +log_agent_settings = node[:monasca][:log_agent] +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +log_agent_dimensions = { + service: "monitoring", + hostname: node["hostname"] +} +# TODO(trebskit) actually it should be retrieved from keystone +log_agent_settings["monasca_log_api_url"] = "http://www.example.org" + +### create user for monasca-log-agent +register_auth_hash = { + user: keystone_settings["admin_user"], + password: keystone_settings["admin_password"], + tenant: keystone_settings["admin_tenant"] +} + +keystone_register "monasca-log-agent wakeup keystone" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + action :wakeup +end + +keystone_register "register monasca-log-agent user" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name keystone_settings["service_user"] + user_password keystone_settings["service_password"] + tenant_name keystone_settings["service_tenant"] + action :add_user +end + +keystone_register "give monasca-log-agent user access" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name keystone_settings["service_user"] + tenant_name keystone_settings["service_tenant"] + role_name "admin" + action :add_access +end + +keystone_register "register monasca-log-agent service" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + service_name "monasca-log-agent" + service_type "monitoring" + service_description "Monasca Log Agent monitoring service" + action :add_service +end + +directory "/var/log/monasca-log-agent/" do + owner log_agent_settings[:user] + owner log_agent_settings[:group] + mode 0o755 + recursive true +end +directory "/etc/monasca-log-agent/" do + owner log_agent_settings[:user] + owner log_agent_settings[:group] + mode 0o755 + recursive true +end + +template "/etc/monasca-log-agent/agent.conf" do + source "log-agent.conf.erb" + owner log_agent_settings[:user] + owner log_agent_settings[:group] + mode 0o640 + variables( + log_agent_settings: log_agent_settings, + log_agent_dimensions: log_agent_dimensions, + keystone_settings: keystone_settings + ) + notifies :reload, "service[monasca-log-agent]" +end + +service "monasca-log-agent" do + service_name log_agent_settings[:service_name] + supports status: true, restart: true, start: true, stop: true + action [:disable, :stop] + ignore_failure true +end + +node.save diff --git a/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb new file mode 100644 index 0000000000..b5d5ee6d6b --- /dev/null +++ b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb @@ -0,0 +1,18 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-log-agent") + include_recipe "monasca::log_agent" +end diff --git a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb new file mode 100644 index 0000000000..4cd16d9d85 --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb @@ -0,0 +1,38 @@ +input { + # TODO(trebskit) should initial list contain some files to be + # monitored ? + file { + add_field => { "dimensions" => { "service" => "system" }} + path => "/var/log/messages" + tags => ["syslog"] + } +} + +filter { + if "syslog" in [tags] { + multiline { + negate => "true" + pattern => "^%{SYSLOGTIMESTAMP}" + what => "previous" + } + } +} + +output { + monasca_log_api { + ### keystone based settings + keystone_api_url => "<%= @keystone_settings['internal_auth_url'] %>" + project_name => "<%= @keystone_settings['service_tenant'] %>" + username => "<%= @keystone_settings['service_user'] %>" + password => "<%= @keystone_settings['service_password'] %>" + user_domain_name => "<%= @keystone_settings['admin_domain'] %>" + project_domain_name => "<%= @keystone_settings['admin_domain'] %>" + ### monasca specific settings + monasca_log_api_url => "<%= @log_agent_settings['monasca_log_api_url'] %>" +<%- unless @log_agent_dimensions.nil? then -%> + dimensions => [<%= @log_agent_dimensions.map{|k,v| "\"#{k}\":\"#{v}\""}.join(',') %>] +<%- else -%> + dimensions => [] +<%- end -%> + } +} diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 5537bc2034..6080acd986 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -27,6 +27,9 @@ "plugin_collect_time_warn": 6, "log_level": "INFO" }, + "log_agent": { + "monasca_log_api_url": "" + }, "api" : { "bind_host" : "*", "bind_port" : 8070, @@ -55,16 +58,19 @@ "schema-revision": 100, "element_states": { "monasca-server": [ "readying", "ready", "applying" ], - "monasca-agent": [ "readying", "ready", "applying" ] + "monasca-agent": [ "readying", "ready", "applying" ], + "monasca-log-agent": [ "readying", "ready", "applying" ] }, "elements": {}, "element_order": [ [ "monasca-server" ], - [ "monasca-agent" ] + [ "monasca-agent" ], + [ "monasca-log-agent" ] ], "element_run_list_order": { "monasca-server": 110, - "monasca-agent": 120 + "monasca-agent": 120, + "monasca-log-agent": 120 }, "config": { "environment": "monasca-base-config", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 0052ea3f8c..aed8e80a9c 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -39,7 +39,14 @@ "plugin_collect_time_warn": { "type": "int", required: true }, "log_level": { "type": "str", required: true } } - }, + }, + "log_agent": { + "required": true, + "type": "map", + "mapping": { + "monasca_log_api_url": { "type": "str", required: true } + } + }, "api": { "required": true, "type": "map", diff --git a/chef/roles/monasca-log-agent.rb b/chef/roles/monasca-log-agent.rb new file mode 100644 index 0000000000..8344514fdd --- /dev/null +++ b/chef/roles/monasca-log-agent.rb @@ -0,0 +1,7 @@ +# # Copyright 2017 FUJITSU LIMITED + +name "monasca-log-agent" +description "Monasca Log Agent Role" +run_list("recipe[monasca::role_monasca_log_agent]") +default_attributes +override_attributes diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index c8ced2b6a5..ef2f01c871 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -1,5 +1,6 @@ # # Copyright 2016, SUSE LINUX GmbH +# Copyright 2017 FUJITSU LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,6 +38,15 @@ def role_constraints "windows" => "/.*/" } }, + "monasca-log-agent" => { + "unique" => false, + "admin" => true, + "count" => -1, + "exclude_platform" => { + "suse" => "< 12.1", + "windows" => "/.*/" + } + }, "monasca-server" => { "unique" => false, "count" => 1, @@ -88,6 +98,7 @@ def create_proposal agent_nodes = nodes base["deployment"][@bc_name]["elements"]["monasca-agent"] = agent_nodes + base["deployment"][@bc_name]["elements"]["monasca-log-agent"] = agent_nodes unless server_nodes.nil? base["deployment"][@bc_name]["elements"] = { "monasca-server" => [server_nodes.first.name] From f55cf0cc41107da34784c9c026da93c1dafde0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Tue, 7 Feb 2017 13:57:47 +0100 Subject: [PATCH 07/52] Use dedicated users for monasca-log-agent (#11) * Use deticate users for monasca-log-agent Following commit provides registering custom project and dedicated user registered inside this project * Added some debug details * Added including monasca:common * Include monasca:common elsewhere * Removed return clause' * Removed action:nothing * Move log outside block and added some variables * Added newline * Removed block * Added new configuration variables into conf template * TNT * Use group instead of 2xowner * Removed extra " from log-agent.conf.erb * Changed way to access values * Added user&group into databad * Moved registering log-agent Since agents (metric&logs) will be configured with single credentials per each node it made sense to put creating them inside the common part. Additionally added randomizing the agents password * Fixing hound error * Fixing hound error * Fixed typo * Addressed matrixik comments * Fixing undefined method in monasca_service.rb * Use empty strings instead of null values * trying to make it work * trying to make it work 2 * trying to make it work 3 * trying to make it work 4 * trying to make it work 5 * trying to make it work 6 * polished * Pleasing Hound --- chef/cookbooks/monasca/attributes/default.rb | 2 + chef/cookbooks/monasca/recipes/common.rb | 114 ++++++++++++++++++ chef/cookbooks/monasca/recipes/log_agent.rb | 68 ++--------- .../monasca/recipes/role_monasca_agent.rb | 3 +- .../monasca/recipes/role_monasca_log_agent.rb | 3 +- .../monasca/recipes/role_monasca_server.rb | 4 +- .../templates/default/log-agent.conf.erb | 8 +- chef/data_bags/crowbar/template-monasca.json | 14 ++- .../data_bags/crowbar/template-monasca.schema | 16 ++- .../app/models/monasca_service.rb | 4 + 10 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 chef/cookbooks/monasca/recipes/common.rb diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index ed99a689cc..e980735642 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -1,5 +1,6 @@ # # Copyright 2016 SUSE Linux GmbH +# Copyright 2017 Fujitsu LIMITED # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,6 +36,7 @@ default[:monasca][:api][:bind_host] = "*" +# log-agent default service settings default[:monasca][:log_agent][:service_name] = monasca_log_agent_service_name default[:monasca][:log_agent][:user] = "monasca-log-agent" default[:monasca][:log_agent][:group] = "logstash" diff --git a/chef/cookbooks/monasca/recipes/common.rb b/chef/cookbooks/monasca/recipes/common.rb new file mode 100644 index 0000000000..b83b63cd3c --- /dev/null +++ b/chef/cookbooks/monasca/recipes/common.rb @@ -0,0 +1,114 @@ +# +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitation. + +monasca_pkgs = node[:monasca][:platform][:packages] +monasca_project = node[:monasca][:service_tenant] +monasca_roles = node[:monasca][:service_roles] + +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +register_auth_hash = { + user: keystone_settings["admin_user"], + password: keystone_settings["admin_password"], + tenant: keystone_settings["admin_tenant"] +} + +log "monasca:common packages are #{monasca_pkgs}" +log "monasca:common project is #{monasca_project}" +log "monasca:common roles are #{monasca_roles}" + +monasca_pkgs.each do |pkg| + package pkg +end + +keystone_register "monasca:common wakeup keystone" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + action :wakeup +end + +keystone_register "monasca:common create tenant #{monasca_project} for monasca" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + tenant_name monasca_project + action :add_tenant +end + +monasca_roles.each do |role| + keystone_register "monasca:common register #{role} role in #{monasca_project} tenant" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + role_name role + action :add_role + end +end + +### recipes agents specific settings +### single credentials all metrics & logs agents +agents_settings = [] + +# once agent is ready, uncomment following lines +# if node["roles"].include?("monasca-agent") +# agents_settings.push(node[:monasca][:agent][:keystone]) +# end +if node["roles"].include?("monasca-log-agent") + la_keystone = node[:monasca][:log_agent][:keystone] + agents_settings.push(la_keystone) +end + +unless agents_settings.empty? + agents_settings.each do |as| + + as_service_user = as["service_user"] + as_service_tenant = as["service_tenant"] + as_service_password = as["service_password"] + as_service_role = as["service_role"] + + keystone_register "monasca:common #{as_service_user} in #{as_service_tenant} project" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name as_service_user + user_password as_service_password + tenant_name as_service_tenant + action :add_user + end + + keystone_register "monasca:common #{as_service_user} assign role #{as_service_role}" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name as_service_user + tenant_name as_service_tenant + role_name as_service_role + action :add_access + end + + end +end +### recipes specific keystone handling diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 69c9f14531..6e47a63a67 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -16,86 +16,34 @@ package "monasca-log-agent" log_agent_settings = node[:monasca][:log_agent] +log_agent_keystone = log_agent_settings[:keystone] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) log_agent_dimensions = { service: "monitoring", hostname: node["hostname"] } -# TODO(trebskit) actually it should be retrieved from keystone -log_agent_settings["monasca_log_api_url"] = "http://www.example.org" - -### create user for monasca-log-agent -register_auth_hash = { - user: keystone_settings["admin_user"], - password: keystone_settings["admin_password"], - tenant: keystone_settings["admin_tenant"] -} - -keystone_register "monasca-log-agent wakeup keystone" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - action :wakeup -end - -keystone_register "register monasca-log-agent user" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name keystone_settings["service_user"] - user_password keystone_settings["service_password"] - tenant_name keystone_settings["service_tenant"] - action :add_user -end - -keystone_register "give monasca-log-agent user access" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - user_name keystone_settings["service_user"] - tenant_name keystone_settings["service_tenant"] - role_name "admin" - action :add_access -end - -keystone_register "register monasca-log-agent service" do - protocol keystone_settings["protocol"] - insecure keystone_settings["insecure"] - host keystone_settings["internal_url_host"] - port keystone_settings["admin_port"] - auth register_auth_hash - service_name "monasca-log-agent" - service_type "monitoring" - service_description "Monasca Log Agent monitoring service" - action :add_service -end directory "/var/log/monasca-log-agent/" do - owner log_agent_settings[:user] - owner log_agent_settings[:group] + owner log_agent_settings["user"] + group log_agent_settings["group"] mode 0o755 recursive true end directory "/etc/monasca-log-agent/" do - owner log_agent_settings[:user] - owner log_agent_settings[:group] + owner log_agent_settings["user"] + group log_agent_settings["group"] mode 0o755 recursive true end template "/etc/monasca-log-agent/agent.conf" do source "log-agent.conf.erb" - owner log_agent_settings[:user] - owner log_agent_settings[:group] + owner log_agent_settings["user"] + group log_agent_settings["group"] mode 0o640 variables( + log_agent_keystone: log_agent_keystone, log_agent_settings: log_agent_settings, log_agent_dimensions: log_agent_dimensions, keystone_settings: keystone_settings diff --git a/chef/cookbooks/monasca/recipes/role_monasca_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_agent.rb index 65203e0745..bf8dd7b22d 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_agent.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_agent.rb @@ -15,5 +15,6 @@ # if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-agent") - include_recipe "monasca::agent" + include_recipe "#{@cookbook_name}::common" + include_recipe "#{@cookbook_name}::agent" end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb index b5d5ee6d6b..49208b603f 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb @@ -14,5 +14,6 @@ # if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-log-agent") - include_recipe "monasca::log_agent" + include_recipe "#{@cookbook_name}::common" + include_recipe "#{@cookbook_name}::log_agent" end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_server.rb b/chef/cookbooks/monasca/recipes/role_monasca_server.rb index b8d0a49328..a10fdde1fb 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_server.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_server.rb @@ -15,6 +15,6 @@ # if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-server") - include_recipe "monasca::server" -# include_recipe "monasca::api" + include_recipe "#{@cookbook_name}::common" + include_recipe "#{@cookbook_name}::server" end diff --git a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb index 4cd16d9d85..42e6db7885 100644 --- a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb +++ b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb @@ -22,15 +22,15 @@ output { monasca_log_api { ### keystone based settings keystone_api_url => "<%= @keystone_settings['internal_auth_url'] %>" - project_name => "<%= @keystone_settings['service_tenant'] %>" - username => "<%= @keystone_settings['service_user'] %>" - password => "<%= @keystone_settings['service_password'] %>" + project_name => "<%= @log_agent_keystone['service_tenant'] %>" + username => "<%= @log_agent_keystone['service_user'] %>" + password => "<%= @log_agent_keystone['service_password'] %>" user_domain_name => "<%= @keystone_settings['admin_domain'] %>" project_domain_name => "<%= @keystone_settings['admin_domain'] %>" ### monasca specific settings monasca_log_api_url => "<%= @log_agent_settings['monasca_log_api_url'] %>" <%- unless @log_agent_dimensions.nil? then -%> - dimensions => [<%= @log_agent_dimensions.map{|k,v| "\"#{k}\":\"#{v}\""}.join(',') %>] + dimensions => [<%= @log_agent_dimensions.map{|k,v| "\"#{k}:#{v}\""}.join(',') %>] <%- else -%> dimensions => [] <%- end -%> diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 6080acd986..dae9a7d77d 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -28,7 +28,15 @@ "log_level": "INFO" }, "log_agent": { - "monasca_log_api_url": "" + "user": "monasca-log-agent", + "group": "logstash", + "monasca_log_api_url": "", + "keystone": { + "service_user": "cmm-log-agent", + "service_password": "", + "service_tenant": "cmm", + "service_role": "cmm-agent" + } }, "api" : { "bind_host" : "*", @@ -48,7 +56,9 @@ "database_instance": "none", "keystone_instance": "none", "service_password": "none", - "service_user": "monasca" + "service_user": "monasca", + "service_tenant": "cmm", + "service_roles": ["cmm-agent"] } }, "deployment": { diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index aed8e80a9c..6478aa27b2 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -44,7 +44,19 @@ "required": true, "type": "map", "mapping": { - "monasca_log_api_url": { "type": "str", required: true } + "user": { "type": "str", required: true }, + "group": { "type": "str", required: true }, + "monasca_log_api_url": { "type": "str", required: true }, + "keystone": { + "required": true, + "type": "map", + "mapping": { + "service_user": { "type": "str", required: true }, + "service_password": { "type": "str", required: true }, + "service_tenant": { "type": "str", required: false }, + "service_role": { "type": "str", required: false } + } + } } }, "api": { @@ -72,6 +84,8 @@ "keystone_instance": { "type": "str", "required": true }, "service_user": { "type": "str", "required": true }, "service_password": { "type": "str", "required": true }, + "service_tenant": { "type": "str", "required": true }, + "service_roles": { "type": "seq", "required": true, "sequence": [ { "type": "str" } ] }, "group": { "required": true, "type": "str" }, "user": { "required": true, "type": "str" } } diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index ef2f01c871..7833a2a86b 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -109,8 +109,12 @@ def create_proposal find_dep_proposal("database") base["attributes"][@bc_name]["keystone_instance"] = find_dep_proposal("keystone") + base["attributes"][@bc_name]["service_password"] = random_password base["attributes"][@bc_name][:db][:password] = random_password + # note(trebskit) once agent is ready, uncomment following line + # base["attributes"][@bc_name][:agent][:keystone][:service_password] = random_password + base["attributes"][@bc_name][:log_agent][:keystone][:service_password] = random_password @logger.debug("Monasca create_proposal: exiting") base From 0d1e8398ead4adefc51357df721719b6b8f8fded Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Mon, 20 Feb 2017 09:00:19 +0100 Subject: [PATCH 08/52] Configure monasca-ui plugin (#6) * Configure monasca-ui plugin * Refactor the code * Addresing review comments * Deploy grafana alongside horizon * Remove obsolete changes to monasca schema * Removing extra blank line --- chef/cookbooks/horizon/libraries/helper.rb | 21 ++++++ chef/cookbooks/horizon/recipes/monasca_ui.rb | 37 +++++++++++ chef/cookbooks/horizon/recipes/server.rb | 9 ++- .../default/_80_monasca_ui_settings.py.erb | 66 +++++++++++++++++++ .../suse/openstack-dashboard.conf.erb | 8 +++ 5 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 chef/cookbooks/horizon/libraries/helper.rb create mode 100644 chef/cookbooks/horizon/recipes/monasca_ui.rb create mode 100644 chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb diff --git a/chef/cookbooks/horizon/libraries/helper.rb b/chef/cookbooks/horizon/libraries/helper.rb new file mode 100644 index 0000000000..893487e21a --- /dev/null +++ b/chef/cookbooks/horizon/libraries/helper.rb @@ -0,0 +1,21 @@ +# Copyright 2016 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MonascaUiHelper + def self.monasca_public_host(node) + ha_enabled = node[:monasca][:ha][:enabled] + ssl_enabled = node[:monasca][:api][:ssl] + CrowbarHelper.get_host_for_public_url(node, ssl_enabled, ha_enabled) + end +end diff --git a/chef/cookbooks/horizon/recipes/monasca_ui.rb b/chef/cookbooks/horizon/recipes/monasca_ui.rb new file mode 100644 index 0000000000..86b60888d5 --- /dev/null +++ b/chef/cookbooks/horizon/recipes/monasca_ui.rb @@ -0,0 +1,37 @@ +# Copyright 2016 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +monasca_servers = search(:node, "roles:monasca-server") +monasca_server = monasca_servers[0] + +template "/srv/www/openstack-dashboard/openstack_dashboard/"\ + "local/local_settings.d/_80_monasca_ui_settings.py" do + source "_80_monasca_ui_settings.py.erb" + variables( + kibana_enabled: true, + kibana_host: MonascaUiHelper.monasca_public_host(monasca_server) + ) + owner "root" + group "root" + mode "0644" + notifies :reload, resources(service: "apache2") +end + +package "grafana-apache" do + action :install +end + +file "/etc/apache2/vhost.d/grafana.conf" do + action :delete +end diff --git a/chef/cookbooks/horizon/recipes/server.rb b/chef/cookbooks/horizon/recipes/server.rb index e1cb4e90bd..311063c900 100644 --- a/chef/cookbooks/horizon/recipes/server.rb +++ b/chef/cookbooks/horizon/recipes/server.rb @@ -17,6 +17,8 @@ include_recipe "apache2::mod_wsgi" include_recipe "apache2::mod_rewrite" +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + if %w(suse).include? node[:platform_family] dashboard_path = "/srv/www/openstack-dashboard" else @@ -165,10 +167,12 @@ unless monasca_ui_pkgname.nil? monasca_servers = search(:node, "roles:monasca-server") || [] unless monasca_servers.empty? + include_recipe "#{@cookbook_name}::monasca_ui" package monasca_ui_pkgname do action :install notifies :reload, resources(service: "apache2") end + grafana_available = true end end @@ -280,8 +284,6 @@ "default-character-set" => "'utf8'" } -keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) - glances = search(:node, "roles:glance-server") || [] if glances.length > 0 glance = glances[0] @@ -508,7 +510,8 @@ use_ssl: node[:horizon][:apache][:ssl], ssl_crt_file: node[:horizon][:apache][:ssl_crt_file], ssl_key_file: node[:horizon][:apache][:ssl_key_file], - ssl_crt_chain_file: node[:horizon][:apache][:ssl_crt_chain_file] + ssl_crt_chain_file: node[:horizon][:apache][:ssl_crt_chain_file], + grafana_available: defined?(grafana_available) ? grafana_available : false ) if ::File.symlink?("#{node[:apache][:dir]}/sites-enabled/openstack-dashboard.conf") || node[:platform_family] == "suse" notifies :reload, resources(service: "apache2") diff --git a/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb new file mode 100644 index 0000000000..7256592d4d --- /dev/null +++ b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb @@ -0,0 +1,66 @@ +# Copyright 2016 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from django.conf import settings +from django.utils.translation import ugettext_lazy as _ + +# Services being monitored +MONITORING_SERVICES = getattr( + settings, + 'MONITORING_SERVICES', + [ + {'name': _('OpenStack Services'), + 'groupBy': 'service'}, + {'name': _('Servers'), + 'groupBy': 'hostname'}, + {'name': _('Log Paths'), + 'groupBy': 'path'} + ] +) + +MONITORING_SERVICE_TYPE = getattr( + settings, 'MONITORING_SERVICE_TYPE', 'monitoring' +) + + +# Grafana button titles/file names (global across all projects): +GRAFANA_LINKS = [ + {'title': 'Dashboard', 'fileName': 'openstack.json'}, + {'title': 'Monasca Health', 'fileName': 'monasca.json'} +] + +DEFAULT_LINKS = GRAFANA_LINKS +DASHBOARDS = getattr(settings, 'GRAFANA_LINKS', GRAFANA_LINKS) + +# +# Per project grafana button titles/file names. If in this form, +# '*' will be applied to all projects not explicitly listed. +# +# Note the above form (flat) is supported for backward compatibility. +# +#GRAFANA_LINKS = [ +# {'admin': [ +# {'title': 'Dashboard', 'fileName': 'openstack.json'}, +# {'title': 'RabbitMQ', 'fileName': 'rabbit.json'}, +# {'title': 'Project Utilization', 'fileName': 'libvirt.json'}]}, +# {'*': [ +# {'title': 'OpenStack Dashboard', 'fileName': 'project.json'}, +# {'title': 'Add New Dashboard', 'fileName': 'empty.json'}]} +#] + +ENABLE_KIBANA_BUTTON = getattr(settings, 'ENABLE_KIBANA_BUTTON', <%= @kibana_enabled ? "True" : "False" %>) +KIBANA_HOST = getattr(settings, 'KIBANA_HOST', '<%= @kibana_host %>') + +OPENSTACK_SSL_NO_VERIFY = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False) +OPENSTACK_SSL_CACERT = getattr(settings, 'OPENSTACK_SSL_CACERT', None) diff --git a/chef/cookbooks/horizon/templates/suse/openstack-dashboard.conf.erb b/chef/cookbooks/horizon/templates/suse/openstack-dashboard.conf.erb index 569161f9ec..852d2f9af6 100644 --- a/chef/cookbooks/horizon/templates/suse/openstack-dashboard.conf.erb +++ b/chef/cookbooks/horizon/templates/suse/openstack-dashboard.conf.erb @@ -46,6 +46,9 @@ DocumentRoot <%= @horizon_dir %> Alias /media <%= @horizon_dir %>/media Alias /static <%= @horizon_dir %>/static + <% if @grafana_available %> + Alias /grafana /srv/www/grafana + <% end %> Timeout 120 @@ -54,6 +57,11 @@ ExpiresActive on ExpiresDefault "access plus 1 month" + <% if @grafana_available %> + + Require all granted + + <% end %> Options None From e6f98fd1cf7b00e8d6a4764217ad2cc2ed5245d5 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Mon, 27 Feb 2017 10:08:27 +0100 Subject: [PATCH 09/52] Add monitoring dashboards to Grafana * Add monitoring dashboards to Grafana * Change template to cookbook_file --- .../horizon/files/grafana-monasca.json | 4751 +++++++++++++++++ .../horizon/files/grafana-openstack.json | 847 +++ chef/cookbooks/horizon/recipes/monasca_ui.rb | 18 + 3 files changed, 5616 insertions(+) create mode 100644 chef/cookbooks/horizon/files/grafana-monasca.json create mode 100644 chef/cookbooks/horizon/files/grafana-openstack.json diff --git a/chef/cookbooks/horizon/files/grafana-monasca.json b/chef/cookbooks/horizon/files/grafana-monasca.json new file mode 100644 index 0000000000..ff75e67de4 --- /dev/null +++ b/chef/cookbooks/horizon/files/grafana-monasca.json @@ -0,0 +1,4751 @@ +{ + "id": "sid-14393838984416185642089112666", + "title": "CMM Service", + "originalTitle": "CMM Service", + "tags": [], + "style": "light", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "title": "CMM Health - metrics", + "height": "100px", + "editable": true, + "collapse": false, + "panels": [ + { + "title": "Metrics API", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 19, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "component", + "condition_value": "monasca-api" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Threshold Engine", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 44, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-thresh" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Persister", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 21, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "component", + "condition_value": "monasca-persister" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Metrics DB", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 51, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "influxd" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Notification Engine", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 42, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-notification" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + } + ], + "showTitle": true + }, + { + "title": "CMM Health - Logs", + "height": "100px", + "editable": true, + "collapse": false, + "panels": [ + { + "title": "Log API", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 20, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "component", + "condition_value": "monasca-log-api" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Log Transformer", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 54, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.transformer" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Log Metrics", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 102, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.metrics" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Log Persister", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 53, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.persister" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Log DB", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 33, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "elasticsearch.cluster_status", + "condition_filter": true, + "condition_key": "", + "condition_value": "" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + }, + { + "value": "9", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Kibana", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 22, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "component", + "condition_value": "kibana" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + } + ], + "showTitle": true + }, + { + "title": "CMM Health - Common", + "height": "100px", + "editable": true, + "collapse": false, + "panels": [ + { + "title": "Kafka", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 38, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "kafka.Kafka" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "ZooKeeper", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 48, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "org.apache.zookeeper.server" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Storm - Nimbus", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 52, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "storm.daemon.nimbus" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "Storm - Supervisor", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 55, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "storm.daemon.supervisor" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + }, + { + "title": "MariaDB", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 66, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "mysqld" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "1", + "op": "=", + "text": "UP" + }, + { + "value": "0", + "op": "=", + "text": "DOWN" + }, + { + "value": "2", + "op": "=", + "text": "UP" + }, + { + "value": "3", + "op": "=", + "text": "UP" + }, + { + "value": "4", + "op": "=", + "text": "UP" + }, + { + "value": "5", + "op": "=", + "text": "UP" + }, + { + "value": "6", + "op": "=", + "text": "UP" + }, + { + "value": "7", + "op": "=", + "text": "UP" + }, + { + "value": "8", + "op": "=", + "text": "UP" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 40, 40, 0.59)", + "rgba(245, 150, 40, 0.73)", + "rgba(71, 212, 59, 0.4)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + } + } + ], + "showTitle": true + }, + { + "title": "System resources", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 23, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": 100, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": false + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "cpu.percent" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.user_perc" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.system_perc" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.wait_perc" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Memory usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 24, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "individual", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "mem.total_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.used_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.swap_total_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.swap_used_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.used_cache" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "Disk usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 25, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": 100, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "disk.space_used_perc", + "condition_filter": true, + "condition_key": "mount_point", + "condition_value": "/boot" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "disk.space_used_perc", + "condition_filter": true, + "condition_key": "device", + "condition_value": "rootfs" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "System load", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 26, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "load.avg_1_min", + "period": "" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "load.avg_5_min" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "load.avg_15_min" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Network Monitoring", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "Network usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 61, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "bps", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "net.in_bytes_sec", + "merge": true, + "dimensions": "", + "period": "" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "net.out_bytes_sec", + "merge": true, + "dimensions": "", + "period": "" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Metrics API", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 63, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 64, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 65, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Threshold Engine", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 73, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-thresh" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 74, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-thresh" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 95, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "storm.daemon.nimbus" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "process.pid_count", + "period": "", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "storm.daemon.supervisor" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "storm.daemon.worker" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Persister", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 67, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 68, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 69, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Metrics DB", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 76, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "influxd" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 77, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "influxd" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 78, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "influxd" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Notification Engine", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 70, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-notification" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 71, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-notification" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 72, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca-notification" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Log API", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 79, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 80, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 81, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.api" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Processing time", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 103, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "monasca.log.processing_time_ms" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Publish time", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 104, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": false + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "monasca.log.publish_time_ms" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "" + } + ], + "showTitle": true + }, + { + "title": "Log Transformer", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 85, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.transformer" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 86, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.transformer" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 87, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.transformer" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Log Metrics", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 105, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.metrics" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 106, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.metrics" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 107, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.metrics" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Log Persister", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 82, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 83, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 84, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "monasca.log.persister" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Log DB", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "Storage size", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 31, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "bytes", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "null", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "elasticsearch.store.size" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Open file descriptors", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 56, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "elasticsearch.process.open_fd", + "condition_filter": true, + "condition_key": "" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Kibana", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "Response time", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 36, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "ms", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": false + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "kibana.resp_time_avg_ms" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "kibana.resp_time_max_ms" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Requests per second", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 37, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "kibana.req_sec" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "Kafka", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 60, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "kafka.Kafka" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 59, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "kafka.Kafka" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "leftYAxisLabel": "MB", + "links": [] + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 89, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "kafka.Kafka" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Consumer lag", + "error": false, + "span": 12, + "editable": true, + "type": "graph", + "id": 58, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "kafka.consumer_lag" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "ZooKeeper", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 90, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "org.apache.zookeeper.server" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 91, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "org.apache.zookeeper.server" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 92, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "org.apache.zookeeper.server" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Outstanding bytes", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 57, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "zookeeper.outstanding_bytes" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Connections count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 49, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "zookeeper.connections_count" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Timings", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 47, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "s", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "zookeeper.avg_latency_sec" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "zookeeper.max_latency_sec" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "zookeeper.min_latency_sec" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + }, + { + "title": "MariaDB", + "height": "250px", + "editable": true, + "collapse": true, + "panels": [ + { + "title": "CPU usage", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 96, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.cpu_perc", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "mysqld" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "Allocated memory", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 97, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.mem.rss_mbytes", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "mysqld" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "PID count", + "error": false, + "span": 4, + "editable": true, + "type": "graph", + "id": 98, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": true, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "process.pid_count", + "condition_filter": true, + "condition_key": "process_name", + "condition_value": "mysqld" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [], + "enable": false + }, + "annotations": { + "enable": false, + "list": [] + }, + "refresh": "30s", + "version": 6, + "hideAllLegends": false +} diff --git a/chef/cookbooks/horizon/files/grafana-openstack.json b/chef/cookbooks/horizon/files/grafana-openstack.json new file mode 100644 index 0000000000..b4e8113ef8 --- /dev/null +++ b/chef/cookbooks/horizon/files/grafana-openstack.json @@ -0,0 +1,847 @@ +{ + "id": "sid-14394701277644387945359960469", + "title": "Openstack Dashboard", + "originalTitle": "Openstack Dashboard", + "tags": [], + "style": "light", + "timezone": "browser", + "editable": true, + "hideControls": false, + "sharedCrosshair": false, + "rows": [ + { + "title": "OpenStack Health", + "height": "100px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "title": "compute (Nova)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 1, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "compute" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + }, + { + "title": "networking (Neutron)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 4, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "networking" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + }, + { + "title": "image-service (Glance)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 5, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "image-service" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + }, + { + "title": "block-storage (Cinder)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 7, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "block-storage" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + }, + { + "title": "object-storage (Swift)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 8, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "object-storage" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + }, + { + "title": "identity-service (Keystone)", + "error": false, + "span": 2, + "editable": true, + "type": "singlestat", + "id": 12, + "links": [], + "maxDataPoints": 100, + "interval": null, + "targets": [ + { + "function": "none", + "column": "value", + "series": "http_status", + "condition_filter": true, + "condition_key": "service", + "condition_value": "identity-service" + } + ], + "cacheTimeout": null, + "format": "none", + "prefix": "", + "postfix": "", + "nullText": null, + "valueMaps": [ + { + "value": "0", + "op": "=", + "text": "UP" + }, + { + "value": "1", + "op": "=", + "text": "DOWN" + } + ], + "nullPointMode": "connected", + "valueName": "current", + "prefixFontSize": "50%", + "valueFontSize": "80%", + "postfixFontSize": "50%", + "thresholds": "-1.0,0.2,0.8", + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(71, 212, 59, 0.4)", + "rgba(245, 150, 40, 0.73)", + "rgba(225, 40, 40, 0.59)" + ], + "sparkline": { + "show": false, + "full": false, + "lineColor": "rgb(31, 120, 193)", + "fillColor": "rgba(31, 118, 189, 0.18)" + }, + "height": "100" + } + ], + "notice": false, + "showTitle": true + }, + { + "title": "System Resources", + "height": "250px", + "editable": true, + "collapse": false, + "collapsable": true, + "panels": [ + { + "span": 6, + "editable": true, + "type": "graph", + "loadingEditor": false, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "scale": 1, + "y_formats": [ + "percent", + "none" + ], + "grid": { + "max": null, + "min": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "leftMax": 100, + "rightMax": null, + "leftMin": 0, + "rightMin": null + }, + "annotate": { + "enable": false + }, + "resolution": 100, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true, + "rightSide": false + }, + "percentage": false, + "zerofill": true, + "nullPointMode": "null", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "query_as_alias": true, + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "cpu.percent", + "condition_filter": false, + "alias": "" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.wait_perc" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.system_perc" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "cpu.user_perc" + } + ], + "aliasColors": {}, + "title": "CPU usage", + "id": 2, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "" + }, + { + "title": "Memory usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 13, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "none" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)", + "thresholdLine": false + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true, + "rightSide": false + }, + "nullPointMode": "null", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "mem.total_mb", + "merge": false, + "dimensions": "" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.swap_used_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.used_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.swap_total_mb" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "mem.used_cache" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [], + "leftYAxisLabel": "MB" + }, + { + "title": "Disk usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 14, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "percent", + "short" + ], + "grid": { + "leftMax": 100, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true, + "rightSide": false, + "sortDesc": null, + "sort": null + }, + "nullPointMode": "null", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "disk.space_used_perc", + "merge": false, + "dimensions": "", + "condition_filter": true, + "condition_key": "device", + "condition_value": "rootfs" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "disk.space_used_perc", + "condition_filter": true, + "condition_key": "mount_point", + "condition_value": "/boot" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "disk.space_used_perc", + "condition_filter": true, + "condition_key": "mount_point", + "condition_value": "/srv/node/swiftloopback" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + }, + { + "title": "System load", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 15, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "none", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "null", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "load.avg_1_min" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "load.avg_5_min" + }, + { + "target": "", + "function": "none", + "column": "value", + "series": "load.avg_15_min" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "notice": false, + "showTitle": true + }, + { + "title": "Network monitoring", + "height": "250px", + "editable": true, + "collapse": false, + "panels": [ + { + "title": "Network usage", + "error": false, + "span": 6, + "editable": true, + "type": "graph", + "id": 16, + "datasource": null, + "renderer": "flot", + "x-axis": true, + "y-axis": true, + "y_formats": [ + "bps", + "short" + ], + "grid": { + "leftMax": null, + "rightMax": null, + "leftMin": 0, + "rightMin": null, + "threshold1": null, + "threshold2": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "lines": true, + "fill": 0, + "linewidth": 1, + "points": false, + "pointradius": 5, + "bars": false, + "stack": false, + "percentage": false, + "legend": { + "show": true, + "values": false, + "min": false, + "max": false, + "current": false, + "total": false, + "avg": false, + "alignAsTable": true + }, + "nullPointMode": "connected", + "steppedLine": false, + "tooltip": { + "value_type": "cumulative", + "shared": true + }, + "targets": [ + { + "function": "none", + "column": "value", + "series": "net.in_bytes_sec", + "merge": true, + "dimensions": "" + }, + { + "target": "", + "function": "none", + "column": "value", + "merge": true, + "dimensions": "", + "series": "net.out_bytes_sec" + } + ], + "aliasColors": {}, + "seriesOverrides": [], + "links": [] + } + ], + "showTitle": true + } + ], + "nav": [ + { + "type": "timepicker", + "collapse": false, + "notice": false, + "enable": true, + "status": "Stable", + "time_options": [ + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "now": true + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [], + "enable": false + }, + "annotations": { + "enable": false, + "list": [] + }, + "refresh": "30s", + "version": 6, + "hideAllLegends": false +} diff --git a/chef/cookbooks/horizon/recipes/monasca_ui.rb b/chef/cookbooks/horizon/recipes/monasca_ui.rb index 86b60888d5..6174ad9c42 100644 --- a/chef/cookbooks/horizon/recipes/monasca_ui.rb +++ b/chef/cookbooks/horizon/recipes/monasca_ui.rb @@ -30,8 +30,26 @@ package "grafana-apache" do action :install + notifies :reload, resources(service: "apache2") end file "/etc/apache2/vhost.d/grafana.conf" do action :delete + notifies :reload, resources(service: "apache2") +end + +cookbook_file "/srv/www/grafana/app/dashboards/openstack.json" do + source "grafana-openstack.json" + owner "root" + group "root" + mode "0644" + notifies :reload, resources(service: "apache2") +end + +cookbook_file "/srv/www/grafana/app/dashboards/monasca.json" do + source "grafana-monasca.json" + owner "root" + group "root" + mode "0644" + notifies :reload, resources(service: "apache2") end From aa7e75aa05a47d41121a48b97b9e50a0dd151c98 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Tue, 28 Feb 2017 10:46:57 +0100 Subject: [PATCH 10/52] Provide Ansible integration * Provide Ansible integration - draft version * Adding parameters to UI * Adding group_all template * Fixing inventory file templating * Making sure Ansible runs successfully (at least once) * Adapt constraints for monasca-server * Set default deployment of monasca roles * Add config field for CMM installer URL * Fixing invalid merge * Fix order of roles * Add grafana database password * Fixing hound violations * Fixing hound violations 2 * Draft of inventory file template for cluster * Adjust monasca-master role to use appropriate inventory file template * Fill in inventory file for cluster * Provide Ansible integration - draft version * Adding parameters to UI * Adding group_all template * Fixing inventory file templating * Making sure Ansible runs successfully (at least once) * Adapt constraints for monasca-server * Set default deployment of monasca roles * Add config field for CMM installer URL * Fixing invalid merge * Fix order of roles * Add grafana database password * Fixing hound violations * Fixing hound violations 2 * Addressing review comments * Fix Hound violations * Fix Hound violations 2 * Fix Hound violations 3 * Addressing review comments * Addressing some additional comments * Addressing some additional comments 2 --- chef/cookbooks/monasca/libraries/helper.rb | 42 +++ chef/cookbooks/monasca/recipes/master.rb | 107 +++++++ .../monasca/recipes/role_monasca_master.rb | 19 ++ .../monasca/templates/default/all_group.erb | 107 +++++++ .../templates/default/cmm-hosts-cluster.erb | 287 ++++++++++++++++++ .../templates/default/cmm-hosts-single.erb | 197 ++++++++++++ .../templates/default/credentials.yml.erb | 47 +++ chef/data_bags/crowbar/template-monasca.json | 16 + .../data_bags/crowbar/template-monasca.schema | 18 ++ chef/roles/monasca-master.rb | 5 + .../app/models/monasca_service.rb | 64 ++-- .../monasca/_edit_attributes.html.haml | 19 ++ .../config/locales/monasca/en.yml | 13 + 13 files changed, 916 insertions(+), 25 deletions(-) create mode 100644 chef/cookbooks/monasca/libraries/helper.rb create mode 100644 chef/cookbooks/monasca/recipes/master.rb create mode 100644 chef/cookbooks/monasca/recipes/role_monasca_master.rb create mode 100644 chef/cookbooks/monasca/templates/default/all_group.erb create mode 100644 chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb create mode 100644 chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb create mode 100644 chef/cookbooks/monasca/templates/default/credentials.yml.erb create mode 100644 chef/roles/monasca-master.rb diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb new file mode 100644 index 0000000000..7dc90f223f --- /dev/null +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -0,0 +1,42 @@ +# +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module MonascaHelper + def self.monasca_public_host(node) + ha_enabled = node[:monasca][:ha][:enabled] + ssl_enabled = node[:monasca][:api][:ssl] + CrowbarHelper.get_host_for_public_url(node, ssl_enabled, ha_enabled) + end + + def self.monasca_admin_host(node) + ha_enabled = node[:monasca][:ha][:enabled] + CrowbarHelper.get_host_for_admin_url(node, ha_enabled) + end + + def self.api_admin_url(node) + host = api_admin_host(node) + protocol = node[:monasca][:api][:ssl] ? "https" : "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end + + def self.monasca_hosts(nodes) + hosts = [] + nodes.each do |n| + hosts.push(CrowbarHelper.get_host_for_admin_url(n)) + end + hosts + end +end diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb new file mode 100644 index 0000000000..c0569a5fe0 --- /dev/null +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -0,0 +1,107 @@ +# +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitation. + +package "ansible" + +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +directory "/opt/fujitsu" do + owner "root" + group "root" + mode "0755" +end + +# TODO: use package "monasca-installer" +remote_file "/opt/fujitsu/cmm-suse.tgz" do + source node[:monasca][:master][:cmm_tarball_url] + owner "root" + group "root" + mode "0644" + action :create_if_missing +end + +execute "extract cmm tarball" do + command "tar xf cmm-suse.tgz" + cwd "/opt/fujitsu/" + not_if { Dir.exist?("/opt/fujitsu/monasca-installer") } + notifies :run, "execute[run ansible]", :delayed +end + +template "/opt/fujitsu/monasca-installer/credentials.yml" do + source "credentials.yml.erb" + owner "root" + group "root" + mode "0600" + variables( + keystone_settings: keystone_settings + ) + notifies :run, "execute[run ansible]", :delayed +end + +monasca_hosts = MonascaHelper.monasca_hosts(search(:node, "roles:monasca-server")) + +raise "no nodes with monasca-server role found" if monasca_hosts.nil? or monasca_hosts.empty? + +hosts_template = + if monasca_hosts.length == 1 + "cmm-hosts-single.erb" + else + "cmm-hosts-cluster.erb" + +template "/opt/fujitsu/monasca-installer/cmm-hosts" do + source hosts_template + owner "root" + group "root" + mode "0644" + variables( + monasca_host: monasca_hosts[0], + monasca_hosts: monasca_hosts, + ansible_ssh_user: "root", + offline_resources_host: CrowbarHelper.get_host_for_admin_url(node), + keystone_host: keystone_settings["public_url_host"] + ) + notifies :run, "execute[run ansible]", :delayed +end + +template "/opt/fujitsu/monasca-installer/group_vars/all_group" do + source "all_group.erb" + owner "root" + group "root" + mode "0644" + variables( + keystone_settings: keystone_settings + ) + notifies :run, "execute[run ansible]", :delayed +end + +# This file is used to mark that ansible installer run successfully. +# Without this, it could happen that the installer was not re-tried +# after a failed run. +file "/opt/fujitsu/monasca-installer/.installed" do + content "cmm installed" + owner "root" + group "root" + mode "0644" + notifies :run, "execute[run ansible]", :delayed + action :create_if_missing +end + +execute "run ansible" do + command "rm -f /opt/fujitsu/monasca-installer/.installed"\ + "&& ansible-playbook -i cmm-hosts monasca.yml"\ + "&& touch /opt/fujitsu/monasca-installer/.installed" + cwd "/opt/fujitsu/monasca-installer" + action :nothing +end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_master.rb b/chef/cookbooks/monasca/recipes/role_monasca_master.rb new file mode 100644 index 0000000000..df275e9eac --- /dev/null +++ b/chef/cookbooks/monasca/recipes/role_monasca_master.rb @@ -0,0 +1,19 @@ +# +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-master") + include_recipe "monasca::master" +end diff --git a/chef/cookbooks/monasca/templates/default/all_group.erb b/chef/cookbooks/monasca/templates/default/all_group.erb new file mode 100644 index 0000000000..c2fad477cd --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/all_group.erb @@ -0,0 +1,107 @@ +--- +# Copyright 2017 FUJITSU LIMITED +# file: group_vars/all_group + +# ports +monasca_api_client_port: 8070 +monasca_api_admin_port: 8081 +persister_app_connector_port: 8190 +persister_admin_connector_port: 8191 +monasca_log_api_port: 5607 +keystone_port: 5000 +keystone_user_port: 35357 +kibana_port: 5601 +elasticsearch_port: 9200 +kafka_port: 9092 +zookeeper_port: 2181 +memcached_port: 11211 +mysql_port: 3306 + +# hosts +offline_host: "{{ hostvars['offline-resources-node']['ansible_ssh_host'] }}" +monasca_api_ip: "{% if groups['monasca_api_group']|length > 1 %} + {{ hostvars['monasca-api-balancer-node']['ansible_ssh_host'] }} + {% else %} + {{ hostvars['monasca-api-node']['ansible_ssh_host'] }} + {% endif %} + " +monasca_log_api_ip: "{% if groups['monasca_log_api_group']|length > 1 %} + {{ hostvars['monasca-log-api-balancer-node']['ansible_ssh_host'] }} + {% else %} + {{ hostvars['monasca-log-api-node']['ansible_ssh_host'] }} + {% endif %} + " +kibana_ip: "{% if groups['kibana_group']|length > 1 %} + {{ hostvars['kibana-balancer-node']['ansible_ssh_host'] }} + {% else %} + {{ hostvars['kibana-node']['ansible_ssh_host'] }} + {% endif %} + " +keystone_host: "{{ hostvars['keystone-node']['ansible_ssh_host'] }}" +keepalived_virtual_ip: "{{ hostvars['keepalived-virtual-node']['ansible_ssh_host'] }}" + +# monsaca settings +monasca_conf_dir: "{{ etc_dir }}/monasca" +monasca_work_dir: "{{ opt_dir }}/monasca" + +# keystone +keystone_version: <%= @keystone_settings['api_version'] %> +keystone_url: "<%= @keystone_settings['public_auth_url'] %>" + +# URL service definition +balls_repo_url: "http://{{ offline_host }}:8888" +world_repo_url: "{{ balls_repo_url }}/world" +monasca_api_url: "http://{{ monasca_api_ip|trim|safe }}:{{ monasca_api_client_port|int }}/v2.0" +monasca_log_api_url: "http://{{ monasca_log_api_ip|trim|safe }}:{{ monasca_log_api_port|int }}/v3.0" +kibana_url: "http://{{ kibana_ip|trim|safe }}:{{ kibana_port }}" + +# Monasca Api CLI - Credentials +monasca_user: + user: cmm-operator + password: "{{ keystone_cmm_operator_user_password }}" + project: cmm + +# Installation / Download directories +opt_dir: "/opt" +etc_dir: "/etc" +venvs_dir: "{{ opt_dir }}" +elk_base_dir: "{{ opt_dir }}" +download_tmp_dir: "{{ opt_dir }}/.download_dir" +monasca_download_dir: /opt/monasca-downloads +clean_up_dirs: + - "{{ download_tmp_dir }}" + +# Installation mode +offline_mode: True + +# Download urls +logstash_agent_download_url: "{{ world_repo_url }}" + +# Keystone Admin credentials +keystone_admin_token: <%= @keystone_settings['admin_token'] %> +keystone_admin: '<%= @keystone_settings['admin_user'] %>' +keystone_admin_project: '<%= @keystone_settings['admin_tenant'] %>' + +# Logging +log_dir: /var/log + +# Data directories +data_dir: /var/data + +# Supported varsions of dependencies +ansible_version_min: + ver: "2.2.0.0" + constraint: ">=" +ansible_version_max: + ver: "2.2.1.0" + constraint: "<=" +pip_version_min: 7.0.3 + +# pip configuration +pip_base_dir: "/root/.pip" +pip_config_file: "{{ pip_base_dir }}/.cmm_pip.conf" +pbr_version: 1.8.1 + +# try to use this variable for each wait_for like task +monasca_wait_for_period: 180 + diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb new file mode 100644 index 0000000000..97fe50792a --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb @@ -0,0 +1,287 @@ +# Copyright 2017 FUJITSU LIMITED + +################################################################################ +# Host definitions. Replace monasca_host, openstack_host, ex_node_[1-3]_host, +# offline_resources_host, ssh_user_monasca, ssh_user_openstack, ssh_user_offline_host +# and ssh_user_ex_node_[1-3] with valid information. +################################################################################ + +localhost ansible_connection=local + +# Offline Host +offline-resources-node ansible_ssh_host=<%= @offline_resources_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Openstack Hosts +keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Load balancer hosts +# TODO +#monasca-log-api-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> +#monasca-api-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> +#kibana-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> + +# ElasticSearch nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "elastic-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Kafka nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "kafka-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Zookeeper nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "zookeeper-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Log persister nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "log-persister-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Log transformer nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "log-transformer-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Kibana nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "kibana-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Log api nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "log-api-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Memcached nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "memcached-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Monasca-api nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "monasca-api-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Monasca-notification nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "monasca-notification-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Monasca-persister nodes +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "monasca-persister-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Monasca-log-metrics nodes for cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "monasca-log-metrics-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Load balancer node +# TODO +#balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> + +#Storm Nimbus nodes +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "storm-nimbus-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Storm Supervisor nodes +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "storm-supervisor-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Monasca-thresh hosts +monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# MariaDB nodes for Galera cluster +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "mariadb-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# InfluxDB nodes +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "influxdb-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# HAProxy nodes +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "haproxy-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Keepalived nodes +# TODO +#keepalived-virtual-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> + +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "keepalived-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +# Elasticsearch Curator +<% @monasca_hosts.each_with_index do |host, index| %> +<%= "elasticsearch-curator-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> +<% end %> + +################################################################################ +# Other group definition +################################################################################ + +[offline_group] +offline-resources-node +localhost + +[openstack_group] +keystone-node + +[monasca_group] +balancer-node + +[monasca_persister_group] +<% @monasca_hosts.length.times do |index| %> +<%= "monasca-persister-node-#{index} influxdb_node_for_persister=keepalived-virtual-node" %> +<% end %> + +[monasca_api_group] +<% @monasca_hosts.length.times do |index| %> +<%= "monasca-api-node-#{index} database_node_for_api=keepalived-virtual-node influxdb_node_for_api=keepalived-virtual-node" %> +<% end %> + +[monasca_notification_group] +<% @monasca_hosts.length.times do |index| %> +<%= "monasca-notification-node-#{index} database_node_for_notification=keepalived-virtual-node" %> +<% end %> + +[memcached_group] +<% @monasca_hosts.length.times do |index| %> +<%= "memcached-node-#{index}" %> +<% end %> + +[kafka_group] +<% @monasca_hosts.length.times do |index| %> +<%= "kafka-node-#{index}" %> +<% end %> + +[zookeeper_group] +<% @monasca_hosts.length.times do |index| %> +<%= "zookeeper-node-#{index}" %> +<% end %> + +[monasca_log_api_group] +<% @monasca_hosts.length.times do |index| %> +<%= "log-api-node-#{index}" %> +<% end %> + +[monasca_log_metrics_group] +<% @monasca_hosts.length.times do |index| %> +<%= "monasca-log-metrics-node-#{index}" %> +<% end %> + +[monasca_log_persister_group] +<% @monasca_hosts.length.times do |index| %> +<%= "log-persister-node-#{index}" %> +<% end %> + +[monasca_log_transformer_group] +<% @monasca_hosts.length.times do |index| %> +<%= "log-transformer-node-#{index}" %> +<% end %> + +[elasticsearch_group] +<% @monasca_hosts.length.times do |index| %> +<%= "elastic-node-#{index}" %> +<% end %> + +# specify host which kibana should use to connect to elasticsearch +[kibana_group] +<% @monasca_hosts.length.times do |index| %> +<%= "kibana-node-#{index} es_node_for_kibana=elastic-node-#{index}" %> +<% end %> + +[balancer_group] +balancer-node + +[storm_nimbus_group] +<% @monasca_hosts.length.times do |index| %> +<%= "storm-nimbus-node-#{index}" %> +<% end %> + +[storm_supervisor_group] +<% @monasca_hosts.length.times do |index| %> +<%= "storm-supervisor-node-#{index}" %> +<% end %> + +[monasca_thresh_group] +monasca-thresh-node database_node_for_thresh=keepalived-virtual-node + +[mariadb_group] +<% @monasca_hosts.length.times do |index| %> +<%= "mariadb-node-#{index}" %> +<% end %> + +[influxdb_group] +influxdb-node-0 +<% (@monasca_hosts.length - 1).times do |index| %> +influxdb-node-<%= index + 1 %> influxdb_master_node_name=influxdb-node-0 +<% end %> + +[haproxy_group] +<% @monasca_hosts.length.times do |index| %> +<%= "haproxy-node-#{index}" %> +<% end %> + +[keepalived_group] +<% @monasca_hosts.length.times do |index| %> +<%= "keepalived-node-#{index}" %> +<% end %> + +[elasticsearch_curator_group] +<% @monasca_hosts.length.times do |index| %> +<%= "elasticsearch-curator-node-#{index}" %> +<% end %> + + +################################################################################ +# Group inheritance +################################################################################ + +[elkstack_group:children] +elasticsearch_group +kibana_group +monasca_log_persister_group +monasca_log_transformer_group +monasca_log_metrics_group +balancer_group +elasticsearch_curator_group + +[storm_group:children] +storm_nimbus_group +storm_supervisor_group + +[all_group:children] +offline_group +openstack_group +monasca_group + +[monasca_group:children] +elkstack_group +memcached_group +kafka_group +zookeeper_group +monasca_log_api_group +balancer_group +monasca_api_group +monasca_notification_group +monasca_persister_group +storm_group +monasca_thresh_group +database_cluster_group +grafana_group + +[database_cluster_group:children] +mariadb_group +influxdb_group +haproxy_group +keepalived_group diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb b/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb new file mode 100644 index 0000000000..673150cb4f --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb @@ -0,0 +1,197 @@ +# Copyright 2017 FUJITSU LIMITED + +################################################################################ +# Host definitions. Replace monasca_host, openstack_host, offline_resources_host, +# ansible_ssh_user, ssh_user_openstack and ssh_user_offline_host with +# valid informations. +################################################################################ + +localhost ansible_connection=local + +# Offline Host +offline-resources-node ansible_ssh_host=<%= @offline_resources_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca host +cmm-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Openstack hosts +keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-log-transformer +log-transformer-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-log-persister +log-persister-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Kibana +kibana-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Elasticsearch +elasticsearch-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Memcached +memcached-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Kafka +kafka-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Zookeeper +zookeeper-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-api +monasca-api-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-log-api +monasca-log-api-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-notification +monasca-notification-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-persister node +monasca-persister-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-log-metric node +monasca-log-metrics-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca Agent Hosts +cmm-monasca-agent ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca Log Agent Hosts +cmm-monasca-log-agent ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Storm Nimbus node +storm-nimbus-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Storm Supervisor node +storm-supervisor-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Monasca-thresh host +monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# MariaDB node +mariadb-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# Elasticsearch Curator +elasticsearch-curator-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +# InfluxDB node +influxdb-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> + +################################################################################ +# Agent and Log-Agent group definition: +# To install additional agents (metrics and logs) add nodes to +# monasca_agent_group or monasca_log_agent_group. Run ansible-playbook +# monasca-agent.yml or ansible-playbook monasca-log-agent.yml to install agents. +################################################################################ + +[monasca_agent_group] +cmm-monasca-agent + +[monasca_log_agent_group] +cmm-monasca-log-agent + +################################################################################ +# Other group definition +################################################################################ + +[openstack_group] +keystone-node + +[offline_group] +offline-resources-node +localhost + +[memcached_group] +memcached-node + +[kafka_group] +kafka-node + +[zookeeper_group] +zookeeper-node + +[monasca_group] +cmm-node + +[monasca_api_group] +monasca-api-node database_node_for_api=cmm-node influxdb_node_for_api=influxdb-node + +[monasca_notification_group] +monasca-notification-node database_node_for_notification=cmm-node + +[monasca_persister_group] +monasca-persister-node influxdb_node_for_persister=influxdb-node + +[monasca_log_api_group] +monasca-log-api-node + +[monasca_log_metrics_group] +monasca-log-metrics-node + +[monasca_log_persister_group] +log-persister-node + +[monasca_log_transformer_group] +log-transformer-node + +[elasticsearch_group] +elasticsearch-node + +[kibana_group] +kibana-node es_node_for_kibana=elasticsearch-node + +[storm_nimbus_group] +storm-nimbus-node + +[storm_supervisor_group] +storm-supervisor-node + +[monasca_thresh_group] +monasca-thresh-node database_node_for_thresh=cmm-node + +[mariadb_group] +mariadb-node + +[elasticsearch_curator_group] +elasticsearch-curator-node + +[influxdb_group] +influxdb-node + +################################################################################ +# Group inheritance +################################################################################ + +[elkstack_group:children] +elasticsearch_group +kibana_group +monasca_log_agent_group +monasca_log_persister_group +monasca_log_transformer_group +monasca_log_metrics_group +elasticsearch_curator_group + +[storm_group:children] +storm_nimbus_group +storm_supervisor_group + +[all_group:children] +offline_group +openstack_group +monasca_group +monasca_agent_group + +[monasca_group:children] +elkstack_group +memcached_group +kafka_group +zookeeper_group +monasca_log_api_group +monasca_api_group +monasca_notification_group +monasca_persister_group +storm_group +monasca_thresh_group +mariadb_group +influxdb_group + diff --git a/chef/cookbooks/monasca/templates/default/credentials.yml.erb b/chef/cookbooks/monasca/templates/default/credentials.yml.erb new file mode 100644 index 0000000000..a9fb27bb84 --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/credentials.yml.erb @@ -0,0 +1,47 @@ +# +# Copyright 2017 Fujitsu LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitation. + +# Important: Please provide secure passwords below + +# group_vars/monasca_group credentials +# used for influxdb_users +influxdb_mon_api_password: <%= node[:monasca][:master][:influxdb_mon_api_password] %> +influxdb_mon_persister_password: <%= node[:monasca][:master][:influxdb_mon_persister_password] %> + +# Used for database_users +database_notification_password: <%= node[:monasca][:master][:database_notification_password] %> +database_monapi_password: <%= node[:monasca][:master][:database_monapi_password] %> +database_thresh_password: <%= node[:monasca][:master][:database_thresh_password] %> +database_logapi_password: <%= node[:monasca][:master][:database_logapi_password] %> + +# Keystone user passwords - used in group_vars/openstack_group/keystone_users +## new created users in keystone: password +keystone_cmm_operator_user_password: <%= node[:monasca][:master][:keystone_cmm_operator_user_password] %> +keystone_cmm_agent_password: <%= node[:monasca][:master][:keystone_cmm_agent_password] %> +keystone_admin_agent_password: <%= node[:monasca][:master][:keystone_admin_agent_password] %> +## existing keystone user. Important: This password must match to an already existing user in keystone. It is used to create new users. +keystone_admin_password: <%= @keystone_settings['admin_password'] %> + +# Monasca-Agent credentials. used in host_vars/ +#cmm_monasca_agent_keystone_password: +#openstack_monasca_agent_keystone_password: +#cmm_monasca_agent_database_password: + +# Monasca-Log-Agent credentials. used in host_vars/ +#cmm_monasca_log_agent_keystone_password: +#openstack_monasca_log_agent_keystone_password: + +database_grafana_password: <%= node[:monasca][:master][:database_grafana_password] %> + diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index dae9a7d77d..3378851103 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -45,6 +45,19 @@ "ssl" : false, "threads" : 10 }, + "master" : { + "cmm_tarball_url": "http://127.0.0.1/cmm/cmm-suse.tgz", + "influxdb_mon_api_password" : "", + "influxdb_mon_persister_password" : "", + "database_notification_password" : "", + "database_monapi_password" : "", + "database_thresh_password" : "", + "database_logapi_password" : "", + "keystone_cmm_operator_user_password" : "", + "keystone_cmm_agent_password" : "", + "keystone_admin_agent_password" : "", + "database_grafana_password" : "" + }, "db" : { "database" : "monasca", "password" : "", @@ -68,17 +81,20 @@ "schema-revision": 100, "element_states": { "monasca-server": [ "readying", "ready", "applying" ], + "monasca-master": [ "readying", "ready", "applying" ], "monasca-agent": [ "readying", "ready", "applying" ], "monasca-log-agent": [ "readying", "ready", "applying" ] }, "elements": {}, "element_order": [ [ "monasca-server" ], + [ "monasca-master" ], [ "monasca-agent" ], [ "monasca-log-agent" ] ], "element_run_list_order": { "monasca-server": 110, + "monasca-master": 115, "monasca-agent": 120, "monasca-log-agent": 120 }, diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 6478aa27b2..21fe675539 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -70,6 +70,24 @@ "threads": { "required": true, "type": "int" } } }, + "master": { + "required": true, + "type": "map", + "mapping": { + "cmm_tarball_url": { "required": true, "type": "str" }, + "influxdb_mon_api_password": { "required": true, "type": "str" }, + "influxdb_mon_persister_password": { "required": true, "type": "str" }, + "database_notification_password": { "required": true, "type": "str" }, + "database_monapi_password": { "required": true, "type": "str" }, + "database_thresh_password": { "required": true, "type": "str" }, + "database_logapi_password": { "required": true, "type": "str" }, + "keystone_cmm_operator_user_password": { "required": true, "type": "str" }, + "keystone_cmm_agent_password": { "required": true, "type": "str" }, + "keystone_admin_agent_password": { "required": true, "type": "str" }, + "database_grafana_password": { "required": true, "type": "str" } + } + }, + "db": { "required": true, "type": "map", diff --git a/chef/roles/monasca-master.rb b/chef/roles/monasca-master.rb new file mode 100644 index 0000000000..d159c2c7a3 --- /dev/null +++ b/chef/roles/monasca-master.rb @@ -0,0 +1,5 @@ +name "monasca-master" +description "Monasca Ansible Master Role" +run_list("recipe[monasca::role_monasca_master]") +default_attributes +override_attributes diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 7833a2a86b..23cad2bbea 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -49,13 +49,23 @@ def role_constraints }, "monasca-server" => { "unique" => false, - "count" => 1, + "count" => -1, "cluster" => true, "admin" => false, "exclude_platform" => { "suse" => "< 12.1", "windows" => "/.*/" } + }, + "monasca-master" => { + "unique" => true, + "count" => 1, + "cluster" => false, + "admin" => true, + "exclude_platform" => { + "suse" => "< 12.1", + "windows" => "/.*/" + } } } end @@ -78,32 +88,22 @@ def create_proposal base = super nodes = NodeObject.all - # FIXME: Putting the monasca backend services on the controller is - # temporary to allow for development right now. We will eventually want the - # commented line for server_roles, i.e. have a dedicated Monitoring node - # role in Crowbar. Adding that role will require changes to: - # - # * https://github.com/crowbar/crowbar-core/blob/master/crowbar_framework/app/helpers/nodes_helper.rb#L366 - # * https://github.com/crowbar/crowbar-core/blob/master/bin/crowbar_machines#L362 - # * https://github.com/crowbar/crowbar-core/blob/master/crowbar_framework/config/locales/crowbar/en.yml - # - # at the very least. - server_nodes = nodes.select { |n| n.intended_role == "controller" } - ### server_nodes = nodes.select { |n| n.intended_role == "monitoring" } + server_nodes = nodes.select { |n| n.intended_role == "monitoring" } server_nodes = [nodes.first] if server_nodes.empty? + server_nodes = server_nodes.map { |n| n.name }.compact - # TODO: do we really want to have the agent on all nodes by - # default? - agent_nodes = nodes + agent_nodes = nodes.map { |n| n.name }.compact - base["deployment"][@bc_name]["elements"]["monasca-agent"] = agent_nodes - base["deployment"][@bc_name]["elements"]["monasca-log-agent"] = agent_nodes - unless server_nodes.nil? - base["deployment"][@bc_name]["elements"] = { - "monasca-server" => [server_nodes.first.name] - } - end + master_nodes = nodes.select { |n| n.intended_role == "admin" || n.name.start_with?("crowbar.") } + master_node = master_nodes.empty? ? nodes.first : master_nodes.first + + base["deployment"][@bc_name]["elements"] = { + "monasca-server" => server_nodes, + "monasca-master" => [master_node.name], + "monasca-agent" => agent_nodes, + "monasca-log-agent" => agent_nodes + } base["attributes"][@bc_name]["database_instance"] = find_dep_proposal("database") @@ -115,14 +115,28 @@ def create_proposal # note(trebskit) once agent is ready, uncomment following line # base["attributes"][@bc_name][:agent][:keystone][:service_password] = random_password base["attributes"][@bc_name][:log_agent][:keystone][:service_password] = random_password + base["attributes"][@bc_name][:master][:influxdb_mon_api_password] = random_password + base["attributes"][@bc_name][:master][:influxdb_mon_persister_password] = random_password + base["attributes"][@bc_name][:master][:database_notification_password] = random_password + base["attributes"][@bc_name][:master][:database_monapi_password] = random_password + base["attributes"][@bc_name][:master][:database_thresh_password] = random_password + base["attributes"][@bc_name][:master][:database_logapi_password] = random_password + base["attributes"][@bc_name][:master][:keystone_cmm_operator_user_password] = random_password + base["attributes"][@bc_name][:master][:keystone_cmm_agent_password] = random_password + base["attributes"][@bc_name][:master][:keystone_admin_agent_password] = random_password + base["attributes"][@bc_name][:master][:database_grafana_password] = random_password @logger.debug("Monasca create_proposal: exiting") base end def validate_proposal_after_save(proposal) - validate_one_for_role proposal, "monasca-server" - + validate_one_for_role proposal, "monasca-master" + nodes = proposal["deployment"][@bc_name]["elements"] + if !nodes.key?("monasca-server") || + (nodes["monasca-server"].length != 1 && nodes["monasca-server"].length != 3) + validation_error("Need either one or three monasca-server node(s).") + end super end diff --git a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml index 7041a1bdd6..9cfb49ca0b 100644 --- a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml @@ -1,3 +1,22 @@ = attributes_for @proposal do .panel-sub = header show_raw_deployment?, true + + .panel-body + + %fieldset + %legend + = t(".master_header") + + = string_field %w(master cmm_tarball_url) + = password_field %w(master influxdb_mon_api_password) + = password_field %w(master influxdb_mon_persister_password) + = password_field %w(master database_notification_password) + = password_field %w(master database_monapi_password) + = password_field %w(master database_thresh_password) + = password_field %w(master database_logapi_password) + = password_field %w(master keystone_cmm_operator_user_password) + = password_field %w(master keystone_cmm_agent_password) + = password_field %w(master keystone_admin_agent_password) + = password_field %w(master database_grafana_password) + diff --git a/crowbar_framework/config/locales/monasca/en.yml b/crowbar_framework/config/locales/monasca/en.yml index dfe0dcc36f..af2bc1a76b 100644 --- a/crowbar_framework/config/locales/monasca/en.yml +++ b/crowbar_framework/config/locales/monasca/en.yml @@ -29,6 +29,19 @@ en: agent_header: 'Agent Settings' agent: url: 'Monasca URL' + master_header: 'Installation parameters' + master: + cmm_tarball_url: 'URL of the CMM installer tarball' + influxdb_mon_api_password: 'InfluxDB password' + influxdb_mon_persister_password: 'Metrics persister password' + database_notification_password: 'Notification database password' + database_monapi_password: 'Metrics API database password' + database_thresh_password: 'Thresh Engine database password' + database_logapi_password: 'Log API database password' + keystone_cmm_operator_user_password: 'keystone_cmm_operator_user_password' + keystone_cmm_agent_password: 'keystone_cmm_agent_password' + keystone_admin_agent_password: 'keystone_admin_agent_password' + database_grafana_password: 'Grafana database password' db_header: 'Database settings' db: database: 'Database' From 627c85e4324f9b8a3f03de192e848da3068b8b6e Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Fri, 3 Mar 2017 11:05:02 +0100 Subject: [PATCH 11/52] Add missing end keyword (#16) * Add missing end keyword * Fix indent --- chef/cookbooks/monasca/recipes/master.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index c0569a5fe0..39ee4ced60 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -56,9 +56,10 @@ hosts_template = if monasca_hosts.length == 1 - "cmm-hosts-single.erb" + "cmm-hosts-single.erb" else "cmm-hosts-cluster.erb" + end template "/opt/fujitsu/monasca-installer/cmm-hosts" do source hosts_template From 212f93fc50753cc300485b6db92632f843489762 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Fri, 3 Mar 2017 11:48:11 +0100 Subject: [PATCH 12/52] Fix json indentation (#17) --- chef/data_bags/crowbar/template-monasca.json | 144 +++++----- .../data_bags/crowbar/template-monasca.schema | 247 +++++++++--------- 2 files changed, 195 insertions(+), 196 deletions(-) diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 3378851103..c915941be7 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -2,80 +2,80 @@ "id": "template-monasca", "description": "Logging and Monitoring Service for OpenStack", "attributes": { - "monasca" : { - "agent" : { - "monasca_url": "", - "username": "", - "password": "", - "keystone_url": "", - "user_domain_id": "", - "user_domain_name": "", - "project_name": "", - "project_domain_id": "", - "project_domain_name": "", - "project_id": "", - "insecure": true, - "ca_file": "", - "max_buffer_size": 1000, - "max_measurement_buffer_size": -1, - "backlog_send_rate": 5, - "amplifier": 0, - "hostname": "", - "check_frequency": 15, - "num_collector_threads": 1, - "pool_full_max_retries": 5, - "plugin_collect_time_warn": 6, - "log_level": "INFO" - }, - "log_agent": { - "user": "monasca-log-agent", - "group": "logstash", - "monasca_log_api_url": "", - "keystone": { - "service_user": "cmm-log-agent", - "service_password": "", - "service_tenant": "cmm", - "service_role": "cmm-agent" - } - }, - "api" : { - "bind_host" : "*", - "bind_port" : 8070, - "processes" : 3, - "ssl" : false, - "threads" : 10 - }, - "master" : { - "cmm_tarball_url": "http://127.0.0.1/cmm/cmm-suse.tgz", - "influxdb_mon_api_password" : "", - "influxdb_mon_persister_password" : "", - "database_notification_password" : "", - "database_monapi_password" : "", - "database_thresh_password" : "", - "database_logapi_password" : "", - "keystone_cmm_operator_user_password" : "", - "keystone_cmm_agent_password" : "", - "keystone_admin_agent_password" : "", - "database_grafana_password" : "" - }, - "db" : { - "database" : "monasca", - "password" : "", - "user" : "monasca" - }, - "debug" : false, - "group" : "monasca", - "user" : "monasca", - "database_instance": "none", - "keystone_instance": "none", - "service_password": "none", - "service_user": "monasca", - "service_tenant": "cmm", - "service_roles": ["cmm-agent"] - } + "monasca": { + "agent": { + "monasca_url": "", + "username": "", + "password": "", + "keystone_url": "", + "user_domain_id": "", + "user_domain_name": "", + "project_name": "", + "project_domain_id": "", + "project_domain_name": "", + "project_id": "", + "insecure": true, + "ca_file": "", + "max_buffer_size": 1000, + "max_measurement_buffer_size": -1, + "backlog_send_rate": 5, + "amplifier": 0, + "hostname": "", + "check_frequency": 15, + "num_collector_threads": 1, + "pool_full_max_retries": 5, + "plugin_collect_time_warn": 6, + "log_level": "INFO" + }, + "log_agent": { + "user": "monasca-log-agent", + "group": "logstash", + "monasca_log_api_url": "", + "keystone": { + "service_user": "cmm-log-agent", + "service_password": "", + "service_tenant": "cmm", + "service_role": "cmm-agent" + } + }, + "api": { + "bind_host": "*", + "bind_port": 8070, + "processes": 3, + "ssl": false, + "threads": 10 + }, + "master": { + "cmm_tarball_url": "http://127.0.0.1/cmm/cmm-suse.tgz", + "influxdb_mon_api_password": "", + "influxdb_mon_persister_password": "", + "database_notification_password": "", + "database_monapi_password": "", + "database_thresh_password": "", + "database_logapi_password": "", + "keystone_cmm_operator_user_password": "", + "keystone_cmm_agent_password": "", + "keystone_admin_agent_password": "", + "database_grafana_password": "" + }, + "db": { + "database": "monasca", + "password": "", + "user": "monasca" + }, + "debug": false, + "group": "monasca", + "user": "monasca", + "database_instance": "none", + "keystone_instance": "none", + "service_password": "none", + "service_user": "monasca", + "service_tenant": "cmm", + "service_roles": ["cmm-agent"] + } }, "deployment": { - "monasca" : { + "monasca": { "crowbar-revision": 0, "crowbar-applied": false, "schema-revision": 100, diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 21fe675539..4f89f658ca 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -16,85 +16,84 @@ "required": true, "type": "map", "mapping": { - "monasca_url": { "type": "str", required: true }, - "username": { "type": "str", required: true }, - "password": { "type": "str", required: true }, - "keystone_url": { "type": "str", required: true }, - "user_domain_id": { "type": "str", required: true }, - "user_domain_name": { "type": "str", required: true }, - "project_name": { "type": "str", required: true }, - "project_domain_id": { "type": "str", required: true }, - "project_domain_name": { "type": "str", required: true }, - "project_id": { "type": "str", required: true }, - "insecure": { "type": "bool", required: true }, - "ca_file": { "type": "str", required: true }, - "max_buffer_size": { "type": "int", required: true }, - "max_measurement_buffer_size": { "type": "int", required: true }, - "backlog_send_rate": { "type": "int", required: true }, - "amplifier": { "type": "int", required: true }, - "hostname": { "type": "str", required: true }, - "check_frequency": { "type": "int", required: true }, - "num_collector_threads": { "type": "int", required: true }, - "pool_full_max_retries": { "type": "int", required: true }, - "plugin_collect_time_warn": { "type": "int", required: true }, - "log_level": { "type": "str", required: true } - } + "monasca_url": { "type": "str", "required": true }, + "username": { "type": "str", "required": true }, + "password": { "type": "str", "required": true }, + "keystone_url": { "type": "str", "required": true }, + "user_domain_id": { "type": "str", "required": true }, + "user_domain_name": { "type": "str", "required": true }, + "project_name": { "type": "str", "required": true }, + "project_domain_id": { "type": "str", "required": true }, + "project_domain_name": { "type": "str", "required": true }, + "project_id": { "type": "str", "required": true }, + "insecure": { "type": "bool", "required": true }, + "ca_file": { "type": "str", "required": true }, + "max_buffer_size": { "type": "int", "required": true }, + "max_measurement_buffer_size": { "type": "int", "required": true }, + "backlog_send_rate": { "type": "int", "required": true }, + "amplifier": { "type": "int", "required": true }, + "hostname": { "type": "str", "required": true }, + "check_frequency": { "type": "int", "required": true }, + "num_collector_threads": { "type": "int", "required": true }, + "pool_full_max_retries": { "type": "int", "required": true }, + "plugin_collect_time_warn": { "type": "int", "required": true }, + "log_level": { "type": "str", "required": true } + } }, "log_agent": { "required": true, "type": "map", "mapping": { - "user": { "type": "str", required: true }, - "group": { "type": "str", required: true }, - "monasca_log_api_url": { "type": "str", required: true }, - "keystone": { - "required": true, - "type": "map", - "mapping": { - "service_user": { "type": "str", required: true }, - "service_password": { "type": "str", required: true }, - "service_tenant": { "type": "str", required: false }, - "service_role": { "type": "str", required: false } - } + "user": { "type": "str", "required": true }, + "group": { "type": "str", "required": true }, + "monasca_log_api_url": { "type": "str", "required": true }, + "keystone": { + "required": true, + "type": "map", + "mapping": { + "service_user": { "type": "str", "required": true }, + "service_password": { "type": "str", "required": true }, + "service_tenant": { "type": "str", "required": false }, + "service_role": { "type": "str", "required": false } } + } } }, "api": { "required": true, "type": "map", "mapping": { - "bind_host": { "required": true, "type": "str" }, - "bind_port": { "required": true, "type": "int" }, - "processes": { "required": true, "type": "int" }, - "ssl": { "required": true, "type": "bool" }, - "threads": { "required": true, "type": "int" } - } - }, + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" }, + "processes": { "required": true, "type": "int" }, + "ssl": { "required": true, "type": "bool" }, + "threads": { "required": true, "type": "int" } + } + }, "master": { "required": true, "type": "map", "mapping": { - "cmm_tarball_url": { "required": true, "type": "str" }, - "influxdb_mon_api_password": { "required": true, "type": "str" }, - "influxdb_mon_persister_password": { "required": true, "type": "str" }, - "database_notification_password": { "required": true, "type": "str" }, - "database_monapi_password": { "required": true, "type": "str" }, - "database_thresh_password": { "required": true, "type": "str" }, - "database_logapi_password": { "required": true, "type": "str" }, - "keystone_cmm_operator_user_password": { "required": true, "type": "str" }, - "keystone_cmm_agent_password": { "required": true, "type": "str" }, - "keystone_admin_agent_password": { "required": true, "type": "str" }, - "database_grafana_password": { "required": true, "type": "str" } - } - }, - + "cmm_tarball_url": { "required": true, "type": "str" }, + "influxdb_mon_api_password": { "required": true, "type": "str" }, + "influxdb_mon_persister_password": { "required": true, "type": "str" }, + "database_notification_password": { "required": true, "type": "str" }, + "database_monapi_password": { "required": true, "type": "str" }, + "database_thresh_password": { "required": true, "type": "str" }, + "database_logapi_password": { "required": true, "type": "str" }, + "keystone_cmm_operator_user_password": { "required": true, "type": "str" }, + "keystone_cmm_agent_password": { "required": true, "type": "str" }, + "keystone_admin_agent_password": { "required": true, "type": "str" }, + "database_grafana_password": { "required": true, "type": "str" } + } + }, "db": { "required": true, "type": "map", "mapping": { - "database": { "required": true, "type": "str" }, - "password": { "required": true, "type": "str" }, - "user": { "required": true, "type": "str" } + "database": { "required": true, "type": "str" }, + "password": { "required": true, "type": "str" }, + "user": { "required": true, "type": "str" } } }, "debug": { "type": "bool" }, @@ -106,75 +105,76 @@ "service_roles": { "type": "seq", "required": true, "sequence": [ { "type": "str" } ] }, "group": { "required": true, "type": "str" }, "user": { "required": true, "type": "str" } - } - } - } - }, - "deployment": { - "type": "map", - "required": true, - "mapping": { - "monasca": { - "type": "map", - "required": true, - "mapping": { - "crowbar-revision": { "type": "int", "required": true }, - "crowbar-committing": { "type": "bool" }, - "crowbar-applied": { "type": "bool" }, - "crowbar-status": { "type": "str" }, - "crowbar-failed": { "type": "str" }, - "crowbar-queued": { "type": "bool" }, - "schema-revision": { "type": "int" }, - "element_states": { - "type": "map", - "mapping": { - = : { - "type": "seq", - "required": true, - "sequence": [ { "type": "str" } ] + } + } + } + }, + "deployment": { + "type": "map", + "required": true, + "mapping": { + "monasca": { + "type": "map", + "required": true, + "mapping": { + "crowbar-revision": { "type": "int", "required": true }, + "crowbar-committing": { "type": "bool" }, + "crowbar-applied": { "type": "bool" }, + "crowbar-status": { "type": "str" }, + "crowbar-failed": { "type": "str" }, + "crowbar-queued": { "type": "bool" }, + "schema-revision": { "type": "int" }, + "element_states": { + "type": "map", + "mapping": { + = : { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } } - } - }, - "elements": { - "type": "map", - "required": true, - "mapping": { - = : { - "type": "seq", - "required": true, - "sequence": [ { "type": "str" } ] + }, + "elements": { + "type": "map", + "required": true, + "mapping": { + = : { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } } - } - }, - "element_order": { - "type": "seq", - "required": true, - "sequence": [ { + }, + "element_order": { "type": "seq", - "sequence": [ { "type": "str" } ] - } ] - }, - "element_run_list_order": { - "type": "map", - "required": false, - "mapping": { - = : { - "type": "int", - "required": true - } - } - }, - "config": { - "type": "map", - "required": true, - "mapping": { - "environment": { "type": "str", "required": true }, - "mode": { "type": "str", "required": true }, - "transitions": { "type": "bool", "required": true }, - "transition_list": { + "required": true, + "sequence": [ { "type": "seq", - "required": true, "sequence": [ { "type": "str" } ] + } ] + }, + "element_run_list_order": { + "type": "map", + "required": false, + "mapping": { + = : { + "type": "int", + "required": true + } + } + }, + "config": { + "type": "map", + "required": true, + "mapping": { + "environment": { "type": "str", "required": true }, + "mode": { "type": "str", "required": true }, + "transitions": { "type": "bool", "required": true }, + "transition_list": { + "type": "seq", + "required": true, + "sequence": [ { "type": "str" } ] + } } } } @@ -182,5 +182,4 @@ } } } - } } From f3fc36ee79beff865627837f7eff23572f281c1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Thu, 9 Mar 2017 11:04:37 +0100 Subject: [PATCH 13/52] Update monasca-agent cookbook and barclamp (#2) * Add monasca-reconfigure and sudoers file * Fix monasca-setup bin path * Fix template variables * Keystone settings for agent * Keystone settings for agent * Keystone settings for agent * Add log * More readable config templates * Remove log * Remove per-node credentials todos * Fix json indentations * Rename agent dimensions variable name * Run monasca-reconfigure every time with chef client * Add firts GUI elements * Add GUI elements * Update for new code arrangement for keystone * Fix yaml for translations * Fix monasca helper module name * Use root user for agent temporally * Update monasca agent GUI parts * WIP: Add custom checks for monasca-agent * Extract variables for templates * Fix testing filepath * Fix template for monasca agent checks * Fix template paths again * Address Hound warning * Add more monasca agent checkers * Fix some variables * Add log api port to data_bag * Build Kibana and log-api urls * Add more info to GUI * Add user_managed: false * Fix yaml indentation * Add some logs * Fix monasca-reconfigure * Fix ruby boolean name * Uncomment agent.yaml template code * Fix providing service variable to template * Fix insecure args for monasca-setup * Fix monasca-reconfigure * Configure more variables with monasca-reconfigure * Add more config to UI * Disable generating agent.yaml It's regenerated every time chef client is run, it's running monasca-reconfigure. * Change default overwrite config to false * Add monasca-agent to sudoers * Run monasca-setup as monasca-agent user * Remove logs from monasca-reconfigure * Rename monasca-agent to monasca-metric-agent And rename all variables to reflect this change. * Fix hound warnings * Address comments * Fix copyright * Use same order in data_bag schema like in json * Restore user mapping in log_agent schema * Change monasca-master unique to false * Fix monasca-metric-agents names in json * Use keystone settings from data_bag for metric agent * Replace `owner` with `user` in execute command * Replace group `monasca-agent` with `monasca` * Remove adding monasca-agent to sudoers Will be added from rpm. * Add port to monasca_api_url for monasca-agent * Use helper for getting api url * Fix bad default config name --- chef/cookbooks/monasca/README.md | 2 +- chef/cookbooks/monasca/attributes/default.rb | 14 +- chef/cookbooks/monasca/libraries/helper.rb | 4 +- chef/cookbooks/monasca/recipes/agent.rb | 51 ------- chef/cookbooks/monasca/recipes/common.rb | 7 +- .../cookbooks/monasca/recipes/metric_agent.rb | 65 ++++++++ ..._agent.rb => role_monasca_metric_agent.rb} | 4 +- .../monasca/templates/default/agent.yaml.erb | 142 ------------------ .../templates/default/monasca-reconfigure.erb | 31 ++++ chef/data_bags/crowbar/template-monasca.json | 39 +++-- .../data_bags/crowbar/template-monasca.schema | 35 +++-- chef/roles/monasca-agent.rb | 5 - chef/roles/monasca-log-agent.rb | 2 +- chef/roles/monasca-metric-agent.rb | 5 + .../app/helpers/barclamp/monasca_helper.rb | 31 ++++ .../app/models/monasca_service.rb | 9 +- .../monasca/_edit_attributes.html.haml | 19 ++- .../config/locales/monasca/en.yml | 19 ++- monasca.yml | 3 +- 19 files changed, 231 insertions(+), 256 deletions(-) delete mode 100644 chef/cookbooks/monasca/recipes/agent.rb create mode 100644 chef/cookbooks/monasca/recipes/metric_agent.rb rename chef/cookbooks/monasca/recipes/{role_monasca_agent.rb => role_monasca_metric_agent.rb} (90%) delete mode 100644 chef/cookbooks/monasca/templates/default/agent.yaml.erb create mode 100644 chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb delete mode 100644 chef/roles/monasca-agent.rb create mode 100644 chef/roles/monasca-metric-agent.rb create mode 100644 crowbar_framework/app/helpers/barclamp/monasca_helper.rb diff --git a/chef/cookbooks/monasca/README.md b/chef/cookbooks/monasca/README.md index d8b5334898..30cba9f1fe 100644 --- a/chef/cookbooks/monasca/README.md +++ b/chef/cookbooks/monasca/README.md @@ -3,7 +3,7 @@ DESCRIPTION Chef Cookbook to install and configure Monasca. This cookbook contains 2 roles: -* monasca-agent: deploys monasca-agent on arbitrary nodes +* monasca-metric-agent: deploys monasca metric agent on arbitrary nodes * monasca-server: deploys Monasca's backend services on dedicated Monasca nodes REQUIREMENTS diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index e980735642..fe83fb889c 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -17,11 +17,15 @@ monasca_debug = false monasca_ha_enabled = false +monasca_metric_agent_service_name = "monasca-agent" monasca_log_agent_service_name = "monasca-log-agent" default[:monasca][:platform] = { packages: [], - services: [monasca_log_agent_service_name] + services: [ + monasca_metric_agent_service_name, + monasca_log_agent_service_name + ] } default[:monasca][:db][:database] = "monasca" @@ -32,10 +36,16 @@ override[:monasca][:user] = "monasca" default[:monasca][:debug] = monasca_debug -default[:monasca][:ha_enabled] = monasca_ha_enabled +default[:monasca][:ha][:enabled] = monasca_ha_enabled default[:monasca][:api][:bind_host] = "*" +# metric-agent default service settings +default[:monasca][:metric_agent][:service_name] = monasca_metric_agent_service_name +default[:monasca][:metric_agent][:user] = "monasca-agent" +default[:monasca][:metric_agent][:group] = "monasca" +default[:monasca][:metric_agent][:debug] = monasca_debug + # log-agent default service settings default[:monasca][:log_agent][:service_name] = monasca_log_agent_service_name default[:monasca][:log_agent][:user] = "monasca-log-agent" diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 7dc90f223f..1bc47f34c9 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -25,8 +25,8 @@ def self.monasca_admin_host(node) CrowbarHelper.get_host_for_admin_url(node, ha_enabled) end - def self.api_admin_url(node) - host = api_admin_host(node) + def self.api_public_url(node) + host = monasca_public_host(node) protocol = node[:monasca][:api][:ssl] ? "https" : "http" port = node[:monasca][:api][:bind_port] "#{protocol}://#{host}:#{port}/v2.0" diff --git a/chef/cookbooks/monasca/recipes/agent.rb b/chef/cookbooks/monasca/recipes/agent.rb deleted file mode 100644 index 5499bf0f8c..0000000000 --- a/chef/cookbooks/monasca/recipes/agent.rb +++ /dev/null @@ -1,51 +0,0 @@ -# -# Copyright 2016 SUSE Linux GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -### TODO: uncomment this once there is a package. -# package "openstack-monasca-agent" - -### FIXME: remove this once there is a package creating this directory - -directory "/etc/monasca/agent/" do - owner "root" - group "root" - mode 0o755 - recursive true - notifies :create, "template[/etc/monasca/agent/agent.yaml]" -end - -### TODO: -# * populate agent_settings with useful values for monasca_url and keystone -# settings -# * generate per-node keystone credentials in -# crowbar_framework/app/models/monasca_service.rb -# * use credentials generated in the previous step to create a keystone account -# here (using keystone_register) - -agent_settings = node[:monasca][:agent][:config] - -template "/etc/monasca/agent/agent.yaml" do - source "agent.yaml.erb" - owner "root" - ### FIXME: Uncomment once we have a package that creates a monasca group - # group node[:monasca][:group] - mode 0o640 - variables agent_settings - ### FIXME: Uncomment once we have a package that creates a monasca-agent service - # notifies :reload, resources(service: "openstack-monasca-agent") -end - -node.save diff --git a/chef/cookbooks/monasca/recipes/common.rb b/chef/cookbooks/monasca/recipes/common.rb index b83b63cd3c..37e66d37f6 100644 --- a/chef/cookbooks/monasca/recipes/common.rb +++ b/chef/cookbooks/monasca/recipes/common.rb @@ -68,10 +68,9 @@ ### single credentials all metrics & logs agents agents_settings = [] -# once agent is ready, uncomment following lines -# if node["roles"].include?("monasca-agent") -# agents_settings.push(node[:monasca][:agent][:keystone]) -# end +if node["roles"].include?("monasca-metric-agent") + agents_settings.push(node[:monasca][:metric_agent][:keystone]) +end if node["roles"].include?("monasca-log-agent") la_keystone = node[:monasca][:log_agent][:keystone] agents_settings.push(la_keystone) diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb new file mode 100644 index 0000000000..558106e899 --- /dev/null +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -0,0 +1,65 @@ +# +# Copyright 2016 SUSE Linux GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +package "openstack-monasca-agent" + +agent_settings = node[:monasca][:metric_agent] +agent_keystone = agent_settings[:keystone] + +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) + +monasca_servers = search(:node, "roles:monasca-server") || [] +monasca_server = monasca_servers[0] + +monasca_api_url = MonascaHelper.api_public_url(monasca_server) + +agent_dimensions = { service: "monitoring" } +service = "monitoring" + +monasca_reconfigure_file = "/usr/local/bin/monasca-reconfigure" + +template monasca_reconfigure_file do + source "monasca-reconfigure.erb" + owner agent_settings["user"] + group agent_settings["group"] + mode 0o750 + variables( + monasca_api_url: monasca_api_url, + service: service, + agent_settings: agent_settings, + agent_keystone: agent_keystone, + keystone_settings: keystone_settings, + agent_dimensions: agent_dimensions, + install_plugins_only: false + ) +end + +execute "monasca-setup detect services" do + command monasca_reconfigure_file + only_if { File.exist?(monasca_reconfigure_file) } + user agent_settings["user"] + group agent_settings["group"] +end + +service "monasca-agent" do + service_name agent_settings[:service_name] + supports status: true, restart: true, start: true, stop: true + action [:disable, :stop] + ignore_failure true + # provider Chef::Provider::CrowbarPacemakerService if ha_enabled +end + +node.save diff --git a/chef/cookbooks/monasca/recipes/role_monasca_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb similarity index 90% rename from chef/cookbooks/monasca/recipes/role_monasca_agent.rb rename to chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb index bf8dd7b22d..640eb9cda5 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_agent.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb @@ -14,7 +14,7 @@ # limitations under the License. # -if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-agent") +if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-metric-agent") include_recipe "#{@cookbook_name}::common" - include_recipe "#{@cookbook_name}::agent" + include_recipe "#{@cookbook_name}::metric_agent" end diff --git a/chef/cookbooks/monasca/templates/default/agent.yaml.erb b/chef/cookbooks/monasca/templates/default/agent.yaml.erb deleted file mode 100644 index 2fd1a11bb3..0000000000 --- a/chef/cookbooks/monasca/templates/default/agent.yaml.erb +++ /dev/null @@ -1,142 +0,0 @@ -Api: - # To configure Keystone correctly, a project-scoped token must be acquired. - # To accomplish this, the configuration must be set up with one of the - # following scenarios: - # Set username and password and you have a default project set in keystone. - # Set username, password and project id. - # Set username, password, project name and (domain id or domain name). - # - # Monitoring API URL: URL for the monitoring API, if undefined it will be pulled from the keystone service catalog - # Example: https://region-a.geo-1.monitoring.hpcloudsvc.com:8080/v2.0 - url: <%= @monasca_url %> - # Keystone Username - username: <%= @username %> - # Keystone Password - password: "<%= @password %>" - # Keystone API URL: URL for the Keystone server to use - # Example: https://region-a.geo-1.identity.hpcloudsvc.com:35357/v3/ - keystone_url: <%= @keystone_url %> - # Domain id to be used to resolve username - user_domain_id: <%= @user_domain_id %> - # Domain name to be used to resolve username - user_domain_name: <%= @user_domain_name %> - # Project name to be used by this agent - project_name: <%= @project_name %> - # Project domain id to be used by this agent - project_domain_id: <%= @project_domain_id %> - # Project domain id to be used by this agent - project_domain_name: <%= @project_domain_name %> - # Project id to be used by this agent - project_id: <%= @project_id %> - # Set whether certificates are used for Keystone - # ******************************************************************************************* - # **** CAUTION ****: The insecure flag should NOT be set to True in a production environment! - # ******************************************************************************************* - # If insecure is set to False, a ca_file name must be set to authenticate with Keystone - insecure: <%= @insecure %> - # Name of the ca certs file - ca_file: <%= @ca_file %> - - # The following 3 options are for handling buffering and reconnection to the monasca-api - # If the agent forwarder is consuming too much memory, you may want to set - # max_measurement_buffer_size to a lower value. If you have a larger system with many agents, - # you may want to throttle the number of messages sent to the API by setting the - # backlog_send_rate to a lower number. - - # DEPRECATED - please use max_measurement_buffer_size instead - # Maximum number of messages (batches of measurements) to buffer when unable to communicate - # with the monasca-api (-1 means no limit) - max_buffer_size: <%= @max_buffer_size %> - # Maximum number of measurements to buffer when unable to communicate with the monasca-api - # (-1 means no limit) - max_measurement_buffer_size: <%= @max_measurement_buffer_size %> - # Maximum number of messages to send at one time when communication with the monasca-api is restored - backlog_send_rate: <%= @backlog_send_rate %> - - # Publish extra metrics to the API by adding this number of 'amplifier' dimensions. - # For load testing purposes only; set to 0 for production use. - amplifier: <%= @amplifier %> - -Main: - # Force the hostname to whatever you want. - hostname: <%= @hostname %> - - # Optional dimensions to be sent with every metric from this node - # They should be in the format name: value - # Example of dimensions below - # dimensions: - # service: nova - # group: group_a - # environment: production - <%- unless @dimensions.nil? or @dimension.empty? then -%> - dimensions: <%= @dimensions %> - <%- @dimensions.each_pair do |key, val| -%> - <%= key %>: <%= val %> - <%- end -%> - <%- end -%> - - # Set the threshold for accepting points to allow anything - # with recent_point_threshold seconds - # Defaults to 30 seconds if no value is provided - #recent_point_threshold: 30 - - # time to wait between collection runs - check_freq: <%= @check_frequency %> - - # Number of Collector Threads to run - num_collector_threads: <%= @num_collector_threads %> - - # Maximum number of collection cycles where all of the threads in the pool are - # still running plugins before the collector will exit - pool_full_max_retries: <%= @pool_full_max_retries %> - - # Threshold value for warning on collection time of each check (in seconds) - sub_collection_warn: <%= @plugin_collect_time_warn %> - - # Collector restart interval (in hours) - collector_restart_interval: 24 - - # Change port the Agent is listening to - # listen_port: 17123 - - # Allow non-local traffic to this Agent - # This is required when using this Agent as a proxy for other Agents - # that might not have an internet connection - # For more information, please see - # https://github.com/DataDog/dd-agent/wiki/Network-Traffic-and-Proxy-Configuration - # non_local_traffic: no - -Statsd: - # ========================================================================== # - # Monasca Statsd configuration # - # ========================================================================== # - # Monasca Statsd is a small server that aggregates your custom app metrics. - - # Make sure your client is sending to the same port. - monasca_statsd_port : 8125 - - ## The monasca_statsd flush period. - # monasca_statsd_interval : 20 - - # If you want to forward every packet received by the monasca_statsd server - # to another statsd server, uncomment these lines. - # WARNING: Make sure that forwarded packets are regular statsd packets and not "monasca_statsd" packets, - # as your other statsd server might not be able to handle them. - # monasca_statsd_forward_host: address_of_own_statsd_server - # monasca_statsd_statsd_forward_port: 8125 - -Logging: - # ========================================================================== # - # Logging - # ========================================================================== # - log_level: <%= @log_level %> - collector_log_file: /var/log/monasca/agent/collector.log - forwarder_log_file: /var/log/monasca/agent/forwarder.log - statsd_log_file: /var/log/monasca/agent/statsd.log - - # if syslog is enabled but a host and port are not set, a local domain socket - # connection will be attempted - # - # log_to_syslog: yes - # syslog_host: - # syslog_port: diff --git a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb new file mode 100644 index 0000000000..e79612b6c1 --- /dev/null +++ b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb @@ -0,0 +1,31 @@ +#!/bin/sh + +'/usr/bin/monasca-setup' \ + -u '<%= @agent_keystone["service_user"] %>' \ + -p '<%= @agent_keystone["service_password"] %>' \ + --project_name '<%= @agent_keystone["service_tenant"] %>' \ + --project_id '<%= @agent_keystone["service_tenant"] %>' \ + <% if defined?(@service) -%> --service '<%= @service %>' <% end -%> \ + --keystone_url '<%= @keystone_settings["internal_auth_url"] %>' \ + --user_domain_id '<%= @keystone_settings["admin_domain_id"] %>' \ + --user_domain_name '<%= @keystone_settings["admin_domain"] %>' \ + --project_domain_id '<%= @keystone_settings["admin_domain_id"] %>' \ + --project_domain_name '<%= @keystone_settings["admin_domain"] %>' \ + --monasca_url '<%= @monasca_api_url %>' \ + --user '<%= @agent_settings["user"] %>' \ + --dimensions '<%= @agent_dimensions.map{|k,v| "#{k}:#{v}"}.join(',') %>' \ + --insecure '<%= @agent_settings["insecure"] %>' \ + <% if @agent_settings["system_only"] -%> --system_only <% end -%> \ + <% if @agent_settings["overwrite_config"] -%> --overwrite <% end -%> \ + <% if @agent_settings["ca_file"].length > 0 -%> --ca_file '<%= @agent_settings["ca_file"] %>' <% end -%> \ + --log_level '<%= @agent_settings["log_level"] %>' \ + --monasca_statsd_port '<%= @agent_settings["statsd_port"].to_i %>' \ + --check_frequency '<%= @agent_settings["check_frequency"].to_i %>' \ + --num_collector_threads '<%= @agent_settings["num_collector_threads"].to_i %>' \ + --pool_full_max_retries '<%= @agent_settings["pool_full_max_retries"].to_i %>' \ + --plugin_collect_time_warn '<%= @agent_settings["plugin_collect_time_warn"].to_i %>' \ + --max_buffer_size '<%= @agent_settings["max_buffer_size"].to_i %>' \ + --max_measurement_buffer_size '<%= @agent_settings["max_measurement_buffer_size"].to_i %>' \ + --backlog_send_rate '<%= @agent_settings["backlog_send_rate"].to_i %>' \ + --amplifier '<%= @agent_settings["amplifier"] %>' \ + <% if @install_plugins_only -%> --install_plugins_only <% end -%> diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index c915941be7..c613c9404b 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -3,29 +3,28 @@ "description": "Logging and Monitoring Service for OpenStack", "attributes": { "monasca": { - "agent": { + "metric_agent": { "monasca_url": "", - "username": "", - "password": "", - "keystone_url": "", - "user_domain_id": "", - "user_domain_name": "", - "project_name": "", - "project_domain_id": "", - "project_domain_name": "", - "project_id": "", + "keystone": { + "service_user": "cmm-agent", + "service_password": "", + "service_tenant": "cmm", + "service_role": "cmm-agent" + }, "insecure": true, + "system_only": false, + "overwrite_config": false, "ca_file": "", - "max_buffer_size": 1000, - "max_measurement_buffer_size": -1, - "backlog_send_rate": 5, - "amplifier": 0, - "hostname": "", + "log_level": "INFO", + "statsd_port": 8125, "check_frequency": 15, "num_collector_threads": 1, "pool_full_max_retries": 5, "plugin_collect_time_warn": 6, - "log_level": "INFO" + "max_buffer_size": 1000, + "max_measurement_buffer_size": -1, + "backlog_send_rate": 1000, + "amplifier": 0 }, "log_agent": { "user": "monasca-log-agent", @@ -82,21 +81,21 @@ "element_states": { "monasca-server": [ "readying", "ready", "applying" ], "monasca-master": [ "readying", "ready", "applying" ], - "monasca-agent": [ "readying", "ready", "applying" ], + "monasca-metric-agent": [ "readying", "ready", "applying" ], "monasca-log-agent": [ "readying", "ready", "applying" ] }, "elements": {}, "element_order": [ [ "monasca-server" ], [ "monasca-master" ], - [ "monasca-agent" ], + [ "monasca-metric-agent" ], [ "monasca-log-agent" ] ], "element_run_list_order": { "monasca-server": 110, "monasca-master": 115, - "monasca-agent": 120, - "monasca-log-agent": 120 + "monasca-metric-agent": 120, + "monasca-log-agent": 130 }, "config": { "environment": "monasca-base-config", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 4f89f658ca..1175f6c3c2 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -12,32 +12,35 @@ "required": true, "type": "map", "mapping": { - "agent": { + "metric_agent": { "required": true, "type": "map", "mapping": { "monasca_url": { "type": "str", "required": true }, - "username": { "type": "str", "required": true }, - "password": { "type": "str", "required": true }, - "keystone_url": { "type": "str", "required": true }, - "user_domain_id": { "type": "str", "required": true }, - "user_domain_name": { "type": "str", "required": true }, - "project_name": { "type": "str", "required": true }, - "project_domain_id": { "type": "str", "required": true }, - "project_domain_name": { "type": "str", "required": true }, - "project_id": { "type": "str", "required": true }, + "keystone": { + "required": true, + "type": "map", + "mapping": { + "service_user": { "type": "str", "required": true }, + "service_password": { "type": "str", "required": true }, + "service_tenant": { "type": "str", "required": false }, + "service_role": { "type": "str", "required": false } + } + }, "insecure": { "type": "bool", "required": true }, + "system_only": { "type": "bool", "required": true }, + "overwrite_config": { "type": "bool", "required": true }, "ca_file": { "type": "str", "required": true }, - "max_buffer_size": { "type": "int", "required": true }, - "max_measurement_buffer_size": { "type": "int", "required": true }, - "backlog_send_rate": { "type": "int", "required": true }, - "amplifier": { "type": "int", "required": true }, - "hostname": { "type": "str", "required": true }, + "log_level": { "type": "str", "required": true }, + "statsd_port": { "type": "int", "required": true }, "check_frequency": { "type": "int", "required": true }, "num_collector_threads": { "type": "int", "required": true }, "pool_full_max_retries": { "type": "int", "required": true }, "plugin_collect_time_warn": { "type": "int", "required": true }, - "log_level": { "type": "str", "required": true } + "max_buffer_size": { "type": "int", "required": true }, + "max_measurement_buffer_size": { "type": "int", "required": true }, + "backlog_send_rate": { "type": "int", "required": true }, + "amplifier": { "type": "int", "required": true } } }, "log_agent": { diff --git a/chef/roles/monasca-agent.rb b/chef/roles/monasca-agent.rb deleted file mode 100644 index 25b98767a8..0000000000 --- a/chef/roles/monasca-agent.rb +++ /dev/null @@ -1,5 +0,0 @@ -name "monasca-agent" -description "Monasca Agent Role" -run_list("recipe[monasca::role_monasca_agent]") -default_attributes -override_attributes diff --git a/chef/roles/monasca-log-agent.rb b/chef/roles/monasca-log-agent.rb index 8344514fdd..8558c0da19 100644 --- a/chef/roles/monasca-log-agent.rb +++ b/chef/roles/monasca-log-agent.rb @@ -1,4 +1,4 @@ -# # Copyright 2017 FUJITSU LIMITED +# Copyright 2017 FUJITSU LIMITED name "monasca-log-agent" description "Monasca Log Agent Role" diff --git a/chef/roles/monasca-metric-agent.rb b/chef/roles/monasca-metric-agent.rb new file mode 100644 index 0000000000..18a900f5ba --- /dev/null +++ b/chef/roles/monasca-metric-agent.rb @@ -0,0 +1,5 @@ +name "monasca-metric-agent" +description "Monasca Metric Agent Role" +run_list("recipe[monasca::role_monasca_metric_agent]") +default_attributes +override_attributes diff --git a/crowbar_framework/app/helpers/barclamp/monasca_helper.rb b/crowbar_framework/app/helpers/barclamp/monasca_helper.rb new file mode 100644 index 0000000000..caa222055b --- /dev/null +++ b/crowbar_framework/app/helpers/barclamp/monasca_helper.rb @@ -0,0 +1,31 @@ +# +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +module Barclamp + module MonascaHelper + def log_levels(selected) + options_for_select( + [ + ["ERROR", "ERROR"], + ["WARNING", "WARNING"], + ["INFO", "INFO"], + ["DEBUG", "DEBUG"] + ], + selected.to_s + ) + end + end +end diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 23cad2bbea..ef1b5ea394 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -29,7 +29,7 @@ def self.allow_multiple_proposals? def role_constraints { - "monasca-agent" => { + "monasca-metric-agent" => { "unique" => false, "admin" => true, "count" => -1, @@ -58,7 +58,7 @@ def role_constraints } }, "monasca-master" => { - "unique" => true, + "unique" => false, "count" => 1, "cluster" => false, "admin" => true, @@ -101,7 +101,7 @@ def create_proposal base["deployment"][@bc_name]["elements"] = { "monasca-server" => server_nodes, "monasca-master" => [master_node.name], - "monasca-agent" => agent_nodes, + "monasca-metric-agent" => agent_nodes, "monasca-log-agent" => agent_nodes } @@ -112,8 +112,7 @@ def create_proposal base["attributes"][@bc_name]["service_password"] = random_password base["attributes"][@bc_name][:db][:password] = random_password - # note(trebskit) once agent is ready, uncomment following line - # base["attributes"][@bc_name][:agent][:keystone][:service_password] = random_password + base["attributes"][@bc_name][:metric_agent][:keystone][:service_password] = random_password base["attributes"][@bc_name][:log_agent][:keystone][:service_password] = random_password base["attributes"][@bc_name][:master][:influxdb_mon_api_password] = random_password base["attributes"][@bc_name][:master][:influxdb_mon_persister_password] = random_password diff --git a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml index 9cfb49ca0b..da9a17ca25 100644 --- a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml @@ -4,6 +4,24 @@ .panel-body + %fieldset + %legend + = t(".metric_agent_header") + + = boolean_field %w(metric_agent insecure) + = boolean_field %w(metric_agent system_only) + = string_field %w(metric_agent ca_file) + = select_field %w(metric_agent log_level), :collection => :log_levels + = integer_field %w(metric_agent statsd_port) + = integer_field %w(metric_agent check_frequency) + = integer_field %w(metric_agent num_collector_threads) + = integer_field %w(metric_agent pool_full_max_retries) + = integer_field %w(metric_agent plugin_collect_time_warn) + = integer_field %w(metric_agent max_buffer_size) + = integer_field %w(metric_agent max_measurement_buffer_size) + = integer_field %w(metric_agent backlog_send_rate) + = integer_field %w(metric_agent amplifier) + %fieldset %legend = t(".master_header") @@ -19,4 +37,3 @@ = password_field %w(master keystone_cmm_agent_password) = password_field %w(master keystone_admin_agent_password) = password_field %w(master database_grafana_password) - diff --git a/crowbar_framework/config/locales/monasca/en.yml b/crowbar_framework/config/locales/monasca/en.yml index af2bc1a76b..182cd3cd61 100644 --- a/crowbar_framework/config/locales/monasca/en.yml +++ b/crowbar_framework/config/locales/monasca/en.yml @@ -26,9 +26,22 @@ en: bind_port: 'Port' processes: 'Number of processes' threads: 'Number of concurrent threads' - agent_header: 'Agent Settings' - agent: - url: 'Monasca URL' + metric_agent_header: 'Metrics Agent Settings' + metric_agent: + monasca_url: 'Monasca URL' + insecure: 'Do you want insecure connection?' + system_only: 'Setting to true will cause Monasca setup to run in `system_only` mode only, configure the base config and system metrics (cpu, disk, load, memory, network)' + ca_file: 'Sets the path to the ca certs file if using certificates. Required only if insecure is set to False (ca_file)' + log_level: 'Log level' + statsd_port: 'Monasca Statsd port' + check_frequency: 'Time to wait between collection runs (check_frequency)' + num_collector_threads: 'Number of Collector Threads to run (num_collector_threads)' + pool_full_max_retries: 'Maximum number of collection cycles where all of the threads in the pool are still running plugins before the collector will exit (pool_full_max_retries)' + plugin_collect_time_warn: 'Threshold value for warning on collection time of each check (in seconds, plugin_collect_time_warn)' + max_buffer_size: 'Maximum number of batches of measurements to buffer while unable to communicate with monasca-api (max_buffer_size)' + max_measurement_buffer_size: 'Maximum number of measurements to buffer when unable to communicate with the monasca-api (-1 means no limit, max_measurement_buffer_size)' + backlog_send_rate: 'Maximum number of messages to send at one time when communication with the monasca-api is restored (backlog_send_rate)' + amplifier: 'Publish extra metrics to the API by adding this number of "amplifier" dimensions. For load testing purposes only; set to 0 for production use (amplifier)' master_header: 'Installation parameters' master: cmm_tarball_url: 'URL of the CMM installer tarball' diff --git a/monasca.yml b/monasca.yml index e487dfbb1c..93a5af8426 100644 --- a/monasca.yml +++ b/monasca.yml @@ -29,7 +29,8 @@ barclamp: display: 'Monasca' description: 'OpenStack Monasca: Logging and Monitoring service for OpenStack' version: 0 - user_managed: false + # TODO: change to false to hide it from default users + user_managed: true requires: - '@crowbar' - 'database' From ab4155dacc70ef2f3677431976b4de2e7b65ebd6 Mon Sep 17 00:00:00 2001 From: Witek Bedyk Date: Tue, 14 Mar 2017 09:03:24 +0100 Subject: [PATCH 14/52] Disable systemd service file creation (#19) Enforce skip_enable option for monasca_setup 'skip_enable' option controls the creation of systemd service file for Monasca Agent. The service file is created in RPM package, so the option has to be enabled here. --- chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb index e79612b6c1..5362d18eb9 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb @@ -28,4 +28,5 @@ --max_measurement_buffer_size '<%= @agent_settings["max_measurement_buffer_size"].to_i %>' \ --backlog_send_rate '<%= @agent_settings["backlog_send_rate"].to_i %>' \ --amplifier '<%= @agent_settings["amplifier"] %>' \ + --skip_enable \ <% if @install_plugins_only -%> --install_plugins_only <% end -%> From fd69cc7ddc7ac3cc68e64c7994de8090cb742b80 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Wed, 15 Mar 2017 09:17:06 +0100 Subject: [PATCH 15/52] Fix invalid files location --- chef/cookbooks/horizon/files/{ => default}/grafana-monasca.json | 0 chef/cookbooks/horizon/files/{ => default}/grafana-openstack.json | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename chef/cookbooks/horizon/files/{ => default}/grafana-monasca.json (100%) rename chef/cookbooks/horizon/files/{ => default}/grafana-openstack.json (100%) diff --git a/chef/cookbooks/horizon/files/grafana-monasca.json b/chef/cookbooks/horizon/files/default/grafana-monasca.json similarity index 100% rename from chef/cookbooks/horizon/files/grafana-monasca.json rename to chef/cookbooks/horizon/files/default/grafana-monasca.json diff --git a/chef/cookbooks/horizon/files/grafana-openstack.json b/chef/cookbooks/horizon/files/default/grafana-openstack.json similarity index 100% rename from chef/cookbooks/horizon/files/grafana-openstack.json rename to chef/cookbooks/horizon/files/default/grafana-openstack.json From 94318dc0992492c6b9c91e60d974676c066b9455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Wed, 15 Mar 2017 09:52:52 +0100 Subject: [PATCH 16/52] Get proper log-api url for monasca-log-agent (#18) * Get proper log-api url for monasca-log-agent * Remove useless monasca urls from data_bag * Remove log_agent user and group from data_bag * Always use http in log-api url * Use only http for monasca-agent url * Remove ability to set `insecure` and `ca_file` options from UI. --- chef/cookbooks/monasca/libraries/helper.rb | 13 ++++++++++++- chef/cookbooks/monasca/recipes/log_agent.rb | 6 ++++++ .../monasca/templates/default/log-agent.conf.erb | 2 +- chef/data_bags/crowbar/template-monasca.json | 8 ++++---- chef/data_bags/crowbar/template-monasca.schema | 12 ++++++++---- .../barclamp/monasca/_edit_attributes.html.haml | 2 -- crowbar_framework/config/locales/monasca/en.yml | 2 +- 7 files changed, 32 insertions(+), 13 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 1bc47f34c9..06ee18a200 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -27,11 +27,22 @@ def self.monasca_admin_host(node) def self.api_public_url(node) host = monasca_public_host(node) - protocol = node[:monasca][:api][:ssl] ? "https" : "http" + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" port = node[:monasca][:api][:bind_port] "#{protocol}://#{host}:#{port}/v2.0" end + def self.log_api_public_url(node) + host = monasca_public_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:log_api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:log_api][:port] + "#{protocol}://#{host}:#{port}/v3.0" + end + def self.monasca_hosts(nodes) hosts = [] nodes.each do |n| diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 6e47a63a67..27586cec69 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -19,6 +19,11 @@ log_agent_keystone = log_agent_settings[:keystone] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) +monasca_servers = search(:node, "roles:monasca-server") +monasca_server = monasca_servers[0] + +monasca_log_api_url = MonascaHelper.log_api_public_url(monasca_server) + log_agent_dimensions = { service: "monitoring", hostname: node["hostname"] @@ -43,6 +48,7 @@ group log_agent_settings["group"] mode 0o640 variables( + monasca_log_api_url: monasca_log_api_url, log_agent_keystone: log_agent_keystone, log_agent_settings: log_agent_settings, log_agent_dimensions: log_agent_dimensions, diff --git a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb index 42e6db7885..c7aadfa134 100644 --- a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb +++ b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb @@ -28,7 +28,7 @@ output { user_domain_name => "<%= @keystone_settings['admin_domain'] %>" project_domain_name => "<%= @keystone_settings['admin_domain'] %>" ### monasca specific settings - monasca_log_api_url => "<%= @log_agent_settings['monasca_log_api_url'] %>" + monasca_log_api_url => "<%= @monasca_log_api_url %>" <%- unless @log_agent_dimensions.nil? then -%> dimensions => [<%= @log_agent_dimensions.map{|k,v| "\"#{k}:#{v}\""}.join(',') %>] <%- else -%> diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index c613c9404b..80b8f7effa 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -4,7 +4,6 @@ "attributes": { "monasca": { "metric_agent": { - "monasca_url": "", "keystone": { "service_user": "cmm-agent", "service_password": "", @@ -27,9 +26,6 @@ "amplifier": 0 }, "log_agent": { - "user": "monasca-log-agent", - "group": "logstash", - "monasca_log_api_url": "", "keystone": { "service_user": "cmm-log-agent", "service_password": "", @@ -44,6 +40,10 @@ "ssl": false, "threads": 10 }, + "log_api": { + "port": 5607, + "ssl": false + }, "master": { "cmm_tarball_url": "http://127.0.0.1/cmm/cmm-suse.tgz", "influxdb_mon_api_password": "", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 1175f6c3c2..1f1fca75a7 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -16,7 +16,6 @@ "required": true, "type": "map", "mapping": { - "monasca_url": { "type": "str", "required": true }, "keystone": { "required": true, "type": "map", @@ -47,9 +46,6 @@ "required": true, "type": "map", "mapping": { - "user": { "type": "str", "required": true }, - "group": { "type": "str", "required": true }, - "monasca_log_api_url": { "type": "str", "required": true }, "keystone": { "required": true, "type": "map", @@ -73,6 +69,14 @@ "threads": { "required": true, "type": "int" } } }, + "log_api": { + "required": true, + "type": "map", + "mapping": { + "port": { "required": true, "type": "int" }, + "ssl": { "required": true, "type": "bool" } + } + }, "master": { "required": true, "type": "map", diff --git a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml index da9a17ca25..3dab9c66c1 100644 --- a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml @@ -8,9 +8,7 @@ %legend = t(".metric_agent_header") - = boolean_field %w(metric_agent insecure) = boolean_field %w(metric_agent system_only) - = string_field %w(metric_agent ca_file) = select_field %w(metric_agent log_level), :collection => :log_levels = integer_field %w(metric_agent statsd_port) = integer_field %w(metric_agent check_frequency) diff --git a/crowbar_framework/config/locales/monasca/en.yml b/crowbar_framework/config/locales/monasca/en.yml index 182cd3cd61..31e83c3676 100644 --- a/crowbar_framework/config/locales/monasca/en.yml +++ b/crowbar_framework/config/locales/monasca/en.yml @@ -29,8 +29,8 @@ en: metric_agent_header: 'Metrics Agent Settings' metric_agent: monasca_url: 'Monasca URL' - insecure: 'Do you want insecure connection?' system_only: 'Setting to true will cause Monasca setup to run in `system_only` mode only, configure the base config and system metrics (cpu, disk, load, memory, network)' + insecure: 'Do you want insecure connection?' ca_file: 'Sets the path to the ca certs file if using certificates. Required only if insecure is set to False (ca_file)' log_level: 'Log level' statsd_port: 'Monasca Statsd port' From 8df9f37ec6a187a525134fdb1c51c7b4aa5479d6 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Thu, 16 Mar 2017 08:23:56 +0100 Subject: [PATCH 17/52] Finish integration of Ansible installer (#15) * Finish integration of Ansible installer * Set credentials in command line * Fix violations * Fix violations 2 * Add support for multiple networks in single node * Pass missing variables to ansible * Fix hound violations * Use monasca.yml as entry point for ansible installer * Fix paths and influxdb url * Remove agent nodes from inventory file --- .../monasca/files/default/ansible.cfg | 233 ++++++++++++++++++ chef/cookbooks/monasca/libraries/helper.rb | 4 + chef/cookbooks/monasca/recipes/master.rb | 86 ++++--- .../templates/default/cmm-hosts-single.erb | 21 -- .../templates/default/credentials.yml.erb | 47 ---- chef/data_bags/crowbar/template-monasca.json | 1 - .../data_bags/crowbar/template-monasca.schema | 1 - .../app/models/monasca_service.rb | 19 +- .../monasca/_edit_attributes.html.haml | 16 -- .../config/locales/monasca/en.yml | 13 - 10 files changed, 298 insertions(+), 143 deletions(-) create mode 100644 chef/cookbooks/monasca/files/default/ansible.cfg delete mode 100644 chef/cookbooks/monasca/templates/default/credentials.yml.erb diff --git a/chef/cookbooks/monasca/files/default/ansible.cfg b/chef/cookbooks/monasca/files/default/ansible.cfg new file mode 100644 index 0000000000..9425aaa602 --- /dev/null +++ b/chef/cookbooks/monasca/files/default/ansible.cfg @@ -0,0 +1,233 @@ +# Copyright 2017 FUJITSU LIMITED + +# config file for ansible -- http://ansible.com/ +# ============================================== + +# nearly all parameters can be overridden in ansible-playbook +# or with command line flags. ansible will read ANSIBLE_CONFIG, +# ansible.cfg in the current working directory, .ansible.cfg in +# the home directory or /etc/ansible/ansible.cfg, whichever it +# finds first + +[defaults] + +# some basic default values... + +inventory = /etc/ansible/hosts +#library = /usr/share/my_modules/ +remote_tmp = $HOME/.ansible/tmp +pattern = * +forks = 5 +poll_interval = 15 +sudo_user = root +#ask_sudo_pass = True +#ask_pass = True +transport = ssh +#remote_port = 22 +module_lang = C + +# plays will gather facts by default, which contain information about +# the remote system. +# +# smart - gather by default, but don't regather if already gathered +# implicit - gather by default, turn off with gather_facts: False +# explicit - do not gather by default, must say gather_facts: True +gathering = implicit + +# additional paths to search for roles in, colon separated +#roles_path = /etc/ansible/roles + +# uncomment this to disable SSH key host checking +#host_key_checking = False + +# change this for alternative sudo implementations +sudo_exe = sudo + +# what flags to pass to sudo +#sudo_flags = -H + +# SSH timeout +timeout = 10 + +# default user to use for playbooks if user is not specified +# (/usr/bin/ansible will use current user as default) +#remote_user = root + +# logging is off by default unless this path is defined +# if so defined, consider logrotate +#log_path = /var/log/ansible.log + +# default module name for /usr/bin/ansible +#module_name = command + +# use this shell for commands executed under sudo +# you may need to change this to bin/bash in rare instances +# if sudo is constrained +#executable = /bin/sh + +# if inventory variables overlap, does the higher precedence one win +# or are hash values merged together? The default is 'replace' but +# this can also be set to 'merge'. +#hash_behaviour = replace + +# list any Jinja2 extensions to enable here: +jinja2_extensions = jinja2.ext.do,jinja2.ext.i18n + +# if set, always use this private key file for authentication, same as +# if passing --private-key to ansible or ansible-playbook +#private_key_file = /path/to/file + +# format of string {{ ansible_managed }} available within Jinja2 +# templates indicates to users editing templates files will be replaced. +# replacing {file}, {host} and {uid} and strftime codes with proper values. +ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} on {host} + +# by default, ansible-playbook will display "Skipping [host]" if it determines a task +# should not be run on a host. Set this to "False" if you don't want to see these "Skipping" +# messages. NOTE: the task header will still be shown regardless of whether or not the +# task is skipped. +#display_skipped_hosts = True + +# by default (as of 1.3), Ansible will raise errors when attempting to dereference +# Jinja2 variables that are not set in templates or action lines. Uncomment this line +# to revert the behavior to pre-1.3. +#error_on_undefined_vars = False + +# by default (as of 1.6), Ansible may display warnings based on the configuration of the +# system running ansible itself. This may include warnings about 3rd party packages or +# other conditions that should be resolved if possible. +# to disable these warnings, set the following value to False: +#system_warnings = True + +# by default (as of 1.4), Ansible may display deprecation warnings for language +# features that should no longer be used and will be removed in future versions. +# to disable these warnings, set the following value to False: +#deprecation_warnings = True + +# (as of 1.8), Ansible can optionally warn when usage of the shell and +# command module appear to be simplified by using a default Ansible module +# instead. These warnings can be silenced by adjusting the following +# setting or adding warn=yes or warn=no to the end of the command line +# parameter string. This will for example suggest using the git module +# instead of shelling out to the git command. +# command_warnings = False + + +# set plugin path directories here, separate with colons +action_plugins = /usr/share/ansible_plugins/action_plugins +callback_plugins = /usr/share/ansible_plugins/callback_plugins +connection_plugins = /usr/share/ansible_plugins/connection_plugins +lookup_plugins = /usr/share/ansible_plugins/lookup_plugins +vars_plugins = /usr/share/ansible_plugins/vars_plugins +filter_plugins = /usr/share/ansible_plugins/filter_plugins + +# by default callbacks are not loaded for /bin/ansible, enable this if you +# want, for example, a notification or logging callback to also apply to +# /bin/ansible runs +#bin_ansible_callbacks = False + + +# don't like cows? that's unfortunate. +# set to 1 if you don't want cowsay support or export ANSIBLE_NOCOWS=1 +#nocows = 1 + +# don't like colors either? +# set to 1 if you don't want colors, or export ANSIBLE_NOCOLOR=1 +#nocolor = 1 + +# the CA certificate path used for validating SSL certs. This path +# should exist on the controlling node, not the target nodes +# common locations: +# RHEL/CentOS: /etc/pki/tls/certs/ca-bundle.crt +# Fedora : /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +# Ubuntu : /usr/share/ca-certificates/cacert.org/cacert.org.crt +#ca_file_path = + +# the http user-agent string to use when fetching urls. Some web server +# operators block the default urllib user agent as it is frequently used +# by malicious attacks/scripts, so we set it to something unique to +# avoid issues. +#http_user_agent = ansible-agent + +# if set to a persistent type (not 'memory', for example 'redis') fact values +# from previous runs in Ansible will be stored. This may be useful when +# wanting to use, for example, IP information from one group of servers +# without having to talk to them in the same playbook run to get their +# current IP information. +fact_caching = memory + + +# retry files +#retry_files_enabled = False +#retry_files_save_path = ~/.ansible-retry + +[privilege_escalation] +#become=True +#become_method=sudo +#become_user=root +#become_ask_pass=False + +[paramiko_connection] + +# uncomment this line to cause the paramiko connection plugin to not record new host +# keys encountered. Increases performance on new host additions. Setting works independently of the +# host key checking setting above. +#record_host_keys=False + +# by default, Ansible requests a pseudo-terminal for commands executed under sudo. Uncomment this +# line to disable this behaviour. +#pty=False + +[ssh_connection] + +# ssh arguments to use +# Leaving off ControlPersist will result in poor performance, so use +# paramiko on older platforms rather than removing it +ssh_args = -o ControlMaster=auto -o BatchMode=yes -o ForwardAgent=yes + +# The path to use for the ControlPath sockets. This defaults to +# "%(directory)s/ansible-ssh-%%h-%%p-%%r", however on some systems with +# very long hostnames or very long path names (caused by long user names or +# deeply nested home directories) this can exceed the character limit on +# file socket names (108 characters for most platforms). In that case, you +# may wish to shorten the string below. +# +# Example: +# control_path = %(directory)s/%%h-%%r +#control_path = %(directory)s/ansible-ssh-%%h-%%p-%%r + +# Enabling pipelining reduces the number of SSH operations required to +# execute a module on the remote server. This can result in a significant +# performance improvement when enabled, however when using "become:" you must +# first disable 'requiretty' in /etc/sudoers +# +# By default, this option is disabled to preserve compatibility with +# sudoers configurations that have requiretty (the default on many distros). +# +#pipelining = False + +# if True, make ansible use scp if the connection type is ssh +# (default is sftp) +#scp_if_ssh = True + +[accelerate] +accelerate_port = 5099 +accelerate_timeout = 30 +accelerate_connect_timeout = 5.0 + +# The daemon timeout is measured in minutes. This time is measured +# from the last activity to the accelerate daemon. +accelerate_daemon_timeout = 30 + +# If set to yes, accelerate_multi_key will allow multiple +# private keys to be uploaded to it, though each user must +# have access to the system via SSH to add a new key. The default +# is "no". +#accelerate_multi_key = yes + +[selinux] +# file systems that require special treatment when dealing with security context +# the default behaviour that copies the existing context or uses the user default +# needs to be changed to use the file system dependant context. +#special_context_filesystems=nfs,vboxsf,fuse + diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 06ee18a200..774cbae868 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -50,4 +50,8 @@ def self.monasca_hosts(nodes) end hosts end + + def self.get_host_for_monitoring_url(node) + Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address + end end diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index 39ee4ced60..e0956ba53d 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -14,42 +14,16 @@ # limitation. package "ansible" +package "monasca-installer" -keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) - -directory "/opt/fujitsu" do - owner "root" - group "root" - mode "0755" -end - -# TODO: use package "monasca-installer" -remote_file "/opt/fujitsu/cmm-suse.tgz" do - source node[:monasca][:master][:cmm_tarball_url] +cookbook_file "/etc/ansible/ansible.cfg" do + source "ansible.cfg" owner "root" group "root" mode "0644" - action :create_if_missing -end - -execute "extract cmm tarball" do - command "tar xf cmm-suse.tgz" - cwd "/opt/fujitsu/" - not_if { Dir.exist?("/opt/fujitsu/monasca-installer") } - notifies :run, "execute[run ansible]", :delayed -end - -template "/opt/fujitsu/monasca-installer/credentials.yml" do - source "credentials.yml.erb" - owner "root" - group "root" - mode "0600" - variables( - keystone_settings: keystone_settings - ) - notifies :run, "execute[run ansible]", :delayed end +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) monasca_hosts = MonascaHelper.monasca_hosts(search(:node, "roles:monasca-server")) raise "no nodes with monasca-server role found" if monasca_hosts.nil? or monasca_hosts.empty? @@ -61,7 +35,7 @@ "cmm-hosts-cluster.erb" end -template "/opt/fujitsu/monasca-installer/cmm-hosts" do +template "/opt/monasca-installer/cmm-hosts" do source hosts_template owner "root" group "root" @@ -76,7 +50,7 @@ notifies :run, "execute[run ansible]", :delayed end -template "/opt/fujitsu/monasca-installer/group_vars/all_group" do +template "/opt/monasca-installer/group_vars/all_group" do source "all_group.erb" owner "root" group "root" @@ -90,7 +64,7 @@ # This file is used to mark that ansible installer run successfully. # Without this, it could happen that the installer was not re-tried # after a failed run. -file "/opt/fujitsu/monasca-installer/.installed" do +file "/opt/monasca-installer/.installed" do content "cmm installed" owner "root" group "root" @@ -99,10 +73,48 @@ action :create_if_missing end +monasca_node = search(:node, "roles:monasca-server")[0] +cmm_net_ip = MonascaHelper.get_host_for_monitoring_url(monasca_node) +pub_net_ip = CrowbarHelper.get_host_for_public_url(monasca_node, false, false) + +ansible_vars = { + influxdb_mon_api_password: node[:monasca][:master][:influxdb_mon_api_password], + influxdb_mon_persister_password: node[:monasca][:master][:influxdb_mon_persister_password], + database_notification_password: node[:monasca][:master][:database_notification_password], + database_monapi_password: node[:monasca][:master][:database_monapi_password], + database_thresh_password: node[:monasca][:master][:database_thresh_password], + database_logapi_password: node[:monasca][:master][:database_logapi_password], + keystone_cmm_operator_user_password: + node[:monasca][:master][:keystone_cmm_operator_user_password], + keystone_cmm_agent_password: node[:monasca][:master][:keystone_cmm_agent_password], + keystone_admin_agent_password: node[:monasca][:master][:keystone_admin_agent_password], + keystone_admin_password: keystone_settings["admin_password"], + database_grafana_password: node[:monasca][:master][:database_grafana_password], + + memcached_listen_ip: cmm_net_ip, + kafka_host: cmm_net_ip, + kibana_host: pub_net_ip, + log_api_bind_host: pub_net_ip, + influxdb_bind_address: cmm_net_ip, + monasca_api_bind_host: pub_net_ip, + elasticsearch_host: cmm_net_ip, + nimbus_host: cmm_net_ip, + zookeeper_hosts: cmm_net_ip, + kafka_hosts: "#{cmm_net_ip}:9092", + mariadb_bind_address: cmm_net_ip, + database_host: cmm_net_ip, + monasca_api_url: "http://#{pub_net_ip}:8070/v2.0", + monasca_log_api_url: "http://#{pub_net_ip}:5607/v2.0", + memcached_nodes: "#{cmm_net_ip}:11211", + influxdb_url: "http://#{cmm_net_ip}:8086", + elasticsearch_nodes: "[#{cmm_net_ip}]", + elasticsearch_hosts: cmm_net_ip +}.to_json + execute "run ansible" do - command "rm -f /opt/fujitsu/monasca-installer/.installed"\ - "&& ansible-playbook -i cmm-hosts monasca.yml"\ - "&& touch /opt/fujitsu/monasca-installer/.installed" - cwd "/opt/fujitsu/monasca-installer" + command "rm -f /opt/monasca-installer/.installed"\ + "&& ansible-playbook -i cmm-hosts -e '#{ansible_vars}' monasca.yml"\ + "&& touch /opt/monasca-installer/.installed" + cwd "/opt/monasca-installer" action :nothing end diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb b/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb index 673150cb4f..162448e50b 100644 --- a/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb +++ b/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb @@ -53,12 +53,6 @@ monasca-persister-node ansible_ssh_host=<%= @monasca_host %> ansible_s # Monasca-log-metric node monasca-log-metrics-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> -# Monasca Agent Hosts -cmm-monasca-agent ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> - -# Monasca Log Agent Hosts -cmm-monasca-log-agent ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> - # Storm Nimbus node storm-nimbus-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -77,19 +71,6 @@ elasticsearch-curator-node ansible_ssh_host=<%= @monasca_host %> ansible_s # InfluxDB node influxdb-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> -################################################################################ -# Agent and Log-Agent group definition: -# To install additional agents (metrics and logs) add nodes to -# monasca_agent_group or monasca_log_agent_group. Run ansible-playbook -# monasca-agent.yml or ansible-playbook monasca-log-agent.yml to install agents. -################################################################################ - -[monasca_agent_group] -cmm-monasca-agent - -[monasca_log_agent_group] -cmm-monasca-log-agent - ################################################################################ # Other group definition ################################################################################ @@ -165,7 +146,6 @@ influxdb-node [elkstack_group:children] elasticsearch_group kibana_group -monasca_log_agent_group monasca_log_persister_group monasca_log_transformer_group monasca_log_metrics_group @@ -179,7 +159,6 @@ storm_supervisor_group offline_group openstack_group monasca_group -monasca_agent_group [monasca_group:children] elkstack_group diff --git a/chef/cookbooks/monasca/templates/default/credentials.yml.erb b/chef/cookbooks/monasca/templates/default/credentials.yml.erb deleted file mode 100644 index a9fb27bb84..0000000000 --- a/chef/cookbooks/monasca/templates/default/credentials.yml.erb +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright 2017 Fujitsu LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitation. - -# Important: Please provide secure passwords below - -# group_vars/monasca_group credentials -# used for influxdb_users -influxdb_mon_api_password: <%= node[:monasca][:master][:influxdb_mon_api_password] %> -influxdb_mon_persister_password: <%= node[:monasca][:master][:influxdb_mon_persister_password] %> - -# Used for database_users -database_notification_password: <%= node[:monasca][:master][:database_notification_password] %> -database_monapi_password: <%= node[:monasca][:master][:database_monapi_password] %> -database_thresh_password: <%= node[:monasca][:master][:database_thresh_password] %> -database_logapi_password: <%= node[:monasca][:master][:database_logapi_password] %> - -# Keystone user passwords - used in group_vars/openstack_group/keystone_users -## new created users in keystone: password -keystone_cmm_operator_user_password: <%= node[:monasca][:master][:keystone_cmm_operator_user_password] %> -keystone_cmm_agent_password: <%= node[:monasca][:master][:keystone_cmm_agent_password] %> -keystone_admin_agent_password: <%= node[:monasca][:master][:keystone_admin_agent_password] %> -## existing keystone user. Important: This password must match to an already existing user in keystone. It is used to create new users. -keystone_admin_password: <%= @keystone_settings['admin_password'] %> - -# Monasca-Agent credentials. used in host_vars/ -#cmm_monasca_agent_keystone_password: -#openstack_monasca_agent_keystone_password: -#cmm_monasca_agent_database_password: - -# Monasca-Log-Agent credentials. used in host_vars/ -#cmm_monasca_log_agent_keystone_password: -#openstack_monasca_log_agent_keystone_password: - -database_grafana_password: <%= node[:monasca][:master][:database_grafana_password] %> - diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 80b8f7effa..61115efc1d 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -45,7 +45,6 @@ "ssl": false }, "master": { - "cmm_tarball_url": "http://127.0.0.1/cmm/cmm-suse.tgz", "influxdb_mon_api_password": "", "influxdb_mon_persister_password": "", "database_notification_password": "", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 1f1fca75a7..8105eb6456 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -81,7 +81,6 @@ "required": true, "type": "map", "mapping": { - "cmm_tarball_url": { "required": true, "type": "str" }, "influxdb_mon_api_password": { "required": true, "type": "str" }, "influxdb_mon_persister_password": { "required": true, "type": "str" }, "database_notification_password": { "required": true, "type": "str" }, diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index ef1b5ea394..3d6c05cb46 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -49,8 +49,11 @@ def role_constraints }, "monasca-server" => { "unique" => false, - "count" => -1, - "cluster" => true, + # TODO: change for cluster + # "count" => -1, + # "cluster" => true, + "count" => 1, + "cluster" => false, "admin" => false, "exclude_platform" => { "suse" => "< 12.1", @@ -131,11 +134,13 @@ def create_proposal def validate_proposal_after_save(proposal) validate_one_for_role proposal, "monasca-master" - nodes = proposal["deployment"][@bc_name]["elements"] - if !nodes.key?("monasca-server") || - (nodes["monasca-server"].length != 1 && nodes["monasca-server"].length != 3) - validation_error("Need either one or three monasca-server node(s).") - end + validate_one_for_role proposal, "monasca-server" + # TODO: uncomment for cluster support + # nodes = proposal["deployment"][@bc_name]["elements"] + # if !nodes.key?("monasca-server") || + # (nodes["monasca-server"].length != 1 && nodes["monasca-server"].length != 3) + # validation_error("Need either one or three monasca-server node(s).") + # end super end diff --git a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml index 3dab9c66c1..31510cfdda 100644 --- a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml @@ -19,19 +19,3 @@ = integer_field %w(metric_agent max_measurement_buffer_size) = integer_field %w(metric_agent backlog_send_rate) = integer_field %w(metric_agent amplifier) - - %fieldset - %legend - = t(".master_header") - - = string_field %w(master cmm_tarball_url) - = password_field %w(master influxdb_mon_api_password) - = password_field %w(master influxdb_mon_persister_password) - = password_field %w(master database_notification_password) - = password_field %w(master database_monapi_password) - = password_field %w(master database_thresh_password) - = password_field %w(master database_logapi_password) - = password_field %w(master keystone_cmm_operator_user_password) - = password_field %w(master keystone_cmm_agent_password) - = password_field %w(master keystone_admin_agent_password) - = password_field %w(master database_grafana_password) diff --git a/crowbar_framework/config/locales/monasca/en.yml b/crowbar_framework/config/locales/monasca/en.yml index 31e83c3676..7ca98ea710 100644 --- a/crowbar_framework/config/locales/monasca/en.yml +++ b/crowbar_framework/config/locales/monasca/en.yml @@ -42,19 +42,6 @@ en: max_measurement_buffer_size: 'Maximum number of measurements to buffer when unable to communicate with the monasca-api (-1 means no limit, max_measurement_buffer_size)' backlog_send_rate: 'Maximum number of messages to send at one time when communication with the monasca-api is restored (backlog_send_rate)' amplifier: 'Publish extra metrics to the API by adding this number of "amplifier" dimensions. For load testing purposes only; set to 0 for production use (amplifier)' - master_header: 'Installation parameters' - master: - cmm_tarball_url: 'URL of the CMM installer tarball' - influxdb_mon_api_password: 'InfluxDB password' - influxdb_mon_persister_password: 'Metrics persister password' - database_notification_password: 'Notification database password' - database_monapi_password: 'Metrics API database password' - database_thresh_password: 'Thresh Engine database password' - database_logapi_password: 'Log API database password' - keystone_cmm_operator_user_password: 'keystone_cmm_operator_user_password' - keystone_cmm_agent_password: 'keystone_cmm_agent_password' - keystone_admin_agent_password: 'keystone_admin_agent_password' - database_grafana_password: 'Grafana database password' db_header: 'Database settings' db: database: 'Database' From f261fe75090bfc9267bc3b2724efcc7bd3128766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Thu, 16 Mar 2017 10:09:39 +0100 Subject: [PATCH 18/52] Use sudo for running monasca-reconfigure (#21) --- chef/cookbooks/monasca/recipes/metric_agent.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb index 558106e899..4a283879e5 100644 --- a/chef/cookbooks/monasca/recipes/metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -48,7 +48,9 @@ end execute "monasca-setup detect services" do - command monasca_reconfigure_file + # Run monasca-setup with `sudo` + # https://github.com/openstack/monasca-agent/blob/master/docs/Agent.md#monasca-setup-recommended + command "sudo #{monasca_reconfigure_file}" only_if { File.exist?(monasca_reconfigure_file) } user agent_settings["user"] group agent_settings["group"] From 4c74e9c79d2447d987c17400479a47960472b2b3 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Tue, 21 Mar 2017 07:47:42 +0100 Subject: [PATCH 19/52] Remove default grafana config file (#22) --- chef/cookbooks/horizon/recipes/monasca_ui.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/horizon/recipes/monasca_ui.rb b/chef/cookbooks/horizon/recipes/monasca_ui.rb index 6174ad9c42..0ed5cbdbf6 100644 --- a/chef/cookbooks/horizon/recipes/monasca_ui.rb +++ b/chef/cookbooks/horizon/recipes/monasca_ui.rb @@ -33,7 +33,7 @@ notifies :reload, resources(service: "apache2") end -file "/etc/apache2/vhost.d/grafana.conf" do +file "/etc/apache2/vhosts.d/grafana.conf" do action :delete notifies :reload, resources(service: "apache2") end From eff582f7fa10fd6986455240d81d5cd6c28239be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Wed, 22 Mar 2017 13:49:54 +0100 Subject: [PATCH 20/52] Fix monasca-log-agent rpm name (#24) --- chef/cookbooks/monasca/recipes/log_agent.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 27586cec69..27ae4922ba 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -13,7 +13,7 @@ # limitations under the License. # -package "monasca-log-agent" +package "openstack-monasca-log-agent" log_agent_settings = node[:monasca][:log_agent] log_agent_keystone = log_agent_settings[:keystone] From 6db37e52415983863e6d8298e391ac877b514ff2 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Thu, 23 Mar 2017 13:53:25 +0100 Subject: [PATCH 21/52] Fix suse (#23) * Register monasca endpoints in Keystone * Fix kibana url * Rearrange helper functions --- .../default/_80_monasca_ui_settings.py.erb | 3 +- chef/cookbooks/monasca/libraries/helper.rb | 18 +++++ chef/cookbooks/monasca/recipes/server.rb | 75 +++++++++++++++---- 3 files changed, 82 insertions(+), 14 deletions(-) diff --git a/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb index 7256592d4d..f5ce10f665 100644 --- a/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb +++ b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb @@ -60,7 +60,8 @@ DASHBOARDS = getattr(settings, 'GRAFANA_LINKS', GRAFANA_LINKS) #] ENABLE_KIBANA_BUTTON = getattr(settings, 'ENABLE_KIBANA_BUTTON', <%= @kibana_enabled ? "True" : "False" %>) -KIBANA_HOST = getattr(settings, 'KIBANA_HOST', '<%= @kibana_host %>') +# TODO: this needs to be fixed for clustered deployment +KIBANA_HOST = getattr(settings, 'KIBANA_HOST', 'http://<%= @kibana_host %>:5601/') OPENSTACK_SSL_NO_VERIFY = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False) OPENSTACK_SSL_CACERT = getattr(settings, 'OPENSTACK_SSL_CACERT', None) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 774cbae868..e5c48922ea 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -34,6 +34,24 @@ def self.api_public_url(node) "#{protocol}://#{host}:#{port}/v2.0" end + def self.api_admin_url(node) + host = monasca_admin_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end + + def self.api_internal_url(node) + host = get_host_for_monitoring_url(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end + def self.log_api_public_url(node) host = monasca_public_host(node) # SSL is not supported at this moment diff --git a/chef/cookbooks/monasca/recipes/server.rb b/chef/cookbooks/monasca/recipes/server.rb index 0b2daedf60..f0103f4715 100644 --- a/chef/cookbooks/monasca/recipes/server.rb +++ b/chef/cookbooks/monasca/recipes/server.rb @@ -12,21 +12,70 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# TODO: Fill this with code that deploys the Monasca backend services -### Example code for retrieving a database URL and keystone settings: +keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) -# keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name), +register_auth_hash = { + user: keystone_settings["admin_user"], + password: keystone_settings["admin_password"], + tenant: keystone_settings["admin_tenant"] +} -# db_settings = fetch_database_settings -# db_conn_scheme = db_settings[:url_scheme] +keystone_register "monasca api wakeup keystone" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + action :wakeup +end -# if db_settings[:backend_name] == "mysql" -# db_conn_scheme = "mysql+pymysql" -# end +keystone_register "register monasca api user" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name keystone_settings["service_user"] + user_password keystone_settings["service_password"] + tenant_name keystone_settings["service_tenant"] + action :add_user +end -# database_connection = "#{db_conn_scheme}://" \ -# "#{node[:monasca][:db][:user]}" \ -# ":#{node[:monasca][:db][:password]}" \ -# "@#{db_settings[:address]}" \ -# "/#{node[:monasca][:db][:database]}" +keystone_register "give monasca api user access" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + user_name keystone_settings["service_user"] + tenant_name keystone_settings["service_tenant"] + role_name "admin" + action :add_access +end + +keystone_register "register monasca api service" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + service_name "monasca" + service_type "monitoring" + service_description "Monasca monitoring service" + action :add_service +end + +keystone_register "register monasca api endpoint" do + protocol keystone_settings["protocol"] + insecure keystone_settings["insecure"] + host keystone_settings["internal_url_host"] + port keystone_settings["admin_port"] + auth register_auth_hash + endpoint_service "monasca" + endpoint_region keystone_settings["endpoint_region"] + endpoint_publicURL MonascaHelper.api_public_url(node) + endpoint_adminURL MonascaHelper.api_admin_url(node) + endpoint_internalURL MonascaHelper.api_internal_url(node) + action :add_endpoint_template +end From fd75ed6e62736857d4efd3aab0e37aed5bd75276 Mon Sep 17 00:00:00 2001 From: Artur Basiak Date: Fri, 24 Mar 2017 14:13:21 +0100 Subject: [PATCH 22/52] Remove unused group (#27) Remove unused grafana_group from cluster template --- chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb index 97fe50792a..440700a8d1 100644 --- a/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb +++ b/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb @@ -278,7 +278,6 @@ monasca_persister_group storm_group monasca_thresh_group database_cluster_group -grafana_group [database_cluster_group:children] mariadb_group From 42226beb8071332814b15f9261f4cf6bf35736be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Tue, 28 Mar 2017 13:09:27 +0200 Subject: [PATCH 23/52] Disable deprecation_warnings for Ansible (#34) They are outputted to STDERR and break applying cookbook. --- chef/cookbooks/monasca/files/default/ansible.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chef/cookbooks/monasca/files/default/ansible.cfg b/chef/cookbooks/monasca/files/default/ansible.cfg index 9425aaa602..e6a48cc8a1 100644 --- a/chef/cookbooks/monasca/files/default/ansible.cfg +++ b/chef/cookbooks/monasca/files/default/ansible.cfg @@ -102,7 +102,7 @@ ansible_managed = Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S by {uid} # by default (as of 1.4), Ansible may display deprecation warnings for language # features that should no longer be used and will be removed in future versions. # to disable these warnings, set the following value to False: -#deprecation_warnings = True +deprecation_warnings = False # (as of 1.8), Ansible can optionally warn when usage of the shell and # command module appear to be simplified by using a default Ansible module @@ -230,4 +230,3 @@ accelerate_daemon_timeout = 30 # the default behaviour that copies the existing context or uses the user default # needs to be changed to use the file system dependant context. #special_context_filesystems=nfs,vboxsf,fuse - From ff3039eb53cc4228cd81de8018e7323054890f23 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Wed, 29 Mar 2017 08:02:39 +0200 Subject: [PATCH 24/52] Agent log dir (#33) * Register monasca endpoints in Keystone * Fix kibana url * Rearrange helper functions * Make agent log dir configurable --- chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb | 1 + chef/data_bags/crowbar/template-monasca.json | 1 + chef/data_bags/crowbar/template-monasca.schema | 1 + .../app/views/barclamp/monasca/_edit_attributes.html.haml | 1 + crowbar_framework/config/locales/monasca/en.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb index 5362d18eb9..30d89c8063 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb @@ -18,6 +18,7 @@ <% if @agent_settings["system_only"] -%> --system_only <% end -%> \ <% if @agent_settings["overwrite_config"] -%> --overwrite <% end -%> \ <% if @agent_settings["ca_file"].length > 0 -%> --ca_file '<%= @agent_settings["ca_file"] %>' <% end -%> \ + --log_dir '<%= @agent_settings["log_dir"] %>' \ --log_level '<%= @agent_settings["log_level"] %>' \ --monasca_statsd_port '<%= @agent_settings["statsd_port"].to_i %>' \ --check_frequency '<%= @agent_settings["check_frequency"].to_i %>' \ diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 61115efc1d..ae5904ea88 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -14,6 +14,7 @@ "system_only": false, "overwrite_config": false, "ca_file": "", + "log_dir": "/var/log/monasca-agent/", "log_level": "INFO", "statsd_port": 8125, "check_frequency": 15, diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 8105eb6456..de8ad99ccb 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -30,6 +30,7 @@ "system_only": { "type": "bool", "required": true }, "overwrite_config": { "type": "bool", "required": true }, "ca_file": { "type": "str", "required": true }, + "log_dir": { "type": "str", "required": true }, "log_level": { "type": "str", "required": true }, "statsd_port": { "type": "int", "required": true }, "check_frequency": { "type": "int", "required": true }, diff --git a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml index 31510cfdda..5f8f92cfc5 100644 --- a/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml +++ b/crowbar_framework/app/views/barclamp/monasca/_edit_attributes.html.haml @@ -9,6 +9,7 @@ = t(".metric_agent_header") = boolean_field %w(metric_agent system_only) + = string_field %w(metric_agent log_dir) = select_field %w(metric_agent log_level), :collection => :log_levels = integer_field %w(metric_agent statsd_port) = integer_field %w(metric_agent check_frequency) diff --git a/crowbar_framework/config/locales/monasca/en.yml b/crowbar_framework/config/locales/monasca/en.yml index 7ca98ea710..7d0c33b2bd 100644 --- a/crowbar_framework/config/locales/monasca/en.yml +++ b/crowbar_framework/config/locales/monasca/en.yml @@ -32,6 +32,7 @@ en: system_only: 'Setting to true will cause Monasca setup to run in `system_only` mode only, configure the base config and system metrics (cpu, disk, load, memory, network)' insecure: 'Do you want insecure connection?' ca_file: 'Sets the path to the ca certs file if using certificates. Required only if insecure is set to False (ca_file)' + log_dir: 'Log directory' log_level: 'Log level' statsd_port: 'Monasca Statsd port' check_frequency: 'Time to wait between collection runs (check_frequency)' From f6d3d861cfc5c2135299b2f54b9186a4d150563e Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Fri, 31 Mar 2017 11:04:19 +0200 Subject: [PATCH 25/52] Configure agent service name for monasca setup (#32) Override default monasca-agent service name with --agent_service_name flag --- chef/cookbooks/monasca/attributes/default.rb | 1 + .../monasca/templates/default/monasca-reconfigure.erb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index fe83fb889c..7a0482b3d5 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -45,6 +45,7 @@ default[:monasca][:metric_agent][:user] = "monasca-agent" default[:monasca][:metric_agent][:group] = "monasca" default[:monasca][:metric_agent][:debug] = monasca_debug +default[:monasca][:metric_agent][:agent_service_name] = "openstack-monasca-agent" # log-agent default service settings default[:monasca][:log_agent][:service_name] = monasca_log_agent_service_name diff --git a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb index 30d89c8063..492cebf18e 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb @@ -30,4 +30,5 @@ --backlog_send_rate '<%= @agent_settings["backlog_send_rate"].to_i %>' \ --amplifier '<%= @agent_settings["amplifier"] %>' \ --skip_enable \ - <% if @install_plugins_only -%> --install_plugins_only <% end -%> + <% if @install_plugins_only -%> --install_plugins_only <% end -%> \ + --agent_service_name '<%= @agent_settings["agent_service_name"] %>' From 1b22220d6f0b6bd296d5f7f4fe45d6103dfa4e76 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Mon, 3 Apr 2017 08:44:46 +0200 Subject: [PATCH 26/52] Address review comments (#35) --- chef/cookbooks/monasca/recipes/api.rb | 148 ---------------- chef/cookbooks/monasca/recipes/common.rb | 1 - chef/cookbooks/monasca/recipes/master.rb | 10 +- .../templates/default/api-config.conf.erb | 159 ------------------ ...-cluster.erb => monasca-hosts-cluster.erb} | 0 ...ts-single.erb => monasca-hosts-single.erb} | 0 .../app/models/monasca_service.rb | 12 +- 7 files changed, 11 insertions(+), 319 deletions(-) delete mode 100644 chef/cookbooks/monasca/recipes/api.rb delete mode 100755 chef/cookbooks/monasca/templates/default/api-config.conf.erb rename chef/cookbooks/monasca/templates/default/{cmm-hosts-cluster.erb => monasca-hosts-cluster.erb} (100%) rename chef/cookbooks/monasca/templates/default/{cmm-hosts-single.erb => monasca-hosts-single.erb} (100%) diff --git a/chef/cookbooks/monasca/recipes/api.rb b/chef/cookbooks/monasca/recipes/api.rb deleted file mode 100644 index 361163b4c9..0000000000 --- a/chef/cookbooks/monasca/recipes/api.rb +++ /dev/null @@ -1,148 +0,0 @@ -# -# Copyright 2016 SUSE Linux GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -include_recipe "apache2" -include_recipe "apache2::mod_wsgi" -include_recipe "apache2::mod_rewrite" - -apache_module "deflate" do - conf false - enable true -end - -apache_site "000-default" do - enable false -end - -### TODO: uncomment this once there is a package. -# package "openstack-monasca-api" - -### FIXME: remove this once there is a package creating this directory - -directory "/etc/monasca/" do - owner "root" - group "root" - mode 0o755 - recursive true - notifies :create, "template[/etc/monasca/api-config.conf]" -end - -ha_enabled = node[:monasca][:ha][:enabled] - -db_settings = fetch_database_settings -db_conn_scheme = db_settings[:url_scheme] - -db_settings[:backend_name] == "mysql" && db_conn_scheme = "mysql+pymysql" - -database_connection = "#{db_conn_scheme}://" \ - "#{node[:monasca][:db][:user]}" \ - ":#{node[:monasca][:db][:password]}" \ - "@#{db_settings[:address]}" \ - "/#{node[:monasca][:db][:database]}" - -crowbar_pacemaker_sync_mark "wait-monasca_database" - -# Create the Monasca Database -database "create #{node[:monasca][:db][:database]} database" do - connection db_settings[:connection] - database_name node[:monasca][:db][:database] - provider db_settings[:provider] - action :create - only_if { !ha_enabled || CrowbarPacemakerHelper.is_cluster_founder?(node) } -end - -database_user "create #{@cookbook_name} database user" do - connection db_settings[:connection] - database_name node[:monasca][:db][:database] - username node[:monasca][:db][:user] - password node[:monasca][:db][:password] - host "%" - provider db_settings[:user_provider] - action :create - only_if { !ha_enabled || CrowbarPacemakerHelper.is_cluster_founder?(node) } -end - -database_user "grant database access for #{@cookbook_name} database user" do - connection db_settings[:connection] - database_name node[:monasca][:db][:database] - username node[:monasca][:db][:user] - password node[:monasca][:db][:password] - host "%" - privileges db_settings[:privs] - provider db_settings[:user_provider] - action :grant - only_if { !ha_enabled || CrowbarPacemakerHelper.is_cluster_founder?(node) } -end - -### FIXME: uncomment this once we have a package that contains a monasca-manage -### command. -# execute "monasca-manage db upgrade" do -# user node[:monasca][:user] -# group node[:monasca][:group] -# command "monasca-manage db upgrade -d #{database_connection} -v head " -# # We only do the sync the first time, and only if we're not doing HA or if we -# # are the founder of the HA cluster (so that it's really only done once). -# only_if do -# !node[:monasca][:db_synced] && -# (!ha_enabled || CrowbarPacemakerHelper.is_cluster_founder?(node)) -# end -# end - -# We want to keep a note that we've done db_sync, so we don't do it again. -# If we were doing that outside a ruby_block, we would add the note in the -# compile phase, before the actual db_sync is done (which is wrong, since it -# could possibly not be reached in case of errors). -ruby_block "mark node for monasca db_sync" do - block do - node.set[:monasca][:db_synced] = true - node.save - end - action :nothing - subscribes :create, "execute[monasca-manage db upgrade]", :immediately -end - -crowbar_pacemaker_sync_mark "create-monasca_database" - -template "/etc/monasca/api-config.conf" do - source "api-config.conf.erb" - owner "root" - ### FIXME: Uncomment once we have a package that creates a monasca group - # group node[:monasca][:group] - mode 0o0640 - variables( - database_connection: database_connection, - keystone_settings: KeystoneHelper.keystone_settings(node, @cookbook_name) - ) - notifies :reload, resources(service: "apache2") -end - -### FIXME: Uncomment once we actually have a runnable WSGI app from a -### monasca-api package -# crowbar_openstack_wsgi "WSGI entry for monasca-api" do -# bind_host bind_host -# bind_port bind_port -# daemon_process "monasca-api" -# user node[:monasca][:user] -# group node[:monasca][:group] -# processes node[:monasca][:api][:processes] -# threads node[:monasca][:api][:threads] -# end -# -# apache_site "monasca-api.conf" do -# enable true -# end - -node.save diff --git a/chef/cookbooks/monasca/recipes/common.rb b/chef/cookbooks/monasca/recipes/common.rb index 37e66d37f6..15706ca1ea 100644 --- a/chef/cookbooks/monasca/recipes/common.rb +++ b/chef/cookbooks/monasca/recipes/common.rb @@ -107,7 +107,6 @@ role_name as_service_role action :add_access end - end end ### recipes specific keystone handling diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index e0956ba53d..7ad81b0277 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -26,16 +26,16 @@ keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) monasca_hosts = MonascaHelper.monasca_hosts(search(:node, "roles:monasca-server")) -raise "no nodes with monasca-server role found" if monasca_hosts.nil? or monasca_hosts.empty? +raise "no nodes with monasca-server role found" if monasca_hosts.nil? || monasca_hosts.empty? hosts_template = if monasca_hosts.length == 1 - "cmm-hosts-single.erb" + "monasca-hosts-single.erb" else - "cmm-hosts-cluster.erb" + "monasca-hosts-cluster.erb" end -template "/opt/monasca-installer/cmm-hosts" do +template "/opt/monasca-installer/monasca-hosts" do source hosts_template owner "root" group "root" @@ -113,7 +113,7 @@ execute "run ansible" do command "rm -f /opt/monasca-installer/.installed"\ - "&& ansible-playbook -i cmm-hosts -e '#{ansible_vars}' monasca.yml"\ + "&& ansible-playbook -i monasca-hosts -e '#{ansible_vars}' monasca.yml"\ "&& touch /opt/monasca-installer/.installed" cwd "/opt/monasca-installer" action :nothing diff --git a/chef/cookbooks/monasca/templates/default/api-config.conf.erb b/chef/cookbooks/monasca/templates/default/api-config.conf.erb deleted file mode 100755 index eee55a5165..0000000000 --- a/chef/cookbooks/monasca/templates/default/api-config.conf.erb +++ /dev/null @@ -1,159 +0,0 @@ -[DEFAULT] -log_config_append=/etc/monasca/api-logging.conf - -# Identifies the region that the Monasca API is running in. -region = useast - -# Dispatchers to be loaded to serve restful APIs -[dispatcher] -versions = monasca_api.v2.reference.versions:Versions -version_2_0 = monasca_api.v2.reference.version_2_0:Version2 -metrics = monasca_api.v2.reference.metrics:Metrics -metrics_measurements = monasca_api.v2.reference.metrics:MetricsMeasurements -metrics_statistics = monasca_api.v2.reference.metrics:MetricsStatistics -metrics_names = monasca_api.v2.reference.metrics:MetricsNames -alarm_definitions = monasca_api.v2.reference.alarm_definitions:AlarmDefinitions -alarms = monasca_api.v2.reference.alarms:Alarms -alarms_count = monasca_api.v2.reference.alarms:AlarmsCount -alarms_state_history = monasca_api.v2.reference.alarms:AlarmsStateHistory -notification_methods = monasca_api.v2.reference.notifications:Notifications -dimension_values = monasca_api.v2.reference.metrics:DimensionValues -dimension_names = monasca_api.v2.reference.metrics:DimensionNames -notification_method_types = monasca_api.v2.reference.notificationstype:NotificationsType - -[security] -# The roles that are allowed full access to the API. -default_authorized_roles = user, domainuser, domainadmin, monasca-user - -# The roles that are allowed to only POST metrics to the API. This role would be used by the Monasca Agent. -agent_authorized_roles = monasca-agent - -# The roles that are allowed to only GET metrics from the API. -read_only_authorized_roles = monasca-read-only-user - -# The roles that are allowed to access the API on behalf of another tenant. -# For example, a service can POST metrics to another tenant if they are a member of the "delegate" role. -delegate_authorized_roles = admin - -[messaging] -# The message queue driver to use -driver = monasca_api.common.messaging.kafka_publisher:KafkaPublisher - -[repositories] -# The driver to use for the metrics repository -# Switches depending on backend database in use. Influxdb or Cassandra. -metrics_driver = monasca_api.common.repositories.influxdb.metrics_repository:MetricsRepository -#metrics_driver = monasca_api.common.repositories.cassandra.metrics_repository:MetricsRepository - -# The driver to use for the alarm definitions repository -alarm_definitions_driver = monasca_api.common.repositories.mysql.alarm_definitions_repository:AlarmDefinitionsRepository - -# The driver to use for the alarms repository -alarms_driver = monasca_api.common.repositories.mysql.alarms_repository:AlarmsRepository - -# The driver to use for the notifications repository -notifications_driver = monasca_api.common.repositories.mysql.notifications_repository:NotificationsRepository - -# The driver to use for the notification method type repository -notification_method_type_driver = monasca_api.common.repositories.sqla.notification_method_type_repository:NotificationMethodTypeRepository - - -[dispatcher] -driver = v2_reference - -[kafka] -# The endpoint to the kafka server -uri = 192.168.10.4:9092 - -# The topic that metrics will be published too -metrics_topic = metrics - -# consumer group name -group = api - -# how many times to try when error occurs -max_retry = 1 - -# wait time between tries when kafka goes down -wait_time = 1 - -# use synchronous or asynchronous connection to kafka -async = False - -# send messages in bulk or send messages one by one. -compact = False - -# How many partitions this connection should listen messages on, this -# parameter is for reading from kafka. If listens on multiple partitions, -# For example, if the client should listen on partitions 1 and 3, then the -# configuration should look like the following: -# partitions = 1 -# partitions = 3 -# default to listen on partition 0. -partitions = 0 - -[influxdb] -# Only needed if Influxdb database is used for backend. -# The IP address of the InfluxDB service. -ip_address = 192.168.10.4 - -# The port number that the InfluxDB service is listening on. -port = 8086 - -# The username to authenticate with. -user = mon_api - -# The password to authenticate with. -password = password - -# The name of the InfluxDB database to use. -database_name = mon - -[cassandra] -# Only needed if Cassandra database is used for backend. -# Comma separated list of Cassandra node IP addresses. No spaces. -cluster_ip_addresses: 192.168.10.6 -keyspace: monasca - -# Below is configuration for database. -# The order of reading configuration for database is: -# 1) [mysql] section -# 2) [database] -# url -# 3) [database] -# host = 127.0.0.1 -# username = monapi -# password = password -# drivername = mysq+pymysql -# port = 3306 -# database = mon -# query = "" -#[mysql] -#database_name = mon -#hostname = 192.168.10.4 -#username = monapi -#password = password -# -# [database] -# url = "mysql+pymysql://monapi:password@127.0.0.1/mon" -# host = 127.0.0.1 -# username = monapi -# password = password -# drivername = mysq+pymysql -# port = 3306 -# database = mon -# query = "" - -[database] -url = <%= @database_connection %> - -[keystone_authtoken] -auth_type = password -user_domain_id = default -project_domain_id = default -auth_url = <%= @keystone_settings['admin_auth_url'] %> -auth_uri = <%= @keystone_settings['public_auth_url'] %> -auth_version = <%= @keystone_settings['api_version_for_middleware'] %> -username = <%= @keystone_settings['service_user'] %> -password = <%= @keystone_settings['service_password'] %> -project_name = <%= @keystone_settings['service_tenant'] %> diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb similarity index 100% rename from chef/cookbooks/monasca/templates/default/cmm-hosts-cluster.erb rename to chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb diff --git a/chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb similarity index 100% rename from chef/cookbooks/monasca/templates/default/cmm-hosts-single.erb rename to chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 3d6c05cb46..3a40c3949e 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -31,19 +31,19 @@ def role_constraints { "monasca-metric-agent" => { "unique" => false, - "admin" => true, + "admin" => false, "count" => -1, "exclude_platform" => { - "suse" => "< 12.1", + "suse" => "< 12.2", "windows" => "/.*/" } }, "monasca-log-agent" => { "unique" => false, - "admin" => true, + "admin" => false, "count" => -1, "exclude_platform" => { - "suse" => "< 12.1", + "suse" => "< 12.2", "windows" => "/.*/" } }, @@ -56,7 +56,7 @@ def role_constraints "cluster" => false, "admin" => false, "exclude_platform" => { - "suse" => "< 12.1", + "suse" => "< 12.2", "windows" => "/.*/" } }, @@ -66,7 +66,7 @@ def role_constraints "cluster" => false, "admin" => true, "exclude_platform" => { - "suse" => "< 12.1", + "suse" => "< 12.2", "windows" => "/.*/" } } From 4ce35876d8c271627140c659f140e09126c11123 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Mon, 3 Apr 2017 08:45:16 +0200 Subject: [PATCH 27/52] Fix agent config (#31) Fix various issues Fix grafana config Fix memcached_nodes config Fix memcached_nodes config 2 Remove commented grafana config --- chef/cookbooks/horizon/libraries/helper.rb | 9 ++++ chef/cookbooks/horizon/recipes/monasca_ui.rb | 11 +++++ .../default/_80_monasca_ui_settings.py.erb | 1 + .../templates/default/grafana-config.js | 48 +++++++++++++++++++ chef/cookbooks/monasca/recipes/master.rb | 3 +- .../templates/default/log-agent.conf.erb | 2 +- .../templates/default/monasca-reconfigure.erb | 7 +-- 7 files changed, 73 insertions(+), 8 deletions(-) create mode 100644 chef/cookbooks/horizon/templates/default/grafana-config.js diff --git a/chef/cookbooks/horizon/libraries/helper.rb b/chef/cookbooks/horizon/libraries/helper.rb index 893487e21a..6fa77db8b8 100644 --- a/chef/cookbooks/horizon/libraries/helper.rb +++ b/chef/cookbooks/horizon/libraries/helper.rb @@ -18,4 +18,13 @@ def self.monasca_public_host(node) ssl_enabled = node[:monasca][:api][:ssl] CrowbarHelper.get_host_for_public_url(node, ssl_enabled, ha_enabled) end + + def self.api_public_url(node) + host = monasca_public_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end end diff --git a/chef/cookbooks/horizon/recipes/monasca_ui.rb b/chef/cookbooks/horizon/recipes/monasca_ui.rb index 0ed5cbdbf6..b54dbd269a 100644 --- a/chef/cookbooks/horizon/recipes/monasca_ui.rb +++ b/chef/cookbooks/horizon/recipes/monasca_ui.rb @@ -38,6 +38,17 @@ notifies :reload, resources(service: "apache2") end +template "/srv/www/grafana/config.js" do + source "grafana-config.js" + variables( + api_url: MonascaUiHelper.api_public_url(monasca_server) + ) + owner "root" + group "www" + mode "0644" + notifies :reload, resources(service: "apache2") +end + cookbook_file "/srv/www/grafana/app/dashboards/openstack.json" do source "grafana-openstack.json" owner "root" diff --git a/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb index f5ce10f665..8bbc210481 100644 --- a/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb +++ b/chef/cookbooks/horizon/templates/default/_80_monasca_ui_settings.py.erb @@ -65,3 +65,4 @@ KIBANA_HOST = getattr(settings, 'KIBANA_HOST', 'http://<%= @kibana_host %>:5601/ OPENSTACK_SSL_NO_VERIFY = getattr(settings, 'OPENSTACK_SSL_NO_VERIFY', False) OPENSTACK_SSL_CACERT = getattr(settings, 'OPENSTACK_SSL_CACERT', None) +OPENSTACK_ENDPOINT_TYPE="publicURL" diff --git a/chef/cookbooks/horizon/templates/default/grafana-config.js b/chef/cookbooks/horizon/templates/default/grafana-config.js new file mode 100644 index 0000000000..54177000d1 --- /dev/null +++ b/chef/cookbooks/horizon/templates/default/grafana-config.js @@ -0,0 +1,48 @@ +define(['settings'], +function (Settings) { + "use strict"; + + return new Settings({ + + datasources: { + mon: { + type: 'mon', + url: "<%= @api_url %>", + default: true, + grafanaDB: true + } + }, + + /* Global configuration options + * ======================================================== + */ + + // specify the limit for dashboard search results + search: { + max_results: 20 + }, + + // default start dashboard + default_route: '/file/default.json', + + // set to false to disable unsaved changes warning + unsaved_changes_warning: true, + + // set the default timespan for the playlist feature + // Example: "1m", "1h" + playlist_timespan: "1m", + + // If you want to specify password before saving, please specify it bellow + // The purpose of this password is not security, but to stop some users from accidentally changing dashboards + admin: { + password: '' + }, + + // Add your own custom pannels + plugins: { + panels: [] + } + + }); +}); + diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index 7ad81b0277..d65b24e522 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -96,6 +96,7 @@ kibana_host: pub_net_ip, log_api_bind_host: pub_net_ip, influxdb_bind_address: cmm_net_ip, + influxdb_host: cmm_net_ip, monasca_api_bind_host: pub_net_ip, elasticsearch_host: cmm_net_ip, nimbus_host: cmm_net_ip, @@ -105,7 +106,7 @@ database_host: cmm_net_ip, monasca_api_url: "http://#{pub_net_ip}:8070/v2.0", monasca_log_api_url: "http://#{pub_net_ip}:5607/v2.0", - memcached_nodes: "#{cmm_net_ip}:11211", + memcached_nodes: ["#{cmm_net_ip}:11211"], influxdb_url: "http://#{cmm_net_ip}:8086", elasticsearch_nodes: "[#{cmm_net_ip}]", elasticsearch_hosts: cmm_net_ip diff --git a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb index c7aadfa134..0c302761da 100644 --- a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb +++ b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb @@ -21,7 +21,7 @@ filter { output { monasca_log_api { ### keystone based settings - keystone_api_url => "<%= @keystone_settings['internal_auth_url'] %>" + keystone_api_url => "<%= @keystone_settings['admin_auth_url'] %>/v3" project_name => "<%= @log_agent_keystone['service_tenant'] %>" username => "<%= @log_agent_keystone['service_user'] %>" password => "<%= @log_agent_keystone['service_password'] %>" diff --git a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb index 492cebf18e..e991832035 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-reconfigure.erb @@ -4,13 +4,8 @@ -u '<%= @agent_keystone["service_user"] %>' \ -p '<%= @agent_keystone["service_password"] %>' \ --project_name '<%= @agent_keystone["service_tenant"] %>' \ - --project_id '<%= @agent_keystone["service_tenant"] %>' \ <% if defined?(@service) -%> --service '<%= @service %>' <% end -%> \ - --keystone_url '<%= @keystone_settings["internal_auth_url"] %>' \ - --user_domain_id '<%= @keystone_settings["admin_domain_id"] %>' \ - --user_domain_name '<%= @keystone_settings["admin_domain"] %>' \ - --project_domain_id '<%= @keystone_settings["admin_domain_id"] %>' \ - --project_domain_name '<%= @keystone_settings["admin_domain"] %>' \ + --keystone_url '<%= @keystone_settings["admin_auth_url"] %>/v3' \ --monasca_url '<%= @monasca_api_url %>' \ --user '<%= @agent_settings["user"] %>' \ --dimensions '<%= @agent_dimensions.map{|k,v| "#{k}:#{v}"}.join(',') %>' \ From 2a7c7bba72a6d946756d557cfb91e292ae842f50 Mon Sep 17 00:00:00 2001 From: Artur Basiak Date: Mon, 3 Apr 2017 12:17:59 +0200 Subject: [PATCH 28/52] Remove offline resources (#36) Remove all variables connected to offline resources. --- chef/cookbooks/monasca/recipes/master.rb | 1 - .../monasca/templates/default/all_group.erb | 10 ---------- .../templates/default/monasca-hosts-cluster.erb | 10 +--------- .../templates/default/monasca-hosts-single.erb | 14 ++------------ 4 files changed, 3 insertions(+), 32 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index d65b24e522..ce7a4cc408 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -44,7 +44,6 @@ monasca_host: monasca_hosts[0], monasca_hosts: monasca_hosts, ansible_ssh_user: "root", - offline_resources_host: CrowbarHelper.get_host_for_admin_url(node), keystone_host: keystone_settings["public_url_host"] ) notifies :run, "execute[run ansible]", :delayed diff --git a/chef/cookbooks/monasca/templates/default/all_group.erb b/chef/cookbooks/monasca/templates/default/all_group.erb index c2fad477cd..8d3ffe2d24 100644 --- a/chef/cookbooks/monasca/templates/default/all_group.erb +++ b/chef/cookbooks/monasca/templates/default/all_group.erb @@ -18,7 +18,6 @@ memcached_port: 11211 mysql_port: 3306 # hosts -offline_host: "{{ hostvars['offline-resources-node']['ansible_ssh_host'] }}" monasca_api_ip: "{% if groups['monasca_api_group']|length > 1 %} {{ hostvars['monasca-api-balancer-node']['ansible_ssh_host'] }} {% else %} @@ -49,8 +48,6 @@ keystone_version: <%= @keystone_settings['api_version'] %> keystone_url: "<%= @keystone_settings['public_auth_url'] %>" # URL service definition -balls_repo_url: "http://{{ offline_host }}:8888" -world_repo_url: "{{ balls_repo_url }}/world" monasca_api_url: "http://{{ monasca_api_ip|trim|safe }}:{{ monasca_api_client_port|int }}/v2.0" monasca_log_api_url: "http://{{ monasca_log_api_ip|trim|safe }}:{{ monasca_log_api_port|int }}/v3.0" kibana_url: "http://{{ kibana_ip|trim|safe }}:{{ kibana_port }}" @@ -71,12 +68,6 @@ monasca_download_dir: /opt/monasca-downloads clean_up_dirs: - "{{ download_tmp_dir }}" -# Installation mode -offline_mode: True - -# Download urls -logstash_agent_download_url: "{{ world_repo_url }}" - # Keystone Admin credentials keystone_admin_token: <%= @keystone_settings['admin_token'] %> keystone_admin: '<%= @keystone_settings['admin_user'] %>' @@ -104,4 +95,3 @@ pbr_version: 1.8.1 # try to use this variable for each wait_for like task monasca_wait_for_period: 180 - diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb index 440700a8d1..9cc1c27075 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb @@ -2,15 +2,12 @@ ################################################################################ # Host definitions. Replace monasca_host, openstack_host, ex_node_[1-3]_host, -# offline_resources_host, ssh_user_monasca, ssh_user_openstack, ssh_user_offline_host +# ssh_user_monasca, ssh_user_openstack # and ssh_user_ex_node_[1-3] with valid information. ################################################################################ localhost ansible_connection=local -# Offline Host -offline-resources-node ansible_ssh_host=<%= @offline_resources_host %> ansible_ssh_user=<%= @ansible_ssh_user %> - # Openstack Hosts keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -129,10 +126,6 @@ monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_s # Other group definition ################################################################################ -[offline_group] -offline-resources-node -localhost - [openstack_group] keystone-node @@ -261,7 +254,6 @@ storm_nimbus_group storm_supervisor_group [all_group:children] -offline_group openstack_group monasca_group diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb index 162448e50b..907ee1dfd2 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb @@ -1,16 +1,12 @@ # Copyright 2017 FUJITSU LIMITED ################################################################################ -# Host definitions. Replace monasca_host, openstack_host, offline_resources_host, -# ansible_ssh_user, ssh_user_openstack and ssh_user_offline_host with -# valid informations. +# Host definitions. Replace monasca_host, openstack_host, +# ansible_ssh_user, ssh_user_openstack with valid informations. ################################################################################ localhost ansible_connection=local -# Offline Host -offline-resources-node ansible_ssh_host=<%= @offline_resources_host %> ansible_ssh_user=<%= @ansible_ssh_user %> - # Monasca host cmm-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -78,10 +74,6 @@ influxdb-node ansible_ssh_host=<%= @monasca_host %> ansible_s [openstack_group] keystone-node -[offline_group] -offline-resources-node -localhost - [memcached_group] memcached-node @@ -156,7 +148,6 @@ storm_nimbus_group storm_supervisor_group [all_group:children] -offline_group openstack_group monasca_group @@ -173,4 +164,3 @@ storm_group monasca_thresh_group mariadb_group influxdb_group - From 3aad3a2f10ca6a9fcd487f7d19545be87b9b33b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Thu, 6 Apr 2017 12:57:47 +0200 Subject: [PATCH 29/52] Run monasca-reconfigure only when the file changes (#38) monasca-reconfigure detection script can run for a long time, so run it only when the file is changed by Chef. As of now user will need to run this script by hand to detect new services if the file is not changed. --- chef/cookbooks/monasca/recipes/metric_agent.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb index 4a283879e5..be56d3e486 100644 --- a/chef/cookbooks/monasca/recipes/metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -45,6 +45,7 @@ agent_dimensions: agent_dimensions, install_plugins_only: false ) + notifies :run, "execute[monasca-setup detect services]", :delayed end execute "monasca-setup detect services" do @@ -54,6 +55,7 @@ only_if { File.exist?(monasca_reconfigure_file) } user agent_settings["user"] group agent_settings["group"] + action :nothing end service "monasca-agent" do From 01f59d9a1a0ebffdd9b871d5683a323dd03c647e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dobros=C5=82aw=20=C5=BBybort?= Date: Tue, 11 Apr 2017 13:12:56 +0200 Subject: [PATCH 30/52] Replace all cmm occurrences with monasca (#37) * Replace all cmm occurrences with monasca * Update monasca_service.rb * Update master.rb * Update all_group.erb * Update template-monasca.json * Update template-monasca.schema --- .../files/default/grafana-monasca.json | 10 ++--- chef/cookbooks/monasca/recipes/master.rb | 38 +++++++++---------- .../monasca/templates/default/all_group.erb | 8 ++-- .../default/monasca-hosts-single.erb | 12 +++--- chef/data_bags/crowbar/template-monasca.json | 20 +++++----- .../data_bags/crowbar/template-monasca.schema | 4 +- .../app/models/monasca_service.rb | 4 +- 7 files changed, 48 insertions(+), 48 deletions(-) diff --git a/chef/cookbooks/horizon/files/default/grafana-monasca.json b/chef/cookbooks/horizon/files/default/grafana-monasca.json index ff75e67de4..36fdb55f50 100644 --- a/chef/cookbooks/horizon/files/default/grafana-monasca.json +++ b/chef/cookbooks/horizon/files/default/grafana-monasca.json @@ -1,7 +1,7 @@ { "id": "sid-14393838984416185642089112666", - "title": "CMM Service", - "originalTitle": "CMM Service", + "title": "Monasca Service", + "originalTitle": "Monasca Service", "tags": [], "style": "light", "timezone": "browser", @@ -10,7 +10,7 @@ "sharedCrosshair": false, "rows": [ { - "title": "CMM Health - metrics", + "title": "Monasca Health - metrics", "height": "100px", "editable": true, "collapse": false, @@ -409,7 +409,7 @@ "showTitle": true }, { - "title": "CMM Health - Logs", + "title": "Monasca Health - Logs", "height": "100px", "editable": true, "collapse": false, @@ -805,7 +805,7 @@ "showTitle": true }, { - "title": "CMM Health - Common", + "title": "Monasca Health - Common", "height": "100px", "editable": true, "collapse": false, diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index ce7a4cc408..8f2b3e141b 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -64,7 +64,7 @@ # Without this, it could happen that the installer was not re-tried # after a failed run. file "/opt/monasca-installer/.installed" do - content "cmm installed" + content "monasca installed" owner "root" group "root" mode "0644" @@ -73,7 +73,7 @@ end monasca_node = search(:node, "roles:monasca-server")[0] -cmm_net_ip = MonascaHelper.get_host_for_monitoring_url(monasca_node) +monasca_net_ip = MonascaHelper.get_host_for_monitoring_url(monasca_node) pub_net_ip = CrowbarHelper.get_host_for_public_url(monasca_node, false, false) ansible_vars = { @@ -83,32 +83,32 @@ database_monapi_password: node[:monasca][:master][:database_monapi_password], database_thresh_password: node[:monasca][:master][:database_thresh_password], database_logapi_password: node[:monasca][:master][:database_logapi_password], - keystone_cmm_operator_user_password: - node[:monasca][:master][:keystone_cmm_operator_user_password], - keystone_cmm_agent_password: node[:monasca][:master][:keystone_cmm_agent_password], + keystone_monasca_operator_password: + node[:monasca][:master][:keystone_monasca_operator_password], + keystone_monasca_agent_password: node[:monasca][:master][:keystone_monasca_agent_password], keystone_admin_agent_password: node[:monasca][:master][:keystone_admin_agent_password], keystone_admin_password: keystone_settings["admin_password"], database_grafana_password: node[:monasca][:master][:database_grafana_password], - memcached_listen_ip: cmm_net_ip, - kafka_host: cmm_net_ip, + memcached_listen_ip: monasca_net_ip, + kafka_host: monasca_net_ip, kibana_host: pub_net_ip, log_api_bind_host: pub_net_ip, - influxdb_bind_address: cmm_net_ip, - influxdb_host: cmm_net_ip, + influxdb_bind_address: monasca_net_ip, + influxdb_host: monasca_net_ip, monasca_api_bind_host: pub_net_ip, - elasticsearch_host: cmm_net_ip, - nimbus_host: cmm_net_ip, - zookeeper_hosts: cmm_net_ip, - kafka_hosts: "#{cmm_net_ip}:9092", - mariadb_bind_address: cmm_net_ip, - database_host: cmm_net_ip, + elasticsearch_host: monasca_net_ip, + nimbus_host: monasca_net_ip, + zookeeper_hosts: monasca_net_ip, + kafka_hosts: "#{monasca_net_ip}:9092", + mariadb_bind_address: monasca_net_ip, + database_host: monasca_net_ip, monasca_api_url: "http://#{pub_net_ip}:8070/v2.0", monasca_log_api_url: "http://#{pub_net_ip}:5607/v2.0", - memcached_nodes: ["#{cmm_net_ip}:11211"], - influxdb_url: "http://#{cmm_net_ip}:8086", - elasticsearch_nodes: "[#{cmm_net_ip}]", - elasticsearch_hosts: cmm_net_ip + memcached_nodes: ["#{monasca_net_ip}:11211"], + influxdb_url: "http://#{monasca_net_ip}:8086", + elasticsearch_nodes: "[#{monasca_net_ip}]", + elasticsearch_hosts: monasca_net_ip }.to_json execute "run ansible" do diff --git a/chef/cookbooks/monasca/templates/default/all_group.erb b/chef/cookbooks/monasca/templates/default/all_group.erb index 8d3ffe2d24..9c3ea70730 100644 --- a/chef/cookbooks/monasca/templates/default/all_group.erb +++ b/chef/cookbooks/monasca/templates/default/all_group.erb @@ -54,9 +54,9 @@ kibana_url: "http://{{ kibana_ip|trim|safe }}:{{ kibana_port }}" # Monasca Api CLI - Credentials monasca_user: - user: cmm-operator - password: "{{ keystone_cmm_operator_user_password }}" - project: cmm + user: monasca-operator + password: "{{ keystone_monasca_operator_password }}" + project: monasca # Installation / Download directories opt_dir: "/opt" @@ -90,7 +90,7 @@ pip_version_min: 7.0.3 # pip configuration pip_base_dir: "/root/.pip" -pip_config_file: "{{ pip_base_dir }}/.cmm_pip.conf" +pip_config_file: "{{ pip_base_dir }}/.monasca_pip.conf" pbr_version: 1.8.1 # try to use this variable for each wait_for like task diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb index 907ee1dfd2..6277970659 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-single.erb @@ -8,7 +8,7 @@ localhost ansible_connection=local # Monasca host -cmm-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> # Openstack hosts keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -47,7 +47,7 @@ monasca-notification-node ansible_ssh_host=<%= @monasca_host %> ansible_s monasca-persister-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> # Monasca-log-metric node -monasca-log-metrics-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-log-metrics-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> # Storm Nimbus node storm-nimbus-node ansible_ssh_host=<%= @monasca_host %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -84,13 +84,13 @@ kafka-node zookeeper-node [monasca_group] -cmm-node +monasca-node [monasca_api_group] -monasca-api-node database_node_for_api=cmm-node influxdb_node_for_api=influxdb-node +monasca-api-node database_node_for_api=monasca-node influxdb_node_for_api=influxdb-node [monasca_notification_group] -monasca-notification-node database_node_for_notification=cmm-node +monasca-notification-node database_node_for_notification=monasca-node [monasca_persister_group] monasca-persister-node influxdb_node_for_persister=influxdb-node @@ -120,7 +120,7 @@ storm-nimbus-node storm-supervisor-node [monasca_thresh_group] -monasca-thresh-node database_node_for_thresh=cmm-node +monasca-thresh-node database_node_for_thresh=monasca-node [mariadb_group] mariadb-node diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index ae5904ea88..b86619b721 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -5,10 +5,10 @@ "monasca": { "metric_agent": { "keystone": { - "service_user": "cmm-agent", + "service_user": "monasca-agent", "service_password": "", - "service_tenant": "cmm", - "service_role": "cmm-agent" + "service_tenant": "monasca", + "service_role": "monasca-agent" }, "insecure": true, "system_only": false, @@ -28,10 +28,10 @@ }, "log_agent": { "keystone": { - "service_user": "cmm-log-agent", + "service_user": "monasca-log-agent", "service_password": "", - "service_tenant": "cmm", - "service_role": "cmm-agent" + "service_tenant": "monasca", + "service_role": "monasca-agent" } }, "api": { @@ -52,8 +52,8 @@ "database_monapi_password": "", "database_thresh_password": "", "database_logapi_password": "", - "keystone_cmm_operator_user_password": "", - "keystone_cmm_agent_password": "", + "keystone_monasca_operator_password": "", + "keystone_monasca_agent_password": "", "keystone_admin_agent_password": "", "database_grafana_password": "" }, @@ -69,8 +69,8 @@ "keystone_instance": "none", "service_password": "none", "service_user": "monasca", - "service_tenant": "cmm", - "service_roles": ["cmm-agent"] + "service_tenant": "monasca", + "service_roles": ["monasca-agent"] } }, "deployment": { diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index de8ad99ccb..23d585b241 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -88,8 +88,8 @@ "database_monapi_password": { "required": true, "type": "str" }, "database_thresh_password": { "required": true, "type": "str" }, "database_logapi_password": { "required": true, "type": "str" }, - "keystone_cmm_operator_user_password": { "required": true, "type": "str" }, - "keystone_cmm_agent_password": { "required": true, "type": "str" }, + "keystone_monasca_operator_password": { "required": true, "type": "str" }, + "keystone_monasca_agent_password": { "required": true, "type": "str" }, "keystone_admin_agent_password": { "required": true, "type": "str" }, "database_grafana_password": { "required": true, "type": "str" } } diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 3a40c3949e..9d340337ae 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -123,8 +123,8 @@ def create_proposal base["attributes"][@bc_name][:master][:database_monapi_password] = random_password base["attributes"][@bc_name][:master][:database_thresh_password] = random_password base["attributes"][@bc_name][:master][:database_logapi_password] = random_password - base["attributes"][@bc_name][:master][:keystone_cmm_operator_user_password] = random_password - base["attributes"][@bc_name][:master][:keystone_cmm_agent_password] = random_password + base["attributes"][@bc_name][:master][:keystone_monasca_operator_password] = random_password + base["attributes"][@bc_name][:master][:keystone_monasca_agent_password] = random_password base["attributes"][@bc_name][:master][:keystone_admin_agent_password] = random_password base["attributes"][@bc_name][:master][:database_grafana_password] = random_password From b6190bd329193e7cd95c6b3a9ce77b04a524356a Mon Sep 17 00:00:00 2001 From: Artur Basiak Date: Tue, 11 Apr 2017 13:47:02 +0200 Subject: [PATCH 31/52] Use node_search_with_cache method (#39) Use node_search_with_cache() method insted of search(). This will work better on nodes that do not have monasca-server role applied. --- chef/cookbooks/horizon/recipes/monasca_ui.rb | 7 +++++-- chef/cookbooks/horizon/recipes/server.rb | 2 +- chef/cookbooks/monasca/recipes/log_agent.rb | 7 +++++-- chef/cookbooks/monasca/recipes/metric_agent.rb | 7 +++++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/chef/cookbooks/horizon/recipes/monasca_ui.rb b/chef/cookbooks/horizon/recipes/monasca_ui.rb index b54dbd269a..4f8e72bc51 100644 --- a/chef/cookbooks/horizon/recipes/monasca_ui.rb +++ b/chef/cookbooks/horizon/recipes/monasca_ui.rb @@ -12,8 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -monasca_servers = search(:node, "roles:monasca-server") -monasca_server = monasca_servers[0] +monasca_server = node_search_with_cache("roles:monasca-server").first +if monasca_server.nil? + Chef::Log.warn("No monasca-server found.") + return +end template "/srv/www/openstack-dashboard/openstack_dashboard/"\ "local/local_settings.d/_80_monasca_ui_settings.py" do diff --git a/chef/cookbooks/horizon/recipes/server.rb b/chef/cookbooks/horizon/recipes/server.rb index 311063c900..8bfedbaac4 100644 --- a/chef/cookbooks/horizon/recipes/server.rb +++ b/chef/cookbooks/horizon/recipes/server.rb @@ -165,7 +165,7 @@ end unless monasca_ui_pkgname.nil? - monasca_servers = search(:node, "roles:monasca-server") || [] + monasca_servers = node_search_with_cache("roles:monasca-server") unless monasca_servers.empty? include_recipe "#{@cookbook_name}::monasca_ui" package monasca_ui_pkgname do diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 27ae4922ba..89dca4d4fc 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -19,8 +19,11 @@ log_agent_keystone = log_agent_settings[:keystone] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) -monasca_servers = search(:node, "roles:monasca-server") -monasca_server = monasca_servers[0] +monasca_server = node_search_with_cache("roles:monasca-server").first +if monasca_server.nil? + Chef::Log.warn("No monasca-server found.") + return +end monasca_log_api_url = MonascaHelper.log_api_public_url(monasca_server) diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb index be56d3e486..8350026ef4 100644 --- a/chef/cookbooks/monasca/recipes/metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -21,8 +21,11 @@ keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) -monasca_servers = search(:node, "roles:monasca-server") || [] -monasca_server = monasca_servers[0] +monasca_server = node_search_with_cache("roles:monasca-server").first +if monasca_server.nil? + Chef::Log.warn("No monasca-server found.") + return +end monasca_api_url = MonascaHelper.api_public_url(monasca_server) From 90525b813e73b95a50da76b5482dc832d677f0a1 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Wed, 12 Apr 2017 08:05:45 +0200 Subject: [PATCH 32/52] Log agent config (#40) * Add config for log agent * Allow manual config of log agent * Remove log agent config from ui and update template * Fix hound violations * Fix hound violations 2 --- chef/cookbooks/monasca/recipes/log_agent.rb | 18 ++++++++++++-- .../templates/default/log-agent.conf.erb | 24 ++++++------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 89dca4d4fc..9416d80850 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -28,7 +28,6 @@ monasca_log_api_url = MonascaHelper.log_api_public_url(monasca_server) log_agent_dimensions = { - service: "monitoring", hostname: node["hostname"] } @@ -45,6 +44,20 @@ recursive true end +log_files = { + "/var/log/messages" => "system", + "/var/log/zypper.log" => "system" +} + +ruby_block "find log files" do + block do + log_dirs = + Dir.entries("/var/log") + .select { |e| File.directory?("/var/log/#{e}") && !(e == "." || e == "..") } + log_dirs.each { |d| log_files["/var/log/#{d}/**/*.log"] = d.downcase } + end +end + template "/etc/monasca-log-agent/agent.conf" do source "log-agent.conf.erb" owner log_agent_settings["user"] @@ -55,7 +68,8 @@ log_agent_keystone: log_agent_keystone, log_agent_settings: log_agent_settings, log_agent_dimensions: log_agent_dimensions, - keystone_settings: keystone_settings + keystone_settings: keystone_settings, + log_files: log_files ) notifies :reload, "service[monasca-log-agent]" end diff --git a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb index 0c302761da..a4c322a5bb 100644 --- a/chef/cookbooks/monasca/templates/default/log-agent.conf.erb +++ b/chef/cookbooks/monasca/templates/default/log-agent.conf.erb @@ -1,21 +1,11 @@ input { - # TODO(trebskit) should initial list contain some files to be - # monitored ? - file { - add_field => { "dimensions" => { "service" => "system" }} - path => "/var/log/messages" - tags => ["syslog"] - } -} - -filter { - if "syslog" in [tags] { - multiline { - negate => "true" - pattern => "^%{SYSLOGTIMESTAMP}" - what => "previous" - } - } +<% @log_files.each_pair do |path, service| %> + <%= "file { + add_field => { \"dimensions\" => { \"service\" => \"#{service}\" }} + path => \"#{path}\" + }" + %> +<% end %> } output { From e30910eaa20f1fa2d1c26e485d14cffee7181a03 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Wed, 12 Apr 2017 11:09:19 +0200 Subject: [PATCH 33/52] Start and enable agents after installation --- chef/cookbooks/monasca/recipes/log_agent.rb | 2 +- chef/cookbooks/monasca/recipes/metric_agent.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 9416d80850..3e09cbd71a 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -77,7 +77,7 @@ service "monasca-log-agent" do service_name log_agent_settings[:service_name] supports status: true, restart: true, start: true, stop: true - action [:disable, :stop] + action [:enable, :start] ignore_failure true end diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb index 8350026ef4..3410f372c0 100644 --- a/chef/cookbooks/monasca/recipes/metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -64,7 +64,7 @@ service "monasca-agent" do service_name agent_settings[:service_name] supports status: true, restart: true, start: true, stop: true - action [:disable, :stop] + action [:enable, :start] ignore_failure true # provider Chef::Provider::CrowbarPacemakerService if ha_enabled end From e86104e9639cd87aaedee453a2d404dba585b0f9 Mon Sep 17 00:00:00 2001 From: Jakub Wachowski Date: Fri, 24 Feb 2017 12:28:03 +0100 Subject: [PATCH 34/52] Add ha to monasca barclamp --- chef/cookbooks/monasca/attributes/default.rb | 5 ++++ chef/cookbooks/monasca/recipes/ha.rb | 26 ++++++++++++++++++++ chef/cookbooks/monasca/recipes/server.rb | 6 +++++ 3 files changed, 37 insertions(+) create mode 100644 chef/cookbooks/monasca/recipes/ha.rb diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index 7a0482b3d5..2f43df0fcf 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -52,3 +52,8 @@ default[:monasca][:log_agent][:user] = "monasca-log-agent" default[:monasca][:log_agent][:group] = "logstash" default[:monasca][:log_agent][:debug] = monasca_debug + +default[:monasca][:ha][:enabled] = false +# Ports to bind to when haproxy is used for the real ports +default[:monasca][:ha][:ports][:plain] = 5580 +default[:monasca][:ha][:ports][:ssl] = 5581 diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb new file mode 100644 index 0000000000..2ce8ad9d76 --- /dev/null +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -0,0 +1,26 @@ +# Copyright 2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include_recipe "crowbar-pacemaker::haproxy" + +haproxy_loadbalancer "monasca" do + address "0.0.0.0" + port 1234 + use_ssl false + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "plain") + action :nothing +end.run_action(:create) + +crowbar_pacemaker_sync_mark "sync-monasca_before_ha" diff --git a/chef/cookbooks/monasca/recipes/server.rb b/chef/cookbooks/monasca/recipes/server.rb index f0103f4715..46b8f4bab1 100644 --- a/chef/cookbooks/monasca/recipes/server.rb +++ b/chef/cookbooks/monasca/recipes/server.rb @@ -79,3 +79,9 @@ endpoint_internalURL MonascaHelper.api_internal_url(node) action :add_endpoint_template end + +ha_enabled = node[:monasca][:ha][:enabled] + +if ha_enabled + include_recipe "monasca::ha" +end From 86db54b6d6f0ad8093fa8cd8b4971d3bfad50f2d Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Mon, 3 Apr 2017 10:34:32 +0200 Subject: [PATCH 35/52] Update HA for monasca barclamp with all services --- chef/cookbooks/monasca/attributes/default.rb | 20 ++- chef/cookbooks/monasca/libraries/helper.rb | 157 ++++++++++++------ chef/cookbooks/monasca/recipes/ha.rb | 46 ++++- chef/cookbooks/monasca/recipes/log_agent.rb | 3 + .../cookbooks/monasca/recipes/metric_agent.rb | 3 +- .../monasca/recipes/role_monasca_log_agent.rb | 1 + .../monasca/recipes/role_monasca_master.rb | 3 +- .../recipes/role_monasca_metric_agent.rb | 1 + .../monasca/recipes/role_monasca_server.rb | 1 + chef/cookbooks/monasca/recipes/server.rb | 4 +- chef/data_bags/crowbar/template-monasca.json | 15 +- .../data_bags/crowbar/template-monasca.schema | 27 ++- 12 files changed, 214 insertions(+), 67 deletions(-) diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index 2f43df0fcf..befedb3a4e 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -36,9 +36,6 @@ override[:monasca][:user] = "monasca" default[:monasca][:debug] = monasca_debug -default[:monasca][:ha][:enabled] = monasca_ha_enabled - -default[:monasca][:api][:bind_host] = "*" # metric-agent default service settings default[:monasca][:metric_agent][:service_name] = monasca_metric_agent_service_name @@ -53,7 +50,18 @@ default[:monasca][:log_agent][:group] = "logstash" default[:monasca][:log_agent][:debug] = monasca_debug -default[:monasca][:ha][:enabled] = false +# default[:monasca][:api][:bind_host] = "*" +# default[:monasca][:log_api][:bind_host] = "*" +# default[:monasca][:kibana][:bind_host] = "*" +# default[:monasca][:mariadb][:bind_host] = "-" +# default[:monasca][:influxdb][:bind_host] = "-" + +default[:monasca][:ha][:enabled] = monasca_ha_enabled # Ports to bind to when haproxy is used for the real ports -default[:monasca][:ha][:ports][:plain] = 5580 -default[:monasca][:ha][:ports][:ssl] = 5581 +# Public network +default[:monasca][:ha][:ports][:api] = 18070 +default[:monasca][:ha][:ports][:log_api] = 15607 +default[:monasca][:ha][:ports][:kibana] = 15601 +# Internal network +default[:monasca][:ha][:ports][:mariadb] = 13306 +default[:monasca][:ha][:ports][:influxdb] = 18086 diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index e5c48922ea..3c69c0530a 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -14,62 +14,121 @@ # limitations under the License. module MonascaHelper - def self.monasca_public_host(node) - ha_enabled = node[:monasca][:ha][:enabled] - ssl_enabled = node[:monasca][:api][:ssl] - CrowbarHelper.get_host_for_public_url(node, ssl_enabled, ha_enabled) - end + class << self + def monasca_public_host(node) + ha_enabled = node[:monasca][:ha][:enabled] + ssl_enabled = node[:monasca][:api][:ssl] + CrowbarHelper.get_host_for_public_url(node, ssl_enabled, ha_enabled) + end - def self.monasca_admin_host(node) - ha_enabled = node[:monasca][:ha][:enabled] - CrowbarHelper.get_host_for_admin_url(node, ha_enabled) - end + def monasca_admin_host(node) + ha_enabled = node[:monasca][:ha][:enabled] + CrowbarHelper.get_host_for_admin_url(node, ha_enabled) + end - def self.api_public_url(node) - host = monasca_public_host(node) - # SSL is not supported at this moment - # protocol = node[:monasca][:api][:ssl] ? "https" : "http" - protocol = "http" - port = node[:monasca][:api][:bind_port] - "#{protocol}://#{host}:#{port}/v2.0" - end + def api_public_url(node) + host = monasca_public_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end - def self.api_admin_url(node) - host = monasca_admin_host(node) - # SSL is not supported at this moment - # protocol = node[:monasca][:api][:ssl] ? "https" : "http" - protocol = "http" - port = node[:monasca][:api][:bind_port] - "#{protocol}://#{host}:#{port}/v2.0" - end + def api_admin_url(node) + host = monasca_admin_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end - def self.api_internal_url(node) - host = get_host_for_monitoring_url(node) - # SSL is not supported at this moment - # protocol = node[:monasca][:api][:ssl] ? "https" : "http" - protocol = "http" - port = node[:monasca][:api][:bind_port] - "#{protocol}://#{host}:#{port}/v2.0" - end + def api_internal_url(node) + host = get_host_for_monitoring_url(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:api][:bind_port] + "#{protocol}://#{host}:#{port}/v2.0" + end - def self.log_api_public_url(node) - host = monasca_public_host(node) - # SSL is not supported at this moment - # protocol = node[:monasca][:log_api][:ssl] ? "https" : "http" - protocol = "http" - port = node[:monasca][:log_api][:port] - "#{protocol}://#{host}:#{port}/v3.0" - end + def log_api_public_url(node) + host = monasca_public_host(node) + # SSL is not supported at this moment + # protocol = node[:monasca][:log_api][:ssl] ? "https" : "http" + protocol = "http" + port = node[:monasca][:log_api][:bind_port] + "#{protocol}://#{host}:#{port}/v3.0" + end - def self.monasca_hosts(nodes) - hosts = [] - nodes.each do |n| - hosts.push(CrowbarHelper.get_host_for_admin_url(n)) + def monasca_hosts(nodes) + hosts = [] + nodes.each do |n| + hosts.push(CrowbarHelper.get_host_for_admin_url(n)) + end + hosts end - hosts - end - def self.get_host_for_monitoring_url(node) - Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address + def get_host_for_monitoring_url(node) + Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address + end + + def network_settings(node) + @ip ||= Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address + @cluster_admin_ip ||= nil + + if node[:monasca][:ha][:enabled] && !@cluster_admin_ip + @cluster_admin_ip = CrowbarPacemakerHelper.cluster_vip(node, "monitoring") + end + + if node[:monasca][:ha][:enabled] + bind_port_api = node[:monasca][:ha][:ports][:api].to_i, + bind_port_log_api = node[:monasca][:ha][:ports][:log_api].to_i, + bind_port_kibana = node[:monasca][:ha][:ports][:kibana].to_i, + bind_port_mariadb = node[:monasca][:ha][:ports][:mariadb].to_i, + bind_port_influxdb = node[:monasca][:ha][:ports][:influxdb].to_i + else + bind_port_api = node[:monasca][:api][:bind_port].to_i + bind_port_log_api = node[:monasca][:log_api][:bind_port].to_i + bind_port_kibana = node[:monasca][:kibana][:bind_port].to_i + bind_port_mariadb = node[:monasca][:mariadb][:bind_port].to_i + bind_port_influxdb = node[:monasca][:influxdb][:bind_port].to_i + end + + @network_settings ||= { + ip: @ip, + + api: { + bind_port: bind_port_api, + ha_bind_host: node[:monasca][:api][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_port: node[:monasca][:api][:bind_port].to_i + }, + + log_api: { + bind_port: bind_port_log_api, + ha_bind_host: node[:monasca][:log_api][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_port: node[:monasca][:log_api][:bind_port].to_i + }, + + kibana: { + bind_port: bind_port_kibana, + ha_bind_host: node[:monasca][:kibana][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_port: node[:monasca][:kibana][:bind_port].to_i + }, + + mariadb: { + bind_port: bind_port_mariadb, + ha_bind_host: node[:monasca][:mariadb][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_port: node[:monasca][:mariadb][:bind_port].to_i + }, + + influxdb: { + bind_port: bind_port_influxdb, + ha_bind_host: node[:monasca][:influxdb][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i + }, + } + end end end diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 2ce8ad9d76..3d23a2afde 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -13,13 +13,49 @@ # limitations under the License. # +unless node[:monasca][:ha][:enabled] + log "HA support for Monasca is not enabled" + return +end + +log "Setting up Monasca HA support" + +network_settings = MonascaHelper.network_settings(node) + include_recipe "crowbar-pacemaker::haproxy" -haproxy_loadbalancer "monasca" do - address "0.0.0.0" - port 1234 - use_ssl false - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "plain") +haproxy_loadbalancer "monasca-api" do + address network_settings[:api][:ha_bind_host] + port network_settings[:api][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "monasca_api") + action :nothing +end.run_action(:create) + +haproxy_loadbalancer "monasca-log-api" do + address network_settings[:log_api][:ha_bind_host] + port network_settings[:log_api][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "monasca_log_api") + action :nothing +end.run_action(:create) + +haproxy_loadbalancer "kibana" do + address network_settings[:kibana][:ha_bind_host] + port network_settings[:kibana][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "kibana") + action :nothing +end.run_action(:create) + +haproxy_loadbalancer "mariadb" do + address network_settings[:mariadb][:ha_bind_host] + port network_settings[:mariadb][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "mariadb") + action :nothing +end.run_action(:create) + +haproxy_loadbalancer "influxdb" do + address network_settings[:influxdb][:ha_bind_host] + port network_settings[:influxdb][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "influxdb") action :nothing end.run_action(:create) diff --git a/chef/cookbooks/monasca/recipes/log_agent.rb b/chef/cookbooks/monasca/recipes/log_agent.rb index 3e09cbd71a..cc711fc25d 100644 --- a/chef/cookbooks/monasca/recipes/log_agent.rb +++ b/chef/cookbooks/monasca/recipes/log_agent.rb @@ -17,7 +17,9 @@ log_agent_settings = node[:monasca][:log_agent] log_agent_keystone = log_agent_settings[:keystone] + keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) +ha_enabled = node[:monasca][:ha][:enabled] monasca_server = node_search_with_cache("roles:monasca-server").first if monasca_server.nil? @@ -79,6 +81,7 @@ supports status: true, restart: true, start: true, stop: true action [:enable, :start] ignore_failure true + provider Chef::Provider::CrowbarPacemakerService if ha_enabled end node.save diff --git a/chef/cookbooks/monasca/recipes/metric_agent.rb b/chef/cookbooks/monasca/recipes/metric_agent.rb index 3410f372c0..2a2dd35129 100644 --- a/chef/cookbooks/monasca/recipes/metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/metric_agent.rb @@ -20,6 +20,7 @@ agent_keystone = agent_settings[:keystone] keystone_settings = KeystoneHelper.keystone_settings(node, @cookbook_name) +ha_enabled = node[:monasca][:ha][:enabled] monasca_server = node_search_with_cache("roles:monasca-server").first if monasca_server.nil? @@ -66,7 +67,7 @@ supports status: true, restart: true, start: true, stop: true action [:enable, :start] ignore_failure true - # provider Chef::Provider::CrowbarPacemakerService if ha_enabled + provider Chef::Provider::CrowbarPacemakerService if ha_enabled end node.save diff --git a/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb index 49208b603f..dcac0a796e 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_log_agent.rb @@ -16,4 +16,5 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-log-agent") include_recipe "#{@cookbook_name}::common" include_recipe "#{@cookbook_name}::log_agent" + include_recipe "#{@cookbook_name}::ha" end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_master.rb b/chef/cookbooks/monasca/recipes/role_monasca_master.rb index df275e9eac..1b9aca5ea9 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_master.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_master.rb @@ -15,5 +15,6 @@ # if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-master") - include_recipe "monasca::master" + include_recipe "#{@cookbook_name}::master" + include_recipe "#{@cookbook_name}::ha" end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb b/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb index 640eb9cda5..01ab0fb06e 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_metric_agent.rb @@ -17,4 +17,5 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-metric-agent") include_recipe "#{@cookbook_name}::common" include_recipe "#{@cookbook_name}::metric_agent" + include_recipe "#{@cookbook_name}::ha" end diff --git a/chef/cookbooks/monasca/recipes/role_monasca_server.rb b/chef/cookbooks/monasca/recipes/role_monasca_server.rb index a10fdde1fb..4efa01d855 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_server.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_server.rb @@ -17,4 +17,5 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-server") include_recipe "#{@cookbook_name}::common" include_recipe "#{@cookbook_name}::server" + include_recipe "#{@cookbook_name}::ha" end diff --git a/chef/cookbooks/monasca/recipes/server.rb b/chef/cookbooks/monasca/recipes/server.rb index 46b8f4bab1..f160f923c3 100644 --- a/chef/cookbooks/monasca/recipes/server.rb +++ b/chef/cookbooks/monasca/recipes/server.rb @@ -82,6 +82,4 @@ ha_enabled = node[:monasca][:ha][:enabled] -if ha_enabled - include_recipe "monasca::ha" -end +include_recipe "monasca::ha" if ha_enabled diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index b86619b721..5ada3d2c40 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -42,9 +42,22 @@ "threads": 10 }, "log_api": { - "port": 5607, + "bind_host": "*", + "bind_port": 5607, "ssl": false }, + "kibana": { + "bind_host": "*", + "bind_port": 5601 + }, + "mariadb": { + "bind_host": "-", + "bind_port": 3306 + }, + "influxdb": { + "bind_host": "-", + "bind_port": 8086 + }, "master": { "influxdb_mon_api_password": "", "influxdb_mon_persister_password": "", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 23d585b241..9decc4cf94 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -74,10 +74,35 @@ "required": true, "type": "map", "mapping": { - "port": { "required": true, "type": "int" }, + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" }, "ssl": { "required": true, "type": "bool" } } }, + "kibana": { + "required": true, + "type": "map", + "mapping": { + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" } + } + }, + "mariadb": { + "required": true, + "type": "map", + "mapping": { + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" } + } + }, + "influxdb": { + "required": true, + "type": "map", + "mapping": { + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" } + } + }, "master": { "required": true, "type": "map", From c25f22ea4a407c75964b0ecb8a6b245924bbe14e Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Mon, 3 Apr 2017 11:14:59 +0200 Subject: [PATCH 36/52] Uncomment code for cluster validation --- .../app/models/monasca_service.rb | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/crowbar_framework/app/models/monasca_service.rb b/crowbar_framework/app/models/monasca_service.rb index 9d340337ae..17fc3348e4 100644 --- a/crowbar_framework/app/models/monasca_service.rb +++ b/crowbar_framework/app/models/monasca_service.rb @@ -49,11 +49,8 @@ def role_constraints }, "monasca-server" => { "unique" => false, - # TODO: change for cluster - # "count" => -1, - # "cluster" => true, - "count" => 1, - "cluster" => false, + "count" => -1, + "cluster" => true, "admin" => false, "exclude_platform" => { "suse" => "< 12.2", @@ -135,12 +132,11 @@ def create_proposal def validate_proposal_after_save(proposal) validate_one_for_role proposal, "monasca-master" validate_one_for_role proposal, "monasca-server" - # TODO: uncomment for cluster support - # nodes = proposal["deployment"][@bc_name]["elements"] - # if !nodes.key?("monasca-server") || - # (nodes["monasca-server"].length != 1 && nodes["monasca-server"].length != 3) - # validation_error("Need either one or three monasca-server node(s).") - # end + nodes = proposal["deployment"][@bc_name]["elements"] + if !nodes.key?("monasca-server") || + (nodes["monasca-server"].length != 1 && nodes["monasca-server"].length != 3) + validation_error("Need either one or three monasca-server node(s).") + end super end From a3ab6a17b2372257ceb2c298d8e46eaae5a36c62 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Mon, 3 Apr 2017 11:25:15 +0200 Subject: [PATCH 37/52] Address Hound comments --- chef/cookbooks/monasca/libraries/helper.rb | 10 +++++----- chef/cookbooks/monasca/recipes/ha.rb | 20 +++++++++++++++----- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 3c69c0530a..c5282bee59 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -83,10 +83,10 @@ def network_settings(node) end if node[:monasca][:ha][:enabled] - bind_port_api = node[:monasca][:ha][:ports][:api].to_i, - bind_port_log_api = node[:monasca][:ha][:ports][:log_api].to_i, - bind_port_kibana = node[:monasca][:ha][:ports][:kibana].to_i, - bind_port_mariadb = node[:monasca][:ha][:ports][:mariadb].to_i, + bind_port_api = node[:monasca][:ha][:ports][:api].to_i + bind_port_log_api = node[:monasca][:ha][:ports][:log_api].to_i + bind_port_kibana = node[:monasca][:ha][:ports][:kibana].to_i + bind_port_mariadb = node[:monasca][:ha][:ports][:mariadb].to_i bind_port_influxdb = node[:monasca][:ha][:ports][:influxdb].to_i else bind_port_api = node[:monasca][:api][:bind_port].to_i @@ -127,7 +127,7 @@ def network_settings(node) bind_port: bind_port_influxdb, ha_bind_host: node[:monasca][:influxdb][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i - }, + } } end end diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 3d23a2afde..9e77f999ff 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -27,35 +27,45 @@ haproxy_loadbalancer "monasca-api" do address network_settings[:api][:ha_bind_host] port network_settings[:api][:ha_bind_port] - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "monasca_api") + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "monasca_api" + ) action :nothing end.run_action(:create) haproxy_loadbalancer "monasca-log-api" do address network_settings[:log_api][:ha_bind_host] port network_settings[:log_api][:ha_bind_port] - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "monasca_log_api") + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "monasca_log_api" + ) action :nothing end.run_action(:create) haproxy_loadbalancer "kibana" do address network_settings[:kibana][:ha_bind_host] port network_settings[:kibana][:ha_bind_port] - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "kibana") + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "kibana" + ) action :nothing end.run_action(:create) haproxy_loadbalancer "mariadb" do address network_settings[:mariadb][:ha_bind_host] port network_settings[:mariadb][:ha_bind_port] - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "mariadb") + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "mariadb" + ) action :nothing end.run_action(:create) haproxy_loadbalancer "influxdb" do address network_settings[:influxdb][:ha_bind_host] port network_settings[:influxdb][:ha_bind_port] - servers CrowbarPacemakerHelper.haproxy_servers_for_service(node, "monasca", "monasca-server", "influxdb") + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "influxdb" + ) action :nothing end.run_action(:create) From b5152f5957430d9bcbc46bb2d670e9fcdd831926 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Tue, 4 Apr 2017 09:41:54 +0200 Subject: [PATCH 38/52] Add influxdb relay config Rename cluster_admin_ip to cluster_monitoring_ip --- chef/cookbooks/monasca/attributes/default.rb | 1 + chef/cookbooks/monasca/libraries/helper.rb | 51 +++++++++++++++---- chef/cookbooks/monasca/recipes/ha.rb | 13 +++++ chef/data_bags/crowbar/template-monasca.json | 4 ++ .../data_bags/crowbar/template-monasca.schema | 8 +++ 5 files changed, 66 insertions(+), 11 deletions(-) diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index befedb3a4e..c1ff5378e0 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -65,3 +65,4 @@ # Internal network default[:monasca][:ha][:ports][:mariadb] = 13306 default[:monasca][:ha][:ports][:influxdb] = 18086 +default[:monasca][:ha][:ports][:influxdb_relay] = 19096 diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index c5282bee59..d6d30ac358 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -71,15 +71,21 @@ def monasca_hosts(nodes) end def get_host_for_monitoring_url(node) - Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address + Chef::Recipe::Barclamp::Inventory.get_network_by_type( + node, "monitoring" + ).address end def network_settings(node) - @ip ||= Chef::Recipe::Barclamp::Inventory.get_network_by_type(node, "monitoring").address - @cluster_admin_ip ||= nil - - if node[:monasca][:ha][:enabled] && !@cluster_admin_ip - @cluster_admin_ip = CrowbarPacemakerHelper.cluster_vip(node, "monitoring") + @ip ||= Chef::Recipe::Barclamp::Inventory.get_network_by_type( + node, "monitoring" + ).address + @cluster_monitoring_ip ||= nil + + if node[:monasca][:ha][:enabled] && !@cluster_monitoring_ip + @cluster_monitoring_ip = CrowbarPacemakerHelper.cluster_vip( + node, "monitoring" + ) end if node[:monasca][:ha][:enabled] @@ -88,12 +94,14 @@ def network_settings(node) bind_port_kibana = node[:monasca][:ha][:ports][:kibana].to_i bind_port_mariadb = node[:monasca][:ha][:ports][:mariadb].to_i bind_port_influxdb = node[:monasca][:ha][:ports][:influxdb].to_i + bind_port_influxdb_relay = node[:monasca][:ha][:ports][:influxdb_relay].to_i else bind_port_api = node[:monasca][:api][:bind_port].to_i bind_port_log_api = node[:monasca][:log_api][:bind_port].to_i bind_port_kibana = node[:monasca][:kibana][:bind_port].to_i bind_port_mariadb = node[:monasca][:mariadb][:bind_port].to_i bind_port_influxdb = node[:monasca][:influxdb][:bind_port].to_i + bind_port_influxdb_relay = node[:monasca][:influxdb][:influxdb_relay].to_i end @network_settings ||= { @@ -101,32 +109,53 @@ def network_settings(node) api: { bind_port: bind_port_api, - ha_bind_host: node[:monasca][:api][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_host: node[:monasca][:api][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, ha_bind_port: node[:monasca][:api][:bind_port].to_i }, log_api: { bind_port: bind_port_log_api, - ha_bind_host: node[:monasca][:log_api][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_host: node[:monasca][:log_api][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, ha_bind_port: node[:monasca][:log_api][:bind_port].to_i }, kibana: { bind_port: bind_port_kibana, - ha_bind_host: node[:monasca][:kibana][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_host: node[:monasca][:kibana][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, ha_bind_port: node[:monasca][:kibana][:bind_port].to_i }, mariadb: { bind_port: bind_port_mariadb, - ha_bind_host: node[:monasca][:mariadb][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_host: node[:monasca][:mariadb][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, ha_bind_port: node[:monasca][:mariadb][:bind_port].to_i }, influxdb: { bind_port: bind_port_influxdb, - ha_bind_host: node[:monasca][:influxdb][:bind_host] == "*" ? "0.0.0.0" : @cluster_admin_ip, + ha_bind_host: node[:monasca][:influxdb][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i + }, + + # FIXME: InfluxDB Relay should be "hidden" on the same port + # as InfluxDB, and all communication that go to /write API endpoint + # should be relayed to InfluxDB Relay + influxdb_relay: { + bind_port: bind_port_influxdb_relay, + ha_bind_host: node[:monasca][:influxdb_relay][:bind_host] == "*" ? + "0.0.0.0" : + @cluster_monitoring_ip, + ha_bind_port: node[:monasca][:influxdb_relay][:bind_port].to_i } } end diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 9e77f999ff..bf9c2b2284 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -69,4 +69,17 @@ action :nothing end.run_action(:create) +# FIXME: InfluxDB Relay should be "hidden" on the same port +# as InfluxDB, and all communication that go to /write API endpoint +# should be relayed to InfluxDB Relay. +# Need to use `frontend` and `backend` HAProxy configuration. +haproxy_loadbalancer "influxdb-relay" do + address network_settings[:influxdb_relay][:ha_bind_host] + port network_settings[:influxdb_relay][:ha_bind_port] + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "influxdb_relay" + ) + action :nothing +end.run_action(:create) + crowbar_pacemaker_sync_mark "sync-monasca_before_ha" diff --git a/chef/data_bags/crowbar/template-monasca.json b/chef/data_bags/crowbar/template-monasca.json index 5ada3d2c40..dfcacf4293 100644 --- a/chef/data_bags/crowbar/template-monasca.json +++ b/chef/data_bags/crowbar/template-monasca.json @@ -58,6 +58,10 @@ "bind_host": "-", "bind_port": 8086 }, + "influxdb_relay": { + "bind_host": "-", + "bind_port": 9096 + }, "master": { "influxdb_mon_api_password": "", "influxdb_mon_persister_password": "", diff --git a/chef/data_bags/crowbar/template-monasca.schema b/chef/data_bags/crowbar/template-monasca.schema index 9decc4cf94..15cf3be708 100644 --- a/chef/data_bags/crowbar/template-monasca.schema +++ b/chef/data_bags/crowbar/template-monasca.schema @@ -103,6 +103,14 @@ "bind_port": { "required": true, "type": "int" } } }, + "influxdb_relay": { + "required": true, + "type": "map", + "mapping": { + "bind_host": { "required": true, "type": "str" }, + "bind_port": { "required": true, "type": "int" } + } + }, "master": { "required": true, "type": "map", From e924c7188bdba282440f885cf334389abdc49368 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Tue, 4 Apr 2017 11:26:03 +0200 Subject: [PATCH 39/52] Address hound comments --- chef/cookbooks/monasca/libraries/helper.rb | 31 +++++++++------------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index d6d30ac358..25a127f6ca 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -104,46 +104,43 @@ def network_settings(node) bind_port_influxdb_relay = node[:monasca][:influxdb][:influxdb_relay].to_i end + is_public_api = node[:monasca][:api][:bind_host] == "*" + is_public_log_api = node[:monasca][:log_api][:bind_host] == "*" + is_public_kibana = node[:monasca][:kibana][:bind_host] == "*" + is_public_mariadb = node[:monasca][:mariadb][:bind_host] == "*" + is_public_influxdb = node[:monasca][:influxdb][:bind_host] == "*" + is_public_influxdb_relay = node[:monasca][:influxdb_relay][:bind_host] == "*" + @network_settings ||= { ip: @ip, api: { bind_port: bind_port_api, - ha_bind_host: node[:monasca][:api][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_api ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:api][:bind_port].to_i }, log_api: { bind_port: bind_port_log_api, - ha_bind_host: node[:monasca][:log_api][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_log_api ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:log_api][:bind_port].to_i }, kibana: { bind_port: bind_port_kibana, - ha_bind_host: node[:monasca][:kibana][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_kibana ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:kibana][:bind_port].to_i }, mariadb: { bind_port: bind_port_mariadb, - ha_bind_host: node[:monasca][:mariadb][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_mariadb ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:mariadb][:bind_port].to_i }, influxdb: { bind_port: bind_port_influxdb, - ha_bind_host: node[:monasca][:influxdb][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_influxdb ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i }, @@ -152,9 +149,7 @@ def network_settings(node) # should be relayed to InfluxDB Relay influxdb_relay: { bind_port: bind_port_influxdb_relay, - ha_bind_host: node[:monasca][:influxdb_relay][:bind_host] == "*" ? - "0.0.0.0" : - @cluster_monitoring_ip, + ha_bind_host: is_public_influxdb_relay ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:influxdb_relay][:bind_port].to_i } } From 5f1ba3ae59ca926df858acd90f6f372d10759274 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 6 Apr 2017 08:47:05 +0200 Subject: [PATCH 40/52] Add frontend for InfluxDB --- chef/cookbooks/monasca/attributes/default.rb | 2 +- chef/cookbooks/monasca/libraries/helper.rb | 9 +++-- chef/cookbooks/monasca/recipes/ha.rb | 37 +++++++++++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/chef/cookbooks/monasca/attributes/default.rb b/chef/cookbooks/monasca/attributes/default.rb index c1ff5378e0..8e00f31a54 100644 --- a/chef/cookbooks/monasca/attributes/default.rb +++ b/chef/cookbooks/monasca/attributes/default.rb @@ -65,4 +65,4 @@ # Internal network default[:monasca][:ha][:ports][:mariadb] = 13306 default[:monasca][:ha][:ports][:influxdb] = 18086 -default[:monasca][:ha][:ports][:influxdb_relay] = 19096 +default[:monasca][:ha][:ports][:influxdb_relay] = 9096 diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 25a127f6ca..50c178d866 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -138,15 +138,18 @@ def network_settings(node) ha_bind_port: node[:monasca][:mariadb][:bind_port].to_i }, + influxdb_cluster: { + bind_port: bind_port_influxdb, + ha_bind_host: is_public_influxdb ? "0.0.0.0" : @cluster_monitoring_ip, + ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i + }, + influxdb: { bind_port: bind_port_influxdb, ha_bind_host: is_public_influxdb ? "0.0.0.0" : @cluster_monitoring_ip, ha_bind_port: node[:monasca][:influxdb][:bind_port].to_i }, - # FIXME: InfluxDB Relay should be "hidden" on the same port - # as InfluxDB, and all communication that go to /write API endpoint - # should be relayed to InfluxDB Relay influxdb_relay: { bind_port: bind_port_influxdb_relay, ha_bind_host: is_public_influxdb_relay ? "0.0.0.0" : @cluster_monitoring_ip, diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index bf9c2b2284..14721aabfb 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -61,21 +61,40 @@ end.run_action(:create) haproxy_loadbalancer "influxdb" do - address network_settings[:influxdb][:ha_bind_host] - port network_settings[:influxdb][:ha_bind_port] + name "influxdb_cluster" + type "frontend" + address network_settings[:influxdb_cluster][:ha_bind_host] + port network_settings[:influxdb_cluster][:ha_bind_port] + mode "tcp" + # acl "write_path path_beg -i /write" + # use_backend "influxdb_relay_back if write_path" + # default_backend "influxdb_back" + servers CrowbarPacemakerHelper.haproxy_servers_for_service( + node, "monasca", "monasca-server", "influxdb_cluster" + ) + action :nothing +end.run_action(:create) + +haproxy_loadbalancer "influxdb-backend" do + name "influxdb_back" + type "backend" + balance "leastconn" + options [ + "httpchk GET /ping" + ] servers CrowbarPacemakerHelper.haproxy_servers_for_service( node, "monasca", "monasca-server", "influxdb" ) action :nothing end.run_action(:create) -# FIXME: InfluxDB Relay should be "hidden" on the same port -# as InfluxDB, and all communication that go to /write API endpoint -# should be relayed to InfluxDB Relay. -# Need to use `frontend` and `backend` HAProxy configuration. -haproxy_loadbalancer "influxdb-relay" do - address network_settings[:influxdb_relay][:ha_bind_host] - port network_settings[:influxdb_relay][:ha_bind_port] +haproxy_loadbalancer "influxdb-relay-backend" do + name "influxdb_relay_back" + type "backend" + balance "leastconn" + options [ + "httpchk GET /ping" + ] servers CrowbarPacemakerHelper.haproxy_servers_for_service( node, "monasca", "monasca-server", "influxdb_relay" ) From 730fc30eb8b2a8f192038122259f2538107c53bf Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 6 Apr 2017 09:13:22 +0200 Subject: [PATCH 41/52] Fix getting port for api and log-api servers --- chef/cookbooks/monasca/recipes/ha.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 14721aabfb..0821cb5d7b 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -28,7 +28,7 @@ address network_settings[:api][:ha_bind_host] port network_settings[:api][:ha_bind_port] servers CrowbarPacemakerHelper.haproxy_servers_for_service( - node, "monasca", "monasca-server", "monasca_api" + node, "monasca", "monasca-server", "api" ) action :nothing end.run_action(:create) @@ -37,7 +37,7 @@ address network_settings[:log_api][:ha_bind_host] port network_settings[:log_api][:ha_bind_port] servers CrowbarPacemakerHelper.haproxy_servers_for_service( - node, "monasca", "monasca-server", "monasca_log_api" + node, "monasca", "monasca-server", "log_api" ) action :nothing end.run_action(:create) From df309210d7ff4a64c593b3843333644c78d416bd Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 6 Apr 2017 09:46:42 +0200 Subject: [PATCH 42/52] Fix loadbalancer for influx parts --- chef/cookbooks/monasca/recipes/ha.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 0821cb5d7b..79e4fa29f8 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -69,8 +69,9 @@ # acl "write_path path_beg -i /write" # use_backend "influxdb_relay_back if write_path" # default_backend "influxdb_back" + # TODO: remove servers from frontend servers CrowbarPacemakerHelper.haproxy_servers_for_service( - node, "monasca", "monasca-server", "influxdb_cluster" + node, "monasca", "monasca-server", "influxdb" ) action :nothing end.run_action(:create) @@ -78,7 +79,10 @@ haproxy_loadbalancer "influxdb-backend" do name "influxdb_back" type "backend" - balance "leastconn" + # TODO: remove address and port from backend + address network_settings[:influxdb][:ha_bind_host] + port network_settings[:influxdb][:ha_bind_port] + # balance "leastconn" options [ "httpchk GET /ping" ] @@ -91,7 +95,10 @@ haproxy_loadbalancer "influxdb-relay-backend" do name "influxdb_relay_back" type "backend" - balance "leastconn" + # TODO: remove address and port from backend + address network_settings[:influxdb_relay][:ha_bind_host] + port network_settings[:influxdb_relay][:ha_bind_port] + # balance "leastconn" options [ "httpchk GET /ping" ] From a4debf5d5b8192641f2f6e150a1775fbc3819268 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 6 Apr 2017 10:39:30 +0200 Subject: [PATCH 43/52] Remove including HA from role_monasca_master --- chef/cookbooks/monasca/recipes/role_monasca_master.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/chef/cookbooks/monasca/recipes/role_monasca_master.rb b/chef/cookbooks/monasca/recipes/role_monasca_master.rb index 1b9aca5ea9..9dd85209bf 100644 --- a/chef/cookbooks/monasca/recipes/role_monasca_master.rb +++ b/chef/cookbooks/monasca/recipes/role_monasca_master.rb @@ -16,5 +16,4 @@ if CrowbarRoleRecipe.node_state_valid_for_role?(node, "monasca", "monasca-master") include_recipe "#{@cookbook_name}::master" - include_recipe "#{@cookbook_name}::ha" end From df0b813ec5eddd29e5952e3a8958aad4ae5f0b7e Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Fri, 7 Apr 2017 08:25:39 +0200 Subject: [PATCH 44/52] Add more config options for HAProxy --- chef/cookbooks/monasca/recipes/ha.rb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 79e4fa29f8..61ca1299bb 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -66,9 +66,10 @@ address network_settings[:influxdb_cluster][:ha_bind_host] port network_settings[:influxdb_cluster][:ha_bind_port] mode "tcp" - # acl "write_path path_beg -i /write" - # use_backend "influxdb_relay_back if write_path" - # default_backend "influxdb_back" + balance "" + acls ["write_path path_beg -i /write"] + use_backends ["influxdb_relay_back if write_path"] + default_backend "influxdb_back" # TODO: remove servers from frontend servers CrowbarPacemakerHelper.haproxy_servers_for_service( node, "monasca", "monasca-server", "influxdb" @@ -82,7 +83,7 @@ # TODO: remove address and port from backend address network_settings[:influxdb][:ha_bind_host] port network_settings[:influxdb][:ha_bind_port] - # balance "leastconn" + balance "leastconn" options [ "httpchk GET /ping" ] @@ -98,7 +99,7 @@ # TODO: remove address and port from backend address network_settings[:influxdb_relay][:ha_bind_host] port network_settings[:influxdb_relay][:ha_bind_port] - # balance "leastconn" + balance "leastconn" options [ "httpchk GET /ping" ] From d5bf6776882a91c1cb2afcf930b7d2b42db628df Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Fri, 7 Apr 2017 12:25:45 +0200 Subject: [PATCH 45/52] Remove unnecessary configuration Also set mode "tcp" for all InfluxDB configs. --- chef/cookbooks/monasca/recipes/ha.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/ha.rb b/chef/cookbooks/monasca/recipes/ha.rb index 61ca1299bb..f77bf150b0 100644 --- a/chef/cookbooks/monasca/recipes/ha.rb +++ b/chef/cookbooks/monasca/recipes/ha.rb @@ -70,19 +70,13 @@ acls ["write_path path_beg -i /write"] use_backends ["influxdb_relay_back if write_path"] default_backend "influxdb_back" - # TODO: remove servers from frontend - servers CrowbarPacemakerHelper.haproxy_servers_for_service( - node, "monasca", "monasca-server", "influxdb" - ) action :nothing end.run_action(:create) haproxy_loadbalancer "influxdb-backend" do name "influxdb_back" type "backend" - # TODO: remove address and port from backend - address network_settings[:influxdb][:ha_bind_host] - port network_settings[:influxdb][:ha_bind_port] + mode "tcp" balance "leastconn" options [ "httpchk GET /ping" @@ -96,9 +90,7 @@ haproxy_loadbalancer "influxdb-relay-backend" do name "influxdb_relay_back" type "backend" - # TODO: remove address and port from backend - address network_settings[:influxdb_relay][:ha_bind_host] - port network_settings[:influxdb_relay][:ha_bind_port] + mode "tcp" balance "leastconn" options [ "httpchk GET /ping" From 541060728420a346fad1cb6f28cfdf980cbb347e Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Mon, 10 Apr 2017 12:50:25 +0200 Subject: [PATCH 46/52] Update hosts-cluster with proper virtual IP --- chef/cookbooks/monasca/libraries/helper.rb | 14 ++++++++------ chef/cookbooks/monasca/recipes/master.rb | 4 ++++ .../templates/default/monasca-hosts-cluster.erb | 13 +++++-------- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 50c178d866..0c70c1ce4d 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -76,16 +76,18 @@ def get_host_for_monitoring_url(node) ).address end - def network_settings(node) - @ip ||= Chef::Recipe::Barclamp::Inventory.get_network_by_type( + def get_vip_for_monitoring_cluster(node) + CrowbarPacemakerHelper.cluster_vip( node, "monitoring" - ).address + ) + end + + def network_settings(node) + @ip ||= get_host_for_monitoring_url(node) @cluster_monitoring_ip ||= nil if node[:monasca][:ha][:enabled] && !@cluster_monitoring_ip - @cluster_monitoring_ip = CrowbarPacemakerHelper.cluster_vip( - node, "monitoring" - ) + @cluster_monitoring_ip = get_vip_for_monitoring_cluster(node) end if node[:monasca][:ha][:enabled] diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index 8f2b3e141b..5ad931e7be 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -35,6 +35,9 @@ "monasca-hosts-cluster.erb" end +monasca_node = search(:node, "roles:monasca-server")[0] +monitoring_vip = MonascaHelper.get_vip_for_monitoring_cluster(monasca_node) + template "/opt/monasca-installer/monasca-hosts" do source hosts_template owner "root" @@ -43,6 +46,7 @@ variables( monasca_host: monasca_hosts[0], monasca_hosts: monasca_hosts, + monitoring_vip: monitoring_vip, ansible_ssh_user: "root", keystone_host: keystone_settings["public_url_host"] ) diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb index 9cc1c27075..50b0114ca9 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb @@ -12,10 +12,9 @@ localhost ansible_connection=local keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> # Load balancer hosts -# TODO -#monasca-log-api-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> -#monasca-api-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> -#kibana-balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-log-api-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-api-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +kibana-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> # ElasticSearch nodes for cluster <% @monasca_hosts.each_with_index do |host, index| %> @@ -78,8 +77,7 @@ keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ <% end %> # Load balancer node -# TODO -#balancer-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> +balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> #Storm Nimbus nodes <% @monasca_hosts.each_with_index do |host, index| %> @@ -110,8 +108,7 @@ monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_s <% end %> # Keepalived nodes -# TODO -#keepalived-virtual-node ansible_ssh_host= ansible_ssh_user=<%= @ansible_ssh_user %> +keepalived-virtual-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> <% @monasca_hosts.each_with_index do |host, index| %> <%= "keepalived-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> From f54d467985946979eee83f513262b3820ad2399f Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Tue, 11 Apr 2017 13:56:23 +0200 Subject: [PATCH 47/52] Use only admin and public networks --- chef/cookbooks/monasca/libraries/helper.rb | 12 +++++++++--- chef/cookbooks/monasca/recipes/master.rb | 8 +++++--- .../templates/default/monasca-hosts-cluster.erb | 10 +++++----- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index 0c70c1ce4d..d9886622f8 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -76,9 +76,15 @@ def get_host_for_monitoring_url(node) ).address end - def get_vip_for_monitoring_cluster(node) + def get_vip_for_public_cluster(node) CrowbarPacemakerHelper.cluster_vip( - node, "monitoring" + node, "public" + ) + end + + def get_vip_for_admin_cluster(node) + CrowbarPacemakerHelper.cluster_vip( + node, "admin" ) end @@ -87,7 +93,7 @@ def network_settings(node) @cluster_monitoring_ip ||= nil if node[:monasca][:ha][:enabled] && !@cluster_monitoring_ip - @cluster_monitoring_ip = get_vip_for_monitoring_cluster(node) + @cluster_monitoring_ip = get_vip_for_admin_cluster(node) end if node[:monasca][:ha][:enabled] diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index 5ad931e7be..df29ec651f 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -1,6 +1,6 @@ -# # Copyright 2017 Fujitsu LIMITED # +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -36,7 +36,8 @@ end monasca_node = search(:node, "roles:monasca-server")[0] -monitoring_vip = MonascaHelper.get_vip_for_monitoring_cluster(monasca_node) +public_vip = MonascaHelper.get_vip_for_public_cluster(monasca_node) +admin_vip = MonascaHelper.get_vip_for_admin_cluster(monasca_node) template "/opt/monasca-installer/monasca-hosts" do source hosts_template @@ -46,7 +47,8 @@ variables( monasca_host: monasca_hosts[0], monasca_hosts: monasca_hosts, - monitoring_vip: monitoring_vip, + public_vip: public_vip, + admin_vip: admin_vip, ansible_ssh_user: "root", keystone_host: keystone_settings["public_url_host"] ) diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb index 50b0114ca9..105a0056b6 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb @@ -12,9 +12,9 @@ localhost ansible_connection=local keystone-node ansible_ssh_host=<%= @keystone_host %> ansible_ssh_user=<%= @ansible_ssh_user %> # Load balancer hosts -monasca-log-api-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> -monasca-api-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> -kibana-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-log-api-balancer-node ansible_ssh_host=<%= @public_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +monasca-api-balancer-node ansible_ssh_host=<%= @public_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +kibana-balancer-node ansible_ssh_host=<%= @public_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> # ElasticSearch nodes for cluster <% @monasca_hosts.each_with_index do |host, index| %> @@ -77,7 +77,7 @@ kibana-balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible <% end %> # Load balancer node -balancer-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +balancer-node ansible_ssh_host=<%= @admin_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> #Storm Nimbus nodes <% @monasca_hosts.each_with_index do |host, index| %> @@ -108,7 +108,7 @@ monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_s <% end %> # Keepalived nodes -keepalived-virtual-node ansible_ssh_host=<%= @monitoring_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> +keepalived-virtual-node ansible_ssh_host=<%= @admin_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> <% @monasca_hosts.each_with_index do |host, index| %> <%= "keepalived-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> From 67dae9e0a66e6e61e06ba4f49b564a3c762c7bd9 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Wed, 12 Apr 2017 14:05:49 +0200 Subject: [PATCH 48/52] Use different ports for services running behind HAProxy --- chef/cookbooks/monasca/recipes/master.rb | 8 +++++++- .../monasca/templates/default/all_group.erb | 14 +++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index df29ec651f..ab202a9a14 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -55,13 +55,19 @@ notifies :run, "execute[run ansible]", :delayed end +network_settings = MonascaHelper.network_settings(node) + template "/opt/monasca-installer/group_vars/all_group" do source "all_group.erb" owner "root" group "root" mode "0644" variables( - keystone_settings: keystone_settings + keystone_settings: keystone_settings, + monasca_api_client_port: network_settings[:api][:bind_port], + monasca_log_api_port: network_settings[:log_api][:bind_port], + kibana_port: network_settings[:kibana][:bind_port], + mysql_port: network_settings[:mariadb][:bind_port], ) notifies :run, "execute[run ansible]", :delayed end diff --git a/chef/cookbooks/monasca/templates/default/all_group.erb b/chef/cookbooks/monasca/templates/default/all_group.erb index 9c3ea70730..c2265ab555 100644 --- a/chef/cookbooks/monasca/templates/default/all_group.erb +++ b/chef/cookbooks/monasca/templates/default/all_group.erb @@ -3,19 +3,19 @@ # file: group_vars/all_group # ports -monasca_api_client_port: 8070 +monasca_api_client_port: <%= @monasca_api_client_port %> monasca_api_admin_port: 8081 persister_app_connector_port: 8190 persister_admin_connector_port: 8191 -monasca_log_api_port: 5607 +log_api_port: <%= @monasca_log_api_port %> keystone_port: 5000 keystone_user_port: 35357 -kibana_port: 5601 +kibana_port: <%= @kibana_port %> elasticsearch_port: 9200 kafka_port: 9092 zookeeper_port: 2181 memcached_port: 11211 -mysql_port: 3306 +mysql_port: <%= @mysql_port %> # hosts monasca_api_ip: "{% if groups['monasca_api_group']|length > 1 %} @@ -24,7 +24,7 @@ monasca_api_ip: "{% if groups['monasca_api_group']|length > 1 %} {{ hostvars['monasca-api-node']['ansible_ssh_host'] }} {% endif %} " -monasca_log_api_ip: "{% if groups['monasca_log_api_group']|length > 1 %} +log_api_ip: "{% if groups['monasca_log_api_group']|length > 1 %} {{ hostvars['monasca-log-api-balancer-node']['ansible_ssh_host'] }} {% else %} {{ hostvars['monasca-log-api-node']['ansible_ssh_host'] }} @@ -49,7 +49,7 @@ keystone_url: "<%= @keystone_settings['public_auth_url'] %>" # URL service definition monasca_api_url: "http://{{ monasca_api_ip|trim|safe }}:{{ monasca_api_client_port|int }}/v2.0" -monasca_log_api_url: "http://{{ monasca_log_api_ip|trim|safe }}:{{ monasca_log_api_port|int }}/v3.0" +monasca_log_api_url: "http://{{ log_api_ip|trim|safe }}:{{ log_api_port|int }}/v3.0" kibana_url: "http://{{ kibana_ip|trim|safe }}:{{ kibana_port }}" # Monasca Api CLI - Credentials @@ -86,7 +86,7 @@ ansible_version_min: ansible_version_max: ver: "2.2.1.0" constraint: "<=" -pip_version_min: 7.0.3 +pip_version_min: 8.1.1 # pip configuration pip_base_dir: "/root/.pip" From 6a9528e129984444a6f6d5ab5266b995a585095a Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 13 Apr 2017 09:27:48 +0200 Subject: [PATCH 49/52] Use proper influxdb ports in cluster --- chef/cookbooks/monasca/recipes/master.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index ab202a9a14..4da4c9c16f 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -108,6 +108,8 @@ log_api_bind_host: pub_net_ip, influxdb_bind_address: monasca_net_ip, influxdb_host: monasca_net_ip, + influxdb_client_port: network_settings[:influxdb][:bind_port], + influxdb_relay_http_port: network_settings[:influxdb_relay][:bind_port], monasca_api_bind_host: pub_net_ip, elasticsearch_host: monasca_net_ip, nimbus_host: monasca_net_ip, From daed0a34e1f87a0aea8bd9abbac550874dd379f3 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Thu, 13 Apr 2017 13:49:13 +0200 Subject: [PATCH 50/52] Remove haproxy and keepalived hosts from hosts-cluster --- .../templates/default/monasca-hosts-cluster.erb | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb index 105a0056b6..3cddd698b3 100644 --- a/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb +++ b/chef/cookbooks/monasca/templates/default/monasca-hosts-cluster.erb @@ -102,18 +102,9 @@ monasca-thresh-node ansible_ssh_host=<%= @monasca_host %> ansible_s <%= "influxdb-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> <% end %> -# HAProxy nodes -<% @monasca_hosts.each_with_index do |host, index| %> -<%= "haproxy-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> -<% end %> - # Keepalived nodes keepalived-virtual-node ansible_ssh_host=<%= @admin_vip %> ansible_ssh_user=<%= @ansible_ssh_user %> -<% @monasca_hosts.each_with_index do |host, index| %> -<%= "keepalived-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> -<% end %> - # Elasticsearch Curator <% @monasca_hosts.each_with_index do |host, index| %> <%= "elasticsearch-curator-node-#{index} ansible_ssh_host=#{host}" %> ansible_ssh_user=<%= @ansible_ssh_user %> @@ -218,14 +209,8 @@ influxdb-node-<%= index + 1 %> influxdb_master_node_name=influxdb-node-0 <% end %> [haproxy_group] -<% @monasca_hosts.length.times do |index| %> -<%= "haproxy-node-#{index}" %> -<% end %> [keepalived_group] -<% @monasca_hosts.length.times do |index| %> -<%= "keepalived-node-#{index}" %> -<% end %> [elasticsearch_curator_group] <% @monasca_hosts.length.times do |index| %> From 0d68a014a2356835513af8c5bc91f113f95fabbe Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Fri, 14 Apr 2017 10:39:03 +0200 Subject: [PATCH 51/52] Add more helpers --- chef/cookbooks/monasca/libraries/helper.rb | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/chef/cookbooks/monasca/libraries/helper.rb b/chef/cookbooks/monasca/libraries/helper.rb index d9886622f8..0999ba1f40 100644 --- a/chef/cookbooks/monasca/libraries/helper.rb +++ b/chef/cookbooks/monasca/libraries/helper.rb @@ -76,15 +76,22 @@ def get_host_for_monitoring_url(node) ).address end + def get_vip_for_admin_cluster(node) + CrowbarPacemakerHelper.cluster_vip( + node, "admin" + ) + end + def get_vip_for_public_cluster(node) CrowbarPacemakerHelper.cluster_vip( node, "public" ) end - def get_vip_for_admin_cluster(node) + # TODO: needs https://github.com/crowbar/crowbar-ha/pull/195 + def get_vip_for_monitoring_cluster(node) CrowbarPacemakerHelper.cluster_vip( - node, "admin" + node, "monitoring" ) end From 1c56481020bc3af3113f516c25f768b7c39459f4 Mon Sep 17 00:00:00 2001 From: Dobroslaw Zybort Date: Fri, 14 Apr 2017 14:16:22 +0200 Subject: [PATCH 52/52] Fix setting proper ports in cluster --- chef/cookbooks/monasca/recipes/master.rb | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/chef/cookbooks/monasca/recipes/master.rb b/chef/cookbooks/monasca/recipes/master.rb index 4da4c9c16f..c3d899aa94 100644 --- a/chef/cookbooks/monasca/recipes/master.rb +++ b/chef/cookbooks/monasca/recipes/master.rb @@ -102,27 +102,28 @@ keystone_admin_password: keystone_settings["admin_password"], database_grafana_password: node[:monasca][:master][:database_grafana_password], + # Stick to the same order like in monasca.yml memcached_listen_ip: monasca_net_ip, + memcached_nodes: ["#{monasca_net_ip}:11211"], + zookeeper_hosts: monasca_net_ip, kafka_host: monasca_net_ip, + kafka_hosts: "#{monasca_net_ip}:9092", + elasticsearch_host: monasca_net_ip, + elasticsearch_hosts: monasca_net_ip, + elasticsearch_nodes: "[#{monasca_net_ip}]", kibana_host: pub_net_ip, log_api_bind_host: pub_net_ip, + monasca_log_api_url: "http://#{pub_net_ip}:#{node[:monasca][:log_api][:bind_port]}/v2.0", + nimbus_host: monasca_net_ip, + database_host: monasca_net_ip, + mariadb_bind_address: monasca_net_ip, influxdb_bind_address: monasca_net_ip, influxdb_host: monasca_net_ip, + influxdb_url: "http://#{monasca_net_ip}:#{network_settings[:influxdb][:bind_port]}", influxdb_client_port: network_settings[:influxdb][:bind_port], influxdb_relay_http_port: network_settings[:influxdb_relay][:bind_port], monasca_api_bind_host: pub_net_ip, - elasticsearch_host: monasca_net_ip, - nimbus_host: monasca_net_ip, - zookeeper_hosts: monasca_net_ip, - kafka_hosts: "#{monasca_net_ip}:9092", - mariadb_bind_address: monasca_net_ip, - database_host: monasca_net_ip, - monasca_api_url: "http://#{pub_net_ip}:8070/v2.0", - monasca_log_api_url: "http://#{pub_net_ip}:5607/v2.0", - memcached_nodes: ["#{monasca_net_ip}:11211"], - influxdb_url: "http://#{monasca_net_ip}:8086", - elasticsearch_nodes: "[#{monasca_net_ip}]", - elasticsearch_hosts: monasca_net_ip + monasca_api_url: "http://#{pub_net_ip}:#{node[:monasca][:api][:bind_port]}/v2.0" }.to_json execute "run ansible" do