From 8dc474afef28686fb9c9d8e17b8e3a41a1d4768a Mon Sep 17 00:00:00 2001 From: Shirshendu Mukherjee Date: Wed, 4 Jul 2018 15:35:14 +0530 Subject: [PATCH 1/5] Add GD2 API call capability --- Gemfile | 2 + Gemfile.lock | 6 + app/controllers/actions_controller.rb | 20 ++ app/controllers/application_controller.rb | 7 + app/controllers/clusters_controller.rb | 246 +++++++++++----------- app/controllers/nodes_controller.rb | 15 +- app/controllers/objects_controller.rb | 25 +++ app/models/cluster.rb | 15 +- app/models/peer.rb | 7 + app/presenters/cluster_presenter.rb | 42 +--- config.ru | 5 + config/initializers/etcd.rb | 13 ++ dependencies/jwt.spec | 81 +++++++ lib/gd2_client.rb | 58 +++++ lib/tendrl.rb | 2 + lib/tendrl/http_response_error_handler.rb | 8 + tendrl-api.spec | 3 +- 17 files changed, 376 insertions(+), 179 deletions(-) create mode 100644 app/controllers/actions_controller.rb create mode 100644 app/controllers/objects_controller.rb create mode 100644 app/models/peer.rb create mode 100644 dependencies/jwt.spec create mode 100644 lib/gd2_client.rb diff --git a/Gemfile b/Gemfile index 3e31fe6..2ce25ef 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,8 @@ gem 'activemodel', '4.2.6', require: 'active_model' gem 'rake', '0.9.6' gem 'puma', '3.6.0' gem 'bcrypt', '3.1.11' +gem 'httparty', '0.15.6' +gem 'jwt', '1.5.6' group :development do gem 'rubocop', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 599c085..f49a9a8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -27,13 +27,17 @@ GEM hashdiff (0.3.1) http-cookie (1.0.3) domain_name (~> 0.5) + httparty (0.15.6) + multi_xml (>= 0.5.2) i18n (0.7.0) json (1.8.3) + jwt (1.5.6) mime-types (3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) minitest (5.9.1) mixlib-log (1.7.1) + multi_xml (0.6.0) netrc (0.11.0) parser (2.3.1.4) ast (~> 2.2) @@ -107,6 +111,8 @@ DEPENDENCIES asciidoctor bcrypt (= 3.1.11) etcd (= 0.3.0) + httparty (= 0.15.6) + jwt (= 1.5.6) puma (= 3.6.0) rack-test rake (= 0.9.6) diff --git a/app/controllers/actions_controller.rb b/app/controllers/actions_controller.rb new file mode 100644 index 0000000..3a67787 --- /dev/null +++ b/app/controllers/actions_controller.rb @@ -0,0 +1,20 @@ +class ActionsController < AuthenticatedUsersController + before '/clusters/:cluster_id/?*?' do + begin + @cluster = Tendrl::Cluster.find(params[:cluster_id]) + rescue Etcd::KeyNotFound => e + e = Tendrl::HttpResponseErrorHandler.new( + e, cause: '/clusters/id', object_id: params[:cluster_id] + ) + halt e.status, e.body.to_json + end + end + + get '/clusters/:cluster_id' do + ClusterPresenter.single(Tendrl::Cluster.find(params[:cluster_id])).to_json + end + + post '/clusters/:cluster_id/jobs' do + HTTParty.post("http://localhost:8000/clusters/#{cluster_id}/jobs", body: { shell: 'echo "foobar"'}.to_json) + end +end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 2bf5e44..a1ddb7b 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -79,6 +79,13 @@ class ApplicationController < Sinatra::Base settings.http_allow_headers.join(',') end + helpers do + def parsed_body + @body ||= request.body.read + JSON.parse(@body) + end + end + options '*' do status 200 end diff --git a/app/controllers/clusters_controller.rb b/app/controllers/clusters_controller.rb index cd2014f..e3b6db4 100644 --- a/app/controllers/clusters_controller.rb +++ b/app/controllers/clusters_controller.rb @@ -4,148 +4,144 @@ class ClustersController < AuthenticatedUsersController { clusters: ClusterPresenter.list(clusters) }.to_json end - get '/clusters/:cluster_id' do - cluster = Tendrl::Cluster.find(params[:cluster_id]) - status 200 - ClusterPresenter.single( - params[:cluster_id] => cluster.attributes - ).to_json + before '/clusters/:cluster_id/?*?' do + @cluster = Tendrl::Cluster.new(params[:cluster_id]) + #@cluster.gd2.get("/ping") end - get '/clusters/:cluster_id/nodes' do - nodes = Tendrl::Node.find_all_by_cluster_id(params[:cluster_id]) - node_list = NodePresenter.list(nodes).map do |node_data| - bricks = Tendrl::Brick.find_all_by_cluster_id_and_node_fqdn( - params[:cluster_id], node_data['fqdn'] - ) - node_data.merge(bricks_count: bricks.size) - end - { nodes: node_list }.to_json + get '/clusters/:cluster_id' do + state = @cluster.gd2.state + state.to_json end - get '/clusters/:cluster_id/nodes/:node_id/bricks' do - node = Tendrl::Node.find_by_cluster_id( - params[:cluster_id], params[:node_id] - ) - halt 404 unless node.present? - bricks = Tendrl::Brick.find_all_by_cluster_id_and_node_fqdn( - params[:cluster_id], node['fqdn'] - ) - { bricks: BrickPresenter.list(bricks) }.to_json + get '/clusters/:cluster_id/peers' do + peers = @cluster.gd2.peers + { peers: peers }.to_json end get '/clusters/:cluster_id/volumes' do - volumes = Tendrl::Volume.find_all_by_cluster_id(params[:cluster_id]) - { volumes: VolumePresenter.list(volumes) }.to_json + volumes = @cluster.gd2.volumes + { volumes: volumes }.to_json end - get '/clusters/:cluster_id/volumes/:volume_id/bricks' do - references = Tendrl::Brick.find_refs_by_cluster_id_and_volume_id( - params[:cluster_id], params[:volume_id] - ) - bricks = Tendrl::Brick.find_by_cluster_id_and_refs(params[:cluster_id], references) - { bricks: BrickPresenter.list(bricks) }.to_json + post '/import' do + new_endpoint = { + gd2_url: parsed_body['gd2_url'], + secret: parsed_body['secret'] + } + gd2 = Gd2Client.new new_endpoint + state = gd2.state + cluster = Tendrl::Cluster.new state['cluster-id'] + unless cluster.endpoints.include? new_endpoint + Tendrl.etcd.create_in_order( + "/clusters/#{state['cluster-id']}/endpoints", + value: new_endpoint.to_json + ) + end + status 201 + state.merge(endpoints: cluster.endpoints).to_json end - get '/clusters/:cluster_id/notifications' do - notifications = Tendrl::Notification.all - NotificationPresenter.list_by_integration_id(notifications, params[:cluster_id]).to_json - end + #get '/clusters/:cluster_id/nodes/:node_id/bricks' do + #node = Tendrl::Node.find_by_cluster_id( + #params[:cluster_id], params[:node_id] + #) + #halt 404 unless node.present? + #bricks = Tendrl::Brick.find_all_by_cluster_id_and_node_fqdn( + #params[:cluster_id], node['fqdn'] + #) + #{ bricks: BrickPresenter.list(bricks) }.to_json + #end - get '/clusters/:cluster_id/jobs' do - begin - jobs = Tendrl::Job.all - rescue Etcd::KeyNotFound - jobs = [] - end - { jobs: JobPresenter.list_by_integration_id(jobs, params[:cluster_id]) }.to_json - end + #get '/clusters/:cluster_id/volumes/:volume_id/bricks' do + #references = Tendrl::Brick.find_refs_by_cluster_id_and_volume_id( + #params[:cluster_id], params[:volume_id] + #) + #bricks = Tendrl::Brick.find_by_cluster_id_and_refs(params[:cluster_id], references) + #{ bricks: BrickPresenter.list(bricks) }.to_json + #end - post '/clusters/:cluster_id/import' do - Tendrl.load_node_definitions - Tendrl::Cluster.exist? params[:cluster_id] - flow = Tendrl::Flow.new('namespace.tendrl', 'ImportCluster') - body = JSON.parse(request.body.read) - body['Cluster.volume_profiling_flag'] = if ['enable', 'disable'].include?(body['Cluster.volume_profiling_flag']) - body['Cluster.volume_profiling_flag'] - else - 'leave-as-is' - end - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id]).create(body) - status 202 - { job_id: job.job_id }.to_json - end + #get '/clusters/:cluster_id/notifications' do + #notifications = Tendrl::Notification.all + #NotificationPresenter.list_by_integration_id(notifications, params[:cluster_id]).to_json + #end - post '/clusters/:cluster_id/unmanage' do - Tendrl.load_node_definitions - flow = Tendrl::Flow.new('namespace.tendrl', 'UnmanageCluster') - body = JSON.parse(request.body.string.present? ? request.body.string : '{}') - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id]).create(body) - status 202 - { job_id: job.job_id }.to_json - end + #get '/clusters/:cluster_id/jobs' do + #begin + #jobs = Tendrl::Job.all + #rescue Etcd::KeyNotFound + #jobs = [] + #end + #{ jobs: JobPresenter.list_by_integration_id(jobs, params[:cluster_id]) }.to_json + #end - post '/clusters/:cluster_id/expand' do - Tendrl.load_node_definitions - flow = Tendrl::Flow.new 'namespace.tendrl', 'ExpandClusterWithDetectedPeers' - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id] - ).create({}) - status 202 - { job_id: job.job_id }.to_json - end + #post '/clusters/:cluster_id/unmanage' do + #Tendrl.load_node_definitions + #flow = Tendrl::Flow.new('namespace.tendrl', 'UnmanageCluster') + #body = JSON.parse(request.body.string.present? ? request.body.string : '{}') + #job = Tendrl::Job.new( + #current_user, + #flow, + #integration_id: params[:cluster_id]).create(body) + #status 202 + #{ job_id: job.job_id }.to_json + #end - post '/clusters/:cluster_id/profiling' do - Tendrl.load_definitions(params[:cluster_id]) - body = JSON.parse(request.body.read) - volume_profiling_flag = if ['enable', 'disable'].include?(body['Cluster.volume_profiling_flag']) - body['Cluster.volume_profiling_flag'] - else - 'leave-as-is' - end - flow = Tendrl::Flow.new('namespace.gluster', 'EnableDisableVolumeProfiling') + #post '/clusters/:cluster_id/expand' do + #Tendrl.load_node_definitions + #flow = Tendrl::Flow.new 'namespace.tendrl', 'ExpandClusterWithDetectedPeers' + #job = Tendrl::Job.new( + #current_user, + #flow, + #integration_id: params[:cluster_id] + #).create({}) + #status 202 + #{ job_id: job.job_id }.to_json + #end - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id], - type: 'sds' - ).create('Cluster.volume_profiling_flag' => volume_profiling_flag) - status 202 - { job_id: job.job_id }.to_json - end + #post '/clusters/:cluster_id/profiling' do + #Tendrl.load_definitions(params[:cluster_id]) + #body = JSON.parse(request.body.read) + #volume_profiling_flag = if ['enable', 'disable'].include?(body['Cluster.volume_profiling_flag']) + #body['Cluster.volume_profiling_flag'] + #else + #'leave-as-is' + #end + #flow = Tendrl::Flow.new('namespace.gluster', 'EnableDisableVolumeProfiling') - post '/clusters/:cluster_id/volumes/:volume_id/start_profiling' do - Tendrl.load_definitions(params[:cluster_id]) - flow = Tendrl::Flow.new('namespace.gluster', 'StartProfiling', 'Volume') - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id], - type: 'sds' - ).create('Volume.vol_id' => params[:volume_id]) - status 202 - { job_id: job.job_id }.to_json - end + #job = Tendrl::Job.new( + #current_user, + #flow, + #integration_id: params[:cluster_id], + #type: 'sds' + #).create('Cluster.volume_profiling_flag' => volume_profiling_flag) + #status 202 + #{ job_id: job.job_id }.to_json + #end - post '/clusters/:cluster_id/volumes/:volume_id/stop_profiling' do - Tendrl.load_definitions(params[:cluster_id]) - flow = Tendrl::Flow.new('namespace.gluster', 'StopProfiling', 'Volume') - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id], - type: 'sds' - ).create('Volume.vol_id' => params[:volume_id]) - status 202 - { job_id: job.job_id }.to_json - end + #post '/clusters/:cluster_id/volumes/:volume_id/start_profiling' do + #Tendrl.load_definitions(params[:cluster_id]) + #flow = Tendrl::Flow.new('namespace.gluster', 'StartProfiling', 'Volume') + #job = Tendrl::Job.new( + #current_user, + #flow, + #integration_id: params[:cluster_id], + #type: 'sds' + #).create('Volume.vol_id' => params[:volume_id]) + #status 202 + #{ job_id: job.job_id }.to_json + #end + + #post '/clusters/:cluster_id/volumes/:volume_id/stop_profiling' do + #Tendrl.load_definitions(params[:cluster_id]) + #flow = Tendrl::Flow.new('namespace.gluster', 'StopProfiling', 'Volume') + #job = Tendrl::Job.new( + #current_user, + #flow, + #integration_id: params[:cluster_id], + #type: 'sds' + #).create('Volume.vol_id' => params[:volume_id]) + #status 202 + #{ job_id: job.job_id }.to_json + #end end diff --git a/app/controllers/nodes_controller.rb b/app/controllers/nodes_controller.rb index b4471a0..2b67106 100644 --- a/app/controllers/nodes_controller.rb +++ b/app/controllers/nodes_controller.rb @@ -1,7 +1,7 @@ class NodesController < AuthenticatedUsersController before do - Tendrl.load_node_definitions + #Tendrl.load_node_definitions end get '/nodes' do @@ -297,19 +297,6 @@ class NodesController < AuthenticatedUsersController { job_id: job.job_id }.to_json end - post '/:flow' do - flow = Tendrl::Flow.find_by_external_name_and_type( - params[:flow], 'node_agent' - ) - halt 404 if flow.nil? - body = JSON.parse(request.body.read) - job = Tendrl::Job.new(current_user, flow).create(body) - - status 202 - { job_id: job.job_id }.to_json - end - - private def detected_cluster_id(node_id) diff --git a/app/controllers/objects_controller.rb b/app/controllers/objects_controller.rb new file mode 100644 index 0000000..3b6f9ab --- /dev/null +++ b/app/controllers/objects_controller.rb @@ -0,0 +1,25 @@ +class ObjectsController < AuthenticatedUsersController + get '/objects/:type' do + type = params[:type].singularize.humanize + object_class = Tendrl.const_get type + presenter_class = Object.const_get("#{type}Presenter") + { params[:type] => presenter_class.list(object_class.all) }.to_json + end + + get '/objects/:type/:id' do + type = params[:type].singularize.humanize + object_class = Tendrl.const_get type + presenter_class = Object.const_get("#{type}Presenter") + { params[:type] => presenter_class.list(object_class.single) }.to_json + end + + get '/objects/:type/:id/:sub_object_type' do + type = params[:type].singularize.humanize + sub_object_type = params[:sub_object_type].singularize.humanize + object_class = Tendrl.const_get type + sub_object_class = Tendrl.const_get sub_object_type + list = sub_object_class.public_send "find_all_by_#{type.snake_case}_id" + presenter_class = Object.const_get("#{sub_object_type}Presenter") + end + #post '/objects/:type/:object_id/:flow_name' +end diff --git a/app/models/cluster.rb b/app/models/cluster.rb index d028e57..dddeb3b 100644 --- a/app/models/cluster.rb +++ b/app/models/cluster.rb @@ -1,10 +1,19 @@ module Tendrl class Cluster + attr_accessor :uuid - attr_accessor :attributes + def initialize(cluster_id) + @uuid = cluster_id + end + + def endpoints + Tendrl.etcd.get( + "/clusters/#{@uuid}/endpoints", recursive: true + ).children.map(&:value).sort.uniq.map { |e| JSON.parse e } + end - def initialize(attributes={}) - @attributes = attributes + def gd2 + @gd2 ||= Gd2Client.from_endpoint endpoints.sample end class << self diff --git a/app/models/peer.rb b/app/models/peer.rb new file mode 100644 index 0000000..cff8204 --- /dev/null +++ b/app/models/peer.rb @@ -0,0 +1,7 @@ +module Tendrl + class Peer + def peers + get("/v1/peers") + end + end +end diff --git a/app/presenters/cluster_presenter.rb b/app/presenters/cluster_presenter.rb index a2c14f5..b248516 100644 --- a/app/presenters/cluster_presenter.rb +++ b/app/presenters/cluster_presenter.rb @@ -11,43 +11,13 @@ def list(cluster_list) end def single(cluster_attributes) - cluster = nil - cluster_attributes.each do |cluster_id, attributes| - context = attributes.delete('tendrlcontext') - break if context.nil? - context['cluster_id'] = cluster_id - attributes.slice!( - 'errors', - 'globaldetails', - 'short_name', - 'nodes', - 'public_network', - 'cluster_network', - 'is_managed', - 'volume_profiling_state', - 'alert_counters', - 'status', - 'current_job' - ) - attributes['errors'] = JSON.parse(attributes['errors']) rescue [] - nodes = attributes.delete('nodes') - cluster_nodes = [] - if nodes.present? - nodes.each do |node_id, values| - next if values['nodecontext'].blank? - values['nodecontext']['tags'] = JSON.parse(values['nodecontext']['tags']) rescue [] - values['nodecontext']['status'] ||= 'DOWN' - cluster_nodes << values['nodecontext'].merge({ node_id: node_id }) - end - end - - if bricks = attributes.delete('bricks') - attributes.merge!(bricks: bricks(bricks)) - end - - cluster = context.merge(attributes).merge(nodes: cluster_nodes) + attributes = cluster_attributes.values[0] + attributes = attributes.merge!('cluster_id' => cluster_attributes.keys[0]) + attributes['errors'] = JSON.parse(attributes['errors']) rescue [] + if bricks = attributes.delete('bricks') + attributes.merge!(bricks: bricks(bricks)) end - cluster + attributes end def bricks(bricks) diff --git a/config.ru b/config.ru index 3f34025..7c6cff2 100644 --- a/config.ru +++ b/config.ru @@ -24,3 +24,8 @@ map('/1.0') { use AuthenticatedUsersController run ApplicationController } + +#map('/2.0') do + #use ObjectsController + #run ApplicationController +#end diff --git a/config/initializers/etcd.rb b/config/initializers/etcd.rb index 8d2c9f3..46eb7b7 100644 --- a/config/initializers/etcd.rb +++ b/config/initializers/etcd.rb @@ -29,6 +29,19 @@ def cached_delete(path, opts = {}) cache.delete path delete path, opts end + + def get(path, opts = {}) + super('/tendrl2' + path, opts) + end + def set(path, *args) + super('/tendrl2' + path, *args) + end + def create_in_order(path, *args) + super('/tendrl2' + path, *args) + end + def delete(path, *args) + super('/tendrl2' + path, *args) + end end end diff --git a/dependencies/jwt.spec b/dependencies/jwt.spec new file mode 100644 index 0000000..66a1f57 --- /dev/null +++ b/dependencies/jwt.spec @@ -0,0 +1,81 @@ +# Generated from jwt-1.5.6.gem by gem2rpm -*- rpm-spec -*- +%global gem_name jwt + +Name: rubygem-%{gem_name} +Version: 1.5.6 +Release: 1%{?dist} +Summary: JSON Web Token implementation in Ruby +License: MIT +URL: http://github.com/jwt/ruby-jwt +Source0: https://rubygems.org/gems/%{gem_name}-%{version}.gem +BuildRequires: ruby(release) +BuildRequires: rubygems-devel +BuildRequires: ruby +# BuildRequires: rubygem(json) < 2.0 +# BuildRequires: rubygem(rspec) +# BuildRequires: rubygem(simplecov) +# BuildRequires: rubygem(simplecov-json) +# BuildRequires: rubygem(codeclimate-test-reporter) +BuildArch: noarch + +%description +A pure ruby implementation of the RFC 7519 OAuth JSON Web Token (JWT) +standard. + + +%package doc +Summary: Documentation for %{name} +Requires: %{name} = %{version}-%{release} +BuildArch: noarch + +%description doc +Documentation for %{name}. + +%prep +%setup -q -n %{gem_name}-%{version} + +%build +# Create the gem as gem install only works on a gem file +gem build ../%{gem_name}-%{version}.gemspec + +# %%gem_install compiles any C extensions and installs the gem into ./%%gem_dir +# by default, so that we can move it into the buildroot in %%install +%gem_install + +%install +mkdir -p %{buildroot}%{gem_dir} +cp -a .%{gem_dir}/* \ + %{buildroot}%{gem_dir}/ + + + +%check +pushd .%{gem_instdir} +# rspec spec +popd + +%files +%dir %{gem_instdir} +%{gem_instdir}/.codeclimate.yml +%exclude %{gem_instdir}/.gitignore +%exclude %{gem_instdir}/.rubocop.yml +%exclude %{gem_instdir}/.travis.yml +%license %{gem_instdir}/LICENSE +%{gem_instdir}/Manifest +%{gem_libdir} +%exclude %{gem_cache} +%{gem_spec} + +%files doc +%doc %{gem_docdir} +%exclude %{gem_instdir}/.rspec +%doc %{gem_instdir}/CHANGELOG.md +%{gem_instdir}/Gemfile +%doc %{gem_instdir}/README.md +%{gem_instdir}/Rakefile +%{gem_instdir}/ruby-jwt.gemspec +%{gem_instdir}/spec + +%changelog +* Wed Sep 12 2018 Shirshendu Mukherjee - 1.5.6-1 +- Initial package diff --git a/lib/gd2_client.rb b/lib/gd2_client.rb new file mode 100644 index 0000000..057c408 --- /dev/null +++ b/lib/gd2_client.rb @@ -0,0 +1,58 @@ +class Gd2Client + include HTTParty + + def self.from_endpoint(endpoint) + new(gd2_url: endpoint['gd2_url'], secret: endpoint['secret']) + end + + def initialize(gd2_url: 'http://localhost:24007', user: 'glustercli', secret: nil) + @gd2_url = gd2_url + @secret = secret || File.read(File.join('var', 'lib', 'glusterd2', 'auth')) + @claim = { + iss: user, + iat: DateTime.now.utc, + exp: DateTime.now.utc + 10.seconds, + qsh: '' + } + end + + def jwt_token + JWT.encode(@claim, @secret, 'HS256') + end + + def respond_to_missing?(method, include_private = false) + %i[get put post delete].include?(method) || super + end + + def method_missing(m, *args, &block) + return http_call(m, *args) if %i[get put post delete].include? m + super + end + + def http_call(method, path, opts = {}) + @claim[:qsh] = Digest::SHA256.hexdigest(method.to_s.upcase + '&' + path) + response = HTTParty.public_send( + method, + @gd2_url + path, + opts.merge(headers: { 'Authorization' => 'Bearer ' + jwt_token }) + ) + return JSON.parse(response.body) if response.success? + raise Tendrl::HttpResponseErrorHandler.new(StandardError.new, cause: 'gd2_api_error', object_id: method.to_s.capitalize + path.to_s) + end + + def peers + get('/v1/peers') + end + + def state + get('/statedump') + end + + def volumes + get('/v1/volumes') + end + + def volume(vol_name) + get("/v1/volumes#{vol_name}") + end +end diff --git a/lib/tendrl.rb b/lib/tendrl.rb index 91a6c8f..cff5199 100644 --- a/lib/tendrl.rb +++ b/lib/tendrl.rb @@ -142,6 +142,7 @@ def unmarshall!(attrs) require './lib/tendrl/atom' require './lib/tendrl/attribute' require './lib/tendrl/http_response_error_handler' +require './lib/gd2_client' # Models require './app/models/user' @@ -152,6 +153,7 @@ def unmarshall!(attrs) require './app/models/job' require './app/models/volume' require './app/models/brick' +require './app/models/peer' # Forms require './app/forms/user_form' diff --git a/lib/tendrl/http_response_error_handler.rb b/lib/tendrl/http_response_error_handler.rb index 5a201a9..85e53db 100644 --- a/lib/tendrl/http_response_error_handler.rb +++ b/lib/tendrl/http_response_error_handler.rb @@ -67,6 +67,14 @@ class HttpResponseErrorHandler < StandardError } } }, + 'gd2_api_error' => { + status: 503, + body: { + errors: { + message: 'GD2 API Error at: %s' + } + } + }, 'uncaught_exception' => { status: 500, body: { diff --git a/tendrl-api.spec b/tendrl-api.spec index 3a7d5a5..8049209 100644 --- a/tendrl-api.spec +++ b/tendrl-api.spec @@ -25,6 +25,7 @@ BuildRequires: systemd Requires: ruby >= 2.0.0 Requires: rubygem-activemodel >= 4.2.6 Requires: rubygem-bcrypt >= 3.1.10 +Requires: rubygem-httparty Requires: rubygem-i18n >= 0.7.0 Requires: rubygem-json Requires: rubygem-minitest >= 5.9.1 @@ -41,7 +42,7 @@ Requires: rubygem-etcd Requires: rubygem-rack-protection >= 1.5.3 Requires: rubygem-activesupport >= 4.2.6 Requires: rubygem-sinatra >= 1.4.5 -Requires: tendrl-node-agent +Requires: rubygem-jwt >= 1.5.6 %description Collection of tendrl api. From fbba4fd51d320bc6eedc442935bab20df91d684d Mon Sep 17 00:00:00 2001 From: Shirshendu Mukherjee Date: Mon, 17 Sep 2018 14:38:41 +0530 Subject: [PATCH 2/5] Improve cluster model, add some detail API calls --- app/controllers/clusters_controller.rb | 49 ++++++++++++------------- app/models/cluster.rb | 51 +++++++++++--------------- app/models/peer.rb | 7 ---- lib/gd2_client.rb | 14 ++++++- lib/tendrl.rb | 1 - 5 files changed, 58 insertions(+), 64 deletions(-) delete mode 100644 app/models/peer.rb diff --git a/app/controllers/clusters_controller.rb b/app/controllers/clusters_controller.rb index e3b6db4..a6f8962 100644 --- a/app/controllers/clusters_controller.rb +++ b/app/controllers/clusters_controller.rb @@ -1,12 +1,16 @@ class ClustersController < AuthenticatedUsersController get '/clusters' do clusters = Tendrl::Cluster.all - { clusters: ClusterPresenter.list(clusters) }.to_json + JSON.generate clusters: clusters end before '/clusters/:cluster_id/?*?' do - @cluster = Tendrl::Cluster.new(params[:cluster_id]) - #@cluster.gd2.get("/ping") + @cluster = Tendrl::Cluster.find(params[:cluster_id]) + raise Tendrl::HttpResponseErrorHandler.new( + StandardError.new, + cause: '/clusters/id', + object_id: params[:cluster_id] + ) end get '/clusters/:cluster_id' do @@ -24,41 +28,34 @@ class ClustersController < AuthenticatedUsersController { volumes: volumes }.to_json end + get '/clusters/:cluster_id/volumes/:volume_id' do + volume = @cluster.gd2.volumes(params[:volume_id]) + { volume: volume }.to_json + end + + get '/clusters/:cluster_id/volumes/:volume_id/bricks' do + bricks = @cluster.gd2.bricks(params[:volume_id]) + { bricks: bricks }.to_json + end + post '/import' do new_endpoint = { - gd2_url: parsed_body['gd2_url'], - secret: parsed_body['secret'] + 'gd2_url' => parsed_body['gd2_url'], + 'secret' => parsed_body['secret'] } - gd2 = Gd2Client.new new_endpoint + gd2 = Gd2Client.new new_endpoint.symbolize_keys state = gd2.state cluster = Tendrl::Cluster.new state['cluster-id'] unless cluster.endpoints.include? new_endpoint - Tendrl.etcd.create_in_order( - "/clusters/#{state['cluster-id']}/endpoints", - value: new_endpoint.to_json - ) + cluster.add_endpoint new_endpoint end status 201 state.merge(endpoints: cluster.endpoints).to_json end #get '/clusters/:cluster_id/nodes/:node_id/bricks' do - #node = Tendrl::Node.find_by_cluster_id( - #params[:cluster_id], params[:node_id] - #) - #halt 404 unless node.present? - #bricks = Tendrl::Brick.find_all_by_cluster_id_and_node_fqdn( - #params[:cluster_id], node['fqdn'] - #) - #{ bricks: BrickPresenter.list(bricks) }.to_json - #end - - #get '/clusters/:cluster_id/volumes/:volume_id/bricks' do - #references = Tendrl::Brick.find_refs_by_cluster_id_and_volume_id( - #params[:cluster_id], params[:volume_id] - #) - #bricks = Tendrl::Brick.find_by_cluster_id_and_refs(params[:cluster_id], references) - #{ bricks: BrickPresenter.list(bricks) }.to_json + #bricks = @cluster.gd2.get("/endpoints") + #{ bricks: bricks }.to_json #end #get '/clusters/:cluster_id/notifications' do diff --git a/app/models/cluster.rb b/app/models/cluster.rb index dddeb3b..ae403d9 100644 --- a/app/models/cluster.rb +++ b/app/models/cluster.rb @@ -7,48 +7,41 @@ def initialize(cluster_id) end def endpoints - Tendrl.etcd.get( + @endpoints ||= Tendrl.etcd.get( "/clusters/#{@uuid}/endpoints", recursive: true ).children.map(&:value).sort.uniq.map { |e| JSON.parse e } + rescue Etcd::KeyNotFound + [] end def gd2 - @gd2 ||= Gd2Client.from_endpoint endpoints.sample + @gd2 ||= endpoints.map { |e| Gd2Client.from_endpoint e }.find(&:ping?) end - class << self - def exist?(cluster_id) - Tendrl.etcd.get "/clusters/#{cluster_id}" - rescue Etcd::KeyNotFound => e - raise Tendrl::HttpResponseErrorHandler.new( - e, cause: '/clusters/id', object_id: cluster_id - ) - end + def to_json(_) + gd2.state.merge(endpoints: endpoints).to_json + end + def add_endpoint(endpoint) + Tendrl.etcd.create_in_order( + "/clusters/#{@uuid}/endpoints", + value: endpoint.to_json + ) + @endpoints = nil + end + + class << self def find(cluster_id) - attributes = {} - begin - attributes = Tendrl.recurse( - Tendrl.etcd.get( - "/clusters/#{cluster_id}", recursive: true - ) - )[cluster_id] - rescue Etcd::KeyNotFound - raise Tendrl::HttpResponseErrorHandler.new( - e, cause: '/clusters/id', object_id: cluster_id - ) - end - new(attributes) + cluster = new(cluster_id) + return nil unless cluster.gd2.present? end def all - begin - Tendrl.etcd.get('/clusters', recursive: true).children.map do |cluster| - Tendrl.recurse(cluster) - end - rescue Etcd::KeyNotFound - [] + Tendrl.etcd.get('/clusters').children.map do |etcd_node| + Cluster.new File.basename(etcd_node.key) end + rescue Etcd::KeyNotFound + [] end end end diff --git a/app/models/peer.rb b/app/models/peer.rb deleted file mode 100644 index cff8204..0000000 --- a/app/models/peer.rb +++ /dev/null @@ -1,7 +0,0 @@ -module Tendrl - class Peer - def peers - get("/v1/peers") - end - end -end diff --git a/lib/gd2_client.rb b/lib/gd2_client.rb index 057c408..d7995f0 100644 --- a/lib/gd2_client.rb +++ b/lib/gd2_client.rb @@ -53,6 +53,18 @@ def volumes end def volume(vol_name) - get("/v1/volumes#{vol_name}") + get("/v1/volumes/#{vol_name}") + end + + def bricks(vol_name) + get("/v1/volumes/#{vol_name}/bricks") + end + + def ping? + HTTParty.get(@gd2_url + "/ping").success? + end + + def endpoints + get('/endpoints') end end diff --git a/lib/tendrl.rb b/lib/tendrl.rb index cff5199..985eb27 100644 --- a/lib/tendrl.rb +++ b/lib/tendrl.rb @@ -153,7 +153,6 @@ def unmarshall!(attrs) require './app/models/job' require './app/models/volume' require './app/models/brick' -require './app/models/peer' # Forms require './app/forms/user_form' From 5a07da4aded4ac160fea538695395091c73efe89 Mon Sep 17 00:00:00 2001 From: Shirshendu Mukherjee Date: Mon, 17 Sep 2018 21:10:16 +0530 Subject: [PATCH 3/5] Add autogeneration of gd2 api methods from endpoints API --- app/controllers/clusters_controller.rb | 38 +++++++------- app/models/cluster.rb | 6 +-- lib/gd2_client.rb | 70 +++++++++++++------------- 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/app/controllers/clusters_controller.rb b/app/controllers/clusters_controller.rb index a6f8962..fd3770a 100644 --- a/app/controllers/clusters_controller.rb +++ b/app/controllers/clusters_controller.rb @@ -1,50 +1,54 @@ class ClustersController < AuthenticatedUsersController get '/clusters' do - clusters = Tendrl::Cluster.all - JSON.generate clusters: clusters + clusters = Tendrl::Cluster.all.map do |c| + c.gd2.statedump.to_h + end + { clusters: clusters }.to_json end before '/clusters/:cluster_id/?*?' do @cluster = Tendrl::Cluster.find(params[:cluster_id]) - raise Tendrl::HttpResponseErrorHandler.new( - StandardError.new, - cause: '/clusters/id', - object_id: params[:cluster_id] - ) + unless @cluster.present? + raise Tendrl::HttpResponseErrorHandler.new( + StandardError.new, + cause: '/clusters/id', + object_id: params[:cluster_id] + ) + end end get '/clusters/:cluster_id' do - state = @cluster.gd2.state - state.to_json + @cluster.gd2.statedump end get '/clusters/:cluster_id/peers' do - peers = @cluster.gd2.peers + peers = @cluster.gd2.get_peers.to_a { peers: peers }.to_json end get '/clusters/:cluster_id/volumes' do - volumes = @cluster.gd2.volumes + volumes = @cluster.gd2.volume_list.to_a { volumes: volumes }.to_json end - get '/clusters/:cluster_id/volumes/:volume_id' do - volume = @cluster.gd2.volumes(params[:volume_id]) + get '/clusters/:cluster_id/volumes/:volname' do + volume = @cluster.gd2.volume_info(params[:volname]).to_a { volume: volume }.to_json end - get '/clusters/:cluster_id/volumes/:volume_id/bricks' do - bricks = @cluster.gd2.bricks(params[:volume_id]) + get '/clusters/:cluster_id/volumes/:volname/bricks' do + bricks = @cluster.gd2.volume_bricks_status(params[:volname]).to_a { bricks: bricks }.to_json end post '/import' do new_endpoint = { 'gd2_url' => parsed_body['gd2_url'], + 'user' => parsed_body['user'], 'secret' => parsed_body['secret'] } - gd2 = Gd2Client.new new_endpoint.symbolize_keys - state = gd2.state + gd2 = Gd2Client.new new_endpoint + state = gd2.statedump cluster = Tendrl::Cluster.new state['cluster-id'] unless cluster.endpoints.include? new_endpoint cluster.add_endpoint new_endpoint diff --git a/app/models/cluster.rb b/app/models/cluster.rb index ae403d9..4509fab 100644 --- a/app/models/cluster.rb +++ b/app/models/cluster.rb @@ -15,11 +15,11 @@ def endpoints end def gd2 - @gd2 ||= endpoints.map { |e| Gd2Client.from_endpoint e }.find(&:ping?) + @gd2 ||= endpoints.map { |e| Gd2Client.new e }.find(&:ping?) end def to_json(_) - gd2.state.merge(endpoints: endpoints).to_json + gd2.statedump.merge(endpoints: endpoints).to_json end def add_endpoint(endpoint) @@ -33,7 +33,7 @@ def add_endpoint(endpoint) class << self def find(cluster_id) cluster = new(cluster_id) - return nil unless cluster.gd2.present? + cluster.gd2.present? ? cluster : nil end def all diff --git a/lib/gd2_client.rb b/lib/gd2_client.rb index d7995f0..26ccbd8 100644 --- a/lib/gd2_client.rb +++ b/lib/gd2_client.rb @@ -1,23 +1,27 @@ class Gd2Client include HTTParty - def self.from_endpoint(endpoint) - new(gd2_url: endpoint['gd2_url'], secret: endpoint['secret']) + UNVERSIONED_APIS = %w[/version /ping /statedump /endpoints] + + def initialize(endpoint) + endpoint = endpoint.with_indifferent_access + @gd2_url = endpoint[:gd2_url] + @user = endpoint[:user].present? ? endpoint[:user] : 'glustercli' + @secret = endpoint[:secret] || File.read(File.join('var', 'lib', 'glusterd2', 'auth')) + generate_api_methods end - def initialize(gd2_url: 'http://localhost:24007', user: 'glustercli', secret: nil) - @gd2_url = gd2_url - @secret = secret || File.read(File.join('var', 'lib', 'glusterd2', 'auth')) - @claim = { - iss: user, + def claim(method, path) + { + iss: @user, iat: DateTime.now.utc, exp: DateTime.now.utc + 10.seconds, - qsh: '' + qsh: Digest::SHA256.hexdigest(method.to_s.upcase + '&' + path) } end - def jwt_token - JWT.encode(@claim, @secret, 'HS256') + def jwt_token(method, path) + JWT.encode(claim(method, path), @secret, 'HS256') end def respond_to_missing?(method, include_private = false) @@ -30,41 +34,39 @@ def method_missing(m, *args, &block) end def http_call(method, path, opts = {}) - @claim[:qsh] = Digest::SHA256.hexdigest(method.to_s.upcase + '&' + path) - response = HTTParty.public_send( + req_data = { headers: { 'Authorization' => 'Bearer ' + jwt_token(method, path) } } + req_data[:body] = args[-1].to_h if %w[put post patch].include?(method) && args[-1].respond_to(:to_h) + HTTParty.public_send( method, @gd2_url + path, - opts.merge(headers: { 'Authorization' => 'Bearer ' + jwt_token }) + opts.merge(req_data) ) - return JSON.parse(response.body) if response.success? - raise Tendrl::HttpResponseErrorHandler.new(StandardError.new, cause: 'gd2_api_error', object_id: method.to_s.capitalize + path.to_s) - end - - def peers - get('/v1/peers') - end - - def state - get('/statedump') - end - - def volumes - get('/v1/volumes') end - def volume(vol_name) - get("/v1/volumes/#{vol_name}") + def prefixed_path(path) + UNVERSIONED_APIS.include?(path) ? path : '/v1' + path end - def bricks(vol_name) - get("/v1/volumes/#{vol_name}/bricks") + def generate_api_methods + apis.each do |api| + method_name = api['name'].split.join('_').underscore + action = api['methods'].downcase + self.class.send(:define_method, method_name) do |*args| + path = api['path'].gsub(/{.*?}/).with_index { |_, i| args[i] } + path = prefixed_path(path) + response = http_call(action, path, args[-1].present? && args[-1].respond_to?(:to_h) ? args[-1] : {}) + return response if response.success? + raise Tendrl::HttpResponseErrorHandler.new(response.body, cause: 'gd2_api_error', object_id: api['methods'] + path.to_s) + end + end + self end def ping? - HTTParty.get(@gd2_url + "/ping").success? + HTTParty.get(@gd2_url + '/ping').success? end - def endpoints - get('/endpoints') + def apis + get('/endpoints').to_a end end From 7efc7a6e8925063513f25e6eff17059b5d68877a Mon Sep 17 00:00:00 2001 From: Shirshendu Mukherjee Date: Tue, 18 Sep 2018 16:20:58 +0530 Subject: [PATCH 4/5] Trim code for lightweight implementation Some code that we used to access etcd data is no longer necessary, with the lightweight implementation keeping a single source of truth for GD2 data, i.e. the GD2 API. --- app/controllers/actions_controller.rb | 20 -- app/controllers/nodes_controller.rb | 308 ---------------------- app/controllers/objects_controller.rb | 25 -- app/models/brick.rb | 51 ---- app/models/node.rb | 68 ----- app/models/volume.rb | 18 -- app/presenters/brick_presenter.rb | 12 - app/presenters/cluster_presenter.rb | 40 --- app/presenters/node_presenter.rb | 21 -- app/presenters/volume_presenter.rb | 16 -- config.ru | 10 +- lib/tendrl.rb | 13 - lib/tendrl/atom.rb | 8 - lib/tendrl/attribute.rb | 25 -- lib/tendrl/errors/invalid_object_error.rb | 4 - lib/tendrl/flow.rb | 171 ------------ lib/tendrl/object.rb | 44 ---- spec/controllers/nodes_controller_spec.rb | 113 -------- spec/tendrl/flow_spec.rb | 67 ----- 19 files changed, 2 insertions(+), 1032 deletions(-) delete mode 100644 app/controllers/actions_controller.rb delete mode 100644 app/controllers/nodes_controller.rb delete mode 100644 app/controllers/objects_controller.rb delete mode 100644 app/models/brick.rb delete mode 100644 app/models/node.rb delete mode 100644 app/models/volume.rb delete mode 100644 app/presenters/brick_presenter.rb delete mode 100644 app/presenters/cluster_presenter.rb delete mode 100644 app/presenters/node_presenter.rb delete mode 100644 app/presenters/volume_presenter.rb delete mode 100644 lib/tendrl/atom.rb delete mode 100644 lib/tendrl/attribute.rb delete mode 100644 lib/tendrl/errors/invalid_object_error.rb delete mode 100644 lib/tendrl/flow.rb delete mode 100644 lib/tendrl/object.rb delete mode 100644 spec/controllers/nodes_controller_spec.rb delete mode 100644 spec/tendrl/flow_spec.rb diff --git a/app/controllers/actions_controller.rb b/app/controllers/actions_controller.rb deleted file mode 100644 index 3a67787..0000000 --- a/app/controllers/actions_controller.rb +++ /dev/null @@ -1,20 +0,0 @@ -class ActionsController < AuthenticatedUsersController - before '/clusters/:cluster_id/?*?' do - begin - @cluster = Tendrl::Cluster.find(params[:cluster_id]) - rescue Etcd::KeyNotFound => e - e = Tendrl::HttpResponseErrorHandler.new( - e, cause: '/clusters/id', object_id: params[:cluster_id] - ) - halt e.status, e.body.to_json - end - end - - get '/clusters/:cluster_id' do - ClusterPresenter.single(Tendrl::Cluster.find(params[:cluster_id])).to_json - end - - post '/clusters/:cluster_id/jobs' do - HTTParty.post("http://localhost:8000/clusters/#{cluster_id}/jobs", body: { shell: 'echo "foobar"'}.to_json) - end -end diff --git a/app/controllers/nodes_controller.rb b/app/controllers/nodes_controller.rb deleted file mode 100644 index 2b67106..0000000 --- a/app/controllers/nodes_controller.rb +++ /dev/null @@ -1,308 +0,0 @@ -class NodesController < AuthenticatedUsersController - - before do - #Tendrl.load_node_definitions - end - - get '/nodes' do - nodes = Tendrl::Node.all - { nodes: NodePresenter.list(nodes) }.to_json - end - - post '/ImportCluster' do - flow = Tendrl::Flow.new('namespace.tendrl', 'ImportCluster') - body = JSON.parse(request.body.read) - - # ImportCluster job structure: - # - # job = { - # "integration_id": "9a4b84e0-17b3-4543-af9f-e42000c52bfc", - # "run": "tendrl.node_agent.flows.import_cluster.ImportCluster", - # "status": "new", - # "type": "node", - # "node_ids": ["3943fab1-9ed2-4eb6-8121-5a69499c4568"], - # "parameters": { - # "TendrlContext.integration_id": "6b4b84e0-17b3-4543-af9f-e42000c52bfc", - # "Node[]": ["3943fab1-9ed2-4eb6-8121-5a69499c4568"], - # "DetectedCluster.sds_pkg_name": "gluster" - # } - # } - # - # Values sent by the UI: - # - # { - # cluster_id: "c221ccdb-51d6-4b57-9f10-bcf30c7fa351" - # hosts: [ - # { - # name: "dhcp43-203.lab.eng.blr.redhat.com", - # release: "ceph 10.2.5", - # role: "Monitor" - # } - # ], - # node_ids: ["3b6eb27f-3e83-4751-9d45-85a989ae2b25"], - # sds_type: "ceph", - # sds_name: "ceph 10.2.5" - # sds_version: "10.2.5" - # } - - # TODO: UI should be sending the parameters as defined in the flows, API - # shouldn't be translating. - - missing_params = [] - ['sds_type', 'node_ids'].each do |param| - missing_params << param unless body[param] and not body[param].empty? - end - halt 422, { errors: { missing: missing_params } }.to_json unless missing_params.empty? - - node_ids = body['node_ids'] - halt 422, { errors: { message: "'node_ids' must be an array with values" } }.to_json unless node_ids.kind_of?(Array) and not node_ids.empty? - detected_cluster_id = detected_cluster_id(node_ids.first) - halt 422, { errors: { message: "Node #{node_ids.first} not found" } }.to_json if detected_cluster_id.nil? - - body['DetectedCluster.detected_cluster_id'] = detected_cluster_id - body['DetectedCluster.sds_pkg_name'] = body['sds_type'] - body['Node[]'] = node_ids - job = Tendrl::Job.new(current_user, flow).create(body) - - status 202 - { job_id: job.job_id }.to_json - end - - post '/CreateCluster' do - flow = Tendrl::Flow.new('namespace.tendrl', 'CreateCluster') - body = JSON.parse(request.body.read) - - # Ceph CreateCluster example - # - # { - # "sds_name": "ceph", - # "sds_version": "10.2.5", - # "sds_parameters": { - # "name": "MyCluster", - # "cluster_id": "140cd3d5-58e4-4935-a954-d946ceff371d", - # "public_network": "192.168.128.0/24", - # "cluster_network": "192.168.220.0/24", - # "conf_overrides": { - # "global": { - # "osd_pool_default_pg_num": 128, - # "pool_default_pgp_num": 1 - # } - # } - # }, - # "node_identifier": "ip", - # "node_configuration": { - # "10.0.0.24": { - # "role": "ceph/mon", - # "provisioning_ip": "10.0.0.24", - # "monitor_interface": "eth0" - # }, - # "10.0.0.29": { - # "role": "ceph/osd", - # "provisioning_ip": "10.0.0.29", - # "journal_size": 5192, - # "journal_colocation": "false", - # "storage_disks": [ - # { - # "device": "/dev/sda", - # "journal": "/dev/sdc" - # }, - # { - # "device": "/dev/sdb", - # "journal": "/dev/sdc" - # } - # ] - # }, - # "10.0.0.30": { - # "role": "ceph/osd", - # "provisioning_ip": "10.0.0.30", - # "journal_colocation": "true", - # "storage_disks": [ - # { - # "device": "/dev/sda" - # }, - # { - # "device": "/dev/sdb" - # } - # ] - # } - # } - # } - # - # Job structure: - # - # { - # "integration_id": "9a4b84e0-17b3-4543-af9f-e42000c52bfc", - # "run": "tendrl.flows.CreateCluster", - # "status": "new", - # "type": "node", - # "node_ids": [], - # "tags": ["provisioner/ceph"], - # "parameters": { - # "TendrlContext.sds_name": "ceph", - # "TendrlContext.sds_version": "10.2.5", - # "TendrlContext.cluster_name": "MyCluster", - # "TendrlContext.cluster_id": "9a4b84e0-17b3-4543-af9f-e42000c52bfc", - # "Node[]": [ - # "3a95fd96-876d-439a-a64d-70332c069aaa", - # "3943fab1-9ed2-4eb6-8121-5a69499c4568", - # "b10e00e9-e444-41c2-9517-df2118b42731" - # ], - # "Cluster.public_network": "192.168.128.0/24", - # "Cluster.cluster_network": "192.168.220.0/24", - # "Cluster.conf_overrides": { - # "global": { - # "osd_pool_default_pg_num": 128, - # "pool_default_pgp_num": 1 - # } - # }, - # "Cluster.node_configuration": { - # "3a95fd96-876d-439a-a64d-70332c069aaa": { - # "role": "ceph/mon", - # "provisioning_ip": "10.0.0.24", - # "monitor_interface": "eth0" - # }, - # "3943fab1-9ed2-4eb6-8121-5a69499c4568": { - # "role": "ceph/osd", - # "provisioning_ip": "10.0.0.29", - # "journal_size": 5192, - # "journal_colocation": "false", - # "storage_disks": [ - # { - # "device": "/dev/sda", - # "journal": "/dev/sdc" - # }, - # { - # "device": "/dev/sdb", - # "journal": "/dev/sdc" - # } - # ] - # }, - # "b10e00e9-e444-41c2-9517-df2118b42731": { - # "role": "ceph/osd", - # "provisioning_ip": "10.0.0.30", - # "journal_colocation": "true", - # "storage_disks": [ - # { - # "device": "/dev/sda" - # }, - # { - # "device": "/dev/sdb" - # } - # ] - # } - # } - # } - # } - - missing_params = [] - ['sds_name', 'node_configuration'].each do |param| - missing_params << param unless body[param] and not body[param].empty? - end - halt 422, { errors: { missing: missing_params } }.to_json unless missing_params.empty? - - node_identifier = body['node_identifier'] - halt 422, { errors: { invalid: "'node_identifier', if specified, must be either 'uuid' or 'ip', provided: '#{node_identifier}'." } }.to_json \ - if node_identifier and \ - not ['uuid','ip'].include? node_identifier - - nodes = {} - node_ids = body['node_configuration'].keys - - unavailable_nodes = [] - node_ids.each do |node_id| - node = case node_identifier - when 'ip' - Tendrl::Node.find_by_ip(node_id) - when 'uuid' - Tendrl::Node.new(uuid) - end - - if node.nil? or not node.exist? - unavailable_nodes << node_id - next - end - - nodes[node.uuid] = body['node_configuration'][node_id] - end - - halt 422, { errors: { missing: "Unavailable nodes: #{unavailable_nodes.join(', ')}." } }.to_json unless unavailable_nodes.empty? - - parameters = {} - ['sds_name', 'sds_version'].each do |param| - parameters["TendrlContext.#{param}"] = body[param] - end - parameters['TendrlContext.cluster_name'] = body['sds_parameters']['name'] - parameters['TendrlContext.cluster_id'] = body['sds_parameters']['cluster_id'] - - parameters['Node[]'] = nodes.keys - - ['public_network', 'cluster_network', 'conf_overrides'].each do |param| - if body['sds_parameters']["#{param}"].present? - parameters["Cluster.#{param}"] = body['sds_parameters']["#{param}"] - end - end - parameters['Cluster.node_configuration'] = nodes - - job = Tendrl::Job.new(current_user, flow).create(parameters) - status 202 - { job_id: job.job_id }.to_json - end - - # Sample job structure - - # { - # "run": "tendrl.flows.ExpandCluster", - # "type": "node", - # "created_from": "API", - # "created_at": "2017-03-09T14:15:14Z", - # "username": "admin", - # "parameters": { - # "Node[]": ["867d6aae-fb98-4060-9f6a-1da4e4988db8"], - # "Cluster.node_configuration": { - # "867d6aae-fb98-4060-9f6a-1da4e4988db8": { - # "role": "glusterfs/node", - # "provisioning_ip": "10.70.43.222", - # }, - # }, - # "TendrlContext.sds_name": "gluster", - # "TendrlContext.integration_id": "d3c644f1-0f94-43e1-946f-e40c4694d703" - # }, - # "tags":["provisioner/d3c644f1-0f94-43e1-946f-e40c4694d703"], - # } - - put '/:cluster_id/ExpandCluster' do - flow = Tendrl::Flow.new('namespace.tendrl', 'ExpandCluster') - halt 404 if flow.nil? - body = JSON.parse(request.body.read) - missing_params = [] - - ['sds_name', 'Cluster.node_configuration'].each do |param| - missing_params << param unless body[param] and not body[param].empty? - end - halt 422, { errors: { missing: missing_params } }.to_json unless missing_params.empty? - - ['sds_name'].each do |param| - body["TendrlContext.#{param}"] = body.delete('sds_name') - end - - body['Node[]'] = body['Cluster.node_configuration'].keys - - job = Tendrl::Job.new( - current_user, - flow, - integration_id: params[:cluster_id] - ).create(body) - - status 202 - { job_id: job.job_id }.to_json - end - - private - - def detected_cluster_id(node_id) - Tendrl.etcd.get("/nodes/#{node_id}/DetectedCluster/detected_cluster_id").value - rescue Etcd::KeyNotFound - nil - end - - end diff --git a/app/controllers/objects_controller.rb b/app/controllers/objects_controller.rb deleted file mode 100644 index 3b6f9ab..0000000 --- a/app/controllers/objects_controller.rb +++ /dev/null @@ -1,25 +0,0 @@ -class ObjectsController < AuthenticatedUsersController - get '/objects/:type' do - type = params[:type].singularize.humanize - object_class = Tendrl.const_get type - presenter_class = Object.const_get("#{type}Presenter") - { params[:type] => presenter_class.list(object_class.all) }.to_json - end - - get '/objects/:type/:id' do - type = params[:type].singularize.humanize - object_class = Tendrl.const_get type - presenter_class = Object.const_get("#{type}Presenter") - { params[:type] => presenter_class.list(object_class.single) }.to_json - end - - get '/objects/:type/:id/:sub_object_type' do - type = params[:type].singularize.humanize - sub_object_type = params[:sub_object_type].singularize.humanize - object_class = Tendrl.const_get type - sub_object_class = Tendrl.const_get sub_object_type - list = sub_object_class.public_send "find_all_by_#{type.snake_case}_id" - presenter_class = Object.const_get("#{sub_object_type}Presenter") - end - #post '/objects/:type/:object_id/:flow_name' -end diff --git a/app/models/brick.rb b/app/models/brick.rb deleted file mode 100644 index 36c1cd8..0000000 --- a/app/models/brick.rb +++ /dev/null @@ -1,51 +0,0 @@ -module Tendrl - class Brick - class << self - def find_all_by_cluster_id_and_node_fqdn(cluster_id, fqdn) - begin - Tendrl.etcd.get("/clusters/#{cluster_id}/Bricks/all/#{fqdn}", recursive: true) - .children.inject([]) do |bricks, node| - brick = Tendrl.recurse node - bricks + brick.values - end - rescue Etcd::KeyNotFound, Etcd::NotDir - {} - end - end - - def find_refs_by_cluster_id_and_volume_id(cluster_id, volume_id) - begin - Tendrl.etcd.get("/clusters/#{cluster_id}/Volumes/#{volume_id}/Bricks", recursive: true) - .children.inject({}) do |subvolume_refs, subvolume| - sv_paths = Tendrl.recurse(subvolume, {}, downcase_keys: false) - sv_paths.each { |name, paths| sv_paths[name] = paths.keys } - subvolume_refs.merge sv_paths - end - rescue Etcd::KeyNotFound, Etcd::NotDir - end - end - - def find_by_cluster_id_and_refs(cluster_id, sub_volumes) - sub_volumes.collect_concat do |sub_volume, paths| - paths.collect_concat do |path| - path = path.sub(':_', '/') - begin - path_brick = Tendrl.recurse( - Tendrl.etcd.get( - "/clusters/#{cluster_id}/Bricks/all/#{path}", - recursive: true - ) - ) - path_brick.each_value do |attrs| - attrs['subvolume'] = sub_volume - end - path_brick.values - rescue Etcd::KeyNotFound, Etcd::NotDir - [] - end - end - end - end - end - end -end diff --git a/app/models/node.rb b/app/models/node.rb deleted file mode 100644 index 009ee17..0000000 --- a/app/models/node.rb +++ /dev/null @@ -1,68 +0,0 @@ -module Tendrl - class Node - - class << self - - # TODO: The interface sucks. Don't like making people use .nil? here and - # .exist? in the actual object. - def self.find_by_ip(ip) - begin - uuid = Tendrl.etcd.get("/indexes/ip/#{ip}").value - rescue ::Etcd::KeyNotFound - return nil - end - new(uuid) - end - - def all - Tendrl.etcd.get('/nodes').children.map do |node| - begin - nodecontext = Tendrl.recurse(Tendrl.etcd.get("#{node.key}/NodeContext")) - tendrlcontext = Tendrl.recurse(Tendrl.etcd.get("#{node.key}/TendrlContext")) - counters = Tendrl.recurse(Tendrl.etcd.get("#{node.key}/alert_counters")) - node_key = node.key.split('/')[-1] - { node_key => nodecontext.merge(tendrlcontext).merge(counters) } - rescue Etcd::KeyNotFound, Etcd::NotDir - end - end.compact - rescue Etcd::KeyNotFound, Etcd::NotDir - [] - end - - def find_all_by_cluster_id(cluster_id) - begin - tendrlcontext = Tendrl.recurse(Tendrl.etcd.get("/clusters/#{cluster_id}/TendrlContext")) - Tendrl.etcd.get("/clusters/#{cluster_id}/nodes", recursive: true) - .children.map do |node| - node = Tendrl.recurse(node) - node.values.first.merge!(tendrlcontext) - node - end - rescue Etcd::KeyNotFound, Etcd::NotDir - [] - end - end - - def find_by_cluster_id(cluster_id, node_id) - Tendrl.recurse(Tendrl.etcd.get("/clusters/#{cluster_id}/nodes/#{node_id}/NodeContext"))['nodecontext'] - rescue Etcd::KeyNotFound - end - end - - attr_reader :uuid - - def initialize(uuid) - @uuid = uuid - @exists = true - begin - @path = Tendrl.etcd.get("/nodes/#{uuid}") - rescue ::Etcd::KeyNotFound - @exists = false - end - end - - def exist? - @exists - end - end -end diff --git a/app/models/volume.rb b/app/models/volume.rb deleted file mode 100644 index 3b43865..0000000 --- a/app/models/volume.rb +++ /dev/null @@ -1,18 +0,0 @@ -module Tendrl - class Volume - class << self - - def find_all_by_cluster_id(cluster_id) - begin - Tendrl.etcd.get("/clusters/#{cluster_id}/Volumes", recursive: true) - .children.map do |node| - Tendrl.recurse(node) - end - rescue Etcd::KeyNotFound, Etcd::NotDir - [] - end - end - - end - end -end diff --git a/app/presenters/brick_presenter.rb b/app/presenters/brick_presenter.rb deleted file mode 100644 index 99959ab..0000000 --- a/app/presenters/brick_presenter.rb +++ /dev/null @@ -1,12 +0,0 @@ -module BrickPresenter - class << self - def list(bricks) - bricks.map do |attributes| - %w[devices partitions pv].each do |key| - attributes[key] = JSON.parse(attributes[key] || '[]') - end - attributes - end - end - end -end diff --git a/app/presenters/cluster_presenter.rb b/app/presenters/cluster_presenter.rb deleted file mode 100644 index b248516..0000000 --- a/app/presenters/cluster_presenter.rb +++ /dev/null @@ -1,40 +0,0 @@ -module ClusterPresenter - - class << self - - def list(cluster_list) - clusters = [] - cluster_list.each do |cluster| - clusters << single(cluster) - end - clusters.compact - end - - def single(cluster_attributes) - attributes = cluster_attributes.values[0] - attributes = attributes.merge!('cluster_id' => cluster_attributes.keys[0]) - attributes['errors'] = JSON.parse(attributes['errors']) rescue [] - if bricks = attributes.delete('bricks') - attributes.merge!(bricks: bricks(bricks)) - end - attributes - end - - def bricks(bricks) - all = bricks.delete('all') || {} - free = {} - used = {} - if all.present? - all.each do |device, attributes| - if bricks['free'].present? && bricks['free'].keys.include?(device) - free[device] = attributes - elsif bricks['used'].present? && bricks['used'].keys.include?(device) - used[device] = attributes - end - end - end - { all: all, used: used, free: free } - end - - end -end diff --git a/app/presenters/node_presenter.rb b/app/presenters/node_presenter.rb deleted file mode 100644 index 3058fcb..0000000 --- a/app/presenters/node_presenter.rb +++ /dev/null @@ -1,21 +0,0 @@ -module NodePresenter - class << self - def list(nodes_list) - nodes = [] - nodes_list.each do |node| - node.each do |_, attributes| - attributes.slice!('nodecontext','tendrlcontext','alert_counters') - node_attr = attributes.delete('nodecontext') - next if node_attr.blank? - node_attr['tags'] = JSON.parse(node_attr['tags']) rescue [] - node_attr['status'] ||= 'DOWN' - if cluster = attributes.delete('tendrlcontext') - cluster.delete('node_id') - end - nodes << node_attr.merge(attributes).merge(cluster: (cluster || {})) - end - end - nodes - end - end -end diff --git a/app/presenters/volume_presenter.rb b/app/presenters/volume_presenter.rb deleted file mode 100644 index 4995e9a..0000000 --- a/app/presenters/volume_presenter.rb +++ /dev/null @@ -1,16 +0,0 @@ -module VolumePresenter - class << self - def list(raw_volumes) - volumes = [] - raw_volumes.each do |volume| - volume.each do |vol_id, attributes| - attributes['vol_id'] = vol_id - attributes.delete('bricks') - attributes.delete('options') - volumes << attributes - end - end - volumes - end - end -end diff --git a/config.ru b/config.ru index 7c6cff2..58d0039 100644 --- a/config.ru +++ b/config.ru @@ -12,20 +12,14 @@ if ENV['ENABLE_PROFILING'] use Rack::RubyProf, path: path end -map('/1.0') { +map('/1.0') do use PingController use SessionsController use AlertingController use NotificationsController use JobsController use UsersController - use NodesController use ClustersController use AuthenticatedUsersController run ApplicationController -} - -#map('/2.0') do - #use ObjectsController - #run ApplicationController -#end +end diff --git a/lib/tendrl.rb b/lib/tendrl.rb index 985eb27..6160ce4 100644 --- a/lib/tendrl.rb +++ b/lib/tendrl.rb @@ -137,44 +137,31 @@ def unmarshall!(attrs) # Tendrl core require './lib/tendrl/version' -require './lib/tendrl/flow' -require './lib/tendrl/object' -require './lib/tendrl/atom' -require './lib/tendrl/attribute' require './lib/tendrl/http_response_error_handler' require './lib/gd2_client' # Models require './app/models/user' -require './app/models/node' require './app/models/cluster' require './app/models/alert' require './app/models/notification' require './app/models/job' -require './app/models/volume' -require './app/models/brick' # Forms require './app/forms/user_form' # Presenters -require './app/presenters/node_presenter' -require './app/presenters/cluster_presenter' require './app/presenters/job_presenter' require './app/presenters/user_presenter' -require './app/presenters/volume_presenter' -require './app/presenters/brick_presenter' require './app/presenters/notification_presenter' # Errors require './lib/tendrl/errors/tendrl_error' -require './lib/tendrl/errors/invalid_object_error' # Contollers require './app/controllers/application_controller' require './app/controllers/ping_controller' require './app/controllers/authenticated_users_controller' -require './app/controllers/nodes_controller' require './app/controllers/clusters_controller' require './app/controllers/jobs_controller' require './app/controllers/users_controller' diff --git a/lib/tendrl/atom.rb b/lib/tendrl/atom.rb deleted file mode 100644 index 78551c2..0000000 --- a/lib/tendrl/atom.rb +++ /dev/null @@ -1,8 +0,0 @@ -module Tendrl - class Atom - def initialize(type, values) - @type = type - @values = values - end - end -end diff --git a/lib/tendrl/attribute.rb b/lib/tendrl/attribute.rb deleted file mode 100644 index 454f066..0000000 --- a/lib/tendrl/attribute.rb +++ /dev/null @@ -1,25 +0,0 @@ -module Tendrl - class Attribute - attr_reader :name, :help, :type, :default, :required - - def initialize(object_type, name, attributes) - attributes ||= {} - @object_type = object_type - @name = name - @help = attributes['help'] - @type = attributes['type'] - @default = attributes['default'] - @required = nil - end - - def to_hash - { - name: "#{@object_type}.#{@name}", - help: @help, - type: @type, - default: @default, - required: @required - } - end - end -end diff --git a/lib/tendrl/errors/invalid_object_error.rb b/lib/tendrl/errors/invalid_object_error.rb deleted file mode 100644 index 1bb713d..0000000 --- a/lib/tendrl/errors/invalid_object_error.rb +++ /dev/null @@ -1,4 +0,0 @@ -module Tendrl - class InvalidObjectError < TendrlError - end -end diff --git a/lib/tendrl/flow.rb b/lib/tendrl/flow.rb deleted file mode 100644 index 110d1aa..0000000 --- a/lib/tendrl/flow.rb +++ /dev/null @@ -1,171 +0,0 @@ -module Tendrl - class Flow - METHOD_MAPPING = { - 'create' => 'POST', - 'update' => 'PUT', - 'delete' => 'DELETE', - 'action' => 'GET' - }.freeze - - attr_reader :namespace, :flow_name - - def initialize(namespace, flow_name, object = nil) - @instance = Tendrl.current_definitions - @namespace = namespace - @flow_name = flow_name - @object = object - @objects = @instance[namespace]['objects'] - flows = @instance[namespace]['flows'] || {} - @flow = flows[flow_name] || @objects[object]['flows'][flow_name] - end - - def objects - @objects.keys.map do |object_name| - Object.new(namespace, object_name) - end - end - - def sds_name - if @namespace.end_with?('gluster') - 'gluster' - elsif @namespace.end_with?('ceph') - 'ceph' - end - end - - def name - "#{sds_name.to_s.capitalize}#{@flow_name}" - end - - def tags(context) - tags = [] - return tags unless @flow['tags'] - @flow['tags'].each do |tag| - finalized_tag = [] - placeholders = tag.split('/') - placeholders.each do |placeholder| - finalized_tag << if placeholder.start_with?('$') - context[placeholder[1..-1]] - else - placeholder - end - end - tags << finalized_tag.join('/') - end - tags - end - - def reference_attributes - attributes = [] - objects.each do |obj| - attributes << obj.attributes - end - attributes - end - - def attributes - mandatory_attributes + optional_attributes - end - - def type - @flow['type'].downcase - end - - def run - @flow['run'] - end - - def method - METHOD_MAPPING[type] || 'POST' - end - - def mandatory_attributes - flow_attributes = [] - mandatory_attributes = @flow['inputs']['mandatory'] || [] - mandatory_attributes.each do |ma| - next if ma.end_with?('cluster_id') - if ma.end_with?('[]') - flow_attributes << { name: ma, type: 'List', - required: true } - else - attribute = Object.find_by_attribute(ma) - attribute[:required] = true - flow_attributes << attribute - end - end - flow_attributes - end - - def optional_attributes - flow_attributes = [] - optional_attributes = @flow['inputs']['optional'] || [] - optional_attributes.each do |ma| - next if ma.end_with?('cluster_id') - if ma.end_with?('[]') - flow_attributes << { - name: ma, - type: 'List', - required: false - } - else - attribute = Object.find_by_attribute(ma) - attribute[:required] = false - flow_attributes << attribute - end - end - flow_attributes - end - - def self.find_all - flows = [] - Tendrl.current_definitions.keys.map do |key| - next unless [ - 'namespace.tendrl', - 'namespace.gluster', - 'namespace.ceph', - 'namespace.node_agent' - ].include?(key) - if Tendrl.current_definitions[key]['flows'] - Tendrl.current_definitions[key]['flows'].keys.each do |fk| - flow = Tendrl::Flow.new(key, fk) - flows << { - name: flow.name, - method: flow.method, - attributes: flow.attributes - } - end - end - Tendrl.current_definitions[key]['objects'].keys.each do |ok| - object_flows = Tendrl.current_definitions[key]['objects'][ok]['flows'] - next if object_flows.nil? - object_flows.keys.each do |fk| - flow = Tendrl::Flow.new(key, fk, ok) - flows << { - name: flow.name, - method: flow.method, - attributes: flow.attributes - } - end - end - end - flows - end - - def self.find_by_external_name_and_type(external_name, type) - if type == 'node' - namespace = 'namespace.tendrl' - flow = external_name - elsif type == 'cluster' - partial_namespace = 'namespace' - sds_parameters = external_name.underscore.split('_') - namespace = "#{partial_namespace}.#{sds_parameters[0].downcase}" - flow = sds_parameters[1..-1].join('_').to_s.camelize - object = sds_parameters[2].capitalize - elsif type == 'node_agent' - namespace = 'namespace.node_agent' - flow = external_name - end - new(namespace, flow, object) - end - end -end diff --git a/lib/tendrl/object.rb b/lib/tendrl/object.rb deleted file mode 100644 index b9d44a6..0000000 --- a/lib/tendrl/object.rb +++ /dev/null @@ -1,44 +0,0 @@ -module Tendrl - class Object - def initialize(namespace, type) - @config = Tendrl.current_definitions - @type = type - @namespace = namespace - @object = @config[namespace]['objects'][type] - end - - def attributes - @object['attrs'].map do |attr_name, values| - Attribute.new(@type, attr_name, values) - end - end - - def atoms - @object['atoms'].map do |atom_name, values| - Atom.new(atom_name, values) - end - end - - def self.find_by_object_name(object_name) - object = nil - Tendrl.current_definitions.keys.each do |key| - next if key == 'tendrl_schema_version' - if objects = Tendrl.current_definitions[key]['objects'] - objects = objects.keys - if objects.include?(object_name) - object = Object.new(key, object_name) - break - end - end - end - object - end - - def self.find_by_attribute(attribute) - object_name, attribute = attribute.split('.') - object = find_by_object_name(object_name) - attribute = object.attributes.find{ |a| a.name == attribute } - attribute.to_hash - end - end -end diff --git a/spec/controllers/nodes_controller_spec.rb b/spec/controllers/nodes_controller_spec.rb deleted file mode 100644 index 1fcecbd..0000000 --- a/spec/controllers/nodes_controller_spec.rb +++ /dev/null @@ -1,113 +0,0 @@ -require 'spec_helper' -require './app/controllers/nodes_controller' - -describe NodesController do - - let(:http_env){ - { - 'HTTP_AUTHORIZATION' => 'Bearer d03ebb195dbe6385a7caeda699f9930ff2e49f29c381ed82dc95aa642a7660b8', - 'CONTENT_TYPE' => 'application/json' - } - } - - before do - stub_user('dwarner') - stub_access_token('dwarner') - stub_definitions - end - - context 'create' do - - end - - # context 'list' do - # - # before do - # stub_nodes - # stub_clusters(false) - # end - # - # it 'nodes' do - # get "/nodes", {}, http_env - # expect(last_response.status).to eq 200 - # end - # - # end - - context 'node agent' do - - before do - stub_nodes - stub_job_creation - end - - it 'generate journal mapping' do - body = { - "Cluster.node_configuration" => { - "c573b8b8-2488-4db7-8033-27b9a468bce3" => { - "storage_disks" => [ - {"device" => "/dev/vdb", "size" => 189372825600, "ssd" => false}, - {"device" => "/dev/vdc", "size" => 80530636800, "ssd" => false}, - {"device" => "/dev/vdd", "size" => 107374182400, "ssd" => false}, - {"device" => "/dev/vde", "size" => 21474836480, "ssd" => false}, - {"device" => "/dev/vdf", "size" => 26843545600, "ssd" => false } - ] - } - } - } - post '/GenerateJournalMapping', body.to_json, http_env - expect(last_response.status). to eq 202 - end - - end - - context 'expand' do - - before do - stub_definitions - stub_job_creation - end - - it 'gluster cluster' do - body = { - 'sds_name' => 'gluster', - 'Cluster.node_configuration' => { - '867d6aae-fb98-4060-9f6a-1da4e4988db8' => { - 'role' => 'glusterfs/node', - 'provisioning_ip' => '0.0.0.0' - } - } - } - put '/6b4b84e0-17b3-4543-af9f-e42000c52bfc/ExpandCluster', body.to_json, http_env - expect(last_response.status).to eq 202 - end - - it 'ceph cluster' do - body = { - 'sds_name' => 'ceph', - 'Cluster.node_configuration' => { - '867d6aae-fb98-4060-9f6a-1da4e4988db8' => { - 'role' => 'ceph/mon', - 'provisioning_ip' => '0.0.0.0', - 'monitor_interface' => 'eth0' - }, - "d41d073f-db4e-4abf-8018-02b30254b912" => { - "role" => "ceph/osd", - "provisioning_ip" => "0.0.0.0", - "journal_size" => 5120, - "journal_colocation" => false, - "storage_disks" => [ - {"device" => "/dev/vdb", "journal" => "/dev/vdc"}, - {"device" => "/dev/vdd", "journal" => "/dev/vde"} - ] - } - } - } - put '/6b4b84e0-17b3-4543-af9f-e42000c52bfc/ExpandCluster', body.to_json, http_env - expect(last_response.status).to eq 202 - end - end - - -end - diff --git a/spec/tendrl/flow_spec.rb b/spec/tendrl/flow_spec.rb deleted file mode 100644 index a8169c0..0000000 --- a/spec/tendrl/flow_spec.rb +++ /dev/null @@ -1,67 +0,0 @@ -require 'spec_helper' - -RSpec.describe Tendrl::Flow do - context 'node' do - before do - Tendrl.node_definitions = YAML.load_file( - 'spec/fixtures/definitions/master.yaml' - ) - end - - it 'ImportCluster' do - flow = Tendrl::Flow.new( - 'namespace.tendrl', - 'ImportCluster' - ) - expect(flow.flow_name).to eq('ImportCluster') - expect(flow.tags({ 'TendrlContext.integration_id' => '12345'})).to eq(['tendrl/integration/12345']) - end - end - - context 'cluster' do - context 'gluster volume' do - before do - Tendrl.cluster_definitions = YAML.load_file( - 'spec/fixtures/definitions/gluster.yaml' - ) - end - - specify 'StartProfiling' do - flow = Tendrl::Flow.new( - 'namespace.gluster', - 'StartProfiling', - 'Volume' - ) - expect(flow.flow_name).to eq('StartProfiling') - expect(flow.tags({ 'TendrlContext.integration_id' => '12345'})).to eq(['provisioner/12345']) - end - - specify 'StopProfiling' do - flow = Tendrl::Flow.new( - 'namespace.gluster', - 'StopProfiling', - 'Volume' - ) - expect(flow.flow_name).to eq('StopProfiling') - expect(flow.tags({ 'TendrlContext.integration_id' => '12345'})).to eq(['provisioner/12345']) - end - end - - context 'ceph' do - before do - Tendrl.cluster_definitions = YAML.load_file( - 'spec/fixtures/definitions/ceph.yaml' - ) - end - - it 'CreatePool' do - flow = Tendrl::Flow.new( - 'namespace.ceph', - 'CreatePool' - ) - expect(flow.flow_name).to eq('CreatePool') - expect(flow.tags('TendrlContext.integration_id' => '12345')).to eq(['tendrl/integration/12345']) - end - end - end -end From 973a50274d87f45d7124d6921fda5c70a7294222 Mon Sep 17 00:00:00 2001 From: Shirshendu Mukherjee Date: Thu, 25 Oct 2018 11:03:46 +0530 Subject: [PATCH 5/5] Reliability fixes, some new APIs --- .ruby-version | 2 +- app/controllers/clusters_controller.rb | 57 +++++++++++++++++++---- app/models/cluster.rb | 45 +++++++++++++----- lib/gd2_client.rb | 7 ++- lib/tendrl/http_response_error_handler.rb | 6 ++- 5 files changed, 91 insertions(+), 26 deletions(-) diff --git a/.ruby-version b/.ruby-version index 503328d..633c00d 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -2.0.0-p598 +2.0.0-p648 diff --git a/app/controllers/clusters_controller.rb b/app/controllers/clusters_controller.rb index fd3770a..dfc88a5 100644 --- a/app/controllers/clusters_controller.rb +++ b/app/controllers/clusters_controller.rb @@ -1,8 +1,6 @@ class ClustersController < AuthenticatedUsersController get '/clusters' do - clusters = Tendrl::Cluster.all.map do |c| - c.gd2.statedump.to_h - end + clusters = Tendrl::Cluster.all { clusters: clusters }.to_json end @@ -18,7 +16,7 @@ class ClustersController < AuthenticatedUsersController end get '/clusters/:cluster_id' do - @cluster.gd2.statedump + @cluster.to_json end get '/clusters/:cluster_id/peers' do @@ -36,6 +34,14 @@ class ClustersController < AuthenticatedUsersController { volume: volume }.to_json end + post '/clusters/:cluster_id/volumes/:volname/start' do + @cluster.gd2.volume_start(params[:volname]).body + end + + post '/clusters/:cluster_id/volumes/:volname/stop' do + @cluster.gd2.volume_stop(params[:volname]).body + end + get '/clusters/:cluster_id/volumes/:volname/bricks' do bricks = @cluster.gd2.volume_bricks_status(params[:volname]).to_a { bricks: bricks }.to_json @@ -48,13 +54,48 @@ class ClustersController < AuthenticatedUsersController 'secret' => parsed_body['secret'] } gd2 = Gd2Client.new new_endpoint - state = gd2.statedump + halt 404, 'Invalid endpoint' unless gd2.ping? && gd2.generate_api_methods + state = gd2.statedump.to_h cluster = Tendrl::Cluster.new state['cluster-id'] - unless cluster.endpoints.include? new_endpoint - cluster.add_endpoint new_endpoint + unless cluster.data['endpoints'].include? new_endpoint + cluster.add_endpoint state['peer-id'], new_endpoint + end + cluster.add_short_name(parsed_body['short_name']) + begin + # TODO Publish webhook to GD2 as below: + # cluster.gd2.events_webhook_add(url: "http:///api/1.0/clusters/#{cluster.uuid}/event_webhook") + # TODO See below for event_webhook API. + rescue Tendrl::HttpResponseErrorHandler => e + details = e.body[:errors][:details] + raise e unless details.present? && details['errors'].first['code'].to_s == '1' end status 201 - state.merge(endpoints: cluster.endpoints).to_json + cluster.to_json + end + + # Server sent events for subscribing to glusterd2 events + # TODO Add https://github.com/remy/polyfills/blob/master/EventSource.js polyfill to UI + # TODO Use http://walterbm.github.io/blog/2015/10/07/sinatra-and-server-sent-events/ as a reference on how to consume this in UI + connections = [] + get '/clusters/:cluster_id/event_stream', provides: 'text/event-stream' do + stream(:keep_open) do |out| + connections << out + connections.reject!(&:closed?) + end + end + + # This API listens to events from this cluster (to be published to GD2 as part of import) + # TODO Publish events to interested clients that have connected to the event_stream API + # TODO This API should also call prometheus alertmanager APIs if necessary + # TODO Add metric scraping jobs to prometheus server config, so that gluster-prometheus data starts appearing in our prometheus instance + post '/clusters/:cluster_id/event_webhook' do + connections.each do |conn| + conn << '{"data": "foobar"}'+"\n\n" + end + end + + get '/clusters/:cluster_id/events' do + { events: @cluster.gd2.events_list.to_a }.to_json end #get '/clusters/:cluster_id/nodes/:node_id/bricks' do diff --git a/app/models/cluster.rb b/app/models/cluster.rb index 4509fab..5aba911 100644 --- a/app/models/cluster.rb +++ b/app/models/cluster.rb @@ -6,30 +6,53 @@ def initialize(cluster_id) @uuid = cluster_id end + def data + @data ||= Tendrl.recurse( + Tendrl.etcd.get( + "/clusters/#{@uuid}", recursive: true + ) + )[@uuid] rescue {'endpoints' => {}, 'short_name' => nil} + end + def endpoints - @endpoints ||= Tendrl.etcd.get( - "/clusters/#{@uuid}/endpoints", recursive: true - ).children.map(&:value).sort.uniq.map { |e| JSON.parse e } - rescue Etcd::KeyNotFound - [] + @endpoints ||= data['endpoints'].values.sort.uniq.map { |e| JSON.parse e } end def gd2 - @gd2 ||= endpoints.map { |e| Gd2Client.new e }.find(&:ping?) + @gd2 ||= endpoints.map { |e| Gd2Client.new e }.find(&:ping?).generate_api_methods + end + + def to_json(_ = nil) + gd2 + .statedump.to_h + .merge(endpoints: endpoints) + .merge(peers: @gd2.get_peers.to_a) + .merge(volumes: @gd2.volume_list.to_a) + .merge(short_name: short_name) + .to_json end - def to_json(_) - gd2.statedump.merge(endpoints: endpoints).to_json + def short_name + data['short_name'] end - def add_endpoint(endpoint) - Tendrl.etcd.create_in_order( - "/clusters/#{@uuid}/endpoints", + def add_endpoint(peer_id, endpoint) + Tendrl.etcd.set( + "/clusters/#{@uuid}/endpoints/#{peer_id}", value: endpoint.to_json ) + @data = nil @endpoints = nil end + def add_short_name(name) + Tendrl.etcd.set( + "/clusters/#{@uuid}/short_name", + value: name.present? ? name : uuid + ) + @data = nil + end + class << self def find(cluster_id) cluster = new(cluster_id) diff --git a/lib/gd2_client.rb b/lib/gd2_client.rb index 26ccbd8..c2c5302 100644 --- a/lib/gd2_client.rb +++ b/lib/gd2_client.rb @@ -8,7 +8,6 @@ def initialize(endpoint) @gd2_url = endpoint[:gd2_url] @user = endpoint[:user].present? ? endpoint[:user] : 'glustercli' @secret = endpoint[:secret] || File.read(File.join('var', 'lib', 'glusterd2', 'auth')) - generate_api_methods end def claim(method, path) @@ -35,7 +34,7 @@ def method_missing(m, *args, &block) def http_call(method, path, opts = {}) req_data = { headers: { 'Authorization' => 'Bearer ' + jwt_token(method, path) } } - req_data[:body] = args[-1].to_h if %w[put post patch].include?(method) && args[-1].respond_to(:to_h) + req_data[:body] = opts.to_json if %w[put post patch].include?(method) && opts.present? HTTParty.public_send( method, @gd2_url + path, @@ -54,9 +53,9 @@ def generate_api_methods self.class.send(:define_method, method_name) do |*args| path = api['path'].gsub(/{.*?}/).with_index { |_, i| args[i] } path = prefixed_path(path) - response = http_call(action, path, args[-1].present? && args[-1].respond_to?(:to_h) ? args[-1] : {}) + response = http_call(action, path, args[-1].present? && args[-1].respond_to?(:to_h) ? args[-1].to_h : {}) return response if response.success? - raise Tendrl::HttpResponseErrorHandler.new(response.body, cause: 'gd2_api_error', object_id: api['methods'] + path.to_s) + raise Tendrl::HttpResponseErrorHandler.new(response.to_h, cause: 'gd2_api_error', object_id: api['methods'] + path.to_s) end end self diff --git a/lib/tendrl/http_response_error_handler.rb b/lib/tendrl/http_response_error_handler.rb index 85e53db..c99d74a 100644 --- a/lib/tendrl/http_response_error_handler.rb +++ b/lib/tendrl/http_response_error_handler.rb @@ -71,7 +71,8 @@ class HttpResponseErrorHandler < StandardError status: 503, body: { errors: { - message: 'GD2 API Error at: %s' + message: 'GD2 API Error at: %s', + details: true } } }, @@ -85,7 +86,7 @@ class HttpResponseErrorHandler < StandardError } }.freeze - def initialize(error, cause: nil, object_id: nil) + def initialize(error, cause: nil, object_id: nil, details: nil) @error = error @cause = cause @object_id = object_id @@ -94,6 +95,7 @@ def initialize(error, cause: nil, object_id: nil) if @object_id @body[:errors][:message] = @body[:errors][:message] % [@object_id] end + @body[:errors][:details] = @error if @body[:errors][:details] @status = @mapping[:status] end