diff --git a/.gitignore b/.gitignore index cec3cb5..a6af7bb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.gem Gemfile.lock +*.swp diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..4e1e0d2 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--color diff --git a/bin/bunka b/bin/bunka index ccdf384..389ceac 100755 --- a/bin/bunka +++ b/bin/bunka @@ -1,20 +1,162 @@ #!/usr/bin/env ruby require 'bunka' -require 'rubygems' require 'thor' class BunkaCommand < Thor map '-t' => :test + desc 'test (-t) COMMAND [QUERY]', 'Execute command on nodes, scoped on' \ + ' the given query if query is given. Query syntax should be' \ + ' the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def test(command, query = 'name:*') + Bunka.test(command, query, options[:timeout], options[:'print-success'], + options[:invert], options[:sequential], options[:threads], + options[:'from-file']) + end + + map '-s' => :serverspec + desc 'serverspec (-s) SERVERSPECFILE [QUERY]', 'Test nodes with serverspec,' \ + ' scoped on the given query if query is given. Query syntax' \ + ' should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :processes, type: :numeric, desc: 'number of processes', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def serverspec(serverspecfile, query = 'name:*') + Bunka.testserverspec(serverspecfile, query, options[:timeout], + options[:'print-success'], options[:invert], + options[:sequential], options[:processes], + options[:'from-file']) + end + + map '-f' => :file + desc 'file (-f) PATH [QUERY]', 'Test existence of a file' \ + ' on nodes, scoped on the given query if query is given.' \ + ' Query syntax should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def file(path, query = 'name:*') + Bunka.findfile(path, query, options[:timeout], options[:'print-success'], + options[:invert], options[:sequential], options[:threads], + options[:'from-file']) + end + + map '-d' => :dir + desc 'dir (-d) PATH [QUERY]', 'Test existence of a directory' \ + ' on nodes, scoped on the given query if query is given.' \ + ' Query syntax should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def dir(path, query = 'name:*') + Bunka.finddir(path, query, options[:timeout], options[:'print-success'], + options[:invert], options[:sequential], options[:threads], + options[:'from-file']) + end + + map '-md5' => :md5sum + desc 'md5sum (-md5) PATH MD5SUM [QUERY]', 'Compare a MD5sum with a' \ + ' MD5sum of a file on nodes, scoped on the given query if query' \ + ' is given. Query syntax should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def md5sum(path, checksum, query = 'name:*') + Bunka.md5sum(path, checksum, query, options[:timeout], + options[:'print-success'], options[:invert], + options[:sequential], options[:threads], + options[:'from-file']) + end + + map '-p' => :port + desc 'port (-p) PORTNUMBER [QUERY]', 'Test if specific port is listening' \ + ' on nodes, scoped on the given query if query is given.' \ + ' Query syntax should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def port(port, query = 'name:*') + Bunka.port(port, query, options[:timeout], options[:'print-success'], + options[:invert], options[:sequential], options[:threads], + options[:'from-file']) + end - desc 'test COMMAND [QUERY]', 'Execute command on nodes, scoped on the given query if query is given. Query syntax should be the same as `knife search` syntax.' - option :sequential, type: :boolean, desc: 'run over nodes sequantially', default: false - option :invert, type: :boolean, desc: 'invert matched results', default: false - option :timeout, type: :numeric, desc: 'timeout interval per ssh connection (default: 15)', default: 15 - option :threads, type: :numeric, desc: 'number of threads (default: 15)', default: 15 - option :'print-success', type: :boolean, desc: 'prints output of successful commands', default: false - option :'from-file', type: :string, desc: 'path to file with list of servers', default: nil - def test(command, query='name:*') - Bunka.test(command, query, options[:timeout], options[:'print-success'], options[:invert], options[:sequential], options[:threads], options[:'from-file']) + map '-s' => :service + desc 'service (-s) NAME [QUERY]', 'Test if specific service is running' \ + ' on nodes, scoped on the given query if query is given.' \ + ' Query syntax should be the same as `knife search` syntax.' + option :sequential, type: :boolean, desc: 'run over nodes sequantially', + default: false + option :invert, type: :boolean, desc: 'invert matched results', + default: false + option :timeout, type: :numeric, desc: 'timeout interval per ssh connection', + default: 15 + option :threads, type: :numeric, desc: 'number of threads', + default: 15 + option :'print-success', type: :boolean, desc: 'prints output of' \ + 'successful commands', default: false + option :'from-file', type: :string, desc: 'path to file with list of' \ + 'servers', default: nil + def service(name, query = 'name:*') + Bunka.service(name, query, options[:timeout], options[:'print-success'], + options[:invert], options[:sequential], options[:threads], + options[:'from-file']) end end diff --git a/bunka.gemspec b/bunka.gemspec index b2ffe19..282fb79 100644 --- a/bunka.gemspec +++ b/bunka.gemspec @@ -3,11 +3,13 @@ Gem::Specification.new do |spec| spec.version = '1.4.0' spec.executables << 'bunka' spec.date = '2013-11-26' - spec.summary = 'Parallel ssh commands over chef servers with rspec-like output' - spec.description = 'A gem to perform command over parallel ssh connections on multiple chef serverspec. Output is rspec-like.' - spec.authors = ['Steven De Coeyer', 'Jeroen Jacobs'] + spec.summary = 'Parallel ssh commands over' \ + 'chef servers with rspec-like output' + spec.description = 'A gem to perform command over parallel ssh' \ + 'connections on multiple chef serverspec. Output is rspec-like.' + spec.authors = ['Steven De Coeyer', 'Jeroen Jacobs', 'Giel De Bleser'] spec.email = 'tech@openminds.be' - spec.files = `git ls-files`.split($\) + spec.files = `git ls-files`.split($OUTPUT_RECORD_SEPARATOR) spec.homepage = 'https://github.com/openminds/bunka' spec.license = 'MIT' @@ -15,6 +17,5 @@ Gem::Specification.new do |spec| spec.add_dependency 'colorize' spec.add_dependency 'net-ssh' spec.add_dependency 'parallel' - spec.add_dependency 'rake' spec.add_dependency 'thor' end diff --git a/lib/bunka.rb b/lib/bunka.rb index 57a6b9b..1fc259e 100755 --- a/lib/bunka.rb +++ b/lib/bunka.rb @@ -1,14 +1,14 @@ require 'parallel' - require 'bunka/bunka' require 'bunka/chef' -require 'bunka/helpers' -require 'bunka/printers' require 'bunka/ssh' +require 'bunka/serverspec' +require 'bunka/socket' class Bunka class << self - def test command, query, timeout_interval, verbose_success, invert, sequential, threads, file=nil + def test(command, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) @command = command @invert = invert @query = query @@ -16,13 +16,97 @@ def test command, query, timeout_interval, verbose_success, invert, sequential, @threads = sequential ? 1 : threads @timeout_interval = timeout_interval @verbose_success = verbose_success - @file = file + @file = file ? File.expand_path(file) : nil + parallel_exec + end - Parallel.map(nodes, in_threads: @threads) do |fqdn| - execute_query fqdn - end + def testserverspec(serverspecfile, query, timeout_interval, + verbose_success, invert, sequential, + processes, file) + @serverspecfile = File.expand_path(serverspecfile) + @query = query + @invert = invert + @sequential = sequential + @processes = sequential ? 1 : processes + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + @failedarray = [] + @successarray = [] + @timeoutarray = [] + socket_delete + start = Time.now + create_sockets + serverspecsetup print_summary + socket_delete + puts Time.now - start + end + + def findfile(path, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) + @command = "test -f '#{path}'" + @invert = invert + @query = query + @sequential = sequential + @threads = sequential ? 1 : threads + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + parallel_exec + end + + def finddir(path, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) + @command = "test -d '#{path}'" + @invert = invert + @query = query + @sequential = sequential + @threads = sequential ? 1 : threads + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + parallel_exec + end + + def md5sum(path, checksum, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) + @command = "md5sum -c - <<<'#{checksum} #{path}'" + @invert = invert + @query = query + @sequential = sequential + @threads = sequential ? 1 : threads + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + parallel_exec + end + + def port(port, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) + @command = "netstat -anp |grep ':#{port}'" + @invert = invert + @query = query + @sequential = sequential + @threads = sequential ? 1 : threads + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + parallel_exec + end + + def service(name, query, timeout_interval, verbose_success, + invert, sequential, threads, file = nil) + @command = "service #{name} status" + @invert = invert + @query = query + @sequential = sequential + @threads = sequential ? 1 : threads + @timeout_interval = timeout_interval + @verbose_success = verbose_success + @file = file ? File.expand_path(file) : nil + parallel_exec end end end diff --git a/lib/bunka/bunka.rb b/lib/bunka/bunka.rb index 0e249c3..eabd9a7 100644 --- a/lib/bunka/bunka.rb +++ b/lib/bunka/bunka.rb @@ -2,6 +2,15 @@ class Bunka class << self + def parallel_exec + start = Time.now + Parallel.map(nodes, in_threads: @threads) do |fqdn| + execute_query fqdn + end + print_summary + puts Time.now - start + end + def nodes if @file File.readlines(@file).collect(&:strip) @@ -10,22 +19,21 @@ def nodes end end - def execute_query fqdn - begin - timeout @timeout_interval do - Net::SSH.start(fqdn, 'root', paranoid: false, forward_agent: true) do |ssh| - output = ssh_exec!(ssh, @command) - parse_output output, fqdn - end + def execute_query(fqdn) + timeout @timeout_interval do + Net::SSH.start(fqdn, 'root', paranoid: false, + forward_agent: true) do |ssh| + output = ssh_exec!(ssh, @command) + parse_output output, fqdn end - rescue TimeoutError, Errno::ETIMEDOUT, SocketError, Errno::EHOSTUNREACH => e - timed_out "#{fqdn}: #{e.message}" - rescue Exception => e - timed_out "#{fqdn}: #{e.inspect}" end + rescue TimeoutError, Errno::ETIMEDOUT, SocketError, Errno::EHOSTUNREACH => e + timed_out "#{fqdn}: #{e.message}" + rescue Exception => e + timed_out "#{fqdn}: #{e.inspect}" end - def parse_output output, fqdn + def parse_output(output, fqdn) if output[2] == 0 && !invert? succeeded "#{fqdn}: #{output[0]}" elsif output[2] != 0 && invert? diff --git a/lib/bunka/chef.rb b/lib/bunka/chef.rb index d8cc184..ebe4d55 100644 --- a/lib/bunka/chef.rb +++ b/lib/bunka/chef.rb @@ -2,7 +2,7 @@ class Bunka class << self - def knife_search query + def knife_search(query) # Monkey patch Chef::Knife::UI to hide stdout Chef::Knife::UI.class_eval do def stdout diff --git a/lib/bunka/helpers.rb b/lib/bunka/helpers.rb index 4183ff5..06083b4 100644 --- a/lib/bunka/helpers.rb +++ b/lib/bunka/helpers.rb @@ -18,15 +18,15 @@ def timed_out(reason) end def timeout_output_stream - @timeout_output_stream ||= Array.new + @timeout_output_stream ||= [] end def failed_output_stream - @failed_output_stream ||= Array.new + @failed_output_stream ||= [] end def success_output_stream - @success_output_stream ||= Array.new + @success_output_stream ||= [] end def verbose_success? @@ -36,5 +36,17 @@ def verbose_success? def invert? @invert end + + def failedspec + print_fail + end + + def successspec + print_success + end + + def timeoutspec + print_timeout + end end end diff --git a/lib/bunka/printers.rb b/lib/bunka/printers.rb index a2cbb7d..c7d7353 100644 --- a/lib/bunka/printers.rb +++ b/lib/bunka/printers.rb @@ -14,36 +14,105 @@ def print_timeout print '*'.yellow end + def print_failed_stream + puts "\nFailures: \n".red + failed_output_stream.each do |output| + puts output.red + end + end + def print_timeout_stream + puts "\nTimed out or unresolved nodes: \n".yellow timeout_output_stream.each do |output| puts output.yellow end end - def print_failed_stream - failed_output_stream.each do |output| + def print_success_stream + puts "\nSuccesses: \n".green + success_output_stream.each do |output| + puts output.green + end + end + + def print_spec_streams + @failedarray.reject! &:empty? + @successarray.reject! &:empty? + @failed = @failedarray.count + @success = @successarray.count - @failedarray.count + @timedout = @timeoutarray.count + @total = @failed + @success + @timedout + specinvert + verbose_success? ? print_successspec_stream : print_failedspec_stream + end + + def print_failedspec_stream + @failedarray.each do |output| puts output.red end end - def print_success_stream - success_output_stream.each do |output| + def print_successspec_stream + @successarray.each do |output| puts output.green end end + def print_timeoutspec_stream + @timeoutarray.each do |output| + puts output.yellow + end + end + + def specinvert + return unless invert? + @dummyarray, @failedarray = @failedarray, @successarray + @successarray = @dummyarray + @dummyint = @failed + @failed = @success + @success = @dummyint + end + def print_summary + @serverspecfile ? print_spec_output : print_output + puts "\n---------------------------------------\n" + @serverspecfile ? print_spec_counts : print_counts + end + + def print_output print "\n" print_timeout_stream print_failed_stream print_success_stream if verbose_success? + end - puts "\n---------------------------------------\n" + def print_spec_output + if !verbose_success? + puts "\n\nFailures: ".red + else + puts "\n\nSuccesses: ".green + end + print_spec_streams + return unless @timeoutarray.count > 0 + puts "\nTimed out or unresolved nodes: \n".yellow + print_timeoutspec_stream + end + + def print_counts + puts "#{'Success:'.green} #{success_output_stream.count}" + puts "#{'Timed out or does not resolve:'.yellow} " \ + "#{timeout_output_stream.count}" + puts "#{'Failed:'.red} #{failed_output_stream.count}" + puts "#{'Total:'.blue} #{success_output_stream.count + + timeout_output_stream.count + + failed_output_stream.count}" + end - puts "#{'Success'.green}: #{success_output_stream.count}" - puts "#{'Timed out or does not resolve'.yellow}: #{timeout_output_stream.count}" - puts "#{'Failed'.red}: #{failed_output_stream.count}" - puts "#{'Total'.blue}: #{success_output_stream.count + timeout_output_stream.count + failed_output_stream.count}" + def print_spec_counts + puts "#{'Success:'.green} " + @success.to_s + puts "#{'Timed out or does not resolve:'.yellow} " + @timedout.to_s + puts "#{'Failed:'.red} " + @failed.to_s + puts "#{'Total:'.blue} " + @total.to_s + "\n" end end end diff --git a/lib/bunka/serverspec.rb b/lib/bunka/serverspec.rb new file mode 100644 index 0000000..111fe0e --- /dev/null +++ b/lib/bunka/serverspec.rb @@ -0,0 +1,149 @@ +require 'rspec' +require 'bunka/helpers' +require 'socket' + +class Bunka + class << self + def serverspecsetup + file_existence + @hosts.each_slice(@processes).each do |h| + Parallel.map(h, in_processes: @processes) do |hostx| + ENV['TARGET_HOST'] = hostx + @hostx = hostx + rspecrunner + RSpec.clear_examples + testresults_to_sockets unless @timedoutbool == true + end + end + end + + def file_existence + check_serverfile_existence + check_serverspecfile_existence + end + + def rspecrunner + rspec_config + config + formatter + # create reporter with json formatter + reporter + config.instance_variable_set(:@reporter, reporter) + # internal hack + # api may not be stable, make sure lock down Rspec version + loader + notifications + reporter.register_listener(formatter, *notifications) + run_tests + hash + end + + def hash + @hash ||= formatter.output_hash + end + + def config + @config ||= RSpec.configuration + end + + def formatter + @formatter ||= + RSpec::Core::Formatters::JsonFormatter.new(config.output_stream) + end + + def reporter + @reporter ||= RSpec::Core::Reporter.new(config) + end + + def loader + @loader ||= config.send(:formatter_loader) + end + + def notifications + @notifications ||= loader.send( + :notifications_for, + RSpec::Core::Formatters::JsonFormatter + ) + end + + def run_tests + timeout @timeout_interval do + Net::SSH.start(@hostx, 'root') + end + RSpec::Core::Runner.run([@serverspecfile]) + rescue TimeoutError, Errno::ETIMEDOUT, SocketError, Errno::EHOSTUNREACH + timeout_to_socket + rescue RuntimeError + puts 'Serverspec failed' + exit + end + + def timeout_to_socket + timeoutspec + @timedoutbool = true + fill_timeout_array + end + + def check_serverfile_existence + if @file + if File.exist?(@file) + @hosts = File.readlines(@file).each &:chomp! + else + puts 'Serverfile not found' + exit + end + elsif @query + @hosts = knife_search @query + end + end + + def check_serverspecfile_existence + return unless File.exist?(@serverspecfile) == false + puts 'Serverspecfile not found' + exit + end + + def testresults_to_sockets + @failed_sock = UNIXSocket.open('/tmp/failed_sock') + @success_sock = UNIXSocket.open('/tmp/success_sock') + @failstatus = false + @successstatus = false + loop_testresults_in_sockets + @failed_sock.close + @success_sock.close + check_invert + end + + def loop_testresults_in_sockets + hash[:examples].each do |x| + if x[:status] == 'failed' + @failstatus = true + @failed_sock.write("\n" + @hostx + ': ' + x[:full_description]) + elsif x[:status] == 'passed' + @success_sock.write("\n" + @hostx + ': ' + x[:full_description]) + end + end + end + + def check_invert + if !invert? && @failstatus == true + failedspec + elsif invert? && @failstatus == false + failedspec + else + successspec + end + end + + def fill_timeout_array + timeout_socket = UNIXSocket.open('/tmp/timeout_sock') + timeout_socket.write(' - ' + @hostx) + end + + def rspec_config + RSpec.configure do |c| + c.output_stream = nil + end + end + end +end diff --git a/lib/bunka/socket.rb b/lib/bunka/socket.rb new file mode 100644 index 0000000..bd7b4b7 --- /dev/null +++ b/lib/bunka/socket.rb @@ -0,0 +1,56 @@ +require 'socket' + +class Bunka + class << self + def create_failed_unix_socket + failed_server = UNIXServer.new('/tmp/failed_sock') # Socket to listen + loop do + Thread.start(failed_server.accept) do |client| + testresult = client.read + @failedarray.push testresult + client.close + end + end + end + + def create_success_unix_socket + success_server = UNIXServer.new('/tmp/success_sock') # Socket to listen + loop do + Thread.start(success_server.accept) do |client| + testresult = client.read + @successarray.push testresult + client.close + end + end + end + + def create_timeout_unix_socket + timeout_server = UNIXServer.new('/tmp/timeout_sock') # Socket to listen + loop do + Thread.start(timeout_server.accept) do |client| + testresult = client.read + @timeoutarray.push testresult + client.close + end + end + end + + def socket_delete + File.delete('/tmp/failed_sock') if File.exist?('/tmp/failed_sock') + File.delete('/tmp/success_sock') if File.exist?('/tmp/success_sock') + File.delete('/tmp/timeout_sock') if File.exist?('/tmp/timeout_sock') + end + + def create_sockets + Thread.new do + create_failed_unix_socket + end + Thread.new do + create_success_unix_socket + end + Thread.new do + create_timeout_unix_socket + end + end + end +end diff --git a/lib/bunka/ssh.rb b/lib/bunka/ssh.rb index 8f378a3..93cc418 100644 --- a/lib/bunka/ssh.rb +++ b/lib/bunka/ssh.rb @@ -8,19 +8,19 @@ def ssh_exec!(ssh, command) exit_code = nil ssh.open_channel do |channel| - channel.exec(command) do |ch, success| + channel.exec(command) do |_ch, success| unless success abort "FAILED: couldn't execute command (ssh.channel.exec)" end - channel.on_data do |ch,data| - stdout_data+=data + channel.on_data do |_ch, data| + stdout_data += data end - channel.on_extended_data do |ch,type,data| - stderr_data+=data + channel.on_extended_data do |_ch, _type, data| + stderr_data += data end - channel.on_request('exit-status') do |ch,data| + channel.on_request('exit-status') do |_ch, data| exit_code = data.read_long end end