From d5bbde8c91db3e1739d9269464fca6dfcf2869e7 Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 1 Jul 2013 16:24:28 -0700 Subject: [PATCH 01/69] s/ssl_ca_path/ssl_ca_file --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 040c5cb..1e000c9 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ This module is puppet 3 tested Installation, make sure service is running and will be started at boot time: class { 'lumberjack': - host => 'logstashhost', - port => '1234', - files => ['/var/log/messages', '/var/log/secure'], - ssl_ca_path => "puppet:///path/to/ca.crt", + host => 'logstashhost', + port => '1234', + files => ['/var/log/messages', '/var/log/secure'], + ssl_ca_file => "puppet:///path/to/ca.crt", } Removal/decommissioning: From 30e3fc703bc98ff262911b9dc5a8c9b75d76d823 Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 1 Jul 2013 16:24:55 -0700 Subject: [PATCH 02/69] syntax cleanup (no functionality changes) --- manifests/instance.pp | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index c5a3de3..a85a7d7 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -57,9 +57,9 @@ require lumberjack File { - owner => 'root', - group => 'root', - mode => '0644' + owner => root, + group => root, + mode => 0644 } if ($run_as_service == true ) { @@ -86,7 +86,7 @@ file { "/etc/init.d/lumberjack-${name}": ensure => $ensure, - mode => '0755', + mode => 0755, content => template("${module_name}/etc/init.d/lumberjack.erb"), notify => $notify_lumberjack } @@ -145,22 +145,19 @@ } } else { - $notify_lumberjack = undef - } - - file { "/etc/lumberjack/${name}": - ensure => directory, - } + file { + "/etc/lumberjack/${name}": + ensure => directory; # Setup certificate files - file { "/etc/lumberjack/${name}/ca.crt": - ensure => $ensure, - source => $ssl_ca_file, - require => File[ "/etc/lumberjack/${name}" ], - notify => $notify_lumberjack + "/etc/lumberjack/${name}/ca.crt": + ensure => $ensure, + source => $ssl_ca_file, + require => File[ "/etc/lumberjack/${name}" ], + notify => $notify_lumberjack } } From 7ebd13e2afe4b6e4fc9cb6d552f9f23b739a40ec Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 1 Jul 2013 16:43:28 -0700 Subject: [PATCH 03/69] no depending on other modules! bad! --- manifests/instance.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index a85a7d7..082a30f 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -51,7 +51,7 @@ $files = undef, $fields = false, $run_as_service = true, - $ensure = $logstash::ensure + $ensure = present ) { require lumberjack From 5f59a37d952a38f3422e4bdafacff8607594b9e1 Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 1 Jul 2013 16:44:17 -0700 Subject: [PATCH 04/69] must create the directory before we start making subdirectories (dependency inheritance is automatic) --- manifests/instance.pp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manifests/instance.pp b/manifests/instance.pp index 082a30f..c4accc6 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -149,6 +149,9 @@ } file { + "/etc/lumberjack": + ensure => directory; + "/etc/lumberjack/${name}": ensure => directory; From d09cad03969e834c37c975bada6a1e9d6b58e0ba Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 1 Jul 2013 17:02:06 -0700 Subject: [PATCH 05/69] correct references -- bad use of generic service names / patterns, must be -${name} --- manifests/instance.pp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index c4accc6..0c18a7a 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -138,10 +138,11 @@ service { "lumberjack-${name}": ensure => $service_ensure, enable => $service_enable, - name => $lumberjack::params::service_name, + #name => $lumberjack::params::service_name, hasstatus => $lumberjack::params::service_hasstatus, hasrestart => $lumberjack::params::service_hasrestart, - pattern => $lumberjack::params::service_pattern, + #pattern => $lumberjack::params::service_pattern, + pattern => "lumberjack-${name}"; } } else { From b46dcb7abac00aba84a1bec7aed52fc58b5616cf Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 2 Jul 2013 09:49:40 -0700 Subject: [PATCH 06/69] change default behavior to restart on init file change --- manifests/params.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/params.pp b/manifests/params.pp index 0c6e751..172ede9 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -41,7 +41,7 @@ $status = 'enabled' # Restart service on change - $restart_on_change = false + $restart_on_change = true #### Internal module values From 76f8823e9af3420443cda3167611d5bc0130cbf9 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 2 Jul 2013 09:50:10 -0700 Subject: [PATCH 07/69] incorrect variable reference fixed --- templates/etc/init.d/lumberjack.erb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/etc/init.d/lumberjack.erb b/templates/etc/init.d/lumberjack.erb index 0d15f67..7530a57 100644 --- a/templates/etc/init.d/lumberjack.erb +++ b/templates/etc/init.d/lumberjack.erb @@ -14,10 +14,10 @@ end DAEMON_PATH="/opt/lumberjack/" DAEMON="bin/lumberjack.sh" -DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @instance %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>" +DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>" -NAME="lumberjack-<%= @instance %>" -DESC="Lumberjack-<%= @instance %>" +NAME="lumberjack-<%= @name %>" +DESC="Lumberjack-<%= @name %>" PIDFILE=/var/run/$NAME.pid SCRIPTNAME=/etc/init.d/$NAME From 5dfe783033e0758804628f97a58c8954692c6ef6 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 2 Jul 2013 10:25:48 -0700 Subject: [PATCH 08/69] wrap /etc/lumberjack creation in an if (!defined...) to allow multiple calls to the lumberjack::instance definition --- manifests/instance.pp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 0c18a7a..2e788e8 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -149,10 +149,14 @@ $notify_lumberjack = undef } - file { - "/etc/lumberjack": - ensure => directory; + if (!defined(File['/etc/lumberjack'])) { + file { + "/etc/lumberjack": + ensure => directory; + } + } + file { "/etc/lumberjack/${name}": ensure => directory; From 6710f2238c77feff40ca579686f8db9367e696ab Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 2 Jul 2013 14:30:48 -0700 Subject: [PATCH 09/69] add file req before service is launched --- manifests/instance.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 2e788e8..035412c 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -138,11 +138,11 @@ service { "lumberjack-${name}": ensure => $service_ensure, enable => $service_enable, - #name => $lumberjack::params::service_name, hasstatus => $lumberjack::params::service_hasstatus, hasrestart => $lumberjack::params::service_hasrestart, #pattern => $lumberjack::params::service_pattern, - pattern => "lumberjack-${name}"; + pattern => "lumberjack-${name}", + require => File["/etc/lumberjack/${name}"]; } } else { From de2f5cab92a8746da6f0038bc50108d0cfe117aa Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 6 Aug 2013 17:03:33 -0700 Subject: [PATCH 10/69] add daemontools run templates --- templates/log/run.erb | 4 ++++ templates/run.erb | 11 +++++++++++ 2 files changed, 15 insertions(+) create mode 100644 templates/log/run.erb create mode 100644 templates/run.erb diff --git a/templates/log/run.erb b/templates/log/run.erb new file mode 100644 index 0000000..24b218d --- /dev/null +++ b/templates/log/run.erb @@ -0,0 +1,4 @@ +#!/bin/sh +[ -d ./main ] || mkdir ./main +chown -Rf <%= @loguser %> . +exec setuidgid <%= @loguser %> multilog t s5000000 ./main diff --git a/templates/run.erb b/templates/run.erb new file mode 100644 index 0000000..b68e5c0 --- /dev/null +++ b/templates/run.erb @@ -0,0 +1,11 @@ +<%- +fields = @fields +if fields + extra_fields = fields.collect { |k,v| "--field #{k}=#{v}" }.join(" ") +else + extra_fields = '' +end +-%> +#!/bin/sh +exec 2>&1 +exec /opt/lumberjack/bin/lumberjack.sh --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %> From fbeda29c0898a06be8fb42286d16eab7743f6eda Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 6 Aug 2013 17:06:47 -0700 Subject: [PATCH 11/69] add support for daemontools instead of crappy, crappy init.d --- manifests/instance.pp | 34 +++++++++++++++++++++++++++++----- spec/defines/instance_spec.rb | 4 ++-- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 035412c..7742281 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -50,8 +50,8 @@ $port = undef, $files = undef, $fields = false, - $run_as_service = true, - $ensure = present + $ensure = present, + $provider = "init.d", ) { require lumberjack @@ -62,8 +62,7 @@ mode => 0644 } - if ($run_as_service == true ) { - + if ($provider != false ) { # Input validation validate_string($host) @@ -77,6 +76,31 @@ if $fields { validate_hash($fields) } + } + + if ($provider == "daemontools" ) { + + file { "/etc/init.d/lumberjack-${name}": ensure => absent; } + service { "lumberjack-${name}": ensure => stopped, enable => false; } + + $user = root + $loguser = root + + daemontools::setup{ + "lumberjack/${name}": + user => $user, ## Needed to read some log files. Sorry. + loguser => $loguser, + run => template("${module_name}/run.erb"), + logrun => template("${module_name}/log/run.erb"); + } + + daemontools::service { + "lumberjack-${name}": + source => "/etc/lumberjack/${name}", + require => Daemontools::Setup["lumberjack/${name}"]; + } + } + elsif ($provider == "init.d" ) { # Setup init file if running as a service $notify_lumberjack = $lumberjack::restart_on_change ? { @@ -96,7 +120,7 @@ # set params: in operation if $lumberjack::ensure == 'present' { - case $lumberjack::status { + case $lumberjack::params::status { # make sure service is currently running, start it on boot 'enabled': { $service_ensure = 'running' diff --git a/spec/defines/instance_spec.rb b/spec/defines/instance_spec.rb index d22594e..1533ce3 100644 --- a/spec/defines/instance_spec.rb +++ b/spec/defines/instance_spec.rb @@ -14,7 +14,7 @@ :port => 1234, :files => [ '/var/log/file1', '/var/log/file2' ], :fields => { 'field1' => 'value1', 'field2' => 'value2' }, - :run_as_service => true + :provider => "init.d" } end it { should contain_file('/etc/init.d/lumberjack-foo') } @@ -27,7 +27,7 @@ let :params do { :ssl_ca_file => 'puppet:///path/to/ca.crt', - :run_as_service => false + :provider => "none" } end it { should_not contain_file('/etc/init.d/lumberjack-foo') } From 5ef8acfd79a93c2a70c6c731d096b29204002c00 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 8 Aug 2013 16:55:46 -0700 Subject: [PATCH 12/69] override type field --- manifests/instance.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/instance.pp b/manifests/instance.pp index 7742281..228f554 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -85,6 +85,7 @@ $user = root $loguser = root + $extra_fields = ["type=${name}"] daemontools::setup{ "lumberjack/${name}": From 722b913430c5a13e892d2196036ae55c51879299 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 8 Aug 2013 17:04:05 -0700 Subject: [PATCH 13/69] override type field (correctly this time, I hope) --- manifests/instance.pp | 1 - 1 file changed, 1 deletion(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 228f554..7742281 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -85,7 +85,6 @@ $user = root $loguser = root - $extra_fields = ["type=${name}"] daemontools::setup{ "lumberjack/${name}": From 807e804b1fee5166f547c14e5b53531fa8da4fdf Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 8 Aug 2013 17:06:37 -0700 Subject: [PATCH 14/69] override type field (correctly this time, I hope, take 2) --- manifests/instance.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 7742281..924e0d7 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -49,7 +49,7 @@ $host = undef, $port = undef, $files = undef, - $fields = false, + $fields = { type => $name }, $ensure = present, $provider = "init.d", ) { From 81d5bdf898c9af96363e3240ad12ff224012c277 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 8 Aug 2013 17:19:03 -0700 Subject: [PATCH 15/69] notify the Daemontools service when the run script changes (duh) --- manifests/instance.pp | 3 ++- templates/run.erb | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 924e0d7..69fee68 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -91,7 +91,8 @@ user => $user, ## Needed to read some log files. Sorry. loguser => $loguser, run => template("${module_name}/run.erb"), - logrun => template("${module_name}/log/run.erb"); + logrun => template("${module_name}/log/run.erb"), + notify => Daemontools::Service["lumberjack-${name}"]; } daemontools::service { diff --git a/templates/run.erb b/templates/run.erb index b68e5c0..21619a5 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -8,4 +8,4 @@ end -%> #!/bin/sh exec 2>&1 -exec /opt/lumberjack/bin/lumberjack.sh --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %> +exec /opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> <%= @logfiles %> From c6ac396236508a35077be96d5b97cbe2c44217eb Mon Sep 17 00:00:00 2001 From: jv Date: Fri, 9 Aug 2013 16:19:13 -0700 Subject: [PATCH 16/69] refactor this a little to correct a syntax problem --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 21619a5..749e615 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,7 +1,7 @@ <%- fields = @fields if fields - extra_fields = fields.collect { |k,v| "--field #{k}=#{v}" }.join(" ") + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") else extra_fields = '' end From 1f3d00b4775a1aa6b2e6cabf38f6c72ecbb31d8e Mon Sep 17 00:00:00 2001 From: jv Date: Fri, 9 Aug 2013 16:22:03 -0700 Subject: [PATCH 17/69] attempting single-quoting to fix '=' being eaten --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 749e615..051f5d8 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,7 +1,7 @@ <%- fields = @fields if fields - extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") + extra_fields = fields.map { |k,v| "--field '" + k + "=" + v + "'" }.join(" ") else extra_fields = '' end From 93648348f64995c9933255888bbfc7b9f10b199e Mon Sep 17 00:00:00 2001 From: jv Date: Fri, 9 Aug 2013 16:28:26 -0700 Subject: [PATCH 18/69] revert --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 051f5d8..749e615 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,7 +1,7 @@ <%- fields = @fields if fields - extra_fields = fields.map { |k,v| "--field '" + k + "=" + v + "'" }.join(" ") + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") else extra_fields = '' end From b0daa39c9ecc650f8da8c5528827f9427cd5faa4 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Fri, 9 Aug 2013 16:51:36 -0700 Subject: [PATCH 19/69] using tail -F as an interface between log files and lumberjack. this should allow lumberjack to work with rotating files and track multiple files correctly --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 749e615..687a372 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -8,4 +8,4 @@ end -%> #!/bin/sh exec 2>&1 -exec /opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> <%= @logfiles %> +exec tail -F <%= @logfiles %> | /opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> - From d3db2d32d397a31c431873364a1d36cc2e019d53 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Mon, 12 Aug 2013 13:18:06 -0700 Subject: [PATCH 20/69] splitting tail and lumberjack into run and log/run to avoid orphaned processes --- templates/log/run.erb | 14 +++++++++++--- templates/run.erb | 11 ++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/templates/log/run.erb b/templates/log/run.erb index 24b218d..59dd42d 100644 --- a/templates/log/run.erb +++ b/templates/log/run.erb @@ -1,4 +1,12 @@ +<%- +fields = @fields +if fields + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") +else + extra_fields = '' +end +-%> #!/bin/sh -[ -d ./main ] || mkdir ./main -chown -Rf <%= @loguser %> . -exec setuidgid <%= @loguser %> multilog t s5000000 ./main +exec 2>&1 +/opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> - > ./main/current + diff --git a/templates/run.erb b/templates/run.erb index 687a372..3177318 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,11 +1,4 @@ -<%- -fields = @fields -if fields - extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") -else - extra_fields = '' -end --%> #!/bin/sh exec 2>&1 -exec tail -F <%= @logfiles %> | /opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> - +exec tail -F <%= @logfiles %> + From 2e0e603013358fea08b7432046a9d791b206455a Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 12 Aug 2013 15:18:23 -0700 Subject: [PATCH 21/69] add new lumberjack config file (not yet functional) --- templates/lumberjack.config.erb | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 templates/lumberjack.config.erb diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb new file mode 100644 index 0000000..2e01a8e --- /dev/null +++ b/templates/lumberjack.config.erb @@ -0,0 +1,52 @@ +<%- +fields = @fields +if fields + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") +else + extra_fields = '' +end +-%>{ + # The network section covers network configuration :) + "network": { + # A list of downstream servers listening for our messages. + # lumberjack will pick one at random and only switch if + # the selected one appears to be dead or unresponsive + "servers": [ "<%= @host %>:<%= @port %>" ], + + # The path to your client ssl certificate (optional) + #"ssl certificate": "./lumberjack.crt", + # The path to your client ssl key (optional) + "ssl key": "./lumberjack.key", + + # The path to your trusted ssl CA file. This is used + # to authenticate your downstream server. + "ssl ca": "./lumberjack_ca.crt" + }, + + # The list of files configurations + "files": [ + # An array of hashes. Each hash tells what paths to watch and + # what fields to annotate on events from those paths. + { + "paths": [ + # single paths are fine + "/var/log/messages", + # globs are fine too, they will be periodically evaluated + # to see if any new files match the wildcard. + "/var/log/*.log" + ], + + # A dictionary of fields to annotate on each event. + "fields": { "type": "syslog" } + }, { + # A path of "-" means stdin. + "paths": [ "-" ], + "fields": { "type": "stdin" } + }, { + "paths": [ + "/var/log/apache/httpd-*.log" + ], + "fields": { "type:" "apache" } + } + ] +} From 2b550dc88853a6c07095ba74c947649e6a4e57c2 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Mon, 12 Aug 2013 17:43:54 -0700 Subject: [PATCH 22/69] overhauling lumberjack module to use latest version --- manifests/instance.pp | 52 +++++------ manifests/watcher.pp | 93 +++++++++++++++++++ templates/log/run.erb | 14 +-- .../lumberjack.config.json-filepiece.erb | 14 +++ .../lumberjack.config.json-footerpiece.erb | 2 + .../lumberjack.config.json-headerpiece.erb | 6 ++ templates/run.erb | 3 +- 7 files changed, 141 insertions(+), 43 deletions(-) create mode 100644 manifests/watcher.pp create mode 100644 templates/lumberjack.config.json-filepiece.erb create mode 100644 templates/lumberjack.config.json-footerpiece.erb create mode 100644 templates/lumberjack.config.json-headerpiece.erb diff --git a/manifests/instance.pp b/manifests/instance.pp index 69fee68..9b744b2 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -16,24 +16,12 @@ # Default value: undef # This variable is optional # -# [*files*] -# Array of files you wish to process -# Value type is array -# Default value: undef -# This variable is optional -# # [*ssl_ca_file*] # File to use for the SSL CA # Value type is string # This variable is mandatory # -# [*fields*] -# Extra fields to send -# Value type is hash -# Default value: false -# This variable is optional -# -# [*run_as_service*] +# [*provider*] # Set this to true if you want to run this as a service. # Set to false if you only want to manage the ssl_ca_file # Value type is boolean @@ -43,14 +31,12 @@ # === Authors # # * Richard Pijnenburg +# * Modified by Jeff Vier # define lumberjack::instance( $ssl_ca_file, $host = undef, $port = undef, - $files = undef, - $fields = { type => $name }, - $ensure = present, $provider = "init.d", ) { @@ -69,13 +55,6 @@ if ! is_numeric($port) { fail("\"${port}\" is not a valid port parameter value") } - - validate_array($files) - $logfiles = join($files,' ') - - if $fields { - validate_hash($fields) - } } if ($provider == "daemontools" ) { @@ -167,30 +146,43 @@ hasrestart => $lumberjack::params::service_hasrestart, #pattern => $lumberjack::params::service_pattern, pattern => "lumberjack-${name}", - require => File["/etc/lumberjack/${name}"]; + require => File["/etc/lumberjack"]; } } else { $notify_lumberjack = undef } - if (!defined(File['/etc/lumberjack'])) { + if (!defined(File["/etc/lumberjack"])) { file { "/etc/lumberjack": ensure => directory; } } + if (!defined(File["/etc/lumberjack/${name}"])) { + file { + "/etc/lumberjack/${name}": + ensure => directory; + } + } file { - "/etc/lumberjack/${name}": - ensure => directory; + "/etc/lumberjack/${name}/pieces/header": + ensure => $ensure, + mode => 0644, + content => template("${module_name}/lumberjack.config.json-headerpiece.erb"), + notify => $notify_lumberjack; + + "/etc/lumberjack/${name}/pieces/footer": + ensure => $ensure, + mode => 0644, + content => template("${module_name}/lumberjack.config.json-footerpiece.erb"), + notify => $notify_lumberjack; - # Setup certificate files "/etc/lumberjack/${name}/ca.crt": ensure => $ensure, source => $ssl_ca_file, - require => File[ "/etc/lumberjack/${name}" ], - notify => $notify_lumberjack + notify => $notify_lumberjack; } } diff --git a/manifests/watcher.pp b/manifests/watcher.pp new file mode 100644 index 0000000..872b87a --- /dev/null +++ b/manifests/watcher.pp @@ -0,0 +1,93 @@ +# Define: lumberjack::instance +# +# This define allows you to setup an instance of lumberjack +# +# === Parameters +# +# [*host*] +# Host name or IP address of the Logstash instance to connect to +# Value type is string +# Default value: undef +# This variable is optional +# +# [*port*] +# Port number of the Logstash instance to connect to +# Value type is number +# Default value: undef +# This variable is optional +# +# [*files*] +# Array of files you wish to process +# Value type is array +# Default value: undef +# This variable is optional +# +# [*ssl_ca_file*] +# File to use for the SSL CA +# Value type is string +# This variable is mandatory +# +# [*fields*] +# Extra fields to send +# Value type is hash +# Default value: false +# This variable is optional +# +# [*run_as_service*] +# Set this to true if you want to run this as a service. +# Set to false if you only want to manage the ssl_ca_file +# Value type is boolean +# Default value: true +# This variable is optional +# +# === Authors +# +# * Richard Pijnenburg +# * Modified by Jeff Vier +# +define lumberjack::watcher( + $part_of, + $files = undef, + $fields = { type => $name }, + $ensure = present, +) { + + require lumberjack + + File { + owner => root, + group => root, + mode => 0644 + } + + if ($provider == present ) { + validate_array($files) + $logfiles = join($files,' ') + + if $fields { + validate_hash($fields) + } + } + + if (!defined(File["/etc/lumberjack/${part_of}"])) { + file { + "/etc/lumberjack/${part_of}": + ensure => directory; + } + } + if (!defined(File["/etc/lumberjack/${part_of}/pieces"])) { + file { + "/etc/lumberjack/${part_of}/pieces": + ensure => directory; + } + } + + file { + "/etc/lumberjack/${part_of}/pieces/filepiece-${name}": + ensure => $ensure, + mode => 0644, + content => template("${module_name}/lumberjack.config.json-filepiece.erb"), + notify => $notify_lumberjack; + } + +} diff --git a/templates/log/run.erb b/templates/log/run.erb index 59dd42d..24b218d 100644 --- a/templates/log/run.erb +++ b/templates/log/run.erb @@ -1,12 +1,4 @@ -<%- -fields = @fields -if fields - extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") -else - extra_fields = '' -end --%> #!/bin/sh -exec 2>&1 -/opt/lumberjack/bin/lumberjack.sh --host <%= @host %> --port <%= @port %> --ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt <%= extra_fields %> - > ./main/current - +[ -d ./main ] || mkdir ./main +chown -Rf <%= @loguser %> . +exec setuidgid <%= @loguser %> multilog t s5000000 ./main diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb new file mode 100644 index 0000000..9bb88ef --- /dev/null +++ b/templates/lumberjack.config.json-filepiece.erb @@ -0,0 +1,14 @@ +<%- +fields = @fields +if fields + extra_fields = fields.map { |k,v| "\"" + k + "\": \"" + v +"\""}.join(", ") +else + extra_fields = '' +end +-%> + { + "paths": [ + "<%= @files %>", + ], + "fields": { <%= @extra_fields %> } + } diff --git a/templates/lumberjack.config.json-footerpiece.erb b/templates/lumberjack.config.json-footerpiece.erb new file mode 100644 index 0000000..4b45020 --- /dev/null +++ b/templates/lumberjack.config.json-footerpiece.erb @@ -0,0 +1,2 @@ + ] +} diff --git a/templates/lumberjack.config.json-headerpiece.erb b/templates/lumberjack.config.json-headerpiece.erb new file mode 100644 index 0000000..21a4130 --- /dev/null +++ b/templates/lumberjack.config.json-headerpiece.erb @@ -0,0 +1,6 @@ +{ + "network": { + "servers": [ "<%= @host %>:<%= @port %>" ], + "ssl ca": "/etc/lumberjack/<%= @name %>/ca.crt" + }, + "files": [ diff --git a/templates/run.erb b/templates/run.erb index 3177318..1c501f9 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,4 +1,3 @@ #!/bin/sh exec 2>&1 -exec tail -F <%= @logfiles %> - +exec /opt/lumberjack/bin/lumberjack.sh -config /etc/lumberjack/<%= @name %>/config.json From 22336b4a864364b5e622eedeb922c29d960eb118 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:52:19 -0700 Subject: [PATCH 23/69] make files arg required --- manifests/watcher.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 872b87a..e87fd78 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -47,8 +47,8 @@ # define lumberjack::watcher( $part_of, - $files = undef, - $fields = { type => $name }, + $files, + $fields = { type => $name, }, $ensure = present, ) { From fa138bd0620b7a222732ca1a6438cb2b7550b472 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:52:57 -0700 Subject: [PATCH 24/69] add the auto-joining of config pieces into a functional config --- templates/run.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/run.erb b/templates/run.erb index 1c501f9..0d6fe90 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -1,3 +1,7 @@ #!/bin/sh exec 2>&1 +cd /etc/lumberjack/<%= @name %>/pieces +cat header > ../config.json +for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json +cat footer >> ../config.json exec /opt/lumberjack/bin/lumberjack.sh -config /etc/lumberjack/<%= @name %>/config.json From 6af39221cf5fee0dc30369b4191b6053bf94b584 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:53:23 -0700 Subject: [PATCH 25/69] fix an errant comma --- templates/lumberjack.config.json-filepiece.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index 9bb88ef..6a720c7 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -1,14 +1,14 @@ <%- fields = @fields if fields - extra_fields = fields.map { |k,v| "\"" + k + "\": \"" + v +"\""}.join(", ") + extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",") else extra_fields = '' end -%> { "paths": [ - "<%= @files %>", + "<%= @files %>" ], "fields": { <%= @extra_fields %> } } From ece424aa19601a0a4e5aec7c8d3221a028c89f63 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:55:03 -0700 Subject: [PATCH 26/69] whitespace --- manifests/watcher.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index e87fd78..cd64962 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -48,8 +48,8 @@ define lumberjack::watcher( $part_of, $files, - $fields = { type => $name, }, - $ensure = present, + $fields = { type => $name, }, + $ensure = present, ) { require lumberjack From b712a36daf0115d677462bb5e53f805a9dbf53b3 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:57:51 -0700 Subject: [PATCH 27/69] split file paths by space --- templates/lumberjack.config.json-filepiece.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index 6a720c7..619c887 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -8,7 +8,7 @@ end -%> { "paths": [ - "<%= @files %>" + "<%= @files.join(' ') %>" ], "fields": { <%= @extra_fields %> } } From 0fc6f33d447d2141c628e370ae75b2ccd19bd25e Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 10:58:00 -0700 Subject: [PATCH 28/69] drop redundant mode --- manifests/instance.pp | 2 -- manifests/watcher.pp | 1 - 2 files changed, 3 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 9b744b2..bd362be 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -169,13 +169,11 @@ file { "/etc/lumberjack/${name}/pieces/header": ensure => $ensure, - mode => 0644, content => template("${module_name}/lumberjack.config.json-headerpiece.erb"), notify => $notify_lumberjack; "/etc/lumberjack/${name}/pieces/footer": ensure => $ensure, - mode => 0644, content => template("${module_name}/lumberjack.config.json-footerpiece.erb"), notify => $notify_lumberjack; diff --git a/manifests/watcher.pp b/manifests/watcher.pp index cd64962..bd89d0a 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -85,7 +85,6 @@ file { "/etc/lumberjack/${part_of}/pieces/filepiece-${name}": ensure => $ensure, - mode => 0644, content => template("${module_name}/lumberjack.config.json-filepiece.erb"), notify => $notify_lumberjack; } From b2a36eca59d70f2a3e4f0f9d0b008deed9e984a9 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 11:02:06 -0700 Subject: [PATCH 29/69] require the parent instance before adding file parts --- manifests/watcher.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index bd89d0a..efa60a9 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -86,6 +86,7 @@ "/etc/lumberjack/${part_of}/pieces/filepiece-${name}": ensure => $ensure, content => template("${module_name}/lumberjack.config.json-filepiece.erb"), + require => Lumberjack::Instance["${part_of}"], notify => $notify_lumberjack; } From 532bb2564b731de40e7c100ddf0d4cb46404cd61 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 13:05:45 -0700 Subject: [PATCH 30/69] make provider a param variable --- manifests/init.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/manifests/init.pp b/manifests/init.pp index 7c63d7a..c075958 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -97,6 +97,7 @@ $autoupgrade = $lumberjack::params::autoupgrade, $status = $lumberjack::params::status, $restart_on_change = $lumberjack::params::restart_on_change, + $provider = $lumberjack::params::provider, $version = false, ) inherits lumberjack::params { From a80734706582abbfed4633b42ffd469d547c9fde Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 13:06:11 -0700 Subject: [PATCH 31/69] split service from instance --- manifests/instance.pp | 109 ++--------------------------- manifests/service.pp | 156 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 102 deletions(-) create mode 100644 manifests/service.pp diff --git a/manifests/instance.pp b/manifests/instance.pp index bd362be..dab837d 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -37,7 +37,7 @@ $ssl_ca_file, $host = undef, $port = undef, - $provider = "init.d", + $provider = "daemontools", ) { require lumberjack @@ -49,108 +49,13 @@ } if ($provider != false ) { - # Input validation - validate_string($host) - - if ! is_numeric($port) { - fail("\"${port}\" is not a valid port parameter value") - } - } - - if ($provider == "daemontools" ) { - - file { "/etc/init.d/lumberjack-${name}": ensure => absent; } - service { "lumberjack-${name}": ensure => stopped, enable => false; } - - $user = root - $loguser = root - - daemontools::setup{ - "lumberjack/${name}": - user => $user, ## Needed to read some log files. Sorry. - loguser => $loguser, - run => template("${module_name}/run.erb"), - logrun => template("${module_name}/log/run.erb"), - notify => Daemontools::Service["lumberjack-${name}"]; - } - - daemontools::service { - "lumberjack-${name}": - source => "/etc/lumberjack/${name}", - require => Daemontools::Setup["lumberjack/${name}"]; - } - } - elsif ($provider == "init.d" ) { - - # Setup init file if running as a service - $notify_lumberjack = $lumberjack::restart_on_change ? { - true => Service["lumberjack-${name}"], - false => undef, + lumberjack::service{ + $name: + ssl_ca_file => $ssl_ca_file, + host => $host, + port => $port, + provider => $provider; } - - file { "/etc/init.d/lumberjack-${name}": - ensure => $ensure, - mode => 0755, - content => template("${module_name}/etc/init.d/lumberjack.erb"), - notify => $notify_lumberjack - } - - #### Service management - - # set params: in operation - if $lumberjack::ensure == 'present' { - - case $lumberjack::params::status { - # make sure service is currently running, start it on boot - 'enabled': { - $service_ensure = 'running' - $service_enable = true - } - # make sure service is currently stopped, do not start it on boot - 'disabled': { - $service_ensure = 'stopped' - $service_enable = false - } - # make sure service is currently running, do not start it on boot - 'running': { - $service_ensure = 'running' - $service_enable = false - } - # do not start service on boot, do not care whether currently running or not - 'unmanaged': { - $service_ensure = undef - $service_enable = false - } - # unknown status - # note: don't forget to update the parameter check in init.pp if you - # add a new or change an existing status. - default: { - fail("\"${lumberjack::status}\" is an unknown service status value") - } - } - - # set params: removal - } else { - - # make sure the service is stopped and disabled (the removal itself will be - # done by package.pp) - $service_ensure = 'stopped' - $service_enable = false - } - - # action - service { "lumberjack-${name}": - ensure => $service_ensure, - enable => $service_enable, - hasstatus => $lumberjack::params::service_hasstatus, - hasrestart => $lumberjack::params::service_hasrestart, - #pattern => $lumberjack::params::service_pattern, - pattern => "lumberjack-${name}", - require => File["/etc/lumberjack"]; - } - - } else { - $notify_lumberjack = undef } if (!defined(File["/etc/lumberjack"])) { diff --git a/manifests/service.pp b/manifests/service.pp new file mode 100644 index 0000000..02f91e6 --- /dev/null +++ b/manifests/service.pp @@ -0,0 +1,156 @@ +# Define: lumberjack::service +# +# This define sets up the lumberjack service +# +# === Parameters +# +# [*host*] +# Host name or IP address of the Logstash instance to connect to +# Value type is string +# Default value: undef +# This variable is optional +# +# [*port*] +# Port number of the Logstash instance to connect to +# Value type is number +# Default value: undef +# This variable is optional +# +# [*ssl_ca_file*] +# File to use for the SSL CA +# Value type is string +# This variable is mandatory +# +# [*provider*] +# Set this to true if you want to run this as a service. +# Set to false if you only want to manage the ssl_ca_file +# Value type is boolean +# Default value: true +# This variable is optional +# +# === Authors +# +# * Richard Pijnenburg +# * Modified by Jeff Vier +# +define lumberjack::service( + $ssl_ca_file, + $host = undef, + $port = undef, + $provider = "daemontools", +) { + + require lumberjack + + File { + owner => root, + group => root, + mode => 0644 + } + + if ($provider != false ) { + # Input validation + validate_string($host) + + if ! is_numeric($port) { + fail("\"${port}\" is not a valid port parameter value") + } + + if ($provider == "daemontools" ) { + + file { "/etc/init.d/lumberjack-${name}": ensure => absent; } + service { "lumberjack-${name}": ensure => stopped, enable => false; } + + $user = root + $loguser = root + + daemontools::setup{ + "lumberjack/${name}": + user => $user, ## Needed to read some log files. Sorry. + loguser => $loguser, + run => template("${module_name}/run.erb"), + logrun => template("${module_name}/log/run.erb"), + notify => Daemontools::Service["lumberjack-${name}"]; + } + + daemontools::service { + "lumberjack-${name}": + source => "/etc/lumberjack/${name}", + require => Daemontools::Setup["lumberjack/${name}"]; + } + } + elsif ($provider == "init.d" ) { + + # Setup init file if running as a service + $notify_lumberjack = $lumberjack::restart_on_change ? { + true => Service["lumberjack-${name}"], + false => undef, + } + + file { "/etc/init.d/lumberjack-${name}": + ensure => $ensure, + mode => 0755, + content => template("${module_name}/etc/init.d/lumberjack.erb"), + notify => $notify_lumberjack + } + + #### Service management + + # set params: in operation + if $lumberjack::ensure == 'present' { + + case $lumberjack::params::status { + # make sure service is currently running, start it on boot + 'enabled': { + $service_ensure = 'running' + $service_enable = true + } + # make sure service is currently stopped, do not start it on boot + 'disabled': { + $service_ensure = 'stopped' + $service_enable = false + } + # make sure service is currently running, do not start it on boot + 'running': { + $service_ensure = 'running' + $service_enable = false + } + # do not start service on boot, do not care whether currently running or not + 'unmanaged': { + $service_ensure = undef + $service_enable = false + } + # unknown status + # note: don't forget to update the parameter check in init.pp if you + # add a new or change an existing status. + default: { + fail("\"${lumberjack::status}\" is an unknown service status value") + } + } + + # set params: removal + } else { + + # make sure the service is stopped and disabled (the removal itself will be + # done by package.pp) + $service_ensure = 'stopped' + $service_enable = false + } + + # action + service { "lumberjack-${name}": + ensure => $service_ensure, + enable => $service_enable, + hasstatus => $lumberjack::params::service_hasstatus, + hasrestart => $lumberjack::params::service_hasrestart, + #pattern => $lumberjack::params::service_pattern, + pattern => "lumberjack-${name}", + require => File["/etc/lumberjack"]; + } + + } else { + $notify_lumberjack = undef + } + } + +} From 1b632061859868143b270f2099acd66735967b3d Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 13:06:33 -0700 Subject: [PATCH 32/69] correct service name + add provider var --- manifests/params.pp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/manifests/params.pp b/manifests/params.pp index 172ede9..6b2292b 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -43,6 +43,8 @@ # Restart service on change $restart_on_change = true + $provider = 'daemontools' + #### Internal module values # packages @@ -64,13 +66,13 @@ # service parameters case $::operatingsystem { 'CentOS', 'Fedora', 'Scientific', 'OracleLinux', 'Amazon', 'RedHat': { - $service_name = 'lumberjack' + $service_name = "lumberjack-${name}" $service_hasrestart = true $service_hasstatus = true $service_pattern = $service_name } 'Debian', 'Ubuntu': { - $service_name = 'lumberjack' + $service_name = "lumberjack-${name}" $service_hasrestart = true $service_hasstatus = true $service_pattern = $service_name From e7f9a07f9a11bde66c1fb1441ddf1d0c2b56d2a0 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 13:07:02 -0700 Subject: [PATCH 33/69] changed filepiece config triggers service restart --- manifests/watcher.pp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index efa60a9..0eb5d4a 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -86,8 +86,7 @@ "/etc/lumberjack/${part_of}/pieces/filepiece-${name}": ensure => $ensure, content => template("${module_name}/lumberjack.config.json-filepiece.erb"), - require => Lumberjack::Instance["${part_of}"], - notify => $notify_lumberjack; + notify => Lumberjack::Service["${part_of}"]; } } From 221a01c50e255864846c8de5e149e301c3d7fb69 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Tue, 13 Aug 2013 13:07:30 -0700 Subject: [PATCH 34/69] some syntax tuning to make json look like what lumberjack wants --- templates/lumberjack.config.json-filepiece.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index 619c887..a755401 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -8,7 +8,7 @@ end -%> { "paths": [ - "<%= @files.join(' ') %>" + <%= @files.map {|f| '"' + f + '"'}.join(',') %> ], - "fields": { <%= @extra_fields %> } + "fields": { <%= extra_fields %> } } From 5cb25f7beb57e9b49aa6e0b39ef62b48b38de067 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 13 Aug 2013 14:17:43 -0700 Subject: [PATCH 35/69] trigger lumberjack to auto-upgrade --- manifests/params.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/params.pp b/manifests/params.pp index 6b2292b..c930c46 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -35,7 +35,7 @@ $ensure = 'present' # autoupgrade - $autoupgrade = false + $autoupgrade = true # service status $status = 'enabled' From bc1bbdb9527527018951e68fd5e38ef1e1562eee Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 13 Aug 2013 17:27:42 -0700 Subject: [PATCH 36/69] update readme example --- README.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1e000c9..9abfbcc 100644 --- a/README.md +++ b/README.md @@ -8,24 +8,33 @@ This module is puppet 3 tested ## Usage -Installation, make sure service is running and will be started at boot time: +### Set up a parent instance. This sets up the "framework" config to add individual file/file-group watchers that can be tagged with individual fields. - class { 'lumberjack': - host => 'logstashhost', - port => '1234', - files => ['/var/log/messages', '/var/log/secure'], - ssl_ca_file => "puppet:///path/to/ca.crt", - } + lumberjack::instance{ + 'core': + host => 'logstash.blah.com', + port => '5005', + ssl_ca_file => 'puppet:///modules/conf/etc/ssl/certs/ca.crt', + provider => 'daemontools'; + } -Removal/decommissioning: +### Add watchers that plug in to the intance you set up. + lumberjack::watcher { + 'syslog': + part_of => 'core', + files => ["/var/log/secure", "/var/log/messages", "/var/log/*.log", ]; - class { 'lumberjack': - ensure => 'absent', - } + 'statsd': + part_of => 'core', + files => ["/data/log/statsd/statsd.log", ]; -Install everything but disable service(s) afterwards: + 'diamond': + part_of => 'core', + files => ["/data/log/diamond/diamond.log", ]; + } + +### Removal/decommissioning: class { 'lumberjack': - status => 'disabled', + ensure => 'absent', } - From 0b6f569565ce2e0388e81a766845915075af3db9 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 13 Aug 2013 17:27:58 -0700 Subject: [PATCH 37/69] drop timestamp, since lumberjack self-stamps --- templates/log/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/log/run.erb b/templates/log/run.erb index 24b218d..9c981b7 100644 --- a/templates/log/run.erb +++ b/templates/log/run.erb @@ -1,4 +1,4 @@ #!/bin/sh [ -d ./main ] || mkdir ./main chown -Rf <%= @loguser %> . -exec setuidgid <%= @loguser %> multilog t s5000000 ./main +exec setuidgid <%= @loguser %> multilog s5000000 ./main From 3706cd27b660c4bacca67ea2ad391f5a66293506 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:22:25 -0700 Subject: [PATCH 38/69] lumberjack fields... if role is an array, join it into a space separated string --- .../lumberjack.config.json-filepiece.erb | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index a755401..e372bc5 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -1,14 +1,24 @@ <%- -fields = @fields -if fields - extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",") -else - extra_fields = '' +@fields ||= {'lumberjack_filepiece_default_key' => 'default_value'} + +if @fields.length < 1 + @fields = {'lumberjack_filepiece_default_key' => 'default_value'} end + +role = @fields['role'] +if role + if role.is_a? Array + @fields['role'] = role.join(' ') + end +end + +data = { + "paths" => @files, + "fields": extra_fields +} + +data_string = JSON.pretty_generate(data) + -%> - { - "paths": [ - <%= @files.map {|f| '"' + f + '"'}.join(',') %> - ], - "fields": { <%= extra_fields %> } - } +<%= data_string %> + From 742ab22ca5733836d6e600f9756d0b6686476039 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:26:00 -0700 Subject: [PATCH 39/69] syntax lumberjack.config.json-filepiece.erb --- templates/lumberjack.config.json-filepiece.erb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index e372bc5..a73d197 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -12,11 +12,7 @@ if role end end -data = { - "paths" => @files, - "fields": extra_fields -} - +data = {"paths" => @files, "fields" => extra_fields} data_string = JSON.pretty_generate(data) -%> From de60d35327e882cb01cefee432b5aac9599a2512 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:27:45 -0700 Subject: [PATCH 40/69] typo lumberjack.config.json-filepiece.erb --- templates/lumberjack.config.json-filepiece.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index a73d197..c5047ee 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -12,7 +12,7 @@ if role end end -data = {"paths" => @files, "fields" => extra_fields} +data = {"paths" => @files, "fields" => @fields} data_string = JSON.pretty_generate(data) -%> From 120bc2b27d0c6ae3939c1035547a7b497bdec89e Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:40:16 -0700 Subject: [PATCH 41/69] reverting lumberjack.config.json-filepiece.erb and converting fields[role] to a space separated string --- .../lumberjack.config.json-filepiece.erb | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index c5047ee..fa3680f 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -1,20 +1,22 @@ <%- -@fields ||= {'lumberjack_filepiece_default_key' => 'default_value'} +fields = @fields -if @fields.length < 1 - @fields = {'lumberjack_filepiece_default_key' => 'default_value'} -end - -role = @fields['role'] -if role - if role.is_a? Array - @fields['role'] = role.join(' ') +if fields + role = fields['role'] + if role + if role.is_a? Array + fields['role'] == role.join(' ') + end end -end - -data = {"paths" => @files, "fields" => @fields} -data_string = JSON.pretty_generate(data) + extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",") +else + extra_fields = '' +end -%> -<%= data_string %> - + { + "paths": [ + <%= @files.map {|f| '"' + f + '"'}.join(',') %> + ], + "fields": { <%= extra_fields %> } + } From bf65b6266cddeffe4710f0a95f7ec00891deef56 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:48:21 -0700 Subject: [PATCH 42/69] protecting lumberjack from field values that are arrays --- templates/lumberjack.config.json-filepiece.erb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index fa3680f..68d1eb5 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -2,11 +2,9 @@ fields = @fields if fields - role = fields['role'] - if role - if role.is_a? Array - fields['role'] == role.join(' ') - end + fields.each_pair do |k,v| + if v.is_a? Array + fields[k] = v.join(' ') end extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",") From 368d7e636e8c505ece384d5931a4e25baaca31ba Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Oct 2013 16:50:15 -0700 Subject: [PATCH 43/69] lumberjack.config.json-filepiece.erb syntax --- templates/lumberjack.config.json-filepiece.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index 68d1eb5..911d6f7 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -3,8 +3,7 @@ fields = @fields if fields fields.each_pair do |k,v| - if v.is_a? Array - fields[k] = v.join(' ') + fields[k] = v.join(' ') if v.is_a? Array end extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",") From be6759e4dd176be3eabb5fdbb90454f3f1b3d418 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 10 Oct 2013 17:05:12 -0700 Subject: [PATCH 44/69] push the field 'name' in if we don't set it explicitly --- manifests/watcher.pp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 0eb5d4a..0109d0c 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -60,13 +60,12 @@ mode => 0644 } - if ($provider == present ) { - validate_array($files) - $logfiles = join($files,' ') + validate_array($files) + $logfiles = join($files,' ') - if $fields { - validate_hash($fields) - } + validate_hash($fields) + if ! $fields['type'] { + $fields['type'] = $name } if (!defined(File["/etc/lumberjack/${part_of}"])) { From 0dc93991594abe4fcf6e517a0efee4e80f7a4448 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Thu, 10 Oct 2013 17:37:17 -0700 Subject: [PATCH 45/69] change field['type'] to field['watcher'] to reduce the overuse of 'type' --- manifests/watcher.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 0109d0c..8d7fcd9 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -48,7 +48,7 @@ define lumberjack::watcher( $part_of, $files, - $fields = { type => $name, }, + $fields = { watcher => $name, }, $ensure = present, ) { @@ -64,8 +64,8 @@ $logfiles = join($files,' ') validate_hash($fields) - if ! $fields['type'] { - $fields['type'] = $name + if ! $fields['watcher'] { + $fields['watcher'] = $name } if (!defined(File["/etc/lumberjack/${part_of}"])) { From 4c0cfca6d47362b3baf83e2acfb8fad9df347c8d Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Thu, 17 Oct 2013 15:34:00 -0700 Subject: [PATCH 46/69] just whitespace --- manifests/service.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/service.pp b/manifests/service.pp index 02f91e6..44a6f76 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -74,9 +74,9 @@ } daemontools::service { - "lumberjack-${name}": - source => "/etc/lumberjack/${name}", - require => Daemontools::Setup["lumberjack/${name}"]; + "lumberjack-${name}": + source => "/etc/lumberjack/${name}", + require => Daemontools::Setup["lumberjack/${name}"]; } } elsif ($provider == "init.d" ) { From 3b4a6e7959323b5db3dd2f923d457430ebda0b22 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 29 Oct 2013 14:45:02 -0700 Subject: [PATCH 47/69] sort fields so that they don't keep re-generating the config when they arrive in a different order --- templates/lumberjack.config.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index 2e01a8e..c043f2f 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -1,7 +1,7 @@ <%- fields = @fields if fields - extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") + extra_fields = fields.sort.map { |k,v| "--field " + k + "=" + v }.join(" ") else extra_fields = '' end From a1b8620b059ac0aff0908a7c06284131698f1e0a Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 29 Oct 2013 15:11:40 -0700 Subject: [PATCH 48/69] move the field sort earlier --- templates/lumberjack.config.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index c043f2f..6f2cb40 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -1,7 +1,7 @@ <%- -fields = @fields +fields = @fields.sort if fields - extra_fields = fields.sort.map { |k,v| "--field " + k + "=" + v }.join(" ") + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") else extra_fields = '' end From 9c21ff9f78baab8c24630ade341fcc96cd5a012d Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 29 Oct 2013 15:47:46 -0700 Subject: [PATCH 49/69] take 3 on the array sort --- templates/lumberjack.config.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index 6f2cb40..5b387a6 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -1,6 +1,7 @@ <%- -fields = @fields.sort +fields = @fields if fields + fileds.sort extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") else extra_fields = '' From 8df88b070929ef400f54e38664a5664fb25e0e19 Mon Sep 17 00:00:00 2001 From: jv Date: Tue, 29 Oct 2013 16:55:31 -0700 Subject: [PATCH 50/69] take 5 on sorting this damn fields hash --- templates/lumberjack.config.erb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index 5b387a6..b273db1 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -1,8 +1,7 @@ <%- fields = @fields if fields - fileds.sort - extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.join(" ") + extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.sort.join(" ") else extra_fields = '' end From 936be3f72d8256aeb66c9d7522b6822c3a90f4b5 Mon Sep 17 00:00:00 2001 From: jv Date: Mon, 11 Nov 2013 18:11:42 -0800 Subject: [PATCH 51/69] just whitespace --- templates/lumberjack.config.json-footerpiece.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lumberjack.config.json-footerpiece.erb b/templates/lumberjack.config.json-footerpiece.erb index 4b45020..858797f 100644 --- a/templates/lumberjack.config.json-footerpiece.erb +++ b/templates/lumberjack.config.json-footerpiece.erb @@ -1,2 +1,2 @@ ] -} +} \ No newline at end of file From 42b81bb79937841cb1206afc6b96b22ef246a5f0 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 21 Nov 2013 11:46:11 -0800 Subject: [PATCH 52/69] ending newline --- templates/lumberjack.config.json-footerpiece.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/lumberjack.config.json-footerpiece.erb b/templates/lumberjack.config.json-footerpiece.erb index 858797f..4b45020 100644 --- a/templates/lumberjack.config.json-footerpiece.erb +++ b/templates/lumberjack.config.json-footerpiece.erb @@ -1,2 +1,2 @@ ] -} \ No newline at end of file +} From c6b079784d8d9ba9b5f3024e226789661cfae12b Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 5 Dec 2013 13:55:40 -0800 Subject: [PATCH 53/69] pointing lumberjack to individual logstash servers --- README.md | 2 +- manifests/init.pp | 4 ++-- manifests/instance.pp | 12 +++++------- manifests/service.pp | 12 +++++------- manifests/watcher.pp | 8 +++----- spec/defines/instance_spec.rb | 2 +- templates/etc/init.d/lumberjack.erb | 2 +- templates/lumberjack.config.erb | 10 ++++++++-- templates/lumberjack.config.json-headerpiece.erb | 9 ++++++++- 9 files changed, 34 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 9abfbcc..6988a0f 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This module is puppet 3 tested lumberjack::instance{ 'core': - host => 'logstash.blah.com', + hosts => ['logstash.blah.com'], port => '5005', ssl_ca_file => 'puppet:///modules/conf/etc/ssl/certs/ca.crt', provider => 'daemontools'; diff --git a/manifests/init.pp b/manifests/init.pp index c075958..f86e3c5 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -52,8 +52,8 @@ # String to set the specific version you want to install. # Defaults to false. # -# [*host*] -# Hostname to connect to +# [*hosts*] +# Array of Hostnames to connect to # # [*port*] # Port to connect to diff --git a/manifests/instance.pp b/manifests/instance.pp index dab837d..3ff5068 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -4,11 +4,9 @@ # # === Parameters # -# [*host*] -# Host name or IP address of the Logstash instance to connect to -# Value type is string -# Default value: undef -# This variable is optional +# [*hosts*] +# Host names or IP addresses of the Logstash instances to connect to +# Value type is array # # [*port*] # Port number of the Logstash instance to connect to @@ -35,7 +33,7 @@ # define lumberjack::instance( $ssl_ca_file, - $host = undef, + $hosts, $port = undef, $provider = "daemontools", ) { @@ -52,7 +50,7 @@ lumberjack::service{ $name: ssl_ca_file => $ssl_ca_file, - host => $host, + hosts => $hosts, port => $port, provider => $provider; } diff --git a/manifests/service.pp b/manifests/service.pp index 44a6f76..56a2ec7 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -4,11 +4,9 @@ # # === Parameters # -# [*host*] -# Host name or IP address of the Logstash instance to connect to -# Value type is string -# Default value: undef -# This variable is optional +# [*hosts*] +# Host names or IP addresses of the Logstash instances to connect to +# Value type is Array # # [*port*] # Port number of the Logstash instance to connect to @@ -35,7 +33,7 @@ # define lumberjack::service( $ssl_ca_file, - $host = undef, + $hosts, $port = undef, $provider = "daemontools", ) { @@ -50,7 +48,7 @@ if ($provider != false ) { # Input validation - validate_string($host) + validate_array($hosts) if ! is_numeric($port) { fail("\"${port}\" is not a valid port parameter value") diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 8d7fcd9..f0808bf 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -4,11 +4,9 @@ # # === Parameters # -# [*host*] -# Host name or IP address of the Logstash instance to connect to -# Value type is string -# Default value: undef -# This variable is optional +# [*hosts*] +# Host names or IP addresses of the Logstash instances to connect to +# Value type is Array # # [*port*] # Port number of the Logstash instance to connect to diff --git a/spec/defines/instance_spec.rb b/spec/defines/instance_spec.rb index 1533ce3..0d66097 100644 --- a/spec/defines/instance_spec.rb +++ b/spec/defines/instance_spec.rb @@ -10,7 +10,7 @@ let :params do { :ssl_ca_file => 'puppet:///path/to/ca.crt', - :host => 'localhost', + :hosts => ['localhost'], :port => 1234, :files => [ '/var/log/file1', '/var/log/file2' ], :fields => { 'field1' => 'value1', 'field2' => 'value2' }, diff --git a/templates/etc/init.d/lumberjack.erb b/templates/etc/init.d/lumberjack.erb index 7530a57..2a67861 100644 --- a/templates/etc/init.d/lumberjack.erb +++ b/templates/etc/init.d/lumberjack.erb @@ -14,7 +14,7 @@ end DAEMON_PATH="/opt/lumberjack/" DAEMON="bin/lumberjack.sh" -DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>" +DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @hosts.first %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>" NAME="lumberjack-<%= @name %>" DESC="Lumberjack-<%= @name %>" diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index b273db1..22f994f 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -5,13 +5,19 @@ if fields else extra_fields = '' end + +hl = [] +@hosts.each do |h| + hl.push "\"#{h}:#{@port}\"" +end +host_list = hl.join(',') -%>{ # The network section covers network configuration :) "network": { # A list of downstream servers listening for our messages. # lumberjack will pick one at random and only switch if # the selected one appears to be dead or unresponsive - "servers": [ "<%= @host %>:<%= @port %>" ], + "servers": [<%= host_list %>], # The path to your client ssl certificate (optional) #"ssl certificate": "./lumberjack.crt", @@ -28,7 +34,7 @@ end # An array of hashes. Each hash tells what paths to watch and # what fields to annotate on events from those paths. { - "paths": [ + "paths": [ # single paths are fine "/var/log/messages", # globs are fine too, they will be periodically evaluated diff --git a/templates/lumberjack.config.json-headerpiece.erb b/templates/lumberjack.config.json-headerpiece.erb index 21a4130..d2f3c37 100644 --- a/templates/lumberjack.config.json-headerpiece.erb +++ b/templates/lumberjack.config.json-headerpiece.erb @@ -1,6 +1,13 @@ +<%- +hl = [] +@hosts.each do |h| + hl.push "\"#{h}:#{@port}\"" +end +host_list = hl.join(',') +-%> { "network": { - "servers": [ "<%= @host %>:<%= @port %>" ], + "servers": [<%= host_list %>], "ssl ca": "/etc/lumberjack/<%= @name %>/ca.crt" }, "files": [ From 0221512e6006aff24c9a73227caba17572136a21 Mon Sep 17 00:00:00 2001 From: jv Date: Thu, 5 Dec 2013 14:26:22 -0800 Subject: [PATCH 54/69] update logstash hosts to an array --- manifests/instance.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 3ff5068..9daf858 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -50,7 +50,7 @@ lumberjack::service{ $name: ssl_ca_file => $ssl_ca_file, - hosts => $hosts, + hosts => $hosts, port => $port, provider => $provider; } From 848827cd7ba1cd8819256e18f4cbc49511b24e2d Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Wed, 5 Mar 2014 11:18:08 -0800 Subject: [PATCH 55/69] convert to logstash-forwarder package + dropping junk comments --- manifests/params.pp | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/manifests/params.pp b/manifests/params.pp index c930c46..b50789c 100644 --- a/manifests/params.pp +++ b/manifests/params.pp @@ -28,39 +28,24 @@ # * Richard Pijnenburg # class lumberjack::params { - - #### Default values for the parameters of the main module class, init.pp - - # ensure $ensure = 'present' - - # autoupgrade $autoupgrade = true - - # service status $status = 'enabled' - - # Restart service on change $restart_on_change = true - $provider = 'daemontools' - #### Internal module values - # packages case $::operatingsystem { 'CentOS', 'Fedora', 'Scientific', 'OracleLinux', 'Amazon', 'RedHat': { - # main application - $package = [ 'lumberjack' ] - } + $package = [ 'logstash-forwarder' ] + } 'Debian', 'Ubuntu': { - # main application - $package = [ 'lumberjack' ] - } + $package = [ 'logstash-forwarder' ] + } default: { fail("\"${module_name}\" provides no package default value for \"${::operatingsystem}\"") - } + } } # service parameters @@ -78,8 +63,7 @@ $service_pattern = $service_name } default: { - fail("\"${module_name}\" provides no service parameters - for \"${::operatingsystem}\"") + fail("\"${module_name}\" doesn't know about OS \"${::operatingsystem}\"") } } From 545b810fd70f8f5904fd319f44d427688cdacce7 Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Wed, 5 Mar 2014 11:18:49 -0800 Subject: [PATCH 56/69] leverage service name from ::params --- manifests/package.pp | 2 -- manifests/service.pp | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/manifests/package.pp b/manifests/package.pp index b6e1b29..6baf48f 100644 --- a/manifests/package.pp +++ b/manifests/package.pp @@ -24,8 +24,6 @@ # class lumberjack::package { - #### Package management - # set params: in operation if $lumberjack::ensure == 'present' { diff --git a/manifests/service.pp b/manifests/service.pp index 56a2ec7..8478eae 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -136,12 +136,11 @@ } # action - service { "lumberjack-${name}": + service { $lumberjack::params::service_name: ensure => $service_ensure, enable => $service_enable, hasstatus => $lumberjack::params::service_hasstatus, hasrestart => $lumberjack::params::service_hasrestart, - #pattern => $lumberjack::params::service_pattern, pattern => "lumberjack-${name}", require => File["/etc/lumberjack"]; } From 664ce0b7700d3f9b15a1f4c550eaea8acc31ec6a Mon Sep 17 00:00:00 2001 From: "jeff@jeffvier.com" Date: Wed, 5 Mar 2014 16:22:00 -0800 Subject: [PATCH 57/69] convert to logstash-forwarder.sh --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 0d6fe90..eb5f6f8 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -4,4 +4,4 @@ cd /etc/lumberjack/<%= @name %>/pieces cat header > ../config.json for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json -exec /opt/lumberjack/bin/lumberjack.sh -config /etc/lumberjack/<%= @name %>/config.json +exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json From 4bbffb34937fd8879a9134ffddb35cd245a84676 Mon Sep 17 00:00:00 2001 From: jv Date: Wed, 5 Mar 2014 16:31:48 -0800 Subject: [PATCH 58/69] add (default) timeout flag --- templates/lumberjack.config.json-headerpiece.erb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/templates/lumberjack.config.json-headerpiece.erb b/templates/lumberjack.config.json-headerpiece.erb index d2f3c37..040e223 100644 --- a/templates/lumberjack.config.json-headerpiece.erb +++ b/templates/lumberjack.config.json-headerpiece.erb @@ -8,6 +8,7 @@ host_list = hl.join(',') { "network": { "servers": [<%= host_list %>], - "ssl ca": "/etc/lumberjack/<%= @name %>/ca.crt" + "ssl ca": "/etc/lumberjack/<%= @name %>/ca.crt", + "timeout": 15 }, "files": [ From 0a07477fadce944ca57c769bc6da15b33ecb583a Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Apr 2014 10:11:33 -0700 Subject: [PATCH 59/69] replacing the hash mutation in watcher.pp --- manifests/watcher.pp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index f0808bf..d31ff18 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -63,7 +63,8 @@ validate_hash($fields) if ! $fields['watcher'] { - $fields['watcher'] = $name + $tmp_fields = {'watcher' => $name} + $fields = merge($fields, $tmp_fields) } if (!defined(File["/etc/lumberjack/${part_of}"])) { From a569d363bb491a1f3f810838f6883fc0dcda8ca4 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Apr 2014 10:17:57 -0700 Subject: [PATCH 60/69] replacing the hash mutation in watcher.pp. keeping fields immutable --- manifests/watcher.pp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index d31ff18..e366104 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -46,7 +46,7 @@ define lumberjack::watcher( $part_of, $files, - $fields = { watcher => $name, }, + $fields1 = { watcher => $name, }, $ensure = present, ) { @@ -61,10 +61,10 @@ validate_array($files) $logfiles = join($files,' ') - validate_hash($fields) - if ! $fields['watcher'] { + validate_hash($fields1) + if ! $fields1['watcher'] { $tmp_fields = {'watcher' => $name} - $fields = merge($fields, $tmp_fields) + $fields = merge($fields1, $tmp_fields) } if (!defined(File["/etc/lumberjack/${part_of}"])) { From e161ade1bbaa6d0106eeaf129957ee3fa5c42ce3 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Apr 2014 10:29:35 -0700 Subject: [PATCH 61/69] replacing the hash mutation in watcher.pp. keeping fields immutable --- manifests/watcher.pp | 7 ++++--- templates/etc/init.d/lumberjack.erb | 2 +- templates/lumberjack.config.erb | 2 +- templates/lumberjack.config.json-filepiece.erb | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index e366104..0d1a98c 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -46,7 +46,7 @@ define lumberjack::watcher( $part_of, $files, - $fields1 = { watcher => $name, }, + $fields = { watcher => $name, }, $ensure = present, ) { @@ -62,9 +62,10 @@ $logfiles = join($files,' ') validate_hash($fields1) - if ! $fields1['watcher'] { + if ! $fields['watcher'] { + # Puppet has decided to Deprecate Data Structure Mutation https://tickets.puppetlabs.com/browse/PUP-864 $tmp_fields = {'watcher' => $name} - $fields = merge($fields1, $tmp_fields) + $fieldsss = merge($fields, $tmp_fields) } if (!defined(File["/etc/lumberjack/${part_of}"])) { diff --git a/templates/etc/init.d/lumberjack.erb b/templates/etc/init.d/lumberjack.erb index 2a67861..678386c 100644 --- a/templates/etc/init.d/lumberjack.erb +++ b/templates/etc/init.d/lumberjack.erb @@ -1,5 +1,5 @@ <%- -fields = @fields +fields = @fieldsss if fields extra_fields = fields.collect { |k,v| "--field #{k}=#{v}" }.join(" ") else diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb index 22f994f..6f517f3 100644 --- a/templates/lumberjack.config.erb +++ b/templates/lumberjack.config.erb @@ -1,5 +1,5 @@ <%- -fields = @fields +fields = @fieldsss if fields extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.sort.join(" ") else diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb index 911d6f7..2682910 100644 --- a/templates/lumberjack.config.json-filepiece.erb +++ b/templates/lumberjack.config.json-filepiece.erb @@ -1,5 +1,5 @@ <%- -fields = @fields +fields = @fieldsss if fields fields.each_pair do |k,v| From 370e3298234481a2b7278b86264d3a3d12baf539 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Apr 2014 10:31:10 -0700 Subject: [PATCH 62/69] replacing the hash mutation in watcher.pp. keeping fields immutable --- manifests/watcher.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 0d1a98c..368c195 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -61,7 +61,7 @@ validate_array($files) $logfiles = join($files,' ') - validate_hash($fields1) + validate_hash($fields) if ! $fields['watcher'] { # Puppet has decided to Deprecate Data Structure Mutation https://tickets.puppetlabs.com/browse/PUP-864 $tmp_fields = {'watcher' => $name} From 269228062bc0d3c0a70d95ef757d49d0f3db7cc2 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Thu, 10 Apr 2014 10:35:25 -0700 Subject: [PATCH 63/69] replacing the hash mutation in watcher.pp. keeping fields immutable --- manifests/watcher.pp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/manifests/watcher.pp b/manifests/watcher.pp index 368c195..3a6c45f 100644 --- a/manifests/watcher.pp +++ b/manifests/watcher.pp @@ -62,7 +62,9 @@ $logfiles = join($files,' ') validate_hash($fields) - if ! $fields['watcher'] { + if $fields['watcher'] { + $fieldsss = $fields + } else { # Puppet has decided to Deprecate Data Structure Mutation https://tickets.puppetlabs.com/browse/PUP-864 $tmp_fields = {'watcher' => $name} $fieldsss = merge($fields, $tmp_fields) From 9ec8890d3e6a4412781a84ed07370e2a9e60de7c Mon Sep 17 00:00:00 2001 From: Don Endres Date: Tue, 15 Apr 2014 17:23:35 -0700 Subject: [PATCH 64/69] trivial change to trigger a lumberjack restart --- templates/run.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/run.erb b/templates/run.erb index eb5f6f8..bd49565 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -4,4 +4,5 @@ cd /etc/lumberjack/<%= @name %>/pieces cat header > ../config.json for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json + exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json From bb1dd397c22e6ae81bea433e22ab02aeb7ad4a8d Mon Sep 17 00:00:00 2001 From: Don Endres Date: Mon, 12 May 2014 20:15:44 +0000 Subject: [PATCH 65/69] restarting lumberjack instances --- templates/run.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/run.erb b/templates/run.erb index bd49565..ea531b0 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -6,3 +6,4 @@ for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json + From ba1bd00555d42a372fe39aabfdbd86a80a3bb226 Mon Sep 17 00:00:00 2001 From: Don Endres Date: Wed, 14 May 2014 19:35:58 +0000 Subject: [PATCH 66/69] triggering lumberjack restart --- templates/run.erb | 1 - 1 file changed, 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index ea531b0..bd49565 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -6,4 +6,3 @@ for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json - From fa6b6b7c1c12ec126a77578a7d1e0300a0eb1eb2 Mon Sep 17 00:00:00 2001 From: jv Date: Wed, 14 May 2014 14:38:04 -0700 Subject: [PATCH 67/69] add support for runtime_opts var --- manifests/instance.pp | 10 ++++++---- manifests/service.pp | 1 + 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/manifests/instance.pp b/manifests/instance.pp index 9daf858..bc7e38c 100644 --- a/manifests/instance.pp +++ b/manifests/instance.pp @@ -35,6 +35,7 @@ $ssl_ca_file, $hosts, $port = undef, + $runtime_opts = '', $provider = "daemontools", ) { @@ -49,10 +50,11 @@ if ($provider != false ) { lumberjack::service{ $name: - ssl_ca_file => $ssl_ca_file, - hosts => $hosts, - port => $port, - provider => $provider; + ssl_ca_file => $ssl_ca_file, + hosts => $hosts, + port => $port, + runtime_opts => $runtime_opts, + provider => $provider; } } diff --git a/manifests/service.pp b/manifests/service.pp index 8478eae..07f1c53 100644 --- a/manifests/service.pp +++ b/manifests/service.pp @@ -35,6 +35,7 @@ $ssl_ca_file, $hosts, $port = undef, + $runtime_opts = '', $provider = "daemontools", ) { From 41de2069edc0c0949e7a40917f843b9d277d96e3 Mon Sep 17 00:00:00 2001 From: jv Date: Wed, 14 May 2014 14:38:14 -0700 Subject: [PATCH 68/69] add support for runtime_opts var --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index bd49565..7872ec5 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -5,4 +5,4 @@ cat header > ../config.json for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json -exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json +exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json <%= @lumberjack_opts %> From 4f790e7ff00cac3c425f4ed6659000a884a2688a Mon Sep 17 00:00:00 2001 From: jv Date: Wed, 14 May 2014 14:40:46 -0700 Subject: [PATCH 69/69] maybe I use the same variable name all the way through, no? --- templates/run.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/run.erb b/templates/run.erb index 7872ec5..8cb24e9 100644 --- a/templates/run.erb +++ b/templates/run.erb @@ -5,4 +5,4 @@ cat header > ../config.json for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json cat footer >> ../config.json -exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json <%= @lumberjack_opts %> +exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json <%= @runtime_opts %>