diff --git a/README.md b/README.md
index 040c5cb..6988a0f 100644
--- a/README.md
+++ b/README.md
@@ -8,24 +8,33 @@ This module is puppet 3 tested
## Usage
-Installation, make sure service is running and will be started at boot time:
+### Set up a parent instance. This sets up the "framework" config to add individual file/file-group watchers that can be tagged with individual fields.
- class { 'lumberjack':
- host => 'logstashhost',
- port => '1234',
- files => ['/var/log/messages', '/var/log/secure'],
- ssl_ca_path => "puppet:///path/to/ca.crt",
- }
+ lumberjack::instance{
+ 'core':
+ hosts => ['logstash.blah.com'],
+ port => '5005',
+ ssl_ca_file => 'puppet:///modules/conf/etc/ssl/certs/ca.crt',
+ provider => 'daemontools';
+ }
-Removal/decommissioning:
+### Add watchers that plug in to the intance you set up.
+ lumberjack::watcher {
+ 'syslog':
+ part_of => 'core',
+ files => ["/var/log/secure", "/var/log/messages", "/var/log/*.log", ];
- class { 'lumberjack':
- ensure => 'absent',
- }
+ 'statsd':
+ part_of => 'core',
+ files => ["/data/log/statsd/statsd.log", ];
-Install everything but disable service(s) afterwards:
+ 'diamond':
+ part_of => 'core',
+ files => ["/data/log/diamond/diamond.log", ];
+ }
+
+### Removal/decommissioning:
class { 'lumberjack':
- status => 'disabled',
+ ensure => 'absent',
}
-
diff --git a/manifests/init.pp b/manifests/init.pp
index 7c63d7a..f86e3c5 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -52,8 +52,8 @@
# String to set the specific version you want to install.
# Defaults to false.
#
-# [*host*]
-# Hostname to connect to
+# [*hosts*]
+# Array of Hostnames to connect to
#
# [*port*]
# Port to connect to
@@ -97,6 +97,7 @@
$autoupgrade = $lumberjack::params::autoupgrade,
$status = $lumberjack::params::status,
$restart_on_change = $lumberjack::params::restart_on_change,
+ $provider = $lumberjack::params::provider,
$version = false,
) inherits lumberjack::params {
diff --git a/manifests/instance.pp b/manifests/instance.pp
index c5a3de3..bc7e38c 100644
--- a/manifests/instance.pp
+++ b/manifests/instance.pp
@@ -4,11 +4,9 @@
#
# === Parameters
#
-# [*host*]
-# Host name or IP address of the Logstash instance to connect to
-# Value type is string
-# Default value: undef
-# This variable is optional
+# [*hosts*]
+# Host names or IP addresses of the Logstash instances to connect to
+# Value type is array
#
# [*port*]
# Port number of the Logstash instance to connect to
@@ -16,24 +14,12 @@
# Default value: undef
# This variable is optional
#
-# [*files*]
-# Array of files you wish to process
-# Value type is array
-# Default value: undef
-# This variable is optional
-#
# [*ssl_ca_file*]
# File to use for the SSL CA
# Value type is string
# This variable is mandatory
#
-# [*fields*]
-# Extra fields to send
-# Value type is hash
-# Default value: false
-# This variable is optional
-#
-# [*run_as_service*]
+# [*provider*]
# Set this to true if you want to run this as a service.
# Set to false if you only want to manage the ssl_ca_file
# Value type is boolean
@@ -43,124 +29,63 @@
# === Authors
#
# * Richard Pijnenburg
+# * Modified by Jeff Vier
#
define lumberjack::instance(
$ssl_ca_file,
- $host = undef,
+ $hosts,
$port = undef,
- $files = undef,
- $fields = false,
- $run_as_service = true,
- $ensure = $logstash::ensure
+ $runtime_opts = '',
+ $provider = "daemontools",
) {
require lumberjack
File {
- owner => 'root',
- group => 'root',
- mode => '0644'
+ owner => root,
+ group => root,
+ mode => 0644
}
- if ($run_as_service == true ) {
-
- # Input validation
- validate_string($host)
-
- if ! is_numeric($port) {
- fail("\"${port}\" is not a valid port parameter value")
- }
-
- validate_array($files)
- $logfiles = join($files,' ')
-
- if $fields {
- validate_hash($fields)
- }
-
- # Setup init file if running as a service
- $notify_lumberjack = $lumberjack::restart_on_change ? {
- true => Service["lumberjack-${name}"],
- false => undef,
+ if ($provider != false ) {
+ lumberjack::service{
+ $name:
+ ssl_ca_file => $ssl_ca_file,
+ hosts => $hosts,
+ port => $port,
+ runtime_opts => $runtime_opts,
+ provider => $provider;
}
+ }
- file { "/etc/init.d/lumberjack-${name}":
- ensure => $ensure,
- mode => '0755',
- content => template("${module_name}/etc/init.d/lumberjack.erb"),
- notify => $notify_lumberjack
- }
-
- #### Service management
-
- # set params: in operation
- if $lumberjack::ensure == 'present' {
-
- case $lumberjack::status {
- # make sure service is currently running, start it on boot
- 'enabled': {
- $service_ensure = 'running'
- $service_enable = true
- }
- # make sure service is currently stopped, do not start it on boot
- 'disabled': {
- $service_ensure = 'stopped'
- $service_enable = false
- }
- # make sure service is currently running, do not start it on boot
- 'running': {
- $service_ensure = 'running'
- $service_enable = false
- }
- # do not start service on boot, do not care whether currently running or not
- 'unmanaged': {
- $service_ensure = undef
- $service_enable = false
- }
- # unknown status
- # note: don't forget to update the parameter check in init.pp if you
- # add a new or change an existing status.
- default: {
- fail("\"${lumberjack::status}\" is an unknown service status value")
- }
- }
-
- # set params: removal
- } else {
-
- # make sure the service is stopped and disabled (the removal itself will be
- # done by package.pp)
- $service_ensure = 'stopped'
- $service_enable = false
+ if (!defined(File["/etc/lumberjack"])) {
+ file {
+ "/etc/lumberjack":
+ ensure => directory;
}
-
- # action
- service { "lumberjack-${name}":
- ensure => $service_ensure,
- enable => $service_enable,
- name => $lumberjack::params::service_name,
- hasstatus => $lumberjack::params::service_hasstatus,
- hasrestart => $lumberjack::params::service_hasrestart,
- pattern => $lumberjack::params::service_pattern,
+ }
+ if (!defined(File["/etc/lumberjack/${name}"])) {
+ file {
+ "/etc/lumberjack/${name}":
+ ensure => directory;
}
-
- } else {
-
- $notify_lumberjack = undef
-
}
+ file {
+ "/etc/lumberjack/${name}/pieces/header":
+ ensure => $ensure,
+ content => template("${module_name}/lumberjack.config.json-headerpiece.erb"),
+ notify => $notify_lumberjack;
- file { "/etc/lumberjack/${name}":
- ensure => directory,
- }
+ "/etc/lumberjack/${name}/pieces/footer":
+ ensure => $ensure,
+ content => template("${module_name}/lumberjack.config.json-footerpiece.erb"),
+ notify => $notify_lumberjack;
- # Setup certificate files
- file { "/etc/lumberjack/${name}/ca.crt":
- ensure => $ensure,
- source => $ssl_ca_file,
- require => File[ "/etc/lumberjack/${name}" ],
- notify => $notify_lumberjack
+ "/etc/lumberjack/${name}/ca.crt":
+ ensure => $ensure,
+ source => $ssl_ca_file,
+ notify => $notify_lumberjack;
}
}
diff --git a/manifests/package.pp b/manifests/package.pp
index b6e1b29..6baf48f 100644
--- a/manifests/package.pp
+++ b/manifests/package.pp
@@ -24,8 +24,6 @@
#
class lumberjack::package {
- #### Package management
-
# set params: in operation
if $lumberjack::ensure == 'present' {
diff --git a/manifests/params.pp b/manifests/params.pp
index 0c6e751..b50789c 100644
--- a/manifests/params.pp
+++ b/manifests/params.pp
@@ -28,56 +28,42 @@
# * Richard Pijnenburg
#
class lumberjack::params {
-
- #### Default values for the parameters of the main module class, init.pp
-
- # ensure
$ensure = 'present'
-
- # autoupgrade
- $autoupgrade = false
-
- # service status
+ $autoupgrade = true
$status = 'enabled'
-
- # Restart service on change
- $restart_on_change = false
-
- #### Internal module values
+ $restart_on_change = true
+ $provider = 'daemontools'
# packages
case $::operatingsystem {
'CentOS', 'Fedora', 'Scientific', 'OracleLinux', 'Amazon', 'RedHat': {
- # main application
- $package = [ 'lumberjack' ]
- }
+ $package = [ 'logstash-forwarder' ]
+ }
'Debian', 'Ubuntu': {
- # main application
- $package = [ 'lumberjack' ]
- }
+ $package = [ 'logstash-forwarder' ]
+ }
default: {
fail("\"${module_name}\" provides no package default value
for \"${::operatingsystem}\"")
- }
+ }
}
# service parameters
case $::operatingsystem {
'CentOS', 'Fedora', 'Scientific', 'OracleLinux', 'Amazon', 'RedHat': {
- $service_name = 'lumberjack'
+ $service_name = "lumberjack-${name}"
$service_hasrestart = true
$service_hasstatus = true
$service_pattern = $service_name
}
'Debian', 'Ubuntu': {
- $service_name = 'lumberjack'
+ $service_name = "lumberjack-${name}"
$service_hasrestart = true
$service_hasstatus = true
$service_pattern = $service_name
}
default: {
- fail("\"${module_name}\" provides no service parameters
- for \"${::operatingsystem}\"")
+ fail("\"${module_name}\" doesn't know about OS \"${::operatingsystem}\"")
}
}
diff --git a/manifests/service.pp b/manifests/service.pp
new file mode 100644
index 0000000..07f1c53
--- /dev/null
+++ b/manifests/service.pp
@@ -0,0 +1,154 @@
+# Define: lumberjack::service
+#
+# This define sets up the lumberjack service
+#
+# === Parameters
+#
+# [*hosts*]
+# Host names or IP addresses of the Logstash instances to connect to
+# Value type is Array
+#
+# [*port*]
+# Port number of the Logstash instance to connect to
+# Value type is number
+# Default value: undef
+# This variable is optional
+#
+# [*ssl_ca_file*]
+# File to use for the SSL CA
+# Value type is string
+# This variable is mandatory
+#
+# [*provider*]
+# Set this to true if you want to run this as a service.
+# Set to false if you only want to manage the ssl_ca_file
+# Value type is boolean
+# Default value: true
+# This variable is optional
+#
+# === Authors
+#
+# * Richard Pijnenburg
+# * Modified by Jeff Vier
+#
+define lumberjack::service(
+ $ssl_ca_file,
+ $hosts,
+ $port = undef,
+ $runtime_opts = '',
+ $provider = "daemontools",
+) {
+
+ require lumberjack
+
+ File {
+ owner => root,
+ group => root,
+ mode => 0644
+ }
+
+ if ($provider != false ) {
+ # Input validation
+ validate_array($hosts)
+
+ if ! is_numeric($port) {
+ fail("\"${port}\" is not a valid port parameter value")
+ }
+
+ if ($provider == "daemontools" ) {
+
+ file { "/etc/init.d/lumberjack-${name}": ensure => absent; }
+ service { "lumberjack-${name}": ensure => stopped, enable => false; }
+
+ $user = root
+ $loguser = root
+
+ daemontools::setup{
+ "lumberjack/${name}":
+ user => $user, ## Needed to read some log files. Sorry.
+ loguser => $loguser,
+ run => template("${module_name}/run.erb"),
+ logrun => template("${module_name}/log/run.erb"),
+ notify => Daemontools::Service["lumberjack-${name}"];
+ }
+
+ daemontools::service {
+ "lumberjack-${name}":
+ source => "/etc/lumberjack/${name}",
+ require => Daemontools::Setup["lumberjack/${name}"];
+ }
+ }
+ elsif ($provider == "init.d" ) {
+
+ # Setup init file if running as a service
+ $notify_lumberjack = $lumberjack::restart_on_change ? {
+ true => Service["lumberjack-${name}"],
+ false => undef,
+ }
+
+ file { "/etc/init.d/lumberjack-${name}":
+ ensure => $ensure,
+ mode => 0755,
+ content => template("${module_name}/etc/init.d/lumberjack.erb"),
+ notify => $notify_lumberjack
+ }
+
+ #### Service management
+
+ # set params: in operation
+ if $lumberjack::ensure == 'present' {
+
+ case $lumberjack::params::status {
+ # make sure service is currently running, start it on boot
+ 'enabled': {
+ $service_ensure = 'running'
+ $service_enable = true
+ }
+ # make sure service is currently stopped, do not start it on boot
+ 'disabled': {
+ $service_ensure = 'stopped'
+ $service_enable = false
+ }
+ # make sure service is currently running, do not start it on boot
+ 'running': {
+ $service_ensure = 'running'
+ $service_enable = false
+ }
+ # do not start service on boot, do not care whether currently running or not
+ 'unmanaged': {
+ $service_ensure = undef
+ $service_enable = false
+ }
+ # unknown status
+ # note: don't forget to update the parameter check in init.pp if you
+ # add a new or change an existing status.
+ default: {
+ fail("\"${lumberjack::status}\" is an unknown service status value")
+ }
+ }
+
+ # set params: removal
+ } else {
+
+ # make sure the service is stopped and disabled (the removal itself will be
+ # done by package.pp)
+ $service_ensure = 'stopped'
+ $service_enable = false
+ }
+
+ # action
+ service { $lumberjack::params::service_name:
+ ensure => $service_ensure,
+ enable => $service_enable,
+ hasstatus => $lumberjack::params::service_hasstatus,
+ hasrestart => $lumberjack::params::service_hasrestart,
+ pattern => "lumberjack-${name}",
+ require => File["/etc/lumberjack"];
+ }
+
+ } else {
+ $notify_lumberjack = undef
+ }
+ }
+
+}
diff --git a/manifests/watcher.pp b/manifests/watcher.pp
new file mode 100644
index 0000000..3a6c45f
--- /dev/null
+++ b/manifests/watcher.pp
@@ -0,0 +1,93 @@
+# Define: lumberjack::instance
+#
+# This define allows you to setup an instance of lumberjack
+#
+# === Parameters
+#
+# [*hosts*]
+# Host names or IP addresses of the Logstash instances to connect to
+# Value type is Array
+#
+# [*port*]
+# Port number of the Logstash instance to connect to
+# Value type is number
+# Default value: undef
+# This variable is optional
+#
+# [*files*]
+# Array of files you wish to process
+# Value type is array
+# Default value: undef
+# This variable is optional
+#
+# [*ssl_ca_file*]
+# File to use for the SSL CA
+# Value type is string
+# This variable is mandatory
+#
+# [*fields*]
+# Extra fields to send
+# Value type is hash
+# Default value: false
+# This variable is optional
+#
+# [*run_as_service*]
+# Set this to true if you want to run this as a service.
+# Set to false if you only want to manage the ssl_ca_file
+# Value type is boolean
+# Default value: true
+# This variable is optional
+#
+# === Authors
+#
+# * Richard Pijnenburg
+# * Modified by Jeff Vier
+#
+define lumberjack::watcher(
+ $part_of,
+ $files,
+ $fields = { watcher => $name, },
+ $ensure = present,
+) {
+
+ require lumberjack
+
+ File {
+ owner => root,
+ group => root,
+ mode => 0644
+ }
+
+ validate_array($files)
+ $logfiles = join($files,' ')
+
+ validate_hash($fields)
+ if $fields['watcher'] {
+ $fieldsss = $fields
+ } else {
+ # Puppet has decided to Deprecate Data Structure Mutation https://tickets.puppetlabs.com/browse/PUP-864
+ $tmp_fields = {'watcher' => $name}
+ $fieldsss = merge($fields, $tmp_fields)
+ }
+
+ if (!defined(File["/etc/lumberjack/${part_of}"])) {
+ file {
+ "/etc/lumberjack/${part_of}":
+ ensure => directory;
+ }
+ }
+ if (!defined(File["/etc/lumberjack/${part_of}/pieces"])) {
+ file {
+ "/etc/lumberjack/${part_of}/pieces":
+ ensure => directory;
+ }
+ }
+
+ file {
+ "/etc/lumberjack/${part_of}/pieces/filepiece-${name}":
+ ensure => $ensure,
+ content => template("${module_name}/lumberjack.config.json-filepiece.erb"),
+ notify => Lumberjack::Service["${part_of}"];
+ }
+
+}
diff --git a/spec/defines/instance_spec.rb b/spec/defines/instance_spec.rb
index d22594e..0d66097 100644
--- a/spec/defines/instance_spec.rb
+++ b/spec/defines/instance_spec.rb
@@ -10,11 +10,11 @@
let :params do {
:ssl_ca_file => 'puppet:///path/to/ca.crt',
- :host => 'localhost',
+ :hosts => ['localhost'],
:port => 1234,
:files => [ '/var/log/file1', '/var/log/file2' ],
:fields => { 'field1' => 'value1', 'field2' => 'value2' },
- :run_as_service => true
+ :provider => "init.d"
} end
it { should contain_file('/etc/init.d/lumberjack-foo') }
@@ -27,7 +27,7 @@
let :params do {
:ssl_ca_file => 'puppet:///path/to/ca.crt',
- :run_as_service => false
+ :provider => "none"
} end
it { should_not contain_file('/etc/init.d/lumberjack-foo') }
diff --git a/templates/etc/init.d/lumberjack.erb b/templates/etc/init.d/lumberjack.erb
index 0d15f67..678386c 100644
--- a/templates/etc/init.d/lumberjack.erb
+++ b/templates/etc/init.d/lumberjack.erb
@@ -1,5 +1,5 @@
<%-
-fields = @fields
+fields = @fieldsss
if fields
extra_fields = fields.collect { |k,v| "--field #{k}=#{v}" }.join(" ")
else
@@ -14,10 +14,10 @@ end
DAEMON_PATH="/opt/lumberjack/"
DAEMON="bin/lumberjack.sh"
-DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @instance %>/ca.crt --host <%= @host %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>"
+DAEMONOPTS="--ssl-ca-path /etc/lumberjack/<%= @name %>/ca.crt --host <%= @hosts.first %> --port <%= @port %> <%= extra_fields %> <%= @logfiles %>"
-NAME="lumberjack-<%= @instance %>"
-DESC="Lumberjack-<%= @instance %>"
+NAME="lumberjack-<%= @name %>"
+DESC="Lumberjack-<%= @name %>"
PIDFILE=/var/run/$NAME.pid
SCRIPTNAME=/etc/init.d/$NAME
diff --git a/templates/log/run.erb b/templates/log/run.erb
new file mode 100644
index 0000000..9c981b7
--- /dev/null
+++ b/templates/log/run.erb
@@ -0,0 +1,4 @@
+#!/bin/sh
+[ -d ./main ] || mkdir ./main
+chown -Rf <%= @loguser %> .
+exec setuidgid <%= @loguser %> multilog s5000000 ./main
diff --git a/templates/lumberjack.config.erb b/templates/lumberjack.config.erb
new file mode 100644
index 0000000..6f517f3
--- /dev/null
+++ b/templates/lumberjack.config.erb
@@ -0,0 +1,58 @@
+<%-
+fields = @fieldsss
+if fields
+ extra_fields = fields.map { |k,v| "--field " + k + "=" + v }.sort.join(" ")
+else
+ extra_fields = ''
+end
+
+hl = []
+@hosts.each do |h|
+ hl.push "\"#{h}:#{@port}\""
+end
+host_list = hl.join(',')
+-%>{
+ # The network section covers network configuration :)
+ "network": {
+ # A list of downstream servers listening for our messages.
+ # lumberjack will pick one at random and only switch if
+ # the selected one appears to be dead or unresponsive
+ "servers": [<%= host_list %>],
+
+ # The path to your client ssl certificate (optional)
+ #"ssl certificate": "./lumberjack.crt",
+ # The path to your client ssl key (optional)
+ "ssl key": "./lumberjack.key",
+
+ # The path to your trusted ssl CA file. This is used
+ # to authenticate your downstream server.
+ "ssl ca": "./lumberjack_ca.crt"
+ },
+
+ # The list of files configurations
+ "files": [
+ # An array of hashes. Each hash tells what paths to watch and
+ # what fields to annotate on events from those paths.
+ {
+ "paths": [
+ # single paths are fine
+ "/var/log/messages",
+ # globs are fine too, they will be periodically evaluated
+ # to see if any new files match the wildcard.
+ "/var/log/*.log"
+ ],
+
+ # A dictionary of fields to annotate on each event.
+ "fields": { "type": "syslog" }
+ }, {
+ # A path of "-" means stdin.
+ "paths": [ "-" ],
+ "fields": { "type": "stdin" }
+ }, {
+ "paths": [
+ "/var/log/apache/httpd-*.log"
+ ],
+ "fields": { "type:" "apache" }
+ }
+ ]
+}
diff --git a/templates/lumberjack.config.json-filepiece.erb b/templates/lumberjack.config.json-filepiece.erb
new file mode 100644
index 0000000..2682910
--- /dev/null
+++ b/templates/lumberjack.config.json-filepiece.erb
@@ -0,0 +1,19 @@
+<%-
+fields = @fieldsss
+
+if fields
+ fields.each_pair do |k,v|
+ fields[k] = v.join(' ') if v.is_a? Array
+ end
+
+ extra_fields = fields.map { |k,v| '"' + k + '": "' + v + '"'}.join(",")
+else
+ extra_fields = ''
+end
+-%>
+ {
+ "paths": [
+ <%= @files.map {|f| '"' + f + '"'}.join(',') %>
+ ],
+ "fields": { <%= extra_fields %> }
+ }
diff --git a/templates/lumberjack.config.json-footerpiece.erb b/templates/lumberjack.config.json-footerpiece.erb
new file mode 100644
index 0000000..4b45020
--- /dev/null
+++ b/templates/lumberjack.config.json-footerpiece.erb
@@ -0,0 +1,2 @@
+ ]
+}
diff --git a/templates/lumberjack.config.json-headerpiece.erb b/templates/lumberjack.config.json-headerpiece.erb
new file mode 100644
index 0000000..040e223
--- /dev/null
+++ b/templates/lumberjack.config.json-headerpiece.erb
@@ -0,0 +1,14 @@
+<%-
+hl = []
+@hosts.each do |h|
+ hl.push "\"#{h}:#{@port}\""
+end
+host_list = hl.join(',')
+-%>
+{
+ "network": {
+ "servers": [<%= host_list %>],
+ "ssl ca": "/etc/lumberjack/<%= @name %>/ca.crt",
+ "timeout": 15
+ },
+ "files": [
diff --git a/templates/run.erb b/templates/run.erb
new file mode 100644
index 0000000..8cb24e9
--- /dev/null
+++ b/templates/run.erb
@@ -0,0 +1,8 @@
+#!/bin/sh
+exec 2>&1
+cd /etc/lumberjack/<%= @name %>/pieces
+cat header > ../config.json
+for x in filepiece-*; do cat $x ; echo ','; done | head -n-1 >> ../config.json
+cat footer >> ../config.json
+
+exec /opt/logstash-forwarder/bin/logstash-forwarder.sh -config /etc/lumberjack/<%= @name %>/config.json <%= @runtime_opts %>