From 18e2220bb96709670953a46d1591088a14c6b9ed Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 2 Feb 2018 12:51:07 -0700 Subject: [PATCH 1/2] adding support for modules --- filebeat/files/filebeat.jinja | 36 +++++++++++++++++++++++++------ pillar.example | 40 ++++++++++++++++++++++++++--------- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/filebeat/files/filebeat.jinja b/filebeat/files/filebeat.jinja index f08ab78..a8d7a39 100644 --- a/filebeat/files/filebeat.jinja +++ b/filebeat/files/filebeat.jinja @@ -1,5 +1,19 @@ {%- set filebeat = salt['pillar.get']('filebeat', {}) %} filebeat: + + modules: +{%- set modules = filebeat.get('modules', {}) %} +{%- for module in modules %} + - module: {{ module.name }} +{%- for detail in module.details %} + {{ detail.name }}: + enabled: {{ detail.get('enabled', 'true') }} +{%- if detail.get('paths') %} + var.paths: ["{{detail.paths|join('", "') }}"] +{%- endif %} +{%- endfor %} +{%- endfor %} + prospectors: {%- set log_paths = filebeat.get('log_paths', ['/var/log/auth.log', '/var/log/syslog']) %} {%- for log_path in log_paths %} @@ -18,6 +32,12 @@ filebeat: force_close_files: {{ log_path.get('force_close_files', 'false') }} fields_under_root: {{ log_path.get('fields_under_root', 'false') }} close_older: {{ log_path.get('close_older', '1h') }} + {%- if log_path.get('json.enabled', False) %} + json.message_key: {{ log_path.get('json.message_key', 'json') }} + json.keys_under_root: {{ log_path.get('json.keys_under_root', 'true') }} + json.add_error_key: {{ log_path.get('json.add_error_key', 'true') }} + {%- endif %} + {%- if 'multiline' in log_path %} multiline: pattern: "{{ log_path.multiline.get('pattern', '') }}" @@ -46,22 +66,24 @@ output: {%- set elasticsearch = filebeat.get('elasticsearch', {}) %} {%- set logstash = filebeat.get('logstash', {}) %} -{%- if filebeat.elasticsearch.get('enabled', False) %} +{%- if elasticsearch.get('enabled', False) %} elasticsearch: - hosts: ["http://{{ filebeat.elasticsearch.server }}"] + hosts: ["http://{{ elasticsearch.server }}"] {%- endif %} -{%- if filebeat.logstash.get('enabled', False) %} +{%- if logstash.get('enabled', False) %} logstash: - hosts: ["{{ filebeat.logstash.server }}"] + hosts: {{ logstash.server }} worker: 1 loadbalance: true index: filebeat -{%- if 'tls' in filebeat.logstash %} -{%- if filebeat.logstash.tls.get('enabled', False) %} +{%- if 'tls' in logstash %} +{%- if logstash.tls.get('enabled', False) %} tls: - certificate_authorities: ["{{ filebeat.logstash.tls.ssl_cert_path }}"] +{%- if logstash.tls.get('ssl_cert_path') %} + certificate_authorities: ["{{ logstash.tls.ssl_cert_path }}"] +{%- endif %} {%- endif %} {%- endif %} {%- endif %} diff --git a/pillar.example b/pillar.example index d274a1a..379a1d6 100644 --- a/pillar.example +++ b/pillar.example @@ -3,6 +3,15 @@ filebeat: config_path: /etc/mycustom/filebeat/filebeat.yml config_source: salt://mycustom/filebeat/filebeat.jinja runlevels_install: True + modules: + - + name: 'system' + details: + - + name: 'syslog' + enabled: 'true' + paths: + - '/path/to/log/syslog*' # if no log_paths specified, generic syslogs are default log_paths: @@ -10,7 +19,7 @@ filebeat: paths: - '/var/log/auth.log' - '/var/log/syslog' - - + - paths: - '/var/log/apache2/access.log' input_type: 'log' @@ -30,21 +39,32 @@ filebeat: fields: - env: my_environment - server_role: webserver + - + paths: + - '/var/log/example/*.json' + json.enabled: true + json.message_key: json + json.keys_under_root: true + json.add_error_key: true - elasticsearch: enabled: False server: 127.0.0.1:9200 - logstash: + logstash: enabled: True - server: 127.0.0.1:5044 + server: + - logstash-shipper1:5044 + - logstash-shipper2:5044 + tls: enabled: True - # this is the public key from your ELK server - # default path is salt://filebeat/files/ca.pem - ssl_cert: salt://mycustom/filebeat/logstash-forwarder.crt + # path to the certificate of your ELK server + # set to empty to use system certificates ssl_cert_path: /etc/pki/tls/certs/logstash-forwarder.crt - - - + # path to the certificate of your ELK server to be installed + # default is salt://filebeat/files/ca.pem + # set to empty to disable + ssl_cert: salt://mycustom/filebeat/logstash-forwarder.crt + # If you want to manage your own certs, set below to False + managed_cert: False From 0636484995b32095b99872512bed4580bf565287 Mon Sep 17 00:00:00 2001 From: Evan Date: Fri, 2 Feb 2018 12:53:02 -0700 Subject: [PATCH 2/2] fixing minor issues with paste --- filebeat/files/filebeat.jinja | 24 +++++++----------------- pillar.example | 32 ++++++++++---------------------- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/filebeat/files/filebeat.jinja b/filebeat/files/filebeat.jinja index a8d7a39..a478348 100644 --- a/filebeat/files/filebeat.jinja +++ b/filebeat/files/filebeat.jinja @@ -1,6 +1,5 @@ {%- set filebeat = salt['pillar.get']('filebeat', {}) %} filebeat: - modules: {%- set modules = filebeat.get('modules', {}) %} {%- for module in modules %} @@ -13,7 +12,6 @@ filebeat: {%- endif %} {%- endfor %} {%- endfor %} - prospectors: {%- set log_paths = filebeat.get('log_paths', ['/var/log/auth.log', '/var/log/syslog']) %} {%- for log_path in log_paths %} @@ -32,12 +30,6 @@ filebeat: force_close_files: {{ log_path.get('force_close_files', 'false') }} fields_under_root: {{ log_path.get('fields_under_root', 'false') }} close_older: {{ log_path.get('close_older', '1h') }} - {%- if log_path.get('json.enabled', False) %} - json.message_key: {{ log_path.get('json.message_key', 'json') }} - json.keys_under_root: {{ log_path.get('json.keys_under_root', 'true') }} - json.add_error_key: {{ log_path.get('json.add_error_key', 'true') }} - {%- endif %} - {%- if 'multiline' in log_path %} multiline: pattern: "{{ log_path.multiline.get('pattern', '') }}" @@ -66,24 +58,22 @@ output: {%- set elasticsearch = filebeat.get('elasticsearch', {}) %} {%- set logstash = filebeat.get('logstash', {}) %} -{%- if elasticsearch.get('enabled', False) %} +{%- if filebeat.elasticsearch.get('enabled', False) %} elasticsearch: - hosts: ["http://{{ elasticsearch.server }}"] + hosts: ["http://{{ filebeat.elasticsearch.server }}"] {%- endif %} -{%- if logstash.get('enabled', False) %} +{%- if filebeat.logstash.get('enabled', False) %} logstash: - hosts: {{ logstash.server }} + hosts: ["{{ filebeat.logstash.server }}"] worker: 1 loadbalance: true index: filebeat -{%- if 'tls' in logstash %} -{%- if logstash.tls.get('enabled', False) %} +{%- if 'tls' in filebeat.logstash %} +{%- if filebeat.logstash.tls.get('enabled', False) %} tls: -{%- if logstash.tls.get('ssl_cert_path') %} - certificate_authorities: ["{{ logstash.tls.ssl_cert_path }}"] -{%- endif %} + certificate_authorities: ["{{ filebeat.logstash.tls.ssl_cert_path }}"] {%- endif %} {%- endif %} {%- endif %} diff --git a/pillar.example b/pillar.example index 379a1d6..ec26a53 100644 --- a/pillar.example +++ b/pillar.example @@ -12,14 +12,13 @@ filebeat: enabled: 'true' paths: - '/path/to/log/syslog*' - # if no log_paths specified, generic syslogs are default log_paths: - paths: - '/var/log/auth.log' - '/var/log/syslog' - - + - paths: - '/var/log/apache2/access.log' input_type: 'log' @@ -39,32 +38,21 @@ filebeat: fields: - env: my_environment - server_role: webserver - - - paths: - - '/var/log/example/*.json' - json.enabled: true - json.message_key: json - json.keys_under_root: true - json.add_error_key: true + elasticsearch: enabled: False server: 127.0.0.1:9200 - logstash: + logstash: enabled: True - server: - - logstash-shipper1:5044 - - logstash-shipper2:5044 - + server: 127.0.0.1:5044 tls: enabled: True - # path to the certificate of your ELK server - # set to empty to use system certificates + # this is the public key from your ELK server + # default path is salt://filebeat/files/ca.pem + ssl_cert: salt://mycustom/filebeat/logstash-forwarder.crt ssl_cert_path: /etc/pki/tls/certs/logstash-forwarder.crt - # path to the certificate of your ELK server to be installed - # default is salt://filebeat/files/ca.pem - # set to empty to disable - ssl_cert: salt://mycustom/filebeat/logstash-forwarder.crt - # If you want to manage your own certs, set below to False - managed_cert: False + + +