From 1b29cf3d8de6eefc6045de8b21646ab90efbce7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rard=20de=20Vos?= Date: Wed, 11 May 2016 14:31:36 +0200 Subject: [PATCH 1/5] Making logstash+TLS more configurable: - client side certs not mandatory - option to use system wide CAs instead of custom --- filebeat/config.sls | 2 +- filebeat/files/filebeat.jinja | 2 ++ pillar.example | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/filebeat/config.sls b/filebeat/config.sls index 952172d..12e1631 100644 --- a/filebeat/config.sls +++ b/filebeat/config.sls @@ -1,6 +1,6 @@ {% from "filebeat/map.jinja" import conf with context %} -{% if salt['pillar.get']('filebeat:logstash:tls:enabled', False) %} +{% if salt['pillar.get']('filebeat:logstash:tls:use_custom_ca', False) %} {{ salt['pillar.get']('filebeat:logstash:tls:ssl_cert_path', '/etc/pki/tls/certs/logstash-forwarder.crt') }}: file.managed: - source: {{ salt['pillar.get']('filebeat:logstash:tls:ssl_cert', 'salt://filebeat/files/ca.pem') }} diff --git a/filebeat/files/filebeat.jinja b/filebeat/files/filebeat.jinja index 05fc670..909a89c 100644 --- a/filebeat/files/filebeat.jinja +++ b/filebeat/files/filebeat.jinja @@ -61,10 +61,12 @@ output: {%- if 'tls' in logstash %} {%- if logstash.tls.get('enabled', False) %} tls: +{%- if 'tls.ssl_cert_path' in logstash %} certificate_authorities: ["{{ logstash.tls.ssl_cert_path }}"] {%- endif %} {%- endif %} {%- endif %} +{%- endif %} shipper: diff --git a/pillar.example b/pillar.example index d274a1a..4ab9929 100644 --- a/pillar.example +++ b/pillar.example @@ -41,6 +41,9 @@ filebeat: server: 127.0.0.1:5044 tls: enabled: True + # set to true to use your own certificate authority + # defaults to False and uses the system CAs + use_custom_ca: True # this is the public key from your ELK server # default path is salt://filebeat/files/ca.pem ssl_cert: salt://mycustom/filebeat/logstash-forwarder.crt From f563d18de26502677e3fdfafce1cd716e0b5b0d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rard=20de=20Vos?= Date: Thu, 12 May 2016 16:23:34 +0200 Subject: [PATCH 2/5] add Comodo RSA Domain Validation CA cert --- filebeat/files/ca.pem | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 filebeat/files/ca.pem diff --git a/filebeat/files/ca.pem b/filebeat/files/ca.pem new file mode 100644 index 0000000..5b57157 --- /dev/null +++ b/filebeat/files/ca.pem @@ -0,0 +1,36 @@ +-----BEGIN CERTIFICATE----- +MIIGCDCCA/CgAwIBAgIQKy5u6tl1NmwUim7bo3yMBzANBgkqhkiG9w0BAQwFADCB +hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV +BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQwMjEy +MDAwMDAwWhcNMjkwMjExMjM1OTU5WjCBkDELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR +Q09NT0RPIENBIExpbWl0ZWQxNjA0BgNVBAMTLUNPTU9ETyBSU0EgRG9tYWluIFZh +bGlkYXRpb24gU2VjdXJlIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAI7CAhnhoFmk6zg1jSz9AdDTScBkxwtiBUUWOqigwAwCfx3M28Sh +bXcDow+G+eMGnD4LgYqbSRutA776S9uMIO3Vzl5ljj4Nr0zCsLdFXlIvNN5IJGS0 +Qa4Al/e+Z96e0HqnU4A7fK31llVvl0cKfIWLIpeNs4TgllfQcBhglo/uLQeTnaG6 +ytHNe+nEKpooIZFNb5JPJaXyejXdJtxGpdCsWTWM/06RQ1A/WZMebFEh7lgUq/51 +UHg+TLAchhP6a5i84DuUHoVS3AOTJBhuyydRReZw3iVDpA3hSqXttn7IzW3uLh0n +c13cRTCAquOyQQuvvUSH2rnlG51/ruWFgqUCAwEAAaOCAWUwggFhMB8GA1UdIwQY +MBaAFLuvfgI9+qbxPISOre44mOzZMjLUMB0GA1UdDgQWBBSQr2o6lFoL2JDqElZz +30O0Oija5zAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNV +HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgG +BmeBDAECATBMBgNVHR8ERTBDMEGgP6A9hjtodHRwOi8vY3JsLmNvbW9kb2NhLmNv +bS9DT01PRE9SU0FDZXJ0aWZpY2F0aW9uQXV0aG9yaXR5LmNybDBxBggrBgEFBQcB +AQRlMGMwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29tb2RvY2EuY29tL0NPTU9E +T1JTQUFkZFRydXN0Q0EuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21v +ZG9jYS5jb20wDQYJKoZIhvcNAQEMBQADggIBAE4rdk+SHGI2ibp3wScF9BzWRJ2p +mj6q1WZmAT7qSeaiNbz69t2Vjpk1mA42GHWx3d1Qcnyu3HeIzg/3kCDKo2cuH1Z/ +e+FE6kKVxF0NAVBGFfKBiVlsit2M8RKhjTpCipj4SzR7JzsItG8kO3KdY3RYPBps +P0/HEZrIqPW1N+8QRcZs2eBelSaz662jue5/DJpmNXMyYE7l3YphLG5SEXdoltMY +dVEVABt0iN3hxzgEQyjpFv3ZBdRdRydg1vs4O2xyopT4Qhrf7W8GjEXCBgCq5Ojc +2bXhc3js9iPc0d1sjhqPpepUfJa3w/5Vjo1JXvxku88+vZbrac2/4EjxYoIQ5QxG +V/Iz2tDIY+3GH5QFlkoakdH368+PUq4NCNk+qKBR6cGHdNXJ93SrLlP7u3r7l+L4 +HyaPs9Kg4DdbKDsx5Q5XLVq4rXmsXiBmGqW5prU5wfWYQ//u+aen/e7KJD2AFsQX +j4rBYKEMrltDR5FL1ZoXX/nUh8HCjLfn4g8wGTeGrODcQgPmlKidrv0PJFGUzpII +0fxQ8ANAe4hZ7Q7drNJ3gjTcBpUC2JD5Leo31Rpg0Gcg19hCC0Wvgmje3WYkN5Ap +lBlGGSW4gNfL1IYoakRwJiNiqZ+Gb7+6kHDSVneFeO/qJakXzlByjAA6quPbYzSf ++AZxAeKCINT+b72x +-----END CERTIFICATE----- + From 50444961b27254e61c7c289ec22326cbafcf4d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rard=20de=20Vos?= Date: Thu, 12 May 2016 16:39:53 +0200 Subject: [PATCH 3/5] it no need to be here --- filebeat/files/ca.pem | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 filebeat/files/ca.pem diff --git a/filebeat/files/ca.pem b/filebeat/files/ca.pem deleted file mode 100644 index 5b57157..0000000 --- a/filebeat/files/ca.pem +++ /dev/null @@ -1,36 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIGCDCCA/CgAwIBAgIQKy5u6tl1NmwUim7bo3yMBzANBgkqhkiG9w0BAQwFADCB -hTELMAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G -A1UEBxMHU2FsZm9yZDEaMBgGA1UEChMRQ09NT0RPIENBIExpbWl0ZWQxKzApBgNV -BAMTIkNPTU9ETyBSU0EgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTQwMjEy -MDAwMDAwWhcNMjkwMjExMjM1OTU5WjCBkDELMAkGA1UEBhMCR0IxGzAZBgNVBAgT -EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEaMBgGA1UEChMR -Q09NT0RPIENBIExpbWl0ZWQxNjA0BgNVBAMTLUNPTU9ETyBSU0EgRG9tYWluIFZh -bGlkYXRpb24gU2VjdXJlIFNlcnZlciBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAI7CAhnhoFmk6zg1jSz9AdDTScBkxwtiBUUWOqigwAwCfx3M28Sh -bXcDow+G+eMGnD4LgYqbSRutA776S9uMIO3Vzl5ljj4Nr0zCsLdFXlIvNN5IJGS0 -Qa4Al/e+Z96e0HqnU4A7fK31llVvl0cKfIWLIpeNs4TgllfQcBhglo/uLQeTnaG6 -ytHNe+nEKpooIZFNb5JPJaXyejXdJtxGpdCsWTWM/06RQ1A/WZMebFEh7lgUq/51 -UHg+TLAchhP6a5i84DuUHoVS3AOTJBhuyydRReZw3iVDpA3hSqXttn7IzW3uLh0n -c13cRTCAquOyQQuvvUSH2rnlG51/ruWFgqUCAwEAAaOCAWUwggFhMB8GA1UdIwQY -MBaAFLuvfgI9+qbxPISOre44mOzZMjLUMB0GA1UdDgQWBBSQr2o6lFoL2JDqElZz -30O0Oija5zAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNV -HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgG -BmeBDAECATBMBgNVHR8ERTBDMEGgP6A9hjtodHRwOi8vY3JsLmNvbW9kb2NhLmNv -bS9DT01PRE9SU0FDZXJ0aWZpY2F0aW9uQXV0aG9yaXR5LmNybDBxBggrBgEFBQcB -AQRlMGMwOwYIKwYBBQUHMAKGL2h0dHA6Ly9jcnQuY29tb2RvY2EuY29tL0NPTU9E -T1JTQUFkZFRydXN0Q0EuY3J0MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21v -ZG9jYS5jb20wDQYJKoZIhvcNAQEMBQADggIBAE4rdk+SHGI2ibp3wScF9BzWRJ2p -mj6q1WZmAT7qSeaiNbz69t2Vjpk1mA42GHWx3d1Qcnyu3HeIzg/3kCDKo2cuH1Z/ -e+FE6kKVxF0NAVBGFfKBiVlsit2M8RKhjTpCipj4SzR7JzsItG8kO3KdY3RYPBps -P0/HEZrIqPW1N+8QRcZs2eBelSaz662jue5/DJpmNXMyYE7l3YphLG5SEXdoltMY -dVEVABt0iN3hxzgEQyjpFv3ZBdRdRydg1vs4O2xyopT4Qhrf7W8GjEXCBgCq5Ojc -2bXhc3js9iPc0d1sjhqPpepUfJa3w/5Vjo1JXvxku88+vZbrac2/4EjxYoIQ5QxG -V/Iz2tDIY+3GH5QFlkoakdH368+PUq4NCNk+qKBR6cGHdNXJ93SrLlP7u3r7l+L4 -HyaPs9Kg4DdbKDsx5Q5XLVq4rXmsXiBmGqW5prU5wfWYQ//u+aen/e7KJD2AFsQX -j4rBYKEMrltDR5FL1ZoXX/nUh8HCjLfn4g8wGTeGrODcQgPmlKidrv0PJFGUzpII -0fxQ8ANAe4hZ7Q7drNJ3gjTcBpUC2JD5Leo31Rpg0Gcg19hCC0Wvgmje3WYkN5Ap -lBlGGSW4gNfL1IYoakRwJiNiqZ+Gb7+6kHDSVneFeO/qJakXzlByjAA6quPbYzSf -+AZxAeKCINT+b72x ------END CERTIFICATE----- - From c05f396d4fcdc6b01818b400cbc8b6efa0e2374f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rard=20de=20Vos?= Date: Thu, 12 May 2016 17:36:21 +0200 Subject: [PATCH 4/5] if use_custom_ca --- filebeat/files/filebeat.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/files/filebeat.jinja b/filebeat/files/filebeat.jinja index 909a89c..6d1631e 100644 --- a/filebeat/files/filebeat.jinja +++ b/filebeat/files/filebeat.jinja @@ -61,7 +61,7 @@ output: {%- if 'tls' in logstash %} {%- if logstash.tls.get('enabled', False) %} tls: -{%- if 'tls.ssl_cert_path' in logstash %} +{%- if logstash.tls.get('use_custom_ca', False) %} certificate_authorities: ["{{ logstash.tls.ssl_cert_path }}"] {%- endif %} {%- endif %} From 85b322a07b001ed6a7c0abb31b8bb0e4902422fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A9rard=20de=20Vos?= Date: Fri, 13 May 2016 11:20:57 +0200 Subject: [PATCH 5/5] add tail_files option --- filebeat/files/filebeat.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/filebeat/files/filebeat.jinja b/filebeat/files/filebeat.jinja index 6d1631e..4717333 100644 --- a/filebeat/files/filebeat.jinja +++ b/filebeat/files/filebeat.jinja @@ -11,6 +11,7 @@ filebeat: input_type: {{ log_path.get('input_type', 'log') }} document_type: {{ log_path.get('document_type', 'syslog') }} ignore_older: {{ log_path.get('ignore_older', '24h') }} + tail_files: {{ log_path.get('tail_files', 'false') }} scan_frequency: {{ log_path.get('scan_frequency', '10s') }} backoff: {{ log_path.get('backoff', '1s') }} max_backoff: {{ log_path.get('max_backoff', '10s') }}