SYS-652 Grafana helm chart, with prometheus and alertmanager

README.md

@@ -42,6 +42,7 @@ The cluster-deployment tools here include helm charts and ansible playbooks to s
 * Pod security policies
 * Automatic certificate issuing/renewal with Letsencrypt
 * PostgreSQL-operator from CrunchyData
+* Grafana with prometheus-based alerting
 
 ### Resource definitions
 
@@ -70,6 +71,7 @@ The cluster-deployment tools here include helm charts and ansible playbooks to s
 | duplicati | [![](https://img.shields.io/docker/v/instantlinux/duplicati?sort=date)](https://hub.docker.com/r/instantlinux/duplicati "Version badge") | backups |
 | ez-ipupdate | [![](https://img.shields.io/docker/v/instantlinux/ez-ipupdate?sort=date)](https://hub.docker.com/r/instantlinux/ez-ipupdate "Version badge") | Dynamic DNS client |
 | haproxy-keepalived | [![](https://img.shields.io/docker/v/instantlinux/haproxy-keepalived?sort=date)](https://hub.docker.com/r/instantlinux/haproxy-keepalived "Version badge") | load balancer |
+| grafana | ** | monitoring dashboard with prometheus-based alerting |
 | guacamole | ** | authenticated remote-desktop server |
 | logspout | ** | central logging for Docker |
 | mysqldump | [![](https://img.shields.io/docker/v/instantlinux/mysqldump?sort=date)](https://hub.docker.com/r/instantlinux/mysqldump "Version badge") | per-database alternative to xtrabackup |

ansible/roles/monitoring_agent/defaults/main.yml

@@ -83,6 +83,7 @@ syslog: "{{ syslog_defaults | combine(syslog_override) }}"
 ubuntu_packages:
   - bc
   - nagios-nrpe-server
+  - prometheus-node-exporter
   - python3-pip
   - rsyslog
   - smartmontools

k8s/Makefile

@@ -95,14 +95,14 @@ $(STACKS)::
 .PHONY: envsubst imports install namespace_config node_labels \
 	persistent remote_volumes secrets sops untaint_master
 
-IMPORTS      = cert-manager flannel metrics
+IMPORTS      = cert-manager flannel
 INSTALL_YAML = $(basename $(wildcard install/*.yaml)) \
           $(addprefix imports/, $(IMPORTS))
 VOLUMES_YAML = $(basename $(wildcard volumes/*.yaml))
 
 install: install/admin-user cluster_network \
 	install/local-storage storage_localdefault imports \
-	install_imports namespace_config install/gitlab-rbac \
+	install_imports namespace_config install/prometheus-rbac \
 	install/k8s-backup install/logspout remote_volumes \
 	sops data-sync-ssh persistent secrets install/ingress-nginx \
 	install/cert-manager
@@ -219,10 +219,14 @@ cert-manager-helm: helm_install
 imports: $(foreach file,$(IMPORTS),imports/$(file).yaml)
 install_imports: $(foreach file, $(IMPORTS), imports/$(file))
 
-imports/metrics.yaml: imports/metrics-$(VERSION_METRICS).yaml
+imports/kube-state-metrics: imports/kube-state-metrics-$(VERSION_METRICS)
 	ln -s $(notdir $<) $@
-imports/metrics-$(VERSION_METRICS).yaml:
-	curl -sLo $@ https://github.com/kubernetes-sigs/metrics-server/releases/download/v$(VERSION_METRICS)/components.yaml
+imports/kube-state-metrics-$(VERSION_METRICS):
+	git clone --dept 1 --branch v$(VERSION_METRICS) \
+	 https://github.com/kubernetes/kube-state-metrics \
+	 imports/kube-state-metrics-$(VERSION_METRICS)
+install_metrics: imports/kube-state-metrics
+	kubectl apply --context=sudo -k $</examples/standard
 
 imports/traefik-prom.yaml:
 	curl -sLo $@ https://raw.githubusercontent.com/mateobur/prometheus-monitoring-guide/master/traefik-prom.yaml

k8s/Makefile.vars

@@ -32,6 +32,8 @@ export TZ                   ?= UTC
 export K8S_INGRESS_NGINX_IP ?= 10.101.1.2
 export AUTHELIA_IP          ?= 10.101.1.5
 export MONITOR_EXT_IP       ?= 192.168.1.20
+# export PROMETHEUS_IP	    ?= 10.101.1.21
+# export PROM_ALERT_IP	    ?= 10.101.1.22
 export RSYSLOGD_IP          ?= 10.101.1.40
 
 # Exposed nodePorts - install/ingress-nginx.yaml

k8s/Makefile.versions

@@ -9,7 +9,7 @@ export VERSION_DEFAULTBACKEND ?= 1.5
 export VERSION_FLANNEL        ?= 0.26.1
 export VERSION_HELM           ?= 3.16.2
 export VERSION_INGRESS_NGINX  ?= 1.11.2
-export VERSION_METRICS        ?= 0.7.2
+export VERSION_METRICS        ?= 2.15.0
 
 # Held back versions - more effort to upgrade
 export VERSION_CALICO        ?= 3.16.5

k8s/helm/grafana/.helmignore

@@ -0,0 +1,2 @@
+*~
+.git

k8s/helm/grafana/Chart.yaml

@@ -0,0 +1,26 @@
+apiVersion: v2
+name: grafana
+description: Grafana metrics visualization and alerting
+home: https://github.com/instantlinux/docker-tools
+sources:
+- https://github.com/instantlinux/docker-tools
+- https://github.com/grafana/grafana
+type: application
+version: 0.1.0
+appVersion: 12.0.0
+dependencies:
+- name: chartlib
+  version: 0.1.8
+  repository: https://instantlinux.github.io/docker-tools
+- name: prometheus
+  version: 0.1.0
+  repository: file://subcharts/prometheus
+  condition: prometheus.enabled
+- name: alertmanager
+  version: 0.1.0
+  repository: file://subcharts/alertmanager
+  condition: alertmanager.enabled
+- name: redis
+  version: 0.1.0
+  repository: file://subcharts/redis
+  condition: redis.enabled

k8s/helm/grafana/subcharts/alertmanager/.helmignore

@@ -0,0 +1,2 @@
+*~
+.git

k8s/helm/grafana/subcharts/alertmanager/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v2
+name: alertmanager
+description: Prometheus alerting
+home: https://github.com/instantlinux/docker-tools
+sources:
+- https://github.com/instantlinux/docker-tools
+- https://hub.docker.com/r/prom/alertmanager
+type: application
+version: 0.1.0
+appVersion: "v0.28.1"
+dependencies:
+- name: chartlib
+  version: 0.1.8
+  repository: https://instantlinux.github.io/docker-tools

k8s/helm/grafana/subcharts/alertmanager/templates/NOTES.txt

@@ -0,0 +1,28 @@
+{{- if hasKey .Values "service" }}
+{{- if or .Values.service.enabled (not (hasKey .Values.service "enabled")) }}
+1. Get the application URL by running these commands:
+{{- if hasKey .Values "ingress" }}
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "local.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "local.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "local.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "local.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s/helm/grafana/subcharts/alertmanager/templates/app.yaml

@@ -0,0 +1,15 @@
+{{- include "chartlib.configmap" . }}
+---
+{{- include "chartlib.deployment" . }}
+---
+{{- include "chartlib.hpa" . }}
+---
+{{- include "chartlib.ingress" . }}
+---
+{{- include "chartlib.ingresstotp" . }}
+---
+{{- include "chartlib.service" . }}
+---
+{{- include "chartlib.serviceaccount" . }}
+---
+{{- include "chartlib.statefulset" . }}

k8s/helm/grafana/subcharts/alertmanager/templates/tests/test-connection.yaml

@@ -0,0 +1,17 @@
+{{- if hasKey .Values "service" }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "local.fullname" . }}-test-connection"
+  labels:
+    {{- include "local.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "local.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never
+{{- end }}

k8s/helm/grafana/subcharts/alertmanager/values.yaml

@@ -0,0 +1,65 @@
+# Default values for alertmanager.
+deployment:
+  command:
+  - /bin/alertmanager
+  - --config.file=/etc/alertmanager/alertmanager.yml
+  - --storage.path=/alertmanager
+  - --web.external-url=http://10.101.1.22:9093
+  containerPorts: [ containerPort: 9093 ]
+  env:
+    smtp_from: alertmanager@example.com
+    smtp_smarthost: smtp:25
+volumeMounts:
+- mountPath: /etc/alertmanager/alertmanager.yml
+  name: config
+  readOnly: true
+  subPath: alertmanager.yml
+- mountPath: /alertmanager
+  name: data
+volumes:
+- name: config
+  configMap:
+    name: grafana-alertmanager
+- name: data
+  hostPath: { path: /var/lib/docker/k8s-volumes/prometheus/alertmanager }
+image:
+  repository: prom/alertmanager
+  pullPolicy: IfNotPresent
+  # tag: default
+
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  enabled: false
+service:
+  clusterIP: 10.101.1.22
+  ports: [ port: 9093 ]
+  type: ClusterIP
+autoscaling:
+  enabled: false
+
+configmap:
+  data:
+    alertmanager.yml: |
+      global:
+        smtp_from: Prometheus Notifications <alertmanager@noreply.example.com>
+        smtp_smarthost: smtp:587
+      route:
+        group_by: [ alertname ]
+        group_wait: 10s
+        group_interval: 1m
+        repeat_interval: 1d
+        receiver: SMTP
+      receivers:
+      - name: SMTP
+        email_configs:
+        - require_tls: true
+          send_resolved: true
+          to: monitor@example.com
+      inhibit_rules:
+      - source_match:
+          severity: critical
+        target_match:
+          severity: warning
+        equal: [ alertname, dev, instance ]

k8s/helm/grafana/subcharts/prometheus/.helmignore

@@ -0,0 +1,2 @@
+*~
+.git

k8s/helm/grafana/subcharts/prometheus/Chart.yaml

@@ -0,0 +1,14 @@
+apiVersion: v2
+name: prometheus
+description: Prometheus metrics and alerting
+home: https://github.com/instantlinux/docker-tools
+sources:
+- https://github.com/instantlinux/docker-tools
+- https://hub.docker.com/r/prom/prometheus
+type: application
+version: 0.1.0
+appVersion: "v3.3.1"
+dependencies:
+- name: chartlib
+  version: 0.1.8
+  repository: https://instantlinux.github.io/docker-tools

k8s/helm/grafana/subcharts/prometheus/templates/NOTES.txt

@@ -0,0 +1,28 @@
+{{- if hasKey .Values "service" }}
+{{- if or .Values.service.enabled (not (hasKey .Values.service "enabled")) }}
+1. Get the application URL by running these commands:
+{{- if hasKey .Values "ingress" }}
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "local.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "local.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "local.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "local.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s/helm/grafana/subcharts/prometheus/templates/app.yaml

@@ -0,0 +1,15 @@
+{{- include "chartlib.configmap" . }}
+---
+{{- include "chartlib.deployment" . }}
+---
+{{- include "chartlib.hpa" . }}
+---
+{{- include "chartlib.ingress" . }}
+---
+{{- include "chartlib.ingresstotp" . }}
+---
+{{- include "chartlib.service" . }}
+---
+{{- include "chartlib.serviceaccount" . }}
+---
+{{- include "chartlib.statefulset" . }}

k8s/helm/grafana/subcharts/prometheus/templates/tests/test-connection.yaml

@@ -0,0 +1,17 @@
+{{- if hasKey .Values "service" }}
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "local.fullname" . }}-test-connection"
+  labels:
+    {{- include "local.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "local.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never
+{{- end }}