diff --git a/images/nut-upsd/Dockerfile b/images/nut-upsd/Dockerfile index 3b9ddac6..cb5f9d31 100644 --- a/images/nut-upsd/Dockerfile +++ b/images/nut-upsd/Dockerfile @@ -1,13 +1,13 @@ FROM alpine:3.20 -MAINTAINER Rich Braun "docker@instantlinux.net" ARG BUILD_DATE ARG VCS_REF -LABEL org.label-schema.build-date=$BUILD_DATE \ +LABEL org.opencontainers.image.authors="Rich Braun docker@instantlinux.net" \ + org.label-schema.build-date=$BUILD_DATE \ org.label-schema.license=GPL-2.0 \ org.label-schema.name=nut-upsd \ org.label-schema.vcs-ref=$VCS_REF \ org.label-schema.vcs-url=https://github.com/instantlinux/docker-tools -ARG NUT_VERSION=2.8.2-r0 +ARG NUT_VERSION=2.8.2-r2 ENV API_USER=upsmon \ API_PASSWORD= \ DESCRIPTION=UPS \ @@ -18,13 +18,13 @@ ENV API_USER=upsmon \ POLLINTERVAL= \ PORT=auto \ SDORDER= \ - SECRET=nut-upsd-password \ + SECRETNAME=nut-upsd-password \ SERIAL= \ SERVER=master \ USER=nut \ VENDORID= HEALTHCHECK CMD upsc $NAME@localhost:3493 2>&1|grep -q stale && \ - kill -SIGTERM -1 || true + killall -TERM upsmon || true RUN echo '@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' \ >>/etc/apk/repositories && \ @@ -38,4 +38,4 @@ RUN echo '@edge http://dl-cdn.alpinelinux.org/alpine/edge/community' \ EXPOSE 3493 COPY entrypoint.sh /usr/local/bin/ -ENTRYPOINT /usr/local/bin/entrypoint.sh +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/images/nut-upsd/README.md b/images/nut-upsd/README.md index 68755688..895f9f04 100644 --- a/images/nut-upsd/README.md +++ b/images/nut-upsd/README.md @@ -48,7 +48,7 @@ NAME | ups | user-assigned config name POLLINTERVAL | | Poll Interval for ups.conf PORT | auto | device port (e.g. /dev/ttyUSB0) on host SDORDER | | UPS shutdown sequence, set to -1 to disable shutdown -SECRET | nut-upsd-password | name of secret to use for API user +SECRETNAME | nut-upsd-password | name of secret to use for API user SERIAL | | hardware serial number of UPS SERVER | master | master or slave priority for scripts USER | nut | local user diff --git a/images/nut-upsd/entrypoint.sh b/images/nut-upsd/entrypoint.sh index ced303d5..ee29ac08 100755 --- a/images/nut-upsd/entrypoint.sh +++ b/images/nut-upsd/entrypoint.sh @@ -1,7 +1,9 @@ #! /bin/sh -e -if [ -d /run/secrets ] && [ -s /run/secrets/$SECRET ]; then - API_PASSWORD=$(cat /run/secrets/$SECRET) +cat /etc/passwd + +if [ -d /run/secrets ] && [ -s /run/secrets/$SECRETNAME ]; then + API_PASSWORD=$(cat /run/secrets/$SECRETNAME) fi if [ ! -e /etc/nut/.setup ]; then @@ -63,10 +65,10 @@ fi chgrp $GROUP /etc/nut/* chmod 640 /etc/nut/* mkdir -p -m 2750 /dev/shm/nut -chown $USER.$GROUP /dev/shm/nut +chown $USER:$GROUP /dev/shm/nut [ -e /var/run/nut ] || ln -s /dev/shm/nut /var/run # Issue #15 - change pid warning message from "No such file" to "Ignoring" -echo 0 > /var/run/nut/upsd.pid && chown $USER.$GROUP /var/run/nut/upsd.pid +echo 0 > /var/run/nut/upsd.pid && chown $USER:$GROUP /var/run/nut/upsd.pid echo 0 > /var/run/upsmon.pid /usr/sbin/upsdrvctl -u root start diff --git a/images/nut-upsd/helm/Chart.yaml b/images/nut-upsd/helm/Chart.yaml index d2e8821a..3d1d7cf4 100644 --- a/images/nut-upsd/helm/Chart.yaml +++ b/images/nut-upsd/helm/Chart.yaml @@ -6,8 +6,8 @@ sources: - https://github.com/instantlinux/docker-tools - https://github.com/networkupstools/nut type: application -version: 0.1.7 -appVersion: "2.8.2-r0" +version: 0.1.8 +appVersion: "2.8.2-r2" dependencies: - name: chartlib version: 0.1.8 diff --git a/k8s/Makefile.instances b/k8s/Makefile.instances index 30880bac..3aedb5f4 100644 --- a/k8s/Makefile.instances +++ b/k8s/Makefile.instances @@ -8,19 +8,6 @@ mariadb-galera:: db00 db01 db02 db03:: etcd -gitlab-runner:: - @echo Invoke 'make gitlab-agent' instead - @exit 1 - -gitlab-agent: ../admin/services/values.yaml ./helm/gitlab-runner/Chart.lock - @echo Installing service with privileged /var/run/docker.sock access - @K8S_NAMESPACE=gitlab make install/limits - @helm upgrade --install -f $< \ - -f ../admin/services/values/gitlab-runner.yaml $(XARGS) \ - gitlab-runner ./helm/gitlab-runner --namespace gitlab \ - --kube-context=kubernetes-admin@$(CLUSTER) - -sops -d secrets/$(CA_SECRET).yml | envsubst | kubectl create $(ADMIN_CTX) -n gitlab -f - - nut-upsd:: @echo Invoke 'make nut-xx' (e.g. nut-01) instead @exit 1 diff --git a/k8s/Makefile.vars b/k8s/Makefile.vars index c40ed953..e9516df0 100644 --- a/k8s/Makefile.vars +++ b/k8s/Makefile.vars @@ -6,11 +6,9 @@ export DOMAIN ?= domain.com -export CA_SECRET ?= instantlinux-ca export CERT_MGR_EMAIL ?= admin@$(DOMAIN) export CLUSTER_NETWORK ?= flannel export DB_HOST ?= db00.$(DOMAIN) -export DOL = $$ export EDITOR ?= vi export K8S_NAMESPACE ?= worker export K8S_NODES ?= kube1.$(DOMAIN) kube2.$(DOMAIN) kube3.$(DOMAIN) @@ -20,12 +18,7 @@ export LIMIT_CPU_DEFAULT ?= 500m export LIMIT_CPU_REQUEST ?= 50m export LIMIT_MEM_DEFAULT ?= 256Mi export LIMIT_MEM_REQUEST ?= 64Mi -export MYDOMAIN ?= $(DOMAIN) export MYTHTV_VOL_SIZE ?= 400Gi -export NAGIOS_FQDN ?= nagios.k8s -export NAGIOS_HOSTNETWORK ?= false -export NAGIOS_MAIL_RELAY ?= smtp.$(DOMAIN):25 -export NAGIOS_USERS ?= nagiosadmin export NAMED_VOLUMES ?= share $(LOCAL_VOLUMES) export NFS_HOST ?= nfs.$(DOMAIN) export POOL_NUM_MEDIUM ?= 0010 @@ -33,8 +26,6 @@ export POOL_NUM_SMALL ?= 0030 export POOL_SIZE_LARGE = 20Gi export POOL_SIZE_MEDIUM = 8Gi export POOL_SIZE_SMALL = 500Mi -export REGISTRY_URI ?= instantlinux -export REGISTRY_LOCAL ?= nexus.$(MYDOMAIN) export TZ ?= UTC # IP addresses - TODO replace static IPs with names, if practical @@ -49,8 +40,19 @@ export NODEPORT_HTTPS ?= 30443 export PORT_DOVECOT_IMAPD ?= 843 export PORT_DOVECOT_IMAPS ?= 993 export PORT_DOVECOT_SMTP ?= 825 -export PORT_GITLAB_SSH ?= 8999 +export PORT_GIT_SSH ?= 8999 export PORT_POSTFIX_INTERNAL ?= 3425 export PORT_POSTFIX_EXTERNAL ?= 3525 # Port configured in install/logspout.yaml export PORT_RSYSLOGD ?= 514 + +# Deprecated +export CA_SECRET ?= instantlinux-ca +export DOL = $$ +export MYDOMAIN ?= $(DOMAIN) +export NAGIOS_FQDN ?= nagios.k8s +export NAGIOS_HOSTNETWORK ?= false +export NAGIOS_MAIL_RELAY ?= smtp.$(DOMAIN):25 +export NAGIOS_USERS ?= nagiosadmin +export REGISTRY_URI ?= instantlinux +export REGISTRY_LOCAL ?= nexus.$(MYDOMAIN) diff --git a/k8s/Makefile.versions b/k8s/Makefile.versions index 3715a073..808e1f3d 100644 --- a/k8s/Makefile.versions +++ b/k8s/Makefile.versions @@ -1,8 +1,3 @@ -# Versions (images within this repo published to docker hub) -# See Chart.yaml appVersion for services defined as helm charts -export VERSION_NAGIOS ?= 4.4.9-2.4.2 -export VERSION_NAGIOSQL ?= 3.4.1-4.4.6 - # Third-party versions - dockerhub export VERSION_DASHBOARD ?= 7.10.0 export VERSION_LOGSPOUT ?= v3.2.14 diff --git a/k8s/README.md b/k8s/README.md index 3580abb0..6b40ea71 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -155,6 +155,8 @@ Set a symlink from a directory under this one (k8s/secrets) to a subdirectory in your local administrative repo. This is where you will store kubernetes secrets, encrypted by a tool called _sops_. +### OpenID + To tighten security by creating API users you will need to set up OpenID / OAuth2. An on-site user directory can be established using the open-source tool [Keycloak](https://www.keycloak.org/) (a docker-compose file is provided under [services/keycloak](https://github.com/instantlinux/docker-tools/tree/main/services/keycloak/docker-compose.yml)) for which a somewhat complicated configuration is required (TODO - I'll write up the procedure in the docs here). Start by downloading [krew](https://github.com/kubernetes-sigs/krew/releases/latest/download/krew-linux_amd64.tar.gz) and adding it to your $PATH. To get a single-user setup working, follow these steps: * Go to your google account and add client-id k8slogin of type desktop, in [credentials dashboard](https://console.cloud.google.com/apis/credentials); @@ -189,6 +191,39 @@ kubectl create clusterrolebinding oidc-cluster-admin \ user: oidc name: user@kubernetes ``` +Alternatively, you can set up Keycloak on a local container, which provides finer-granularity group permissions. Setting that up is beyond scope of this README (and a warning, Keycloak's documentation is not easy to follow). Once the user and group is set up, verify with: +``` +PW= +export TOKEN=$(curl -d username=$USER -d "password=$PW" \ + -d grant_type=password \ + -d client_id=k8s-access \ + -d client_secret=$CLIENT_SECRET \ + https://oidc.instantlinux.net/realms/k8s/protocol/openid-connect/token | \ + jq -r '.access_token') +echo $TOKEN +curl -X GET https://oidc.instantlinux.net/realms/k8s/protocol/openid-connect/userinfo \ + -H "Accept: application/json" \ + -H "Authorization: Bearer $TOKEN" | jq . +``` +and the response will look similar to this: +``` +{ + "sub": "bdeec4c0-5070-4c6a-ac25-1fb0f26ccc1b", + "email_verified": true, + "name": "Rich Braun", + "groups": [ + "/instantlinux" + ], + "preferred_username": "richb", + "given_name": "Rich", + "family_name": "Braun", + "email": "richb@pioneer.ci.net", + "username": "richb" +} +``` +Look in the k8s/install subdirectory for resources in namespace-user.yaml for examples of how to map oidc username from Keycloak to k8s Role and ClusterRole permissions. + +### Installation To configure k8s resources, invoke the following in this directory ([k8s](https://github.com/instantlinux/docker-tools/tree/main/k8s)): ``` @@ -244,6 +279,8 @@ _make secrets/keyname_. Manage their contents and lifecycle using the _sops_ command. This tool also supports cloud key-managers like KMS, but gpg is suitable for bare-metal data center setups. +### Certificate Manager + Cert-manager installation is part of the above _make install_; to start the issuer invoke: ``` @@ -341,10 +378,12 @@ debacles: flannel and calico installed, a major conflict. * Installation procedure for cert-manager is 100% different from 5 - months ago. That took me about 3 hours to resolve. And I'd become - over-reliant on cert-manager: without valid TLS certificates, my - local Docker registry wouldn't come up. Without the registry, most - services wind up in ImagePullBackoff failure state. + months ago (in 2022). That took me about 3 hours to resolve. And I'd + become over-reliant on cert-manager: without valid TLS certificates, + my local Docker registry wouldn't come up. Without the registry, + most services wind up in ImagePullBackoff failure state. (Update in + 2024 -- almost services I run are now on docker hub or + registry.k8s.io, so they depend only on Internet and DNS.) * When restoring cert-manager, get ingress-nginx working first. @@ -354,11 +393,13 @@ debacles: the community-supplied installer (kubeadm) finishes, then it's quite likely whatever script or resource definition you created to automate such manual processes *won't* work next time you need to do disaster- - recovery or routine upgrades. + recovery or routine upgrades. Make sure your kubeadm-config.yaml + defines all the flags required for the control plane under + /etc/kubernetes/manifests. * One thing I'd done that compromised availability in the interest of - security was to encrypt etcd key-value storage. If I revisit that in - the future, make sure to practice backup/restore a couple times, and - make sure to document in an obvious place what the restore procedure - is and where to get the decyption codes. I probably won't revisit - this until the feature is fully GA. + security was to encrypt etcd key-value storage. Make sure to + practice backup/restore a couple times, and document in an obvious + place what the restore procedure is and where to get the decyption + codes. The k8s-cplane ansible playbook here should help. + diff --git a/k8s/install/ingress-nginx.yaml b/k8s/install/ingress-nginx.yaml index 2b9febd5..352f75cd 100644 --- a/k8s/install/ingress-nginx.yaml +++ b/k8s/install/ingress-nginx.yaml @@ -13,7 +13,7 @@ spec: ports: - { port: 80, targetPort: 80, name: http } - { port: 443, targetPort: 443, name: https } - - { port: $PORT_GITLAB_SSH, name: git-ssh } + - { port: $PORT_GIT_SSH, name: git-ssh } sessionAffinity: ClientIP --- apiVersion: v1 @@ -128,7 +128,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: default-http-backend - image: gcr.io/google_containers/defaultbackend-amd64:$VERSION_DEFAULTBACKEND + image: registry.k8s.io/defaultbackend-amd64:$VERSION_DEFAULTBACKEND ports: - containerPort: 8080 --- @@ -271,9 +271,8 @@ metadata: name: ingress-tcp-services namespace: $K8S_NAMESPACE data: - ## TODO document this - # $PORT_GITLAB_SSH: $K8S_NAMESPACE/gitlab:$PORT_GITLAB_SSH - $PORT_GITLAB_SSH: $K8S_NAMESPACE/gitea:$PORT_GITLAB_SSH + # Routing for custom TCP ports served by this ingress + $PORT_GIT_SSH: $K8S_NAMESPACE/gitea:$PORT_GIT_SSH $PORT_POSTFIX_EXTERNAL: $K8S_NAMESPACE/postfix:$PORT_POSTFIX_EXTERNAL $PORT_DOVECOT_IMAPD: $K8S_NAMESPACE/dovecot:$PORT_DOVECOT_IMAPD $PORT_DOVECOT_IMAPS: $K8S_NAMESPACE/dovecot:$PORT_DOVECOT_IMAPS