diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml new file mode 100644 index 000000000..33055ccc1 --- /dev/null +++ b/.github/workflows/_helm.yml @@ -0,0 +1,71 @@ +name: Package helm charts + +on: + workflow_call: + +env: + HELM_VERSION_TO_INSTALL: 3.14.3 + +jobs: + package-helm-charts: + name: Package and Push Helm Chart + runs-on: ubuntu-latest + environment: release + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install helm + uses: Azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION_TO_INSTALL }} + + # Check that alpha/beta versions have the form X.Y.Z-alpha.A requried by Helm. + # An early check saves waiting for the entire build before finding a problem. + - name: Check helm version tag + if: ${{ github.ref_type == 'tag' }} + env: + VERSION: "${{ github.ref_name }}" + run: | + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-alpha|-beta|-rc).*?$ ]]; then + echo "Valid version format: ${VERSION}" + else + echo "Invalid version: ${VERSION}. Expected: X.Y.Z or X.Y.Z-beta.1 or X.Y.Z-alpha.1" + exit 1 + fi + + - name: Package helm charts + env: + VERSION: "${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}" + run: | + set -xe + + mkdir -p charts + for i in Charts/*; do + if [[ ${i} =~ ^.*-ioc$ ]]; then + echo "Skipping IOC schema chart: ${i}" + continue + fi + echo "Packaging chart: ${i}" + helm package -u --app-version ${VERSION} --version ${VERSION} ${i} + mv $(basename ${i})-*.tgz charts/ + done + + - name: Upload helm chart values schemas + uses: actions/upload-artifact@v4 + with: + name: helm-chart-schemas + path: schemas/ + + - name: Push tagged helm chart to registry + if: ${{ github.ref_type == 'tag' }} + run: | + set -x + + echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io/${{ github.repository_owner }} --username ${{ github.repository_owner }} --password-stdin + REGISTRY=oci://ghcr.io/diamondlightsource/charts + for i in charts/*.tgz; do + helm push "${i}" ${REGISTRY} + done diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 3e5879cb9..f652d4145 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -50,9 +50,6 @@ jobs: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" - - name: Update setuptools - run: pip install --upgrade setuptools - - name: Run tests run: tox -e tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9529f09e4..6f661fd49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,12 @@ jobs: if: needs.check.outputs.branch-pr == '' uses: ./.github/workflows/_dist.yml + helm: + uses: ./.github/workflows/_helm.yml + permissions: + contents: read + packages: write + pypi: if: github.ref_type == 'tag' needs: dist @@ -53,7 +59,7 @@ jobs: release: if: github.ref_type == 'tag' - needs: [dist, docs] + needs: [dist, docs, helm] uses: ./.github/workflows/_release.yml permissions: contents: write diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60fc23f9a..deebb103e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,10 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-yaml + exclude: ^Charts/ - id: check-merge-conflict - id: end-of-file-fixer @@ -22,3 +23,10 @@ repos: entry: ruff format --force-exclude types: [python] require_serial: true + + - id: helm-schema + name: Generate Helm schema + entry: ./generate-schemas.sh + language: system + types: [yaml] + require_serial: true diff --git a/Charts/fastcs-ioc/Chart.yaml b/Charts/fastcs-ioc/Chart.yaml new file mode 100644 index 000000000..80f4400f3 --- /dev/null +++ b/Charts/fastcs-ioc/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: fastcs-ioc +version: 0.1.0 +# This chart allows generating a schema for the fastcs-ioc values file. +# fastcs-ioc schema is used by IOC instances which have fastcs +# as a dependency. +# +# This chart is never used except by helm-schema. diff --git a/Charts/fastcs-ioc/values.schema.json b/Charts/fastcs-ioc/values.schema.json new file mode 100644 index 000000000..67827a5a4 --- /dev/null +++ b/Charts/fastcs-ioc/values.schema.json @@ -0,0 +1,300 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "fastcs": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "affinity": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity", + "description": "Affinity for the pod", + "required": [], + "title": "affinity", + "type": "object" + }, + "args": { + "description": "arguments to pass to the above command", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "args", + "type": "array" + }, + "baseIp": { + "default": "10.96.0.0/12", + "description": "Used by allocateIpFromName to allocate a fixed cluster IP for the service.\nThe default is the same for all DLS clusters.\n", + "pattern": "^(\\d{1,3}\\.){3}\\d{1,3}\\/\\d{1,2}$", + "required": [], + "title": "CIDR for services addresses.", + "type": "string" + }, + "ca_server_port": { + "default": 5064, + "description": "service port for Channel Access", + "required": [], + "title": "ca_server_port", + "type": "integer" + }, + "clusterIP": { + "default": "", + "format": "ipv4", + "required": [], + "title": "Override for the cluster IP - only needed if allocateIpFromName clashes", + "type": [ + "string", + "null" + ] + }, + "command": { + "description": "Command to run in the container (as array of arguments)", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "command", + "type": "array" + }, + "debugCommand": { + "description": "Command to run in the debug version of the container", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "debugCommand", + "type": "array" + }, + "developerMode": { + "default": false, + "description": "If true, the container will run in debug mode meaning:-\n - the entrypoint is sleep\n - a PVC is created and mounted over /venv with /venv copied into it\n - adds '-debug' suffix to the image name\n", + "required": [], + "title": "developerMode", + "type": "boolean" + }, + "extra_containers": { + "description": "adds addtional containers specified by image and command", + "items": { + "additionalProperties": false, + "properties": { + "command": { + "description": "Command to run in the container (as array of arguments)", + "required": [], + "type": "array" + }, + "image": { + "description": "Container image URI", + "format": "image", + "required": [], + "type": "string" + }, + "name": { + "description": "A name for the additional container", + "required": [], + "type": "string" + } + }, + "required": [ + "name", + "image", + "command" + ], + "type": "object" + }, + "required": [], + "title": "extra_containers", + "type": "array" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + }, + "image": { + "additionalProperties": false, + "properties": { + "pullPolicy": { + "default": "IfNotPresent", + "required": [], + "title": "pullPolicy", + "type": "string" + }, + "repository": { + "default": "", + "required": [], + "title": "repository", + "type": "string" + }, + "tag": { + "default": "", + "required": [], + "title": "tag", + "type": "string" + } + }, + "required": [ + "repository", + "pullPolicy", + "tag" + ], + "title": "image", + "type": "object" + }, + "initCommand": { + "description": "Command to run in the init container.\nDefaults to copying workspace and venv.\n", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "initCommand", + "type": "array" + }, + "iocConfig": { + "default": "/epics/ioc/config", + "description": "location of config folder (defaults to be the same as C++ IOCs)", + "required": [], + "title": "iocConfig", + "type": "string" + }, + "livenessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "livenessProbe", + "type": "object" + }, + "nodeSelector": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector", + "description": "Node selector for the pod", + "required": [], + "title": "nodeSelector", + "type": [ + "object", + "null" + ] + }, + "podAnnotations": { + "additionalProperties": false, + "required": [], + "title": "Pod Annotations", + "type": "object" + }, + "podLabels": { + "additionalProperties": false, + "required": [], + "title": "Pod Labels", + "type": "object" + }, + "podSecurityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Pod Security Context", + "type": "object" + }, + "pva_server_port": { + "default": 5075, + "description": "service port for PV Access", + "required": [], + "title": "pva_server_port", + "type": "integer" + }, + "readinessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "readinessProbe", + "type": "object" + }, + "resources": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "required": [], + "title": "Resource limits and requests", + "type": "object" + }, + "securityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Container Security Context", + "type": "object" + }, + "service": { + "additionalProperties": false, + "description": "The service will be configured for Channel Access and PVA. Here you can override\nthe ports and also make this a LoadBalancer service if required.\n", + "properties": { + "ca_port": { + "default": 5064, + "required": [], + "title": "ca_port", + "type": "integer" + }, + "pva_port": { + "default": 5075, + "required": [], + "title": "pva_port", + "type": "integer" + }, + "type": { + "default": "ClusterIP", + "required": [], + "title": "type", + "type": "string" + } + }, + "required": [ + "type", + "ca_port", + "pva_port" + ], + "title": "service", + "type": "object" + }, + "tolerations": { + "description": "Tolerations for the pod", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration", + "required": [], + "type": "object" + }, + "required": [], + "title": "tolerations", + "type": "array" + }, + "volumeMounts": { + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumeMounts", + "type": "array" + }, + "volumes": { + "description": "Additional volumes to mount in the output Deployment definition.", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumes", + "type": "array" + } + }, + "required": [], + "title": "fastcs", + "type": "object" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + } + }, + "required": [], + "type": "object" +} diff --git a/Charts/fastcs-ioc/values.yaml b/Charts/fastcs-ioc/values.yaml new file mode 100644 index 000000000..5a346bb65 --- /dev/null +++ b/Charts/fastcs-ioc/values.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=values.schema.json + +# @schema +# title: FastCS IOC Values +# description: Values for the FastCS IOC instance +# type: object +# $ref: ../fastcs/values.schema.json +# @schema +fastcs: {} diff --git a/Charts/fastcs/.helmignore b/Charts/fastcs/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/Charts/fastcs/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/Charts/fastcs/Chart.yaml b/Charts/fastcs/Chart.yaml new file mode 100644 index 000000000..a88bdc1d4 --- /dev/null +++ b/Charts/fastcs/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: fastcs +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/Charts/fastcs/templates/_helpers.tpl b/Charts/fastcs/templates/_helpers.tpl new file mode 100644 index 000000000..3a598e647 --- /dev/null +++ b/Charts/fastcs/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "fastcs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fastcs.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fastcs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "fastcs.labels" -}} +helm.sh/chart: {{ include "fastcs.chart" . }} +{{ include "fastcs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "fastcs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fastcs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/Charts/fastcs/templates/clusterIP.tpl b/Charts/fastcs/templates/clusterIP.tpl new file mode 100644 index 000000000..e1b43bec9 --- /dev/null +++ b/Charts/fastcs/templates/clusterIP.tpl @@ -0,0 +1,73 @@ + +{{- define "allocateIpFromName" -}} + {{- $name := printf "%s.%s" .name .namespace -}} + {{- $baseIpWithCIDR := .baseIp -}} + + {{- $startIp := .startIp | int -}} + {{- $conversion := atoi (adler32sum $name) -}} + + {{- $baseIpParts := split "/" $baseIpWithCIDR -}} + {{- $baseIp := index $baseIpParts "_0" -}} + {{- $cidrRange := index $baseIpParts "_1" | int -}} + + {{- $octets := split "." $baseIp -}} + {{- $firstOctet := index $octets "_0" | int -}} + {{- $secondOctet := index $octets "_1" | int -}} + {{- $thirdOctet := index $octets "_2" | int -}} + {{- $fourthOctet := index $octets "_3" | int -}} + + + {{- $totalIps := 1 }} + {{- $loopcnt:= sub 32 $cidrRange -}} + {{- range $i,$k := until ($loopcnt | int) }} + {{- $totalIps = mul $totalIps 2 }} + {{- end }} + + {{- $ipSuffix := add $startIp (mod $conversion $totalIps) -}} + + {{- $secondOctet := add $secondOctet (div $ipSuffix 65536) -}} + {{- $ipSuffix = mod $ipSuffix 65536 -}} + {{- $thirdOctet := add $thirdOctet (div $ipSuffix 256) -}} + {{- $fourthOctet := mod $ipSuffix 256 -}} + + {{- if gt $fourthOctet 255 }} + {{- $fourthOctet = mod $fourthOctet 256 -}} + {{- end }} + {{- if gt $thirdOctet 255 }} + {{- $thirdOctet = mod $thirdOctet 256 -}} + {{- $secondOctet = add $secondOctet 1 -}} + {{- end }} + {{- if gt $secondOctet 255 }} + {{- $secondOctet = mod $secondOctet 256 -}} + {{- end }} + + {{- printf "%d.%d.%d.%d" $firstOctet $secondOctet $thirdOctet $fourthOctet -}} +{{- end -}} + + +{{- define "allocateIpFromNames" -}} + {{- $name := printf "%s.%s" .name .namespace -}} + {{- $baseIpWithCIDR := .baseIp -}} + + {{- $startIp := .startIp | int -}} + {{- $conversion := atoi (adler32sum $name) -}} + + {{- $baseIpParts := split "/" $baseIpWithCIDR -}} + {{- $baseIp := index $baseIpParts "_0" -}} + {{- $cidrRange := index $baseIpParts "_1" | int -}} + + {{- $octets := split "." $baseIp -}} + {{- $firstOctet := index $octets "_0" | int -}} + {{- $secondOctet := index $octets "_1" | int -}} + {{- $thirdOctet := index $octets "_2" | int -}} + {{- $fourthOctet := index $octets "_3" | int -}} + + {{- $totalIps := 1 }} + {{- $loopcnt:= sub 32 $cidrRange -}} + {{- range $i,$k := until ($loopcnt | int) }} + {{- $totalIps = mul $totalIps 2 }} + {{- end }} + {{- printf "CIDR %d IPs %d" $cidrRange $totalIps -}} + + +{{- end -}} diff --git a/Charts/fastcs/templates/pvcs.yaml b/Charts/fastcs/templates/pvcs.yaml new file mode 100644 index 000000000..4e1e05358 --- /dev/null +++ b/Charts/fastcs/templates/pvcs.yaml @@ -0,0 +1,20 @@ +# PVC for debugging volume +{{ if .Values.developerMode }} +{{- $location := default .Values.global.location .Values.location | required "ERROR - You must supply location or global.location" -}} +{{- $ioc_group := default .Values.global.ioc_group .Values.ioc_group | required "ERROR - You must supply ioc_group or global.ioc_group" -}} + +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-develop + labels: + app: {{ .Release.Name }} + location: {{ $location }} + ioc_group: {{ $ioc_group }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1Gi" +{{- end }} diff --git a/Charts/fastcs/templates/service.yaml b/Charts/fastcs/templates/service.yaml new file mode 100644 index 000000000..4073818fc --- /dev/null +++ b/Charts/fastcs/templates/service.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + labels: + {{- include "fastcs.labels" . | nindent 4 }} + app: {{ .Release.Name }} + location: {{ .Values.global.location }} + ioc_group: {{ .Values.global.ioc_group }} + is_ioc: "true" +spec: + selector: + app: {{ .Release.Name }} + {{- include "fastcs.selectorLabels" . | nindent 4 }} + type: {{ .Values.service.type }} + # allocate a fixed clusterIP for this service based on the service name + {{- $alloc_args := dict "name" .Release.Name "namespace" .Release.Namespace "baseIp" .Values.baseIp "startIp" .Values.startIp }} + clusterIP: {{ .Values.clusterIP | default (include "allocateIpFromName" $alloc_args) }} + ports: + - name: ca-server-tcp + port: {{ .Values.ca_server_port | default 5064 }} + targetPort: {{ .Values.ca_server_port | default 5064 }} + protocol: TCP + - name: ca-server-udp + port: {{ .Values.ca_server_port | default 5064 }} + targetPort: {{ .Values.ca_server_port | default 5064 }} + protocol: UDP + - name: ca-repeater-tcp + port: {{ add1 (.Values.ca_server_port | default 5064) }} + targetPort: {{ add1 (.Values.ca_server_port | default 5064) }} + protocol: TCP + - name: ca-repeater-udp + port: {{ add1 (.Values.ca_server_port | default 5064) }} + targetPort: {{ add1 (.Values.ca_server_port | default 5064) }} + protocol: UDP + - name: pva-server-tcp + port: {{ .Values.pva_server_port | default 5075 }} + targetPort: {{ .Values.pva_server_port | default 5075 }} + protocol: TCP + - name: pva-server-udp + port: {{ .Values.pva_server_port | default 5075 }} + targetPort: {{ .Values.pva_server_port | default 5075 }} + protocol: UDP + - name: pva-broadcast-tcp + port: {{ add1 (.Values.pva_server_port | default 5075) }} + targetPort: {{ add1 (.Values.pva_server_port | default 5075) }} + protocol: TCP + - name: pva-broadcast-udp + port: {{ add1 (.Values.pva_server_port | default 5075) }} + targetPort: {{ add1 (.Values.pva_server_port | default 5075) }} + protocol: UDP diff --git a/Charts/fastcs/templates/statefulset.yaml b/Charts/fastcs/templates/statefulset.yaml new file mode 100644 index 000000000..393a4f8ab --- /dev/null +++ b/Charts/fastcs/templates/statefulset.yaml @@ -0,0 +1,178 @@ +{{- /* +Default the derivable substitution values. + +This keeps the length of the values.txt file for each individual IOC +to a minimum +*/ -}} +{{- $location := default .Values.global.location .Values.location | required "ERROR - You must supply location or global.location" -}} +{{- $ioc_group := default .Values.global.ioc_group .Values.ioc_group | required "ERROR - You must supply ioc_group or global.ioc_group" -}} +{{- $opisClaim := default (print $ioc_group "-opi-claim") .Values.opisClaim -}} +{{- $runtimeClaim := default (print $ioc_group "-runtime-claim") .Values.runtimeClaim -}} +{{- $autosaveClaim := default (print $ioc_group "-autosave-claim") .Values.autosaveClaim -}} +{{- $image := .Values.image | required "ERROR - You must supply image." -}} + +{{- $enabled := eq .Values.global.enabled false | ternary false true -}} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + labels: + location: {{ $location }} + ioc_group: {{ $ioc_group }} + enabled: {{ $enabled | quote }} + is_ioc: "true" + {{- include "fastcs.labels" . | nindent 4 }} +spec: + replicas: {{ $enabled | ternary 1 0 }} + selector: + matchLabels: + {{- include "fastcs.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + location: {{ $location }} + ioc_group: {{ $ioc_group }} + enabled: {{ $enabled | quote }} + is_ioc: "true" + {{- include "fastcs.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + # re-deploy in case the configMap has changed - use a random value + # unless the Commit Hash is supplied (by ArgoCD or helm command line) + rollme: {{ .Values.global.commitHash | default (randAlphaNum 5) | quote }} + spec: + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.developerMode }} + # add in an init container to copy out virtual environment and workspaces + initContainers: + - name: {{ .Release.Name }}-init-debug + image: "{{ .Values.image.repository }}-debug:{{ .Values.image.tag }}" + imagePullPolicy: IfNotPresent + resources: + {{- .Values.initResources | default .Values.resources | toYaml | nindent 12 }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.initCommand }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: {{ .Release.Name }}-develop + mountPath: /dest + {{- end }} + containers: + {{- range .Values.extra_containers }} + - name: {{ .name }} + image: {{ .image }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with $.Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .command }} + command: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{- with $.Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: opis-volume + mountPath: /epics/opi + subPath: "{{ $.Release.Name }}" + - name: config-volume + mountPath: {{ $.Values.iocConfig }} + {{- if $.Values.developerMode }} + - name: {{ $.Release.Name }}-develop + mountPath: /dest + {{- end }} + {{- end }} + - name: {{ .Chart.Name }} + image: '{{ .Values.image.repository }}{{ ternary "-debug" "" .Values.developerMode }}:{{ .Values.image.tag }}' + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.developerMode }} + {{- with .Values.debugCommand }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: opis-volume + mountPath: /epics/opi + subPath: "{{ .Release.Name }}" + - name: config-volume + mountPath: {{ .Values.iocConfig }} + {{- if .Values.developerMode }} + - name: {{ .Release.Name }}-develop + mountPath: /workspaces + subPath: workspaces + - name: {{ .Release.Name }}-develop + mountPath: /venv + subPath: venv + {{- end }} + volumes: + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: opis-volume + persistentVolumeClaim: + claimName: {{ $opisClaim }} + - name: config-volume + configMap: + name: {{ .Release.Name }}-config + {{- if .Values.developerMode }} + - name: {{ .Release.Name }}-develop + persistentVolumeClaim: + claimName: {{ .Release.Name }}-develop + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/Charts/fastcs/values.schema.json b/Charts/fastcs/values.schema.json new file mode 100644 index 000000000..96933397a --- /dev/null +++ b/Charts/fastcs/values.schema.json @@ -0,0 +1,285 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "affinity": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity", + "description": "Affinity for the pod", + "required": [], + "title": "affinity", + "type": "object" + }, + "args": { + "description": "arguments to pass to the above command", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "args", + "type": "array" + }, + "baseIp": { + "default": "10.96.0.0/12", + "description": "Used by allocateIpFromName to allocate a fixed cluster IP for the service.\nThe default is the same for all DLS clusters.\n", + "pattern": "^(\\d{1,3}\\.){3}\\d{1,3}\\/\\d{1,2}$", + "required": [], + "title": "CIDR for services addresses.", + "type": "string" + }, + "ca_server_port": { + "default": 5064, + "description": "service port for Channel Access", + "required": [], + "title": "ca_server_port", + "type": "integer" + }, + "clusterIP": { + "default": "", + "format": "ipv4", + "required": [], + "title": "Override for the cluster IP - only needed if allocateIpFromName clashes", + "type": [ + "string", + "null" + ] + }, + "command": { + "description": "Command to run in the container (as array of arguments)", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "command", + "type": "array" + }, + "debugCommand": { + "description": "Command to run in the debug version of the container", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "debugCommand", + "type": "array" + }, + "developerMode": { + "default": false, + "description": "If true, the container will run in debug mode meaning:-\n - the entrypoint is sleep\n - a PVC is created and mounted over /venv with /venv copied into it\n - adds '-debug' suffix to the image name\n", + "required": [], + "title": "developerMode", + "type": "boolean" + }, + "extra_containers": { + "description": "adds addtional containers specified by image and command", + "items": { + "additionalProperties": false, + "properties": { + "command": { + "description": "Command to run in the container (as array of arguments)", + "required": [], + "type": "array" + }, + "image": { + "description": "Container image URI", + "format": "image", + "required": [], + "type": "string" + }, + "name": { + "description": "A name for the additional container", + "required": [], + "type": "string" + } + }, + "required": [ + "name", + "image", + "command" + ], + "type": "object" + }, + "required": [], + "title": "extra_containers", + "type": "array" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + }, + "image": { + "additionalProperties": false, + "properties": { + "pullPolicy": { + "default": "IfNotPresent", + "required": [], + "title": "pullPolicy", + "type": "string" + }, + "repository": { + "default": "", + "required": [], + "title": "repository", + "type": "string" + }, + "tag": { + "default": "", + "required": [], + "title": "tag", + "type": "string" + } + }, + "required": [ + "repository", + "pullPolicy", + "tag" + ], + "title": "image", + "type": "object" + }, + "initCommand": { + "description": "Command to run in the init container.\nDefaults to copying workspace and venv.\n", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "initCommand", + "type": "array" + }, + "iocConfig": { + "default": "/epics/ioc/config", + "description": "location of config folder (defaults to be the same as C++ IOCs)", + "required": [], + "title": "iocConfig", + "type": "string" + }, + "livenessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "livenessProbe", + "type": "object" + }, + "nodeSelector": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector", + "description": "Node selector for the pod", + "required": [], + "title": "nodeSelector", + "type": [ + "object", + "null" + ] + }, + "podAnnotations": { + "additionalProperties": false, + "required": [], + "title": "Pod Annotations", + "type": "object" + }, + "podLabels": { + "additionalProperties": false, + "required": [], + "title": "Pod Labels", + "type": "object" + }, + "podSecurityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Pod Security Context", + "type": "object" + }, + "pva_server_port": { + "default": 5075, + "description": "service port for PV Access", + "required": [], + "title": "pva_server_port", + "type": "integer" + }, + "readinessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "readinessProbe", + "type": "object" + }, + "resources": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "required": [], + "title": "Resource limits and requests", + "type": "object" + }, + "securityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Container Security Context", + "type": "object" + }, + "service": { + "additionalProperties": false, + "description": "The service will be configured for Channel Access and PVA. Here you can override\nthe ports and also make this a LoadBalancer service if required.\n", + "properties": { + "ca_port": { + "default": 5064, + "required": [], + "title": "ca_port", + "type": "integer" + }, + "pva_port": { + "default": 5075, + "required": [], + "title": "pva_port", + "type": "integer" + }, + "type": { + "default": "ClusterIP", + "required": [], + "title": "type", + "type": "string" + } + }, + "required": [ + "type", + "ca_port", + "pva_port" + ], + "title": "service", + "type": "object" + }, + "tolerations": { + "description": "Tolerations for the pod", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration", + "required": [], + "type": "object" + }, + "required": [], + "title": "tolerations", + "type": "array" + }, + "volumeMounts": { + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumeMounts", + "type": "array" + }, + "volumes": { + "description": "Additional volumes to mount in the output Deployment definition.", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumes", + "type": "array" + } + }, + "required": [], + "type": "object" +} diff --git a/Charts/fastcs/values.yaml b/Charts/fastcs/values.yaml new file mode 100644 index 000000000..b2e46c8cd --- /dev/null +++ b/Charts/fastcs/values.yaml @@ -0,0 +1,248 @@ +# yaml-language-server: $schema=values.schema.json + +# Default values for fastcs-instance chart. +# +# helm-schema annotations describe the schema for values files. +# see https://github.com/dadav/helm-schema +# Generate the values schema with `generate-schema.sh` +# +# This file itself adheres to the schema it describes (to help writing the annotations). + +# @schema +# title: image +# type: object +# @schema +image: + repository: "" + pullPolicy: IfNotPresent + tag: "" + +# @schema +# description: Command to run in the container (as array of arguments) +# type: array +# items: +# type: string +# required: false +# @schema +command: [] + +# @schema +# description: arguments to pass to the above command +# type: array +# items: +# type: string +# required: false +# @schema +args: [] + +# @schema +# description: Command to run in the debug version of the container +# type: array +# items: +# type: string +# required: false +# @schema +debugCommand: + - /bin/bash + - -c + - "sleep infinity" + +# @schema +# description: | +# Command to run in the init container. +# Defaults to copying workspace and venv. +# type: array +# items: +# type: string +# required: false +# @schema +initCommand: + - /bin/bash + - -c + - | + echo "running as account"; id + if [ -d /dest/venv ]; then + echo "Virtual environment already exists, skipping copy" + else + echo "Copying virtual env to the debugging volume" + cp -r /venv /dest/venv + echo "Copying workspaces to the debugging volume" + cp -r /workspaces /dest/workspaces + echo "Setting permissions on the debugging volume" + chmod -R o+rwX /dest/venv /dest/workspaces + fi + echo "Init container completed successfully" + +# @schema +# description: | +# If true, the container will run in debug mode meaning:- +# - the entrypoint is sleep +# - a PVC is created and mounted over /venv with /venv copied into it +# - adds '-debug' suffix to the image name +# type: boolean +# @schema +developerMode: false + +# @schema +# description: location of config folder (defaults to be the same as C++ IOCs) +# type: string +# required: false +# @schema +iocConfig: /epics/ioc/config + +# @schema +# description: service port for Channel Access +# type: integer +# required: false +# @schema +ca_server_port: 5064 +# @schema +# description: service port for PV Access +# type: integer +# required: false +# @schema +pva_server_port: 5075 + +# @schema +# title: CIDR for services addresses. +# description: | +# Used by allocateIpFromName to allocate a fixed cluster IP for the service. +# The default is the same for all DLS clusters. +# type: string +# pattern: ^(\d{1,3}\.){3}\d{1,3}\/\d{1,2}$ +# required: false +# @schema +baseIp: 10.96.0.0/12 +# @schema +# title: Override for the cluster IP - only needed if allocateIpFromName clashes +# type: [string, null] +# format: ipv4 +# required: false +# @schema +clusterIP: + +# @schema +# description: | +# The service will be configured for Channel Access and PVA. Here you can override +# the ports and also make this a LoadBalancer service if required. +# type: object +# required: false +# @schema +service: + type: ClusterIP + ca_port: 5064 + pva_port: 5075 + +# @schema +# description: adds addtional containers specified by image and command +# type: array +# items: +# type: object +# properties: +# name: +# type: string +# description: A name for the additional container +# image: +# type: string +# format: image +# description: Container image URI +# command: +# type: array +# description: Command to run in the container (as array of arguments) +# required: [name, image, command] +# additionalProperties: false +# @schema +extra_containers: [] + +# @schema +# title: Pod Annotations +# type: object +# @schema +podAnnotations: {} +# @schema +# title: Pod Labels +# type: object +# @schema +podLabels: {} + +# @schema +# title: Pod Security Context +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext +# @schema +podSecurityContext: {} + +# @schema +# title: Container Security Context +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext +# @schema +securityContext: {} + +# @schema +# title: Resource limits and requests +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements +# @schema +resources: {} + +# @schema +# title: livenessProbe +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe +# @schema +livenessProbe: {} +# @schema +# title: readinessProbe +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe +# @schema +readinessProbe: {} + +# @schema +# title: volumes +# description: Additional volumes to mount in the output Deployment definition. +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume +# @schema +volumes: [] + +# @schema +# title: volumeMounts +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount +# @schema +volumeMounts: [] + +# TODO - I had problems with schema checking at helm chart ArgoCD deploy time +# TODO - it seemed that {} for nodeSelector was illegal, so I added null as an option +# TODO - I'm not sure why this was needed but not for other object fields with k8s schemas +# @schema +# title: nodeSelector +# description: Node selector for the pod +# type: [object, null] +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector +# @schema +nodeSelector: + +# @schema +# title: tolerations +# description: Tolerations for the pod +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration +# @schema +tolerations: [] + +# @schema +# title: affinity +# description: Affinity for the pod +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity +# @schema +affinity: {} diff --git a/generate-schemas.sh b/generate-schemas.sh new file mode 100755 index 000000000..abe52114b --- /dev/null +++ b/generate-schemas.sh @@ -0,0 +1,15 @@ +#!/bin/env bash + +set -euo pipefail + +this_dir=$(dirname "$0") +cd "$this_dir" + +helm schema -v || helm plugin install https://github.com/dadav/helm-schema + +for chart in Charts/*; do + helm schema -c $chart + ln -fs ../$chart/values.schema.json schemas/$(basename $chart).schema.json + # add a newline to the end of the schema file for pre-commit compliance + echo >> schemas/$(basename $chart).schema.json +done diff --git a/pyproject.toml b/pyproject.toml index 6cc7cca65..53af0c339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ "sphinx-design", "tox-direct", "types-mock", + "setuptools>=70.1", # https://github.com/DiamondLightSource/aioca/issues/71 "aioca", "p4p", "httpx", diff --git a/schemas/fastcs-ioc.schema.json b/schemas/fastcs-ioc.schema.json new file mode 120000 index 000000000..ada6d0539 --- /dev/null +++ b/schemas/fastcs-ioc.schema.json @@ -0,0 +1 @@ +../Charts/fastcs-ioc/values.schema.json \ No newline at end of file diff --git a/schemas/fastcs.schema.json b/schemas/fastcs.schema.json new file mode 120000 index 000000000..80f78888e --- /dev/null +++ b/schemas/fastcs.schema.json @@ -0,0 +1 @@ +../Charts/fastcs/values.schema.json \ No newline at end of file diff --git a/src/fastcs/attributes.py b/src/fastcs/attributes.py index ab0494e89..a0db6ce32 100644 --- a/src/fastcs/attributes.py +++ b/src/fastcs/attributes.py @@ -9,6 +9,9 @@ from .datatypes import ATTRIBUTE_TYPES, AttrCallback, DataType, T +ONCE = float("inf") +"""Special value to indicate that an attribute should be updated once on start up.""" + class AttrMode(Enum): """Access mode of an ``Attribute``.""" @@ -187,11 +190,7 @@ def __init__( ) self._process_callbacks: list[AttrCallback[T]] | None = None self._write_display_callbacks: list[AttrCallback[T]] | None = None - - if handler is not None: - self._setter = handler - else: - self._setter = SimpleAttrHandler() + self._setter = handler async def process(self, value: T) -> None: await self.process_without_display_update(value) @@ -221,7 +220,7 @@ def add_write_display_callback(self, callback: AttrCallback[T]) -> None: self._write_display_callbacks.append(callback) @property - def sender(self) -> AttrHandlerW: + def sender(self) -> AttrHandlerW | None: return self._setter @@ -241,7 +240,7 @@ def __init__( datatype, # type: ignore access_mode, group=group, - handler=handler, + handler=handler if handler else SimpleAttrHandler(), initial_value=initial_value, description=description, ) diff --git a/src/fastcs/backend.py b/src/fastcs/backend.py index cc2bf8c5f..9fefc4519 100644 --- a/src/fastcs/backend.py +++ b/src/fastcs/backend.py @@ -1,11 +1,11 @@ import asyncio from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Coroutine from fastcs.cs_methods import Command, Put, Scan from fastcs.datatypes import T -from .attributes import AttrHandlerR, AttrHandlerW, AttrR, AttrW +from .attributes import ONCE, AttrHandlerR, AttrHandlerW, AttrR, AttrW from .controller import BaseController, Controller from .controller_api import ControllerAPI from .exceptions import FastCSException @@ -40,18 +40,19 @@ def __del__(self): self._stop_scan_tasks() async def serve(self): + scans, initials = _get_scan_and_initial_coros(self.controller_api) + self._initial_coros += initials await self._run_initial_coros() - await self._start_scan_tasks() + await self._start_scan_tasks(scans) async def _run_initial_coros(self): for coro in self._initial_coros: await coro() - async def _start_scan_tasks(self): - self._scan_tasks = { - self._loop.create_task(coro()) - for coro in _get_scan_coros(self.controller_api) - } + async def _start_scan_tasks( + self, coros: list[Callable[[], Coroutine[None, None, None]]] + ): + self._scan_tasks = {self._loop.create_task(coro()) for coro in coros} def _stop_scan_tasks(self): for task in self._scan_tasks: @@ -96,15 +97,18 @@ async def callback(value): return callback -def _get_scan_coros(root_controller_api: ControllerAPI) -> list[Callable]: +def _get_scan_and_initial_coros( + root_controller_api: ControllerAPI, +) -> tuple[list[Callable], list[Callable]]: scan_dict: dict[float, list[Callable]] = defaultdict(list) + initial_coros: list[Callable] = [] for controller_api in root_controller_api.walk_api(): _add_scan_method_tasks(scan_dict, controller_api) - _add_attribute_updater_tasks(scan_dict, controller_api) + _add_attribute_updater_tasks(scan_dict, initial_coros, controller_api) scan_coros = _get_periodic_scan_coros(scan_dict) - return scan_coros + return scan_coros, initial_coros def _add_scan_method_tasks( @@ -115,13 +119,17 @@ def _add_scan_method_tasks( def _add_attribute_updater_tasks( - scan_dict: dict[float, list[Callable]], controller_api: ControllerAPI + scan_dict: dict[float, list[Callable]], + initial_coros: list[Callable], + controller_api: ControllerAPI, ): for attribute in controller_api.attributes.values(): match attribute: case AttrR(updater=AttrHandlerR(update_period=update_period)) as attribute: callback = _create_updater_callback(attribute) - if update_period is not None: + if update_period is ONCE: + initial_coros.append(callback) + elif update_period is not None: scan_dict[update_period].append(callback) diff --git a/src/fastcs/datatypes.py b/src/fastcs/datatypes.py index 9580c3574..498de64fe 100644 --- a/src/fastcs/datatypes.py +++ b/src/fastcs/datatypes.py @@ -5,7 +5,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from functools import cached_property -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar import numpy as np from numpy.typing import DTypeLike @@ -36,12 +36,23 @@ class DataType(Generic[T]): def dtype(self) -> type[T]: # Using property due to lack of Generic ClassVars pass - def validate(self, value: T) -> T: - """Validate a value against fields in the datatype.""" - if not isinstance(value, self.dtype): - raise ValueError(f"Value '{value}' is not of type {self.dtype}") + def validate(self, value: Any) -> T: + """Validate a value against the datatype. - return value + The base implementation is to try the cast and raise a useful error if it fails. + + Child classes can implement logic before calling ``super.validate(value)`` to + modify the value passed in and help the cast succeed or after to perform further + validation of the coerced type. + + """ + if isinstance(value, self.dtype): + return value + + try: + return self.dtype(value) + except (ValueError, TypeError) as e: + raise ValueError(f"Failed to cast {value} to type {self.dtype}") from e @property @abstractmethod diff --git a/src/fastcs/transport/epics/ca/ioc.py b/src/fastcs/transport/epics/ca/ioc.py index c0cae76d2..9653328e0 100644 --- a/src/fastcs/transport/epics/ca/ioc.py +++ b/src/fastcs/transport/epics/ca/ioc.py @@ -187,7 +187,7 @@ def _make_record( ) def datatype_updater(datatype: DataType): - for name, value in record_metadata_from_datatype(datatype).items(): + for name, value in record_metadata_from_datatype(datatype, out_record).items(): record.set_field(name, value) attribute.add_update_datatype_callback(datatype_updater) diff --git a/src/fastcs/transport/epics/ca/util.py b/src/fastcs/transport/epics/ca/util.py index 271b3d939..9f009b34c 100644 --- a/src/fastcs/transport/epics/ca/util.py +++ b/src/fastcs/transport/epics/ca/util.py @@ -62,6 +62,11 @@ def record_metadata_from_datatype( if field in DATATYPE_FIELD_TO_RECORD_FIELD } + if not out_record: + # in type records don't have DRVL/DRVH fields + arguments.pop("DRVL", None) + arguments.pop("DRVH", None) + match datatype: case Waveform(): if len(datatype.shape) != 1: diff --git a/tests/example_p4p_ioc.py b/tests/example_p4p_ioc.py index 5fff09915..d7ce94745 100644 --- a/tests/example_p4p_ioc.py +++ b/tests/example_p4p_ioc.py @@ -3,7 +3,7 @@ import numpy as np -from fastcs.attributes import AttrR, AttrRW, AttrW +from fastcs.attributes import AttrHandlerW, AttrR, AttrRW, AttrW from fastcs.controller import Controller, SubController from fastcs.datatypes import Bool, Enum, Float, Int, Table, Waveform from fastcs.launch import FastCS @@ -14,6 +14,11 @@ from fastcs.wrappers import command, scan +class SimpleAttributeSetter(AttrHandlerW): + async def put(self, attr, value): + await attr.update_display_without_process(value) + + class FEnum(enum.Enum): A = 0 B = 1 @@ -25,7 +30,7 @@ class FEnum(enum.Enum): class ParentController(Controller): description = "some controller" a: AttrRW = AttrRW(Int(max=400_000, max_alarm=40_000)) - b: AttrW = AttrW(Float(min=-1, min_alarm=-0.5)) + b: AttrW = AttrW(Float(min=-1, min_alarm=-0.5), handler=SimpleAttributeSetter()) table: AttrRW = AttrRW( Table([("A", np.int32), ("B", "i"), ("C", "?"), ("D", np.float64)]) @@ -34,7 +39,7 @@ class ParentController(Controller): class ChildController(SubController): fail_on_next_e = True - c: AttrW = AttrW(Int()) + c: AttrW = AttrW(Int(), handler=SimpleAttributeSetter()) @command() async def d(self): diff --git a/tests/test_attribute.py b/tests/test_attribute.py index 2a8e788c2..325fca792 100644 --- a/tests/test_attribute.py +++ b/tests/test_attribute.py @@ -1,11 +1,16 @@ from functools import partial -import numpy as np import pytest from pytest_mock import MockerFixture -from fastcs.attributes import AttrHandlerR, AttrHandlerRW, AttrR, AttrRW, AttrW -from fastcs.datatypes import Enum, Float, Int, String, Waveform +from fastcs.attributes import ( + AttrHandlerR, + AttrHandlerRW, + AttrR, + AttrRW, + AttrW, +) +from fastcs.datatypes import Int, String @pytest.mark.asyncio @@ -35,29 +40,23 @@ async def device_add(): assert ui["number"] == 2 -@pytest.mark.asyncio -async def test_simple_handler_w(mocker: MockerFixture): - attr = AttrW(Int()) - update_display_mock = mocker.patch.object(attr, "update_display_without_process") - - # This is called by the transport when it receives a put - await attr.sender.put(attr, 1) - - # The callback to update the transport display should be called - update_display_mock.assert_called_once_with(1) - - @pytest.mark.asyncio async def test_simple_handler_rw(mocker: MockerFixture): attr = AttrRW(Int()) - update_display_mock = mocker.patch.object(attr, "update_display_without_process") - set_mock = mocker.patch.object(attr, "set") + attr.update_display_without_process = mocker.MagicMock( + wraps=attr.update_display_without_process + ) + attr.set = mocker.MagicMock(wraps=attr.set) + + assert attr.sender + # This is called by the transport when it receives a put await attr.sender.put(attr, 1) - update_display_mock.assert_called_once_with(1) # The Sender of the attribute should just set the value on the attribute - set_mock.assert_awaited_once_with(1) + attr.update_display_without_process.assert_called_once_with(1) + attr.set.assert_called_once_with(1) + assert attr.get() == 1 class SimpleUpdater(AttrHandlerR): @@ -87,22 +86,3 @@ async def test_handler_initialise(mocker: MockerFixture): # Assert no error in calling initialise on the TestUpdater handler await attr.initialise(mocker.ANY) - - -@pytest.mark.parametrize( - ["datatype", "init_args", "value"], - [ - (Int, {"min": 1}, 0), - (Int, {"max": -1}, 0), - (Float, {"min": 1}, 0.0), - (Float, {"max": -1}, 0.0), - (Float, {}, 0), - (String, {}, 0), - (Enum, {"enum_cls": int}, 0), - (Waveform, {"array_dtype": "U64", "shape": (1,)}, np.ndarray([1])), - (Waveform, {"array_dtype": "float64", "shape": (1, 1)}, np.ndarray([1])), - ], -) -def test_validate(datatype, init_args, value): - with pytest.raises(ValueError): - datatype(**init_args).validate(value) diff --git a/tests/test_backend.py b/tests/test_backend.py index 2d578c547..55b8624da 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,6 +1,7 @@ import asyncio +from dataclasses import dataclass -from fastcs.attributes import AttrRW +from fastcs.attributes import ONCE, AttrHandlerR, AttrR, AttrRW from fastcs.backend import Backend, build_controller_api from fastcs.controller import Controller from fastcs.cs_methods import Command @@ -89,3 +90,40 @@ async def test_wrapper(): await backend.controller_api.command_methods["do_nothing_dynamic"]() loop.run_until_complete(test_wrapper()) + + +def test_update_periods(): + @dataclass + class AttrHandlerTimesCalled(AttrHandlerR): + update_period: float | None + _times_called = 0 + + async def update(self, attr): + self._times_called += 1 + await attr.set(self._times_called) + + class MyController(Controller): + update_once = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=ONCE)) + update_quickly = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=0.1)) + update_never = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=None)) + + controller = MyController() + loop = asyncio.get_event_loop() + + backend = Backend(controller, loop) + + assert controller.update_quickly.get() == 0 + assert controller.update_once.get() == 0 + assert controller.update_never.get() == 0 + + async def test_wrapper(): + loop.create_task(backend.serve()) + await asyncio.sleep(1) + + loop.run_until_complete(test_wrapper()) + assert controller.update_quickly.get() > 1 + assert controller.update_once.get() == 1 + assert controller.update_never.get() == 0 + + assert len(backend._scan_tasks) == 1 + assert len(backend._initial_coros) == 2 diff --git a/tests/test_datatypes.py b/tests/test_datatypes.py new file mode 100644 index 000000000..4ff7ed16f --- /dev/null +++ b/tests/test_datatypes.py @@ -0,0 +1,42 @@ +from enum import IntEnum + +import numpy as np +import pytest + +from fastcs.datatypes import DataType, Enum, Float, Int, Waveform + + +def test_base_validate(): + class TestInt(DataType[int]): + @property + def dtype(self) -> type[int]: + return int + + class MyIntEnum(IntEnum): + A = 0 + B = 1 + + test_int = TestInt() + + assert test_int.validate("0") == 0 + assert test_int.validate(MyIntEnum.B) == 1 + + with pytest.raises(ValueError, match="Failed to cast"): + test_int.validate("foo") + + +@pytest.mark.parametrize( + ["datatype", "init_args", "value"], + [ + (Int, {"min": 1}, 0), + (Int, {"max": -1}, 0), + (Float, {"min": 1}, 0.0), + (Float, {"max": -1}, 0.0), + (Enum, {"enum_cls": int}, 0), + (Waveform, {"array_dtype": "U64", "shape": (1,)}, np.ndarray([1])), + (Waveform, {"array_dtype": "float64", "shape": (1, 1)}, np.ndarray([1])), + ], +) +def test_validate(datatype, init_args, value): + with pytest.raises(ValueError): + datatype(**init_args).validate(value) diff --git a/tests/transport/epics/ca/test_softioc.py b/tests/transport/epics/ca/test_softioc.py index d240c6c70..64805bffd 100644 --- a/tests/transport/epics/ca/test_softioc.py +++ b/tests/transport/epics/ca/test_softioc.py @@ -181,7 +181,7 @@ def test_make_output_record( pv = "PV" _make_record(pv, attribute, on_update=update, out_record=True) - kwargs.update(record_metadata_from_datatype(attribute.datatype)) + kwargs.update(record_metadata_from_datatype(attribute.datatype, out_record=True)) kwargs.update(record_metadata_from_attribute(attribute)) kwargs.update({"always_update": True, "on_update": update}) @@ -266,7 +266,8 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): epics_controller_api.attributes["read_write_float"] ), **record_metadata_from_datatype( - epics_controller_api.attributes["read_write_float"].datatype + epics_controller_api.attributes["read_write_float"].datatype, + out_record=True, ), ) builder.longIn.assert_any_call( @@ -286,7 +287,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): epics_controller_api.attributes["read_write_int"] ), **record_metadata_from_datatype( - epics_controller_api.attributes["read_write_int"].datatype + epics_controller_api.attributes["read_write_int"].datatype, out_record=True ), ) builder.mbbIn.assert_called_once_with( @@ -302,7 +303,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): on_update=mocker.ANY, **record_metadata_from_attribute(epics_controller_api.attributes["enum"]), **record_metadata_from_datatype( - epics_controller_api.attributes["enum"].datatype + epics_controller_api.attributes["enum"].datatype, out_record=True ), ) builder.boolOut.assert_called_once_with( @@ -311,7 +312,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): on_update=mocker.ANY, **record_metadata_from_attribute(epics_controller_api.attributes["write_bool"]), **record_metadata_from_datatype( - epics_controller_api.attributes["write_bool"].datatype + epics_controller_api.attributes["write_bool"].datatype, out_record=True ), ) ioc_builder.Action.assert_any_call( @@ -452,7 +453,8 @@ def test_long_pv_names_discarded(mocker: MockerFixture): always_update=True, on_update=mocker.ANY, **record_metadata_from_datatype( - long_name_controller_api.attributes["attr_rw_short_name"].datatype + long_name_controller_api.attributes["attr_rw_short_name"].datatype, + out_record=True, ), **record_metadata_from_attribute( long_name_controller_api.attributes["attr_rw_short_name"] @@ -528,9 +530,9 @@ def test_update_datatype(mocker: MockerFixture): **record_metadata_from_datatype(attr_r.datatype), ) record_r.set_field.assert_not_called() - attr_r.update_datatype(Int(units="m", min=-3)) + attr_r.update_datatype(Int(units="m", min_alarm=-3)) record_r.set_field.assert_any_call("EGU", "m") - record_r.set_field.assert_any_call("DRVL", -3) + record_r.set_field.assert_any_call("LOPR", -3) with pytest.raises( ValueError, @@ -539,7 +541,7 @@ def test_update_datatype(mocker: MockerFixture): attr_r.update_datatype(String()) # type: ignore attr_w = AttrW(Int()) - record_w = _make_record(pv_name, attr_w, on_update=mocker.ANY) + record_w = _make_record(pv_name, attr_w, on_update=mocker.ANY, out_record=True) builder.longIn.assert_called_once_with( pv_name, @@ -547,8 +549,9 @@ def test_update_datatype(mocker: MockerFixture): **record_metadata_from_datatype(attr_w.datatype), ) record_w.set_field.assert_not_called() - attr_w.update_datatype(Int(units="m", min=-3)) + attr_w.update_datatype(Int(units="m", min_alarm=-1, min=-3)) record_w.set_field.assert_any_call("EGU", "m") + record_w.set_field.assert_any_call("LOPR", -1) record_w.set_field.assert_any_call("DRVL", -3) with pytest.raises( diff --git a/tests/transport/epics/ca/test_util.py b/tests/transport/epics/ca/test_util.py index 25e3e7c1d..14cd56065 100644 --- a/tests/transport/epics/ca/test_util.py +++ b/tests/transport/epics/ca/test_util.py @@ -9,6 +9,7 @@ builder_callable_from_attribute, cast_from_epics_type, cast_to_epics_type, + record_metadata_from_datatype, ) @@ -90,14 +91,8 @@ def test_casting_to_epics(datatype, input, output): @pytest.mark.parametrize( "datatype, input", [ - (object(), 0), # TODO cover Waveform and Table cases - (Enum(ShortEnum), 0), # can't use index (Enum(ShortEnum), LongEnum.TOO), # wrong enum.Enum class - (Int(), 4.0), - (Float(), 1), - (Bool(), None), - (String(), 10), ], ) def test_cast_to_epics_validations(datatype, input): @@ -154,3 +149,18 @@ def test_builder_callable_enum_types(datatype, in_record, out_record): attr = AttrRW(datatype) assert builder_callable_from_attribute(attr, False) == out_record assert builder_callable_from_attribute(attr, True) == in_record + + +def test_drive_metadata_from_datatype(): + dtype = Float(units="mm", min=-10.0, max=10.0, min_alarm=-5, max_alarm=5, prec=3) + out_arguments = record_metadata_from_datatype(dtype, True) + assert out_arguments == { + "DRVH": 10.0, + "DRVL": -10.0, + "EGU": "mm", + "HOPR": 5, + "LOPR": -5, + "PREC": 3, + } + in_arguments = record_metadata_from_datatype(dtype, False) + assert in_arguments == {"EGU": "mm", "HOPR": 5, "LOPR": -5, "PREC": 3}