From 6d52511d28b3c248746634e0b53f90ae802e071f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 8 Aug 2025 14:16:43 +0000 Subject: [PATCH] Add Helm Chart with Debug Features first pass at helm chart for fastcs instances fix service name better port exposure for service.yaml add helm chart publishing remove service account creation from helm switch to using Charts folder with subfolders for charts add schema for fastcs-instance chart values.yaml publish chart values schemas publish chart values schemas add schema generation to pre-commit fixing the fastcs-instance chart schema Add setuptools to dev requirements for aioca Remove bodge from CI Update DataType.validate to attempt cast (#182) Do not propagate DRVL and DRVH to in type records in CA transport (#185) Use SimpleHandler on AttrRW only (#186) Add API to update attributes once when Backend.serve() called (#188) Add ONCE constant to update attribute only when serve called add debug features to helm chart repair incorrect helm chart debug now working fix non-debug mode fix incorrect terminationGracePeriodSeconds add debug entrypoint move initCommand into values rename charts to remove "instance" --- .github/workflows/_helm.yml | 71 ++++++ .github/workflows/_test.yml | 3 - .github/workflows/ci.yml | 8 +- .pre-commit-config.yaml | 10 +- Charts/fastcs-ioc/Chart.yaml | 8 + Charts/fastcs-ioc/values.schema.json | 300 +++++++++++++++++++++++ Charts/fastcs-ioc/values.yaml | 9 + Charts/fastcs/.helmignore | 23 ++ Charts/fastcs/Chart.yaml | 24 ++ Charts/fastcs/templates/_helpers.tpl | 51 ++++ Charts/fastcs/templates/clusterIP.tpl | 73 ++++++ Charts/fastcs/templates/pvcs.yaml | 20 ++ Charts/fastcs/templates/service.yaml | 51 ++++ Charts/fastcs/templates/statefulset.yaml | 178 ++++++++++++++ Charts/fastcs/values.schema.json | 285 +++++++++++++++++++++ Charts/fastcs/values.yaml | 248 +++++++++++++++++++ generate-schemas.sh | 15 ++ pyproject.toml | 1 + schemas/fastcs-ioc.schema.json | 1 + schemas/fastcs.schema.json | 1 + src/fastcs/attributes.py | 13 +- src/fastcs/backend.py | 34 ++- src/fastcs/datatypes.py | 23 +- src/fastcs/transport/epics/ca/ioc.py | 2 +- src/fastcs/transport/epics/ca/util.py | 5 + tests/example_p4p_ioc.py | 11 +- tests/test_attribute.py | 56 ++--- tests/test_backend.py | 40 ++- tests/test_datatypes.py | 42 ++++ tests/transport/epics/ca/test_softioc.py | 23 +- tests/transport/epics/ca/test_util.py | 22 +- 31 files changed, 1561 insertions(+), 90 deletions(-) create mode 100644 .github/workflows/_helm.yml create mode 100644 Charts/fastcs-ioc/Chart.yaml create mode 100644 Charts/fastcs-ioc/values.schema.json create mode 100644 Charts/fastcs-ioc/values.yaml create mode 100644 Charts/fastcs/.helmignore create mode 100644 Charts/fastcs/Chart.yaml create mode 100644 Charts/fastcs/templates/_helpers.tpl create mode 100644 Charts/fastcs/templates/clusterIP.tpl create mode 100644 Charts/fastcs/templates/pvcs.yaml create mode 100644 Charts/fastcs/templates/service.yaml create mode 100644 Charts/fastcs/templates/statefulset.yaml create mode 100644 Charts/fastcs/values.schema.json create mode 100644 Charts/fastcs/values.yaml create mode 100755 generate-schemas.sh create mode 120000 schemas/fastcs-ioc.schema.json create mode 120000 schemas/fastcs.schema.json create mode 100644 tests/test_datatypes.py diff --git a/.github/workflows/_helm.yml b/.github/workflows/_helm.yml new file mode 100644 index 000000000..33055ccc1 --- /dev/null +++ b/.github/workflows/_helm.yml @@ -0,0 +1,71 @@ +name: Package helm charts + +on: + workflow_call: + +env: + HELM_VERSION_TO_INSTALL: 3.14.3 + +jobs: + package-helm-charts: + name: Package and Push Helm Chart + runs-on: ubuntu-latest + environment: release + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install helm + uses: Azure/setup-helm@v3 + with: + version: ${{ env.HELM_VERSION_TO_INSTALL }} + + # Check that alpha/beta versions have the form X.Y.Z-alpha.A requried by Helm. + # An early check saves waiting for the entire build before finding a problem. + - name: Check helm version tag + if: ${{ github.ref_type == 'tag' }} + env: + VERSION: "${{ github.ref_name }}" + run: | + if [[ "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-alpha|-beta|-rc).*?$ ]]; then + echo "Valid version format: ${VERSION}" + else + echo "Invalid version: ${VERSION}. Expected: X.Y.Z or X.Y.Z-beta.1 or X.Y.Z-alpha.1" + exit 1 + fi + + - name: Package helm charts + env: + VERSION: "${{ github.ref_type == 'tag' && github.ref_name || '0.0.0' }}" + run: | + set -xe + + mkdir -p charts + for i in Charts/*; do + if [[ ${i} =~ ^.*-ioc$ ]]; then + echo "Skipping IOC schema chart: ${i}" + continue + fi + echo "Packaging chart: ${i}" + helm package -u --app-version ${VERSION} --version ${VERSION} ${i} + mv $(basename ${i})-*.tgz charts/ + done + + - name: Upload helm chart values schemas + uses: actions/upload-artifact@v4 + with: + name: helm-chart-schemas + path: schemas/ + + - name: Push tagged helm chart to registry + if: ${{ github.ref_type == 'tag' }} + run: | + set -x + + echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io/${{ github.repository_owner }} --username ${{ github.repository_owner }} --password-stdin + REGISTRY=oci://ghcr.io/diamondlightsource/charts + for i in charts/*.tgz; do + helm push "${i}" ${REGISTRY} + done diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 3e5879cb9..f652d4145 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -50,9 +50,6 @@ jobs: python-version: ${{ inputs.python-version }} pip-install: ".[dev]" - - name: Update setuptools - run: pip install --upgrade setuptools - - name: Run tests run: tox -e tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9529f09e4..6f661fd49 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,12 @@ jobs: if: needs.check.outputs.branch-pr == '' uses: ./.github/workflows/_dist.yml + helm: + uses: ./.github/workflows/_helm.yml + permissions: + contents: read + packages: write + pypi: if: github.ref_type == 'tag' needs: dist @@ -53,7 +59,7 @@ jobs: release: if: github.ref_type == 'tag' - needs: [dist, docs] + needs: [dist, docs, helm] uses: ./.github/workflows/_release.yml permissions: contents: write diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 60fc23f9a..deebb103e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,10 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: check-added-large-files - id: check-yaml + exclude: ^Charts/ - id: check-merge-conflict - id: end-of-file-fixer @@ -22,3 +23,10 @@ repos: entry: ruff format --force-exclude types: [python] require_serial: true + + - id: helm-schema + name: Generate Helm schema + entry: ./generate-schemas.sh + language: system + types: [yaml] + require_serial: true diff --git a/Charts/fastcs-ioc/Chart.yaml b/Charts/fastcs-ioc/Chart.yaml new file mode 100644 index 000000000..80f4400f3 --- /dev/null +++ b/Charts/fastcs-ioc/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: fastcs-ioc +version: 0.1.0 +# This chart allows generating a schema for the fastcs-ioc values file. +# fastcs-ioc schema is used by IOC instances which have fastcs +# as a dependency. +# +# This chart is never used except by helm-schema. diff --git a/Charts/fastcs-ioc/values.schema.json b/Charts/fastcs-ioc/values.schema.json new file mode 100644 index 000000000..67827a5a4 --- /dev/null +++ b/Charts/fastcs-ioc/values.schema.json @@ -0,0 +1,300 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "fastcs": { + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "affinity": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity", + "description": "Affinity for the pod", + "required": [], + "title": "affinity", + "type": "object" + }, + "args": { + "description": "arguments to pass to the above command", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "args", + "type": "array" + }, + "baseIp": { + "default": "10.96.0.0/12", + "description": "Used by allocateIpFromName to allocate a fixed cluster IP for the service.\nThe default is the same for all DLS clusters.\n", + "pattern": "^(\\d{1,3}\\.){3}\\d{1,3}\\/\\d{1,2}$", + "required": [], + "title": "CIDR for services addresses.", + "type": "string" + }, + "ca_server_port": { + "default": 5064, + "description": "service port for Channel Access", + "required": [], + "title": "ca_server_port", + "type": "integer" + }, + "clusterIP": { + "default": "", + "format": "ipv4", + "required": [], + "title": "Override for the cluster IP - only needed if allocateIpFromName clashes", + "type": [ + "string", + "null" + ] + }, + "command": { + "description": "Command to run in the container (as array of arguments)", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "command", + "type": "array" + }, + "debugCommand": { + "description": "Command to run in the debug version of the container", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "debugCommand", + "type": "array" + }, + "developerMode": { + "default": false, + "description": "If true, the container will run in debug mode meaning:-\n - the entrypoint is sleep\n - a PVC is created and mounted over /venv with /venv copied into it\n - adds '-debug' suffix to the image name\n", + "required": [], + "title": "developerMode", + "type": "boolean" + }, + "extra_containers": { + "description": "adds addtional containers specified by image and command", + "items": { + "additionalProperties": false, + "properties": { + "command": { + "description": "Command to run in the container (as array of arguments)", + "required": [], + "type": "array" + }, + "image": { + "description": "Container image URI", + "format": "image", + "required": [], + "type": "string" + }, + "name": { + "description": "A name for the additional container", + "required": [], + "type": "string" + } + }, + "required": [ + "name", + "image", + "command" + ], + "type": "object" + }, + "required": [], + "title": "extra_containers", + "type": "array" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + }, + "image": { + "additionalProperties": false, + "properties": { + "pullPolicy": { + "default": "IfNotPresent", + "required": [], + "title": "pullPolicy", + "type": "string" + }, + "repository": { + "default": "", + "required": [], + "title": "repository", + "type": "string" + }, + "tag": { + "default": "", + "required": [], + "title": "tag", + "type": "string" + } + }, + "required": [ + "repository", + "pullPolicy", + "tag" + ], + "title": "image", + "type": "object" + }, + "initCommand": { + "description": "Command to run in the init container.\nDefaults to copying workspace and venv.\n", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "initCommand", + "type": "array" + }, + "iocConfig": { + "default": "/epics/ioc/config", + "description": "location of config folder (defaults to be the same as C++ IOCs)", + "required": [], + "title": "iocConfig", + "type": "string" + }, + "livenessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "livenessProbe", + "type": "object" + }, + "nodeSelector": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector", + "description": "Node selector for the pod", + "required": [], + "title": "nodeSelector", + "type": [ + "object", + "null" + ] + }, + "podAnnotations": { + "additionalProperties": false, + "required": [], + "title": "Pod Annotations", + "type": "object" + }, + "podLabels": { + "additionalProperties": false, + "required": [], + "title": "Pod Labels", + "type": "object" + }, + "podSecurityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Pod Security Context", + "type": "object" + }, + "pva_server_port": { + "default": 5075, + "description": "service port for PV Access", + "required": [], + "title": "pva_server_port", + "type": "integer" + }, + "readinessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "readinessProbe", + "type": "object" + }, + "resources": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "required": [], + "title": "Resource limits and requests", + "type": "object" + }, + "securityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Container Security Context", + "type": "object" + }, + "service": { + "additionalProperties": false, + "description": "The service will be configured for Channel Access and PVA. Here you can override\nthe ports and also make this a LoadBalancer service if required.\n", + "properties": { + "ca_port": { + "default": 5064, + "required": [], + "title": "ca_port", + "type": "integer" + }, + "pva_port": { + "default": 5075, + "required": [], + "title": "pva_port", + "type": "integer" + }, + "type": { + "default": "ClusterIP", + "required": [], + "title": "type", + "type": "string" + } + }, + "required": [ + "type", + "ca_port", + "pva_port" + ], + "title": "service", + "type": "object" + }, + "tolerations": { + "description": "Tolerations for the pod", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration", + "required": [], + "type": "object" + }, + "required": [], + "title": "tolerations", + "type": "array" + }, + "volumeMounts": { + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumeMounts", + "type": "array" + }, + "volumes": { + "description": "Additional volumes to mount in the output Deployment definition.", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumes", + "type": "array" + } + }, + "required": [], + "title": "fastcs", + "type": "object" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + } + }, + "required": [], + "type": "object" +} diff --git a/Charts/fastcs-ioc/values.yaml b/Charts/fastcs-ioc/values.yaml new file mode 100644 index 000000000..5a346bb65 --- /dev/null +++ b/Charts/fastcs-ioc/values.yaml @@ -0,0 +1,9 @@ +# yaml-language-server: $schema=values.schema.json + +# @schema +# title: FastCS IOC Values +# description: Values for the FastCS IOC instance +# type: object +# $ref: ../fastcs/values.schema.json +# @schema +fastcs: {} diff --git a/Charts/fastcs/.helmignore b/Charts/fastcs/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/Charts/fastcs/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/Charts/fastcs/Chart.yaml b/Charts/fastcs/Chart.yaml new file mode 100644 index 000000000..a88bdc1d4 --- /dev/null +++ b/Charts/fastcs/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: fastcs +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/Charts/fastcs/templates/_helpers.tpl b/Charts/fastcs/templates/_helpers.tpl new file mode 100644 index 000000000..3a598e647 --- /dev/null +++ b/Charts/fastcs/templates/_helpers.tpl @@ -0,0 +1,51 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "fastcs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "fastcs.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "fastcs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "fastcs.labels" -}} +helm.sh/chart: {{ include "fastcs.chart" . }} +{{ include "fastcs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "fastcs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "fastcs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/Charts/fastcs/templates/clusterIP.tpl b/Charts/fastcs/templates/clusterIP.tpl new file mode 100644 index 000000000..e1b43bec9 --- /dev/null +++ b/Charts/fastcs/templates/clusterIP.tpl @@ -0,0 +1,73 @@ + +{{- define "allocateIpFromName" -}} + {{- $name := printf "%s.%s" .name .namespace -}} + {{- $baseIpWithCIDR := .baseIp -}} + + {{- $startIp := .startIp | int -}} + {{- $conversion := atoi (adler32sum $name) -}} + + {{- $baseIpParts := split "/" $baseIpWithCIDR -}} + {{- $baseIp := index $baseIpParts "_0" -}} + {{- $cidrRange := index $baseIpParts "_1" | int -}} + + {{- $octets := split "." $baseIp -}} + {{- $firstOctet := index $octets "_0" | int -}} + {{- $secondOctet := index $octets "_1" | int -}} + {{- $thirdOctet := index $octets "_2" | int -}} + {{- $fourthOctet := index $octets "_3" | int -}} + + + {{- $totalIps := 1 }} + {{- $loopcnt:= sub 32 $cidrRange -}} + {{- range $i,$k := until ($loopcnt | int) }} + {{- $totalIps = mul $totalIps 2 }} + {{- end }} + + {{- $ipSuffix := add $startIp (mod $conversion $totalIps) -}} + + {{- $secondOctet := add $secondOctet (div $ipSuffix 65536) -}} + {{- $ipSuffix = mod $ipSuffix 65536 -}} + {{- $thirdOctet := add $thirdOctet (div $ipSuffix 256) -}} + {{- $fourthOctet := mod $ipSuffix 256 -}} + + {{- if gt $fourthOctet 255 }} + {{- $fourthOctet = mod $fourthOctet 256 -}} + {{- end }} + {{- if gt $thirdOctet 255 }} + {{- $thirdOctet = mod $thirdOctet 256 -}} + {{- $secondOctet = add $secondOctet 1 -}} + {{- end }} + {{- if gt $secondOctet 255 }} + {{- $secondOctet = mod $secondOctet 256 -}} + {{- end }} + + {{- printf "%d.%d.%d.%d" $firstOctet $secondOctet $thirdOctet $fourthOctet -}} +{{- end -}} + + +{{- define "allocateIpFromNames" -}} + {{- $name := printf "%s.%s" .name .namespace -}} + {{- $baseIpWithCIDR := .baseIp -}} + + {{- $startIp := .startIp | int -}} + {{- $conversion := atoi (adler32sum $name) -}} + + {{- $baseIpParts := split "/" $baseIpWithCIDR -}} + {{- $baseIp := index $baseIpParts "_0" -}} + {{- $cidrRange := index $baseIpParts "_1" | int -}} + + {{- $octets := split "." $baseIp -}} + {{- $firstOctet := index $octets "_0" | int -}} + {{- $secondOctet := index $octets "_1" | int -}} + {{- $thirdOctet := index $octets "_2" | int -}} + {{- $fourthOctet := index $octets "_3" | int -}} + + {{- $totalIps := 1 }} + {{- $loopcnt:= sub 32 $cidrRange -}} + {{- range $i,$k := until ($loopcnt | int) }} + {{- $totalIps = mul $totalIps 2 }} + {{- end }} + {{- printf "CIDR %d IPs %d" $cidrRange $totalIps -}} + + +{{- end -}} diff --git a/Charts/fastcs/templates/pvcs.yaml b/Charts/fastcs/templates/pvcs.yaml new file mode 100644 index 000000000..4e1e05358 --- /dev/null +++ b/Charts/fastcs/templates/pvcs.yaml @@ -0,0 +1,20 @@ +# PVC for debugging volume +{{ if .Values.developerMode }} +{{- $location := default .Values.global.location .Values.location | required "ERROR - You must supply location or global.location" -}} +{{- $ioc_group := default .Values.global.ioc_group .Values.ioc_group | required "ERROR - You must supply ioc_group or global.ioc_group" -}} + +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Release.Name }}-develop + labels: + app: {{ .Release.Name }} + location: {{ $location }} + ioc_group: {{ $ioc_group }} +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: "1Gi" +{{- end }} diff --git a/Charts/fastcs/templates/service.yaml b/Charts/fastcs/templates/service.yaml new file mode 100644 index 000000000..4073818fc --- /dev/null +++ b/Charts/fastcs/templates/service.yaml @@ -0,0 +1,51 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }} + labels: + {{- include "fastcs.labels" . | nindent 4 }} + app: {{ .Release.Name }} + location: {{ .Values.global.location }} + ioc_group: {{ .Values.global.ioc_group }} + is_ioc: "true" +spec: + selector: + app: {{ .Release.Name }} + {{- include "fastcs.selectorLabels" . | nindent 4 }} + type: {{ .Values.service.type }} + # allocate a fixed clusterIP for this service based on the service name + {{- $alloc_args := dict "name" .Release.Name "namespace" .Release.Namespace "baseIp" .Values.baseIp "startIp" .Values.startIp }} + clusterIP: {{ .Values.clusterIP | default (include "allocateIpFromName" $alloc_args) }} + ports: + - name: ca-server-tcp + port: {{ .Values.ca_server_port | default 5064 }} + targetPort: {{ .Values.ca_server_port | default 5064 }} + protocol: TCP + - name: ca-server-udp + port: {{ .Values.ca_server_port | default 5064 }} + targetPort: {{ .Values.ca_server_port | default 5064 }} + protocol: UDP + - name: ca-repeater-tcp + port: {{ add1 (.Values.ca_server_port | default 5064) }} + targetPort: {{ add1 (.Values.ca_server_port | default 5064) }} + protocol: TCP + - name: ca-repeater-udp + port: {{ add1 (.Values.ca_server_port | default 5064) }} + targetPort: {{ add1 (.Values.ca_server_port | default 5064) }} + protocol: UDP + - name: pva-server-tcp + port: {{ .Values.pva_server_port | default 5075 }} + targetPort: {{ .Values.pva_server_port | default 5075 }} + protocol: TCP + - name: pva-server-udp + port: {{ .Values.pva_server_port | default 5075 }} + targetPort: {{ .Values.pva_server_port | default 5075 }} + protocol: UDP + - name: pva-broadcast-tcp + port: {{ add1 (.Values.pva_server_port | default 5075) }} + targetPort: {{ add1 (.Values.pva_server_port | default 5075) }} + protocol: TCP + - name: pva-broadcast-udp + port: {{ add1 (.Values.pva_server_port | default 5075) }} + targetPort: {{ add1 (.Values.pva_server_port | default 5075) }} + protocol: UDP diff --git a/Charts/fastcs/templates/statefulset.yaml b/Charts/fastcs/templates/statefulset.yaml new file mode 100644 index 000000000..393a4f8ab --- /dev/null +++ b/Charts/fastcs/templates/statefulset.yaml @@ -0,0 +1,178 @@ +{{- /* +Default the derivable substitution values. + +This keeps the length of the values.txt file for each individual IOC +to a minimum +*/ -}} +{{- $location := default .Values.global.location .Values.location | required "ERROR - You must supply location or global.location" -}} +{{- $ioc_group := default .Values.global.ioc_group .Values.ioc_group | required "ERROR - You must supply ioc_group or global.ioc_group" -}} +{{- $opisClaim := default (print $ioc_group "-opi-claim") .Values.opisClaim -}} +{{- $runtimeClaim := default (print $ioc_group "-runtime-claim") .Values.runtimeClaim -}} +{{- $autosaveClaim := default (print $ioc_group "-autosave-claim") .Values.autosaveClaim -}} +{{- $image := .Values.image | required "ERROR - You must supply image." -}} + +{{- $enabled := eq .Values.global.enabled false | ternary false true -}} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + labels: + location: {{ $location }} + ioc_group: {{ $ioc_group }} + enabled: {{ $enabled | quote }} + is_ioc: "true" + {{- include "fastcs.labels" . | nindent 4 }} +spec: + replicas: {{ $enabled | ternary 1 0 }} + selector: + matchLabels: + {{- include "fastcs.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + location: {{ $location }} + ioc_group: {{ $ioc_group }} + enabled: {{ $enabled | quote }} + is_ioc: "true" + {{- include "fastcs.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + # re-deploy in case the configMap has changed - use a random value + # unless the Commit Hash is supplied (by ArgoCD or helm command line) + rollme: {{ .Values.global.commitHash | default (randAlphaNum 5) | quote }} + spec: + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.developerMode }} + # add in an init container to copy out virtual environment and workspaces + initContainers: + - name: {{ .Release.Name }}-init-debug + image: "{{ .Values.image.repository }}-debug:{{ .Values.image.tag }}" + imagePullPolicy: IfNotPresent + resources: + {{- .Values.initResources | default .Values.resources | toYaml | nindent 12 }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.initCommand }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: {{ .Release.Name }}-develop + mountPath: /dest + {{- end }} + containers: + {{- range .Values.extra_containers }} + - name: {{ .name }} + image: {{ .image }} + imagePullPolicy: {{ $.Values.image.pullPolicy }} + {{- with $.Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .command }} + command: + {{- . | toYaml | nindent 12 }} + {{- end }} + volumeMounts: + {{- with $.Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: opis-volume + mountPath: /epics/opi + subPath: "{{ $.Release.Name }}" + - name: config-volume + mountPath: {{ $.Values.iocConfig }} + {{- if $.Values.developerMode }} + - name: {{ $.Release.Name }}-develop + mountPath: /dest + {{- end }} + {{- end }} + - name: {{ .Chart.Name }} + image: '{{ .Values.image.repository }}{{ ternary "-debug" "" .Values.developerMode }}:{{ .Values.image.tag }}' + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.developerMode }} + {{- with .Values.debugCommand }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- else }} + {{- with .Values.command }} + command: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.args }} + args: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- with .livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + - name: opis-volume + mountPath: /epics/opi + subPath: "{{ .Release.Name }}" + - name: config-volume + mountPath: {{ .Values.iocConfig }} + {{- if .Values.developerMode }} + - name: {{ .Release.Name }}-develop + mountPath: /workspaces + subPath: workspaces + - name: {{ .Release.Name }}-develop + mountPath: /venv + subPath: venv + {{- end }} + volumes: + {{- with .Values.volumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + - name: opis-volume + persistentVolumeClaim: + claimName: {{ $opisClaim }} + - name: config-volume + configMap: + name: {{ .Release.Name }}-config + {{- if .Values.developerMode }} + - name: {{ .Release.Name }}-develop + persistentVolumeClaim: + claimName: {{ .Release.Name }}-develop + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/Charts/fastcs/values.schema.json b/Charts/fastcs/values.schema.json new file mode 100644 index 000000000..96933397a --- /dev/null +++ b/Charts/fastcs/values.schema.json @@ -0,0 +1,285 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "additionalProperties": false, + "properties": { + "affinity": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity", + "description": "Affinity for the pod", + "required": [], + "title": "affinity", + "type": "object" + }, + "args": { + "description": "arguments to pass to the above command", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "args", + "type": "array" + }, + "baseIp": { + "default": "10.96.0.0/12", + "description": "Used by allocateIpFromName to allocate a fixed cluster IP for the service.\nThe default is the same for all DLS clusters.\n", + "pattern": "^(\\d{1,3}\\.){3}\\d{1,3}\\/\\d{1,2}$", + "required": [], + "title": "CIDR for services addresses.", + "type": "string" + }, + "ca_server_port": { + "default": 5064, + "description": "service port for Channel Access", + "required": [], + "title": "ca_server_port", + "type": "integer" + }, + "clusterIP": { + "default": "", + "format": "ipv4", + "required": [], + "title": "Override for the cluster IP - only needed if allocateIpFromName clashes", + "type": [ + "string", + "null" + ] + }, + "command": { + "description": "Command to run in the container (as array of arguments)", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "command", + "type": "array" + }, + "debugCommand": { + "description": "Command to run in the debug version of the container", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "debugCommand", + "type": "array" + }, + "developerMode": { + "default": false, + "description": "If true, the container will run in debug mode meaning:-\n - the entrypoint is sleep\n - a PVC is created and mounted over /venv with /venv copied into it\n - adds '-debug' suffix to the image name\n", + "required": [], + "title": "developerMode", + "type": "boolean" + }, + "extra_containers": { + "description": "adds addtional containers specified by image and command", + "items": { + "additionalProperties": false, + "properties": { + "command": { + "description": "Command to run in the container (as array of arguments)", + "required": [], + "type": "array" + }, + "image": { + "description": "Container image URI", + "format": "image", + "required": [], + "type": "string" + }, + "name": { + "description": "A name for the additional container", + "required": [], + "type": "string" + } + }, + "required": [ + "name", + "image", + "command" + ], + "type": "object" + }, + "required": [], + "title": "extra_containers", + "type": "array" + }, + "global": { + "description": "Global values are values that can be accessed from any chart or subchart by exactly the same name.", + "required": [], + "title": "global", + "type": "object" + }, + "image": { + "additionalProperties": false, + "properties": { + "pullPolicy": { + "default": "IfNotPresent", + "required": [], + "title": "pullPolicy", + "type": "string" + }, + "repository": { + "default": "", + "required": [], + "title": "repository", + "type": "string" + }, + "tag": { + "default": "", + "required": [], + "title": "tag", + "type": "string" + } + }, + "required": [ + "repository", + "pullPolicy", + "tag" + ], + "title": "image", + "type": "object" + }, + "initCommand": { + "description": "Command to run in the init container.\nDefaults to copying workspace and venv.\n", + "items": { + "required": [], + "type": "string" + }, + "required": [], + "title": "initCommand", + "type": "array" + }, + "iocConfig": { + "default": "/epics/ioc/config", + "description": "location of config folder (defaults to be the same as C++ IOCs)", + "required": [], + "title": "iocConfig", + "type": "string" + }, + "livenessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "livenessProbe", + "type": "object" + }, + "nodeSelector": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector", + "description": "Node selector for the pod", + "required": [], + "title": "nodeSelector", + "type": [ + "object", + "null" + ] + }, + "podAnnotations": { + "additionalProperties": false, + "required": [], + "title": "Pod Annotations", + "type": "object" + }, + "podLabels": { + "additionalProperties": false, + "required": [], + "title": "Pod Labels", + "type": "object" + }, + "podSecurityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Pod Security Context", + "type": "object" + }, + "pva_server_port": { + "default": 5075, + "description": "service port for PV Access", + "required": [], + "title": "pva_server_port", + "type": "integer" + }, + "readinessProbe": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe", + "required": [], + "title": "readinessProbe", + "type": "object" + }, + "resources": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements", + "required": [], + "title": "Resource limits and requests", + "type": "object" + }, + "securityContext": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext", + "required": [], + "title": "Container Security Context", + "type": "object" + }, + "service": { + "additionalProperties": false, + "description": "The service will be configured for Channel Access and PVA. Here you can override\nthe ports and also make this a LoadBalancer service if required.\n", + "properties": { + "ca_port": { + "default": 5064, + "required": [], + "title": "ca_port", + "type": "integer" + }, + "pva_port": { + "default": 5075, + "required": [], + "title": "pva_port", + "type": "integer" + }, + "type": { + "default": "ClusterIP", + "required": [], + "title": "type", + "type": "string" + } + }, + "required": [ + "type", + "ca_port", + "pva_port" + ], + "title": "service", + "type": "object" + }, + "tolerations": { + "description": "Tolerations for the pod", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration", + "required": [], + "type": "object" + }, + "required": [], + "title": "tolerations", + "type": "array" + }, + "volumeMounts": { + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumeMounts", + "type": "array" + }, + "volumes": { + "description": "Additional volumes to mount in the output Deployment definition.", + "items": { + "$ref": "https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume", + "required": [], + "type": "object" + }, + "required": [], + "title": "volumes", + "type": "array" + } + }, + "required": [], + "type": "object" +} diff --git a/Charts/fastcs/values.yaml b/Charts/fastcs/values.yaml new file mode 100644 index 000000000..b2e46c8cd --- /dev/null +++ b/Charts/fastcs/values.yaml @@ -0,0 +1,248 @@ +# yaml-language-server: $schema=values.schema.json + +# Default values for fastcs-instance chart. +# +# helm-schema annotations describe the schema for values files. +# see https://github.com/dadav/helm-schema +# Generate the values schema with `generate-schema.sh` +# +# This file itself adheres to the schema it describes (to help writing the annotations). + +# @schema +# title: image +# type: object +# @schema +image: + repository: "" + pullPolicy: IfNotPresent + tag: "" + +# @schema +# description: Command to run in the container (as array of arguments) +# type: array +# items: +# type: string +# required: false +# @schema +command: [] + +# @schema +# description: arguments to pass to the above command +# type: array +# items: +# type: string +# required: false +# @schema +args: [] + +# @schema +# description: Command to run in the debug version of the container +# type: array +# items: +# type: string +# required: false +# @schema +debugCommand: + - /bin/bash + - -c + - "sleep infinity" + +# @schema +# description: | +# Command to run in the init container. +# Defaults to copying workspace and venv. +# type: array +# items: +# type: string +# required: false +# @schema +initCommand: + - /bin/bash + - -c + - | + echo "running as account"; id + if [ -d /dest/venv ]; then + echo "Virtual environment already exists, skipping copy" + else + echo "Copying virtual env to the debugging volume" + cp -r /venv /dest/venv + echo "Copying workspaces to the debugging volume" + cp -r /workspaces /dest/workspaces + echo "Setting permissions on the debugging volume" + chmod -R o+rwX /dest/venv /dest/workspaces + fi + echo "Init container completed successfully" + +# @schema +# description: | +# If true, the container will run in debug mode meaning:- +# - the entrypoint is sleep +# - a PVC is created and mounted over /venv with /venv copied into it +# - adds '-debug' suffix to the image name +# type: boolean +# @schema +developerMode: false + +# @schema +# description: location of config folder (defaults to be the same as C++ IOCs) +# type: string +# required: false +# @schema +iocConfig: /epics/ioc/config + +# @schema +# description: service port for Channel Access +# type: integer +# required: false +# @schema +ca_server_port: 5064 +# @schema +# description: service port for PV Access +# type: integer +# required: false +# @schema +pva_server_port: 5075 + +# @schema +# title: CIDR for services addresses. +# description: | +# Used by allocateIpFromName to allocate a fixed cluster IP for the service. +# The default is the same for all DLS clusters. +# type: string +# pattern: ^(\d{1,3}\.){3}\d{1,3}\/\d{1,2}$ +# required: false +# @schema +baseIp: 10.96.0.0/12 +# @schema +# title: Override for the cluster IP - only needed if allocateIpFromName clashes +# type: [string, null] +# format: ipv4 +# required: false +# @schema +clusterIP: + +# @schema +# description: | +# The service will be configured for Channel Access and PVA. Here you can override +# the ports and also make this a LoadBalancer service if required. +# type: object +# required: false +# @schema +service: + type: ClusterIP + ca_port: 5064 + pva_port: 5075 + +# @schema +# description: adds addtional containers specified by image and command +# type: array +# items: +# type: object +# properties: +# name: +# type: string +# description: A name for the additional container +# image: +# type: string +# format: image +# description: Container image URI +# command: +# type: array +# description: Command to run in the container (as array of arguments) +# required: [name, image, command] +# additionalProperties: false +# @schema +extra_containers: [] + +# @schema +# title: Pod Annotations +# type: object +# @schema +podAnnotations: {} +# @schema +# title: Pod Labels +# type: object +# @schema +podLabels: {} + +# @schema +# title: Pod Security Context +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext +# @schema +podSecurityContext: {} + +# @schema +# title: Container Security Context +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.SecurityContext +# @schema +securityContext: {} + +# @schema +# title: Resource limits and requests +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.ResourceRequirements +# @schema +resources: {} + +# @schema +# title: livenessProbe +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe +# @schema +livenessProbe: {} +# @schema +# title: readinessProbe +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Probe +# @schema +readinessProbe: {} + +# @schema +# title: volumes +# description: Additional volumes to mount in the output Deployment definition. +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Volume +# @schema +volumes: [] + +# @schema +# title: volumeMounts +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.VolumeMount +# @schema +volumeMounts: [] + +# TODO - I had problems with schema checking at helm chart ArgoCD deploy time +# TODO - it seemed that {} for nodeSelector was illegal, so I added null as an option +# TODO - I'm not sure why this was needed but not for other object fields with k8s schemas +# @schema +# title: nodeSelector +# description: Node selector for the pod +# type: [object, null] +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.NodeSelector +# @schema +nodeSelector: + +# @schema +# title: tolerations +# description: Tolerations for the pod +# type: array +# items: +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Toleration +# @schema +tolerations: [] + +# @schema +# title: affinity +# description: Affinity for the pod +# type: object +# $ref: https://kubernetesjsonschema.dev/v1.18.1/_definitions.json#/definitions/io.k8s.api.core.v1.Affinity +# @schema +affinity: {} diff --git a/generate-schemas.sh b/generate-schemas.sh new file mode 100755 index 000000000..abe52114b --- /dev/null +++ b/generate-schemas.sh @@ -0,0 +1,15 @@ +#!/bin/env bash + +set -euo pipefail + +this_dir=$(dirname "$0") +cd "$this_dir" + +helm schema -v || helm plugin install https://github.com/dadav/helm-schema + +for chart in Charts/*; do + helm schema -c $chart + ln -fs ../$chart/values.schema.json schemas/$(basename $chart).schema.json + # add a newline to the end of the schema file for pre-commit compliance + echo >> schemas/$(basename $chart).schema.json +done diff --git a/pyproject.toml b/pyproject.toml index 6cc7cca65..53af0c339 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dev = [ "sphinx-design", "tox-direct", "types-mock", + "setuptools>=70.1", # https://github.com/DiamondLightSource/aioca/issues/71 "aioca", "p4p", "httpx", diff --git a/schemas/fastcs-ioc.schema.json b/schemas/fastcs-ioc.schema.json new file mode 120000 index 000000000..ada6d0539 --- /dev/null +++ b/schemas/fastcs-ioc.schema.json @@ -0,0 +1 @@ +../Charts/fastcs-ioc/values.schema.json \ No newline at end of file diff --git a/schemas/fastcs.schema.json b/schemas/fastcs.schema.json new file mode 120000 index 000000000..80f78888e --- /dev/null +++ b/schemas/fastcs.schema.json @@ -0,0 +1 @@ +../Charts/fastcs/values.schema.json \ No newline at end of file diff --git a/src/fastcs/attributes.py b/src/fastcs/attributes.py index ab0494e89..a0db6ce32 100644 --- a/src/fastcs/attributes.py +++ b/src/fastcs/attributes.py @@ -9,6 +9,9 @@ from .datatypes import ATTRIBUTE_TYPES, AttrCallback, DataType, T +ONCE = float("inf") +"""Special value to indicate that an attribute should be updated once on start up.""" + class AttrMode(Enum): """Access mode of an ``Attribute``.""" @@ -187,11 +190,7 @@ def __init__( ) self._process_callbacks: list[AttrCallback[T]] | None = None self._write_display_callbacks: list[AttrCallback[T]] | None = None - - if handler is not None: - self._setter = handler - else: - self._setter = SimpleAttrHandler() + self._setter = handler async def process(self, value: T) -> None: await self.process_without_display_update(value) @@ -221,7 +220,7 @@ def add_write_display_callback(self, callback: AttrCallback[T]) -> None: self._write_display_callbacks.append(callback) @property - def sender(self) -> AttrHandlerW: + def sender(self) -> AttrHandlerW | None: return self._setter @@ -241,7 +240,7 @@ def __init__( datatype, # type: ignore access_mode, group=group, - handler=handler, + handler=handler if handler else SimpleAttrHandler(), initial_value=initial_value, description=description, ) diff --git a/src/fastcs/backend.py b/src/fastcs/backend.py index cc2bf8c5f..9fefc4519 100644 --- a/src/fastcs/backend.py +++ b/src/fastcs/backend.py @@ -1,11 +1,11 @@ import asyncio from collections import defaultdict -from collections.abc import Callable +from collections.abc import Callable, Coroutine from fastcs.cs_methods import Command, Put, Scan from fastcs.datatypes import T -from .attributes import AttrHandlerR, AttrHandlerW, AttrR, AttrW +from .attributes import ONCE, AttrHandlerR, AttrHandlerW, AttrR, AttrW from .controller import BaseController, Controller from .controller_api import ControllerAPI from .exceptions import FastCSException @@ -40,18 +40,19 @@ def __del__(self): self._stop_scan_tasks() async def serve(self): + scans, initials = _get_scan_and_initial_coros(self.controller_api) + self._initial_coros += initials await self._run_initial_coros() - await self._start_scan_tasks() + await self._start_scan_tasks(scans) async def _run_initial_coros(self): for coro in self._initial_coros: await coro() - async def _start_scan_tasks(self): - self._scan_tasks = { - self._loop.create_task(coro()) - for coro in _get_scan_coros(self.controller_api) - } + async def _start_scan_tasks( + self, coros: list[Callable[[], Coroutine[None, None, None]]] + ): + self._scan_tasks = {self._loop.create_task(coro()) for coro in coros} def _stop_scan_tasks(self): for task in self._scan_tasks: @@ -96,15 +97,18 @@ async def callback(value): return callback -def _get_scan_coros(root_controller_api: ControllerAPI) -> list[Callable]: +def _get_scan_and_initial_coros( + root_controller_api: ControllerAPI, +) -> tuple[list[Callable], list[Callable]]: scan_dict: dict[float, list[Callable]] = defaultdict(list) + initial_coros: list[Callable] = [] for controller_api in root_controller_api.walk_api(): _add_scan_method_tasks(scan_dict, controller_api) - _add_attribute_updater_tasks(scan_dict, controller_api) + _add_attribute_updater_tasks(scan_dict, initial_coros, controller_api) scan_coros = _get_periodic_scan_coros(scan_dict) - return scan_coros + return scan_coros, initial_coros def _add_scan_method_tasks( @@ -115,13 +119,17 @@ def _add_scan_method_tasks( def _add_attribute_updater_tasks( - scan_dict: dict[float, list[Callable]], controller_api: ControllerAPI + scan_dict: dict[float, list[Callable]], + initial_coros: list[Callable], + controller_api: ControllerAPI, ): for attribute in controller_api.attributes.values(): match attribute: case AttrR(updater=AttrHandlerR(update_period=update_period)) as attribute: callback = _create_updater_callback(attribute) - if update_period is not None: + if update_period is ONCE: + initial_coros.append(callback) + elif update_period is not None: scan_dict[update_period].append(callback) diff --git a/src/fastcs/datatypes.py b/src/fastcs/datatypes.py index 9580c3574..498de64fe 100644 --- a/src/fastcs/datatypes.py +++ b/src/fastcs/datatypes.py @@ -5,7 +5,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from functools import cached_property -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar import numpy as np from numpy.typing import DTypeLike @@ -36,12 +36,23 @@ class DataType(Generic[T]): def dtype(self) -> type[T]: # Using property due to lack of Generic ClassVars pass - def validate(self, value: T) -> T: - """Validate a value against fields in the datatype.""" - if not isinstance(value, self.dtype): - raise ValueError(f"Value '{value}' is not of type {self.dtype}") + def validate(self, value: Any) -> T: + """Validate a value against the datatype. - return value + The base implementation is to try the cast and raise a useful error if it fails. + + Child classes can implement logic before calling ``super.validate(value)`` to + modify the value passed in and help the cast succeed or after to perform further + validation of the coerced type. + + """ + if isinstance(value, self.dtype): + return value + + try: + return self.dtype(value) + except (ValueError, TypeError) as e: + raise ValueError(f"Failed to cast {value} to type {self.dtype}") from e @property @abstractmethod diff --git a/src/fastcs/transport/epics/ca/ioc.py b/src/fastcs/transport/epics/ca/ioc.py index c0cae76d2..9653328e0 100644 --- a/src/fastcs/transport/epics/ca/ioc.py +++ b/src/fastcs/transport/epics/ca/ioc.py @@ -187,7 +187,7 @@ def _make_record( ) def datatype_updater(datatype: DataType): - for name, value in record_metadata_from_datatype(datatype).items(): + for name, value in record_metadata_from_datatype(datatype, out_record).items(): record.set_field(name, value) attribute.add_update_datatype_callback(datatype_updater) diff --git a/src/fastcs/transport/epics/ca/util.py b/src/fastcs/transport/epics/ca/util.py index 271b3d939..9f009b34c 100644 --- a/src/fastcs/transport/epics/ca/util.py +++ b/src/fastcs/transport/epics/ca/util.py @@ -62,6 +62,11 @@ def record_metadata_from_datatype( if field in DATATYPE_FIELD_TO_RECORD_FIELD } + if not out_record: + # in type records don't have DRVL/DRVH fields + arguments.pop("DRVL", None) + arguments.pop("DRVH", None) + match datatype: case Waveform(): if len(datatype.shape) != 1: diff --git a/tests/example_p4p_ioc.py b/tests/example_p4p_ioc.py index 5fff09915..d7ce94745 100644 --- a/tests/example_p4p_ioc.py +++ b/tests/example_p4p_ioc.py @@ -3,7 +3,7 @@ import numpy as np -from fastcs.attributes import AttrR, AttrRW, AttrW +from fastcs.attributes import AttrHandlerW, AttrR, AttrRW, AttrW from fastcs.controller import Controller, SubController from fastcs.datatypes import Bool, Enum, Float, Int, Table, Waveform from fastcs.launch import FastCS @@ -14,6 +14,11 @@ from fastcs.wrappers import command, scan +class SimpleAttributeSetter(AttrHandlerW): + async def put(self, attr, value): + await attr.update_display_without_process(value) + + class FEnum(enum.Enum): A = 0 B = 1 @@ -25,7 +30,7 @@ class FEnum(enum.Enum): class ParentController(Controller): description = "some controller" a: AttrRW = AttrRW(Int(max=400_000, max_alarm=40_000)) - b: AttrW = AttrW(Float(min=-1, min_alarm=-0.5)) + b: AttrW = AttrW(Float(min=-1, min_alarm=-0.5), handler=SimpleAttributeSetter()) table: AttrRW = AttrRW( Table([("A", np.int32), ("B", "i"), ("C", "?"), ("D", np.float64)]) @@ -34,7 +39,7 @@ class ParentController(Controller): class ChildController(SubController): fail_on_next_e = True - c: AttrW = AttrW(Int()) + c: AttrW = AttrW(Int(), handler=SimpleAttributeSetter()) @command() async def d(self): diff --git a/tests/test_attribute.py b/tests/test_attribute.py index 2a8e788c2..325fca792 100644 --- a/tests/test_attribute.py +++ b/tests/test_attribute.py @@ -1,11 +1,16 @@ from functools import partial -import numpy as np import pytest from pytest_mock import MockerFixture -from fastcs.attributes import AttrHandlerR, AttrHandlerRW, AttrR, AttrRW, AttrW -from fastcs.datatypes import Enum, Float, Int, String, Waveform +from fastcs.attributes import ( + AttrHandlerR, + AttrHandlerRW, + AttrR, + AttrRW, + AttrW, +) +from fastcs.datatypes import Int, String @pytest.mark.asyncio @@ -35,29 +40,23 @@ async def device_add(): assert ui["number"] == 2 -@pytest.mark.asyncio -async def test_simple_handler_w(mocker: MockerFixture): - attr = AttrW(Int()) - update_display_mock = mocker.patch.object(attr, "update_display_without_process") - - # This is called by the transport when it receives a put - await attr.sender.put(attr, 1) - - # The callback to update the transport display should be called - update_display_mock.assert_called_once_with(1) - - @pytest.mark.asyncio async def test_simple_handler_rw(mocker: MockerFixture): attr = AttrRW(Int()) - update_display_mock = mocker.patch.object(attr, "update_display_without_process") - set_mock = mocker.patch.object(attr, "set") + attr.update_display_without_process = mocker.MagicMock( + wraps=attr.update_display_without_process + ) + attr.set = mocker.MagicMock(wraps=attr.set) + + assert attr.sender + # This is called by the transport when it receives a put await attr.sender.put(attr, 1) - update_display_mock.assert_called_once_with(1) # The Sender of the attribute should just set the value on the attribute - set_mock.assert_awaited_once_with(1) + attr.update_display_without_process.assert_called_once_with(1) + attr.set.assert_called_once_with(1) + assert attr.get() == 1 class SimpleUpdater(AttrHandlerR): @@ -87,22 +86,3 @@ async def test_handler_initialise(mocker: MockerFixture): # Assert no error in calling initialise on the TestUpdater handler await attr.initialise(mocker.ANY) - - -@pytest.mark.parametrize( - ["datatype", "init_args", "value"], - [ - (Int, {"min": 1}, 0), - (Int, {"max": -1}, 0), - (Float, {"min": 1}, 0.0), - (Float, {"max": -1}, 0.0), - (Float, {}, 0), - (String, {}, 0), - (Enum, {"enum_cls": int}, 0), - (Waveform, {"array_dtype": "U64", "shape": (1,)}, np.ndarray([1])), - (Waveform, {"array_dtype": "float64", "shape": (1, 1)}, np.ndarray([1])), - ], -) -def test_validate(datatype, init_args, value): - with pytest.raises(ValueError): - datatype(**init_args).validate(value) diff --git a/tests/test_backend.py b/tests/test_backend.py index 2d578c547..55b8624da 100644 --- a/tests/test_backend.py +++ b/tests/test_backend.py @@ -1,6 +1,7 @@ import asyncio +from dataclasses import dataclass -from fastcs.attributes import AttrRW +from fastcs.attributes import ONCE, AttrHandlerR, AttrR, AttrRW from fastcs.backend import Backend, build_controller_api from fastcs.controller import Controller from fastcs.cs_methods import Command @@ -89,3 +90,40 @@ async def test_wrapper(): await backend.controller_api.command_methods["do_nothing_dynamic"]() loop.run_until_complete(test_wrapper()) + + +def test_update_periods(): + @dataclass + class AttrHandlerTimesCalled(AttrHandlerR): + update_period: float | None + _times_called = 0 + + async def update(self, attr): + self._times_called += 1 + await attr.set(self._times_called) + + class MyController(Controller): + update_once = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=ONCE)) + update_quickly = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=0.1)) + update_never = AttrR(Int(), handler=AttrHandlerTimesCalled(update_period=None)) + + controller = MyController() + loop = asyncio.get_event_loop() + + backend = Backend(controller, loop) + + assert controller.update_quickly.get() == 0 + assert controller.update_once.get() == 0 + assert controller.update_never.get() == 0 + + async def test_wrapper(): + loop.create_task(backend.serve()) + await asyncio.sleep(1) + + loop.run_until_complete(test_wrapper()) + assert controller.update_quickly.get() > 1 + assert controller.update_once.get() == 1 + assert controller.update_never.get() == 0 + + assert len(backend._scan_tasks) == 1 + assert len(backend._initial_coros) == 2 diff --git a/tests/test_datatypes.py b/tests/test_datatypes.py new file mode 100644 index 000000000..4ff7ed16f --- /dev/null +++ b/tests/test_datatypes.py @@ -0,0 +1,42 @@ +from enum import IntEnum + +import numpy as np +import pytest + +from fastcs.datatypes import DataType, Enum, Float, Int, Waveform + + +def test_base_validate(): + class TestInt(DataType[int]): + @property + def dtype(self) -> type[int]: + return int + + class MyIntEnum(IntEnum): + A = 0 + B = 1 + + test_int = TestInt() + + assert test_int.validate("0") == 0 + assert test_int.validate(MyIntEnum.B) == 1 + + with pytest.raises(ValueError, match="Failed to cast"): + test_int.validate("foo") + + +@pytest.mark.parametrize( + ["datatype", "init_args", "value"], + [ + (Int, {"min": 1}, 0), + (Int, {"max": -1}, 0), + (Float, {"min": 1}, 0.0), + (Float, {"max": -1}, 0.0), + (Enum, {"enum_cls": int}, 0), + (Waveform, {"array_dtype": "U64", "shape": (1,)}, np.ndarray([1])), + (Waveform, {"array_dtype": "float64", "shape": (1, 1)}, np.ndarray([1])), + ], +) +def test_validate(datatype, init_args, value): + with pytest.raises(ValueError): + datatype(**init_args).validate(value) diff --git a/tests/transport/epics/ca/test_softioc.py b/tests/transport/epics/ca/test_softioc.py index d240c6c70..64805bffd 100644 --- a/tests/transport/epics/ca/test_softioc.py +++ b/tests/transport/epics/ca/test_softioc.py @@ -181,7 +181,7 @@ def test_make_output_record( pv = "PV" _make_record(pv, attribute, on_update=update, out_record=True) - kwargs.update(record_metadata_from_datatype(attribute.datatype)) + kwargs.update(record_metadata_from_datatype(attribute.datatype, out_record=True)) kwargs.update(record_metadata_from_attribute(attribute)) kwargs.update({"always_update": True, "on_update": update}) @@ -266,7 +266,8 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): epics_controller_api.attributes["read_write_float"] ), **record_metadata_from_datatype( - epics_controller_api.attributes["read_write_float"].datatype + epics_controller_api.attributes["read_write_float"].datatype, + out_record=True, ), ) builder.longIn.assert_any_call( @@ -286,7 +287,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): epics_controller_api.attributes["read_write_int"] ), **record_metadata_from_datatype( - epics_controller_api.attributes["read_write_int"].datatype + epics_controller_api.attributes["read_write_int"].datatype, out_record=True ), ) builder.mbbIn.assert_called_once_with( @@ -302,7 +303,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): on_update=mocker.ANY, **record_metadata_from_attribute(epics_controller_api.attributes["enum"]), **record_metadata_from_datatype( - epics_controller_api.attributes["enum"].datatype + epics_controller_api.attributes["enum"].datatype, out_record=True ), ) builder.boolOut.assert_called_once_with( @@ -311,7 +312,7 @@ def test_ioc(mocker: MockerFixture, epics_controller_api: ControllerAPI): on_update=mocker.ANY, **record_metadata_from_attribute(epics_controller_api.attributes["write_bool"]), **record_metadata_from_datatype( - epics_controller_api.attributes["write_bool"].datatype + epics_controller_api.attributes["write_bool"].datatype, out_record=True ), ) ioc_builder.Action.assert_any_call( @@ -452,7 +453,8 @@ def test_long_pv_names_discarded(mocker: MockerFixture): always_update=True, on_update=mocker.ANY, **record_metadata_from_datatype( - long_name_controller_api.attributes["attr_rw_short_name"].datatype + long_name_controller_api.attributes["attr_rw_short_name"].datatype, + out_record=True, ), **record_metadata_from_attribute( long_name_controller_api.attributes["attr_rw_short_name"] @@ -528,9 +530,9 @@ def test_update_datatype(mocker: MockerFixture): **record_metadata_from_datatype(attr_r.datatype), ) record_r.set_field.assert_not_called() - attr_r.update_datatype(Int(units="m", min=-3)) + attr_r.update_datatype(Int(units="m", min_alarm=-3)) record_r.set_field.assert_any_call("EGU", "m") - record_r.set_field.assert_any_call("DRVL", -3) + record_r.set_field.assert_any_call("LOPR", -3) with pytest.raises( ValueError, @@ -539,7 +541,7 @@ def test_update_datatype(mocker: MockerFixture): attr_r.update_datatype(String()) # type: ignore attr_w = AttrW(Int()) - record_w = _make_record(pv_name, attr_w, on_update=mocker.ANY) + record_w = _make_record(pv_name, attr_w, on_update=mocker.ANY, out_record=True) builder.longIn.assert_called_once_with( pv_name, @@ -547,8 +549,9 @@ def test_update_datatype(mocker: MockerFixture): **record_metadata_from_datatype(attr_w.datatype), ) record_w.set_field.assert_not_called() - attr_w.update_datatype(Int(units="m", min=-3)) + attr_w.update_datatype(Int(units="m", min_alarm=-1, min=-3)) record_w.set_field.assert_any_call("EGU", "m") + record_w.set_field.assert_any_call("LOPR", -1) record_w.set_field.assert_any_call("DRVL", -3) with pytest.raises( diff --git a/tests/transport/epics/ca/test_util.py b/tests/transport/epics/ca/test_util.py index 25e3e7c1d..14cd56065 100644 --- a/tests/transport/epics/ca/test_util.py +++ b/tests/transport/epics/ca/test_util.py @@ -9,6 +9,7 @@ builder_callable_from_attribute, cast_from_epics_type, cast_to_epics_type, + record_metadata_from_datatype, ) @@ -90,14 +91,8 @@ def test_casting_to_epics(datatype, input, output): @pytest.mark.parametrize( "datatype, input", [ - (object(), 0), # TODO cover Waveform and Table cases - (Enum(ShortEnum), 0), # can't use index (Enum(ShortEnum), LongEnum.TOO), # wrong enum.Enum class - (Int(), 4.0), - (Float(), 1), - (Bool(), None), - (String(), 10), ], ) def test_cast_to_epics_validations(datatype, input): @@ -154,3 +149,18 @@ def test_builder_callable_enum_types(datatype, in_record, out_record): attr = AttrRW(datatype) assert builder_callable_from_attribute(attr, False) == out_record assert builder_callable_from_attribute(attr, True) == in_record + + +def test_drive_metadata_from_datatype(): + dtype = Float(units="mm", min=-10.0, max=10.0, min_alarm=-5, max_alarm=5, prec=3) + out_arguments = record_metadata_from_datatype(dtype, True) + assert out_arguments == { + "DRVH": 10.0, + "DRVL": -10.0, + "EGU": "mm", + "HOPR": 5, + "LOPR": -5, + "PREC": 3, + } + in_arguments = record_metadata_from_datatype(dtype, False) + assert in_arguments == {"EGU": "mm", "HOPR": 5, "LOPR": -5, "PREC": 3}