feat(#607, #608): add support for labels with audio and video

.changeset/moody-otters-taste.md

@@ -0,0 +1,6 @@
+---
+'@getodk/web-forms': minor
+'@getodk/common': minor
+---
+
+Adds support for labels with audio and video

.gitattributes

@@ -3,3 +3,5 @@
 *.jpg -text
 
 *.xlsx binary
+*.mp3 binary
+*.mp4 binary

README.md

@@ -35,7 +35,7 @@ This section is auto generated. Please update `feature-matrix.json` and then run
   <summary>
 
 <!-- prettier-ignore -->
-  ##### Question types (basic functionality)<br/>🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩⬜⬜⬜⬜⬜ 70\%
+  ##### Question types (basic functionality)<br/>🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩🟩⬜⬜⬜⬜ 76\%
 
   </summary>
   <br/>
@@ -58,9 +58,9 @@ This section is auto generated. Please update `feature-matrix.json` and then run
 | range                      |    ✅    |
 | image                      |    ✅    |
 | barcode                    |          |
-| audio                      |          |
+| audio                      |    ✅    |
 | background-audio           |          |
-| video                      |          |
+| video                      |    ✅    |
 | file                       |          |
 | date                       |    🚧    |
 | time                       |          |

feature-matrix.json

@@ -16,9 +16,9 @@
     "range": "✅",
     "image": "✅",
     "barcode": "",
-    "audio": "",
+    "audio": "✅",
     "background-audio": "",
-    "video": "",
+    "video": "✅",
     "file": "",
     "date": "🚧",
     "time": "",

packages/common/src/fixtures/itext/media-label.xml

@@ -0,0 +1,159 @@
+<?xml version="1.0"?>
+<h:html xmlns="http://www.w3.org/2002/xforms"
+        xmlns:h="http://www.w3.org/1999/xhtml"
+        xmlns:jr="http://openrosa.org/javarosa">
+  <h:head>
+    <h:title>Media Label Test</h:title>
+    <model>
+      <itext>
+        <translation lang="en" default="true()">
+          <text id="/data/audio_question:label">
+            <value>Question with audio label</value>
+            <value form="audio">jr://audio/test-audio.mp3</value>
+          </text>
+          <text id="/data/video_question:label">
+            <value>Question with video label</value>
+            <value form="video">jr://video/test-video.mp4</value>
+          </text>
+          <text id="/data/image_question:label">
+            <value>Question with image label</value>
+            <value form="image">jr://images/test-image.jpg</value>
+          </text>
+          <text id="/data/combined_question:label">
+            <value>Edge case: question with all media in label</value>
+            <value form="audio">jr://audio/test-audio.mp3</value>
+            <value form="video">jr://video/test-video.mp4</value>
+            <value form="image">jr://images/test-image.jpg</value>
+          </text>
+          <text id="/data/select_one_media:label">
+            <value>Select one with media options (special style when it contains images)</value>
+          </text>
+          <text id="/data/select_multiple_media:label">
+            <value>Select multiple with media options (special style when it contains images)</value>
+          </text>
+          <text id="/data/select_one_no_image:label">
+            <value>Select one audio or video</value>
+          </text>
+          <text id="/data/select_multiple_no_image:label">
+            <value>Select multiple audio or video</value>
+          </text>
+          <text id="choice_audio">
+            <value>Option 1: Audio</value>
+            <value form="audio">jr://audio/test-audio.mp3</value>
+          </text>
+          <text id="choice_video">
+            <value>Option 2: Video</value>
+            <value form="video">jr://video/test-video.mp4</value>
+          </text>
+          <text id="choice_image">
+            <value>Option 3: Image</value>
+            <value form="image">jr://images/test-image.jpg</value>
+          </text>
+          <text id="choice_image_audio">
+            <value>Option 4: Audio + Image</value>
+            <value form="image">jr://images/test-image.jpg</value>
+            <value form="audio">jr://audio/test-audio.mp3</value>
+          </text>
+        </translation>
+      </itext>
+      <instance>
+        <data id="media-test">
+          <meta>
+            <instanceID/>
+          </meta>
+          <audio_question/>
+          <video_question/>
+          <image_question/>
+          <combined_question/>
+          <select_one_media/>
+          <select_multiple_media/>
+          <select_one_no_image/>
+          <select_multiple_no_image/>
+        </data>
+      </instance>
+      <bind nodeset="/data/meta/instanceID" type="string" readonly="true()" jr:preload="uid"/>
+      <bind nodeset="/data/audio_question" type="string"/>
+      <bind nodeset="/data/video_question" type="string"/>
+      <bind nodeset="/data/image_question" type="string"/>
+      <bind nodeset="/data/combined_question" type="string"/>
+      <bind nodeset="/data/select_one_media" type="select1"/>
+      <bind nodeset="/data/select_multiple_media" type="select"/>
+      <bind nodeset="/data/select_one_no_image" type="select1"/>
+      <bind nodeset="/data/select_multiple_no_image" type="select"/>
+    </model>
+  </h:head>
+  <h:body>
+    <input ref="/data/audio_question">
+      <label ref="jr:itext('/data/audio_question:label')"/>
+    </input>
+    <input ref="/data/video_question">
+      <label ref="jr:itext('/data/video_question:label')"/>
+    </input>
+    <input ref="/data/image_question">
+      <label ref="jr:itext('/data/image_question:label')"/>
+    </input>
+    <input ref="/data/combined_question">
+      <label ref="jr:itext('/data/combined_question:label')"/>
+    </input>
+    <select1 ref="/data/select_one_media">
+      <label ref="jr:itext('/data/select_one_media:label')"/>
+      <item>
+        <label ref="jr:itext('choice_audio')"/>
+        <value>audio</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_video')"/>
+        <value>video</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_image')"/>
+        <value>image</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_image_audio')"/>
+        <value>image-audio</value>
+      </item>
+    </select1>
+    <select ref="/data/select_multiple_media">
+      <label ref="jr:itext('/data/select_multiple_media:label')"/>
+      <item>
+        <label ref="jr:itext('choice_audio')"/>
+        <value>audio</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_video')"/>
+        <value>video</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_image')"/>
+        <value>image</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_image_audio')"/>
+        <value>image-audio</value>
+      </item>
+    </select>
+    <select1 ref="/data/select_one_no_image">
+      <label ref="jr:itext('/data/select_one_no_image:label')"/>
+      <item>
+        <label ref="jr:itext('choice_audio')"/>
+        <value>audio</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_video')"/>
+        <value>video</value>
+      </item>
+    </select1>
+    <select ref="/data/select_multiple_no_image">
+      <label ref="jr:itext('/data/select_multiple_no_image:label')"/>
+      <item>
+        <label ref="jr:itext('choice_audio')"/>
+        <value>audio</value>
+      </item>
+      <item>
+        <label ref="jr:itext('choice_video')"/>
+        <value>video</value>
+      </item>
+    </select>
+  </h:body>
+</h:html>

packages/common/src/fixtures/xform-attachments.ts

@@ -28,6 +28,11 @@ const xformAttachmentFileExtensions = [
 	'.jpeg',
 	'.gif',
 	'.svg',
+	'.mp4',
+	'.mp3',
+	'.m4a',
+	'.wav',
+	'.3gp',
 ] as const;
 
 type XFormAttachmentFileExtensions = typeof xformAttachmentFileExtensions;
@@ -108,6 +113,26 @@ export class XFormAttachmentFixture {
 				this.mimeType = 'image/svg+xml';
 				break;
 
+			case '.mp4':
+				this.mimeType = 'video/mp4';
+				break;
+
+			case '.mp3':
+				this.mimeType = 'audio/mpeg';
+				break;
+
+			case '.m4a':
+				this.mimeType = 'audio/mp4';
+				break;
+
+			case '.wav':
+				this.mimeType = 'audio/wav';
+				break;
+
+			case '.3gp':
+				this.mimeType = 'video/3gpp';
+				break;
+
 			default:
 				throw new UnreachableError(fileExtension);
 		}

packages/common/src/fixtures/xforms.ts

@@ -102,7 +102,7 @@ export class XFormResource<Type extends XFormResourceType> {
 				const service = new JRResourceService();
 				const parentPath = localPath.replace(/\/[^/]+$/, '');
 
-				service.activateFixtures(parentPath, ['file', 'file-csv', 'images']);
+				service.activateFixtures(parentPath, ['file', 'file-csv', 'images', 'audio', 'video']);
 
 				return service;
 			},
@@ -119,7 +119,7 @@ export class XFormResource<Type extends XFormResourceType> {
 			const service = new JRResourceService();
 			const parentPath = resourceURL.pathname.replace(/\/[^/]+$/, '');
 
-			service.activateFixtures(parentPath, ['file', 'file-csv', 'images']);
+			service.activateFixtures(parentPath, ['file', 'file-csv', 'images', 'audio', 'video']);
 
 			return service;
 		};

packages/web-forms/src/assets/styles/select-options.scss

@@ -1,3 +1,5 @@
+@use 'primeflex/core/_variables.scss' as pf;
+
 .value-option {
 	--text-content-margin: 15px;
 }
@@ -55,10 +57,12 @@
 	}
 
 	:deep(.media-content) {
-		background: var(--odk-base-background-color);
+		background: transparent;
 		height: calc(100% - (var(--odk-base-font-size) + (var(--text-content-margin) * 2)));
-		width: 100%;
-		justify-items: center;
+		min-height: 54px;
+		flex: 0 1 auto;
+		margin-left: auto;
+		align-content: center;
 	}
 
 	&:has(.media-content) {
@@ -71,8 +75,11 @@
 		}
 
 		:deep(.text-content) {
+			width: fit-content;
+			min-width: unset;
 			max-width: calc(100% - 70px);
 			margin: var(--text-content-margin) 20px;
+			flex: 1 1 auto;
 		}
 	}
 }
@@ -94,4 +101,27 @@
 	background: var(--odk-light-background-color);
 	align-content: flex-start;
 	flex-wrap: wrap;
+
+	:deep(.media-content) {
+		display: flex;
+		flex-direction: column;
+		align-items: center;
+		width: 100%;
+		justify-content: flex-start;
+		gap: 10px;
+
+		.media-block {
+			border-radius: 0;
+		}
+	}
+}
+
+@media screen and (max-width: #{pf.$sm}) {
+	:not(.select-with-images) .value-option {
+		flex-wrap: wrap;
+
+		:deep(.media-content) {
+			margin-left: unset;
+		}
+	}
 }

packages/web-forms/src/components/OdkWebForm.vue

@@ -5,7 +5,7 @@ import FormHeader from '@/components/form-layout/FormHeader.vue';
 import QuestionList from '@/components/form-layout/QuestionList.vue';
 import { waitAllTasksToFinish } from '@/lib/async/event-loop.ts';
 import {
-	FORM_IMAGE_CACHE,
+	FORM_MEDIA_CACHE,
 	FORM_OPTIONS,
 	IS_FORM_EDIT_MODE,
 	SUBMIT_PRESSED,
@@ -192,7 +192,7 @@ const formOptions = readonly<FormOptions>({
 	geolocationProvider: { getLocation: () => getLocation() },
 });
 provide(FORM_OPTIONS, formOptions);
-provide(FORM_IMAGE_CACHE, new Map<JRResourceURLString, ObjectURL>());
+provide(FORM_MEDIA_CACHE, new Map<JRResourceURLString, ObjectURL>());
 
 const state = initializeFormState();
 const submitPressed = ref(false);

packages/web-forms/src/components/common/CheckboxWidget.vue

@@ -40,7 +40,7 @@ const selectValues = (values: readonly string[]) => {
 			@update:model-value="selectValues"
 			@change="$emit('change')"
 		/>
-		<TextMedia :label="option.label" />
+		<TextMedia :label="option.label" :audio-icons-only="question.currentState.isSelectWithImages" />
 	</label>
 </template>
 

packages/web-forms/src/components/common/IconSVG.vue

@@ -28,11 +28,13 @@ import {
 	mdiPlus,
 	mdiPrinter,
 	mdiRefresh,
+	mdiStopCircleOutline,
 	mdiTrashCanOutline,
 	mdiUnfoldMoreHorizontal,
 	mdiUpload,
 	mdiVectorPolygon,
 	mdiVectorPolyline,
+	mdiVolumeHigh,
 	mdiWeb,
 } from '@mdi/js';
 
@@ -65,11 +67,13 @@ const iconMap: Record<string, string> = {
 	mdiPlus,
 	mdiPrinter,
 	mdiRefresh,
+	mdiStopCircleOutline,
 	mdiTrashCanOutline,
 	mdiUnfoldMoreHorizontal,
 	mdiUpload,
 	mdiVectorPolygon,
 	mdiVectorPolyline,
+	mdiVolumeHigh,
 	mdiWeb,
 };