From 3b3834bcb664dcbc71fc984ce7d9378773758f13 Mon Sep 17 00:00:00 2001 From: z-khan Date: Wed, 5 Feb 2025 20:20:05 +1100 Subject: [PATCH 1/6] add bayer formats --- av/video/frame.pyx | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/av/video/frame.pyx b/av/video/frame.pyx index 36f4deae1..f3b834885 100644 --- a/av/video/frame.pyx +++ b/av/video/frame.pyx @@ -317,6 +317,18 @@ cdef class VideoFrame(Frame): itemsize, dtype = { "abgr": (4, "uint8"), "argb": (4, "uint8"), + "bayer_bggr8": (1, "uint8"), + "bayer_gbrg8": (1, "uint8"), + "bayer_grbg8": (1, "uint8"), + "bayer_rggb8": (1, "uint8"), + "bayer_bggr16le": (2, "uint16"), + "bayer_bggr16be": (2, "uint16"), + "bayer_gbrg16le": (2, "uint16"), + "bayer_gbrg16be": (2, "uint16"), + "bayer_grbg16le": (2, "uint16"), + "bayer_grbg16be": (2, "uint16"), + "bayer_rggb16le": (2, "uint16"), + "bayer_rggb16be": (2, "uint16"), "bgr24": (3, "uint8"), "bgr8": (1, "uint8"), "bgra": (4, "uint8"), @@ -451,6 +463,14 @@ cdef class VideoFrame(Frame): else: # Planes where U and V are interleaved have the same stride as Y. linesizes = (array.strides[0], array.strides[0]) + elif format in ("bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"): + dtype = np.uint8 if format.endswith("8") else np.uint16 + check_ndarray(array, dtype.__name__, 2) + + if array.strides[1] != (1 if format.endswith("8") else 2): + raise ValueError("provided array does not have C_CONTIGUOUS rows") + + linesizes = (array.strides[0],) else: raise ValueError(f"Conversion from numpy array with format `{format}` is not yet supported") @@ -554,7 +574,19 @@ cdef class VideoFrame(Frame): "yuv444p16be": (3, 2, "uint16"), "yuv444p16le": (3, 2, "uint16"), "yuva444p16be": (4, 2, "uint16"), - "yuva444p16le": (4, 2, "uint16"), + "yuva444p16le": (4, 2, "uint16"), + "bayer_bggr8": (1, 1, "uint8"), + "bayer_rggb8": (1, 1, "uint8"), + "bayer_grbg8": (1, 1, "uint8"), + "bayer_gbrg8": (1, 1, "uint8"), + "bayer_bggr16be": (1, 2, "uint16"), + "bayer_bggr16le": (1, 2, "uint16"), + "bayer_rggb16be": (1, 2, "uint16"), + "bayer_rggb16le": (1, 2, "uint16"), + "bayer_grbg16be": (1, 2, "uint16"), + "bayer_grbg16le": (1, 2, "uint16"), + "bayer_gbrg16be": (1, 2, "uint16"), + "bayer_gbrg16le": (1, 2, "uint16"), }.get(format, (None, None, None)) if channels is not None: if array.ndim == 2: # (height, width) -> (height, width, 1) @@ -631,6 +663,13 @@ cdef class VideoFrame(Frame): copy_array_to_plane(flat[:uv_start], frame.planes[0], 1) copy_array_to_plane(flat[uv_start:], frame.planes[1], 2) return frame + elif format in {"bayer_bggr8", "bayer_rggb8", "bayer_grbg8", "bayer_gbrg8", "bayer_bggr16be", "bayer_bggr16le", "bayer_rggb16be", "bayer_rggb16le", "bayer_grbg16be", "bayer_grbg16le", "bayer_gbrg16be", "bayer_gbrg16le"}: + check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) + check_ndarray_shape(array, array.shape[0] % 2 == 0) + check_ndarray_shape(array, array.shape[1] % 2 == 0) + frame = VideoFrame(array.shape[1], array.shape[0], format) + copy_array_to_plane(array if format.endswith("8") else byteswap_array(array, format.endswith("be")), frame.planes[0], 1 if format.endswith("8") else 2) + return frame else: raise ValueError(f"Conversion from numpy array with format `{format}` is not yet supported") @@ -644,6 +683,8 @@ cdef class VideoFrame(Frame): frame = VideoFrame(width, height, format) if format == "rgba": copy_bytes_to_plane(img_bytes, frame.planes[0], 4, flip_horizontal, flip_vertical) + elif format in ("bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"): + copy_bytes_to_plane(img_bytes, frame.planes[0], 1 if format.endswith("8") else 2, flip_horizontal, flip_vertical) else: raise NotImplementedError(f"Format '{format}' is not supported.") return frame From c3f69068f7a37a86a42b5ee7ce3fcf4b0a24860e Mon Sep 17 00:00:00 2001 From: z-khan Date: Wed, 5 Feb 2025 21:34:37 +1100 Subject: [PATCH 2/6] fix dtype --- av/video/frame.pyx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/av/video/frame.pyx b/av/video/frame.pyx index f3b834885..32b135cce 100644 --- a/av/video/frame.pyx +++ b/av/video/frame.pyx @@ -464,8 +464,9 @@ cdef class VideoFrame(Frame): # Planes where U and V are interleaved have the same stride as Y. linesizes = (array.strides[0], array.strides[0]) elif format in ("bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"): - dtype = np.uint8 if format.endswith("8") else np.uint16 - check_ndarray(array, dtype.__name__, 2) + check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) + check_ndarray_shape(array, array.shape[0] % 2 == 0) + check_ndarray_shape(array, array.shape[1] % 2 == 0) if array.strides[1] != (1 if format.endswith("8") else 2): raise ValueError("provided array does not have C_CONTIGUOUS rows") From abec273fb500356f9cac7e736543b9a3cff092ee Mon Sep 17 00:00:00 2001 From: z-khan Date: Wed, 5 Feb 2025 22:35:33 +1100 Subject: [PATCH 3/6] remove excess bayer specific code --- av/video/frame.pyx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/av/video/frame.pyx b/av/video/frame.pyx index 32b135cce..fff5a82aa 100644 --- a/av/video/frame.pyx +++ b/av/video/frame.pyx @@ -464,9 +464,7 @@ cdef class VideoFrame(Frame): # Planes where U and V are interleaved have the same stride as Y. linesizes = (array.strides[0], array.strides[0]) elif format in ("bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"): - check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) - check_ndarray_shape(array, array.shape[0] % 2 == 0) - check_ndarray_shape(array, array.shape[1] % 2 == 0) + check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) if array.strides[1] != (1 if format.endswith("8") else 2): raise ValueError("provided array does not have C_CONTIGUOUS rows") @@ -664,13 +662,6 @@ cdef class VideoFrame(Frame): copy_array_to_plane(flat[:uv_start], frame.planes[0], 1) copy_array_to_plane(flat[uv_start:], frame.planes[1], 2) return frame - elif format in {"bayer_bggr8", "bayer_rggb8", "bayer_grbg8", "bayer_gbrg8", "bayer_bggr16be", "bayer_bggr16le", "bayer_rggb16be", "bayer_rggb16le", "bayer_grbg16be", "bayer_grbg16le", "bayer_gbrg16be", "bayer_gbrg16le"}: - check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) - check_ndarray_shape(array, array.shape[0] % 2 == 0) - check_ndarray_shape(array, array.shape[1] % 2 == 0) - frame = VideoFrame(array.shape[1], array.shape[0], format) - copy_array_to_plane(array if format.endswith("8") else byteswap_array(array, format.endswith("be")), frame.planes[0], 1 if format.endswith("8") else 2) - return frame else: raise ValueError(f"Conversion from numpy array with format `{format}` is not yet supported") From 784f7b480c96d1c01999030596ea9623b3d9d077 Mon Sep 17 00:00:00 2001 From: z-khan Date: Wed, 5 Feb 2025 23:08:00 +1100 Subject: [PATCH 4/6] expose Bayer flag with is_bayer --- av/video/format.pyx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/av/video/format.pyx b/av/video/format.pyx index 6ae66c3a2..d5c88ae5e 100644 --- a/av/video/format.pyx +++ b/av/video/format.pyx @@ -104,7 +104,12 @@ cdef class VideoFormat: def is_rgb(self): """The pixel format contains RGB-like data (as opposed to YUV/grayscale).""" return bool(self.ptr.flags & lib.AV_PIX_FMT_FLAG_RGB) + + @property + def is_bayer(self): + """The pixel format contains Bayer data (as opposed to RGB).""" + return bool(self.ptr.flags & lib.AV_PIX_FMT_FLAG_BAYER) cpdef chroma_width(self, int luma_width=0): """chroma_width(luma_width=0) From 6f86aa16cde7767a6bed71f0f1aafa05a8830fd0 Mon Sep 17 00:00:00 2001 From: z-khan Date: Wed, 5 Feb 2025 23:20:36 +1100 Subject: [PATCH 5/6] fix comment for is_bayer --- av/video/format.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/av/video/format.pyx b/av/video/format.pyx index d5c88ae5e..8779f7151 100644 --- a/av/video/format.pyx +++ b/av/video/format.pyx @@ -108,7 +108,7 @@ cdef class VideoFormat: @property def is_bayer(self): - """The pixel format contains Bayer data (as opposed to RGB).""" + """The pixel format contains Bayer data.""" return bool(self.ptr.flags & lib.AV_PIX_FMT_FLAG_BAYER) cpdef chroma_width(self, int luma_width=0): From ac1152fd2e1f7fbd2bf0b844f3028247853beaa6 Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Wed, 5 Feb 2025 20:51:54 -0500 Subject: [PATCH 6/6] Corrections --- av/video/format.pyi | 5 ++++- av/video/frame.pyx | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/av/video/format.pyi b/av/video/format.pyi index ee16b85b8..e102ef4c0 100644 --- a/av/video/format.pyi +++ b/av/video/format.pyi @@ -6,7 +6,10 @@ class VideoFormat: has_palette: bool is_bit_stream: bool is_planar: bool - is_rgb: bool + @property + def is_rgb(self) -> bool: ... + @property + def is_bayer(self) -> bool: ... width: int height: int components: tuple[VideoFormatComponent, ...] diff --git a/av/video/frame.pyx b/av/video/frame.pyx index fff5a82aa..b74d58d1c 100644 --- a/av/video/frame.pyx +++ b/av/video/frame.pyx @@ -463,7 +463,7 @@ cdef class VideoFrame(Frame): else: # Planes where U and V are interleaved have the same stride as Y. linesizes = (array.strides[0], array.strides[0]) - elif format in ("bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"): + elif format in {"bayer_bggr8", "bayer_rggb8", "bayer_gbrg8", "bayer_grbg8","bayer_bggr16le", "bayer_rggb16le", "bayer_gbrg16le", "bayer_grbg16le","bayer_bggr16be", "bayer_rggb16be", "bayer_gbrg16be", "bayer_grbg16be"}: check_ndarray(array, "uint8" if format.endswith("8") else "uint16", 2) if array.strides[1] != (1 if format.endswith("8") else 2):