Skip to content

fix: Handle Bayer/ProRes RAW formats on iPhone 16/17 for video recording #3706

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mihailShumilov wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: Handle Bayer/ProRes RAW formats on iPhone 16/17 for video recording #3706

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 113 additions & 4 deletions package/ios/Core/CameraSession+Configuration.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,29 @@ extension CameraSession {

// pragma MARK: Format

// Bayer/ProRes RAW formats that cannot be encoded with standard codecs.
// These formats require ProRes RAW and SMPTE RDD18 metadata.
private static let bayerFormats: Set<OSType> = [
0x6274_7032, // btp2 - Bayer To ProRes 2
0x6274_7033, // btp3 - Bayer To ProRes 3
0x6270_3136, // bp16 - 16-bit Bayer
0x6270_3234, // bp24 - 24-bit Bayer
0x6270_3332, // bp32 - 32-bit Bayer
]

// Standard 8-bit formats (most compatible for HEVC/H.264 recording)
private static let standard8BitFormats: Set<OSType> = [
kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, // 420v
kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, // 420f
kCVPixelFormatType_32BGRA, // BGRA
]

// Standard 10-bit formats (require HEVC Main10 profile)
private static let standard10BitFormats: Set<OSType> = [
kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, // x420
kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, // xf20
]

/**
Configures the active format (`format`)
*/
Expand All @@ -193,16 +216,102 @@ extension CameraSession {
return
}

// Find matching format (JS Dictionary -> strongly typed Swift class)
let format = device.formats.first { targetFormat.isEqualTo(format: $0) }
guard let format else {
// Find all matching formats (JS Dictionary -> strongly typed Swift class)
let matchingFormats = device.formats.filter { targetFormat.isEqualTo(format: $0) }
guard !matchingFormats.isEmpty else {
throw CameraError.format(.invalidFormat)
}

// Helper to get pixel format type from format description
func getPixelFormatType(_ fmt: AVCaptureDevice.Format) -> OSType {
return CMFormatDescriptionGetMediaSubType(fmt.formatDescription)
}

// Filter out Bayer-only formats (like iPhone 16/17's special formats)
// These formats cannot be used for standard video recording
let recordableFormats = matchingFormats.filter { fmt in
let pixelFormat = getPixelFormatType(fmt)
let isBayer = CameraSession.bayerFormats.contains(pixelFormat)
if isBayer {
VisionLogger.log(level: .warning,
message: "Filtering out Bayer format: \(pixelFormat.fourCCString) " +
"(\(fmt.videoDimensions.width)x\(fmt.videoDimensions.height))")
}
return !isBayer
}

VisionLogger.log(level: .info,
message: "Found \(matchingFormats.count) matching formats, " +
"\(recordableFormats.count) are recordable (non-Bayer)")

// Prefer 8-bit formats first (most compatible), then 10-bit, then others.
let format: AVCaptureDevice.Format
if let format8Bit = recordableFormats.first(where: {
CameraSession.standard8BitFormats.contains(getPixelFormatType($0))
}) {
format = format8Bit
VisionLogger.log(level: .info,
message: "Selected 8-bit format: \(getPixelFormatType(format).fourCCString) " +
"(\(format.videoDimensions.width)x\(format.videoDimensions.height))")
} else if let format10Bit = recordableFormats.first(where: {
CameraSession.standard10BitFormats.contains(getPixelFormatType($0))
}) {
format = format10Bit
VisionLogger.log(level: .info,
message: "Selected 10-bit format: \(getPixelFormatType(format).fourCCString) " +
"(\(format.videoDimensions.width)x\(format.videoDimensions.height))")
} else if !recordableFormats.isEmpty {
// No standard format found among recordable formats, use first recordable
format = recordableFormats[0]
VisionLogger.log(level: .warning,
message: "No standard pixel format found. Using first recordable: " +
"\(getPixelFormatType(format).fourCCString)")
} else if !matchingFormats.isEmpty {
// FALLBACK: All matching formats are Bayer - find alternative at similar resolution
VisionLogger.log(level: .error,
message: "All \(matchingFormats.count) matching formats are Bayer! " +
"Looking for alternative resolution...")

let targetWidth = matchingFormats[0].videoDimensions.width
let targetHeight = matchingFormats[0].videoDimensions.height

let alternativeFormats = device.formats.filter { fmt in
let pixelFormat = getPixelFormatType(fmt)
return !CameraSession.bayerFormats.contains(pixelFormat) &&
(CameraSession.standard8BitFormats.contains(pixelFormat) ||
CameraSession.standard10BitFormats.contains(pixelFormat))
}.sorted { a, b in
// Sort by how close they are to the target resolution
let aArea = Int(a.videoDimensions.width) * Int(a.videoDimensions.height)
let bArea = Int(b.videoDimensions.width) * Int(b.videoDimensions.height)
let targetArea = Int(targetWidth) * Int(targetHeight)
return abs(aArea - targetArea) < abs(bArea - targetArea)
}

if let alternative = alternativeFormats.first {
format = alternative
VisionLogger.log(level: .warning,
message: "Using alternative format: \(getPixelFormatType(format).fourCCString) " +
"(\(format.videoDimensions.width)x\(format.videoDimensions.height)) " +
"instead of Bayer \(targetWidth)x\(targetHeight)")
} else {
// Last resort: use the Bayer format and hope for the best
format = matchingFormats[0]
VisionLogger.log(level: .error,
message: "No alternative found! Using Bayer format: " +
"\(getPixelFormatType(format).fourCCString) - recording will likely fail")
}
} else {
throw CameraError.format(.invalidFormat)
}

// Set new device Format
device.activeFormat = format

VisionLogger.log(level: .info, message: "Successfully configured Format!")
VisionLogger.log(level: .info,
message: "Successfully configured Format (mediaSubType: " +
"\(getPixelFormatType(format).fourCCString), " +
"dimensions: \(format.videoDimensions.width)x\(format.videoDimensions.height))")
}

func configureVideoOutputFormat(configuration: CameraConfiguration) {
Expand Down
157 changes: 111 additions & 46 deletions package/ios/Core/CameraSession+Video.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,15 @@ import UIKit

private let INSUFFICIENT_STORAGE_ERROR_CODE = -11807

// Bayer/ProRes RAW formats that cannot be encoded directly
private let kBayerFormats: Set<OSType> = [
0x6274_7032, // btp2 - Bayer To ProRes 2
0x6274_7033, // btp3 - Bayer To ProRes 3
0x6270_3136, // bp16 - 16-bit Bayer
0x6270_3234, // bp24 - 24-bit Bayer
0x6270_3332, // bp32 - 32-bit Bayer
]

extension CameraSession {
/**
Starts a video + audio recording with a custom Asset Writer.
Expand Down Expand Up @@ -39,51 +48,49 @@ extension CameraSession {

// Callback for when the recording ends
let onFinish = { (recordingSession: RecordingSession, status: AVAssetWriter.Status, error: Error?) in
defer {
// Disable Audio Session again
if enableAudio {
CameraQueues.audioQueue.async {
self.deactivateAudioSession()
CameraQueues.cameraQueue.async {
defer {
if enableAudio {
CameraQueues.audioQueue.async {
self.deactivateAudioSession()
}
}
}
}

self.recordingSession = nil
self.recordingSizeTimer?.cancel()
self.recordingSizeTimer = nil

if self.didCancelRecording {
VisionLogger.log(level: .info, message: "RecordingSession finished because the recording was canceled.")
onError(.capture(.recordingCanceled))
do {
VisionLogger.log(level: .info, message: "Deleting temporary video file...")
try FileManager.default.removeItem(at: recordingSession.url)
} catch {
self.delegate?.onError(.capture(.fileError(cause: error)))
self.recordingSession = nil
self.recordingSizeTimer?.cancel()
self.recordingSizeTimer = nil

if self.didCancelRecording {
VisionLogger.log(level: .info, message: "RecordingSession finished because the recording was canceled.")
onError(.capture(.recordingCanceled))
do {
VisionLogger.log(level: .info, message: "Deleting temporary video file...")
try FileManager.default.removeItem(at: recordingSession.url)
} catch {
self.delegate?.onError(.capture(.fileError(cause: error)))
}
return
}
return
}

VisionLogger.log(level: .info, message: "RecordingSession finished with status \(status.descriptor).")
VisionLogger.log(level: .info, message: "RecordingSession finished with status \(status.descriptor).")

if let error = error as NSError? {
VisionLogger.log(level: .error, message: "RecordingSession Error \(error.code): \(error.description)")
// Something went wrong, we have an error
if error.code == INSUFFICIENT_STORAGE_ERROR_CODE {
onError(.capture(.insufficientStorage))
} else {
onError(.capture(.unknown(message: "An unknown recording error occured! \(error.code) \(error.description)")))
}
} else {
if status == .completed {
// Recording was successfully saved
let video = Video(path: recordingSession.url.absoluteString,
duration: recordingSession.duration,
size: recordingSession.size)
onVideoRecorded(video)
if let error = error as NSError? {
VisionLogger.log(level: .error, message: "RecordingSession Error \(error.code): \(error.description)")
if error.code == INSUFFICIENT_STORAGE_ERROR_CODE {
onError(.capture(.insufficientStorage))
} else {
onError(.capture(.unknown(message: "An unknown recording error occured! \(error.code) \(error.description)")))
}
} else {
// Recording wasn't saved and we don't have an error either.
onError(.unknown(message: "AVAssetWriter completed with status: \(status.descriptor)"))
if status == .completed {
let video = Video(path: recordingSession.url.absoluteString,
duration: recordingSession.duration,
size: recordingSession.size)
onVideoRecorded(video)
} else {
onError(.unknown(message: "AVAssetWriter completed with status: \(status.descriptor)"))
}
}
}
}
Expand All @@ -102,12 +109,11 @@ extension CameraSession {
orientation: orientation,
completion: onFinish)

// Init Audio + Activate Audio Session (optional)
// Init Audio
if enableAudio,
let audioOutput = self.audioOutput,
let audioInput = self.audioDeviceInput {
VisionLogger.log(level: .info, message: "Enabling Audio for Recording...")
// Activate Audio Session asynchronously
CameraQueues.audioQueue.async {
do {
try self.activateAudioSession()
Expand All @@ -116,17 +122,74 @@ extension CameraSession {
}
}

// Initialize audio asset writer
let audioSettings = audioOutput.recommendedAudioSettingsForAssetWriter(writingTo: options.fileType)
try recordingSession.initializeAudioTrack(withSettings: audioSettings,
format: audioInput.device.activeFormat.formatDescription)
}

// Init Video
let videoSettings = try videoOutput.recommendedVideoSettings(forOptions: options)
// Get device's native pixel format to detect Bayer formats
let devicePixelFormat: OSType? = self.videoDeviceInput.map { input in
CMFormatDescriptionGetMediaSubType(input.device.activeFormat.formatDescription)
}

// Check if device format is a Bayer/ProRes RAW format
let isBayerFormat = devicePixelFormat.map { kBayerFormats.contains($0) } ?? false

// Get available output formats
let availableFormats = videoOutput.availableVideoPixelFormatTypes

// Check what standard formats are available
let has420f = availableFormats.contains(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange)
let has420v = availableFormats.contains(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)
let hasBGRA = availableFormats.contains(kCVPixelFormatType_32BGRA)

// Select compatible output format - always force a compatible format for recording
var selectedFormat: OSType?
var forceH264 = false

// Priority: 420f > 420v > BGRA (for best compatibility)
if has420f {
selectedFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
} else if has420v {
selectedFormat = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
} else if hasBGRA {
selectedFormat = kCVPixelFormatType_32BGRA
forceH264 = true // BGRA works better with H.264
} else if let firstFormat = availableFormats.first {
selectedFormat = firstFormat
}

// Apply the selected format to video output
if let format = selectedFormat {
videoOutput.videoSettings = [
String(kCVPixelBufferPixelFormatTypeKey): format,
]
VisionLogger.log(level: .info,
message: "Set output format to: \(format.fourCCString) " +
"(deviceFormat: \(devicePixelFormat?.fourCCString ?? "nil"), isBayer: \(isBayerFormat))")
}

// Get the actual pixel format that will be used
var actualPixelFormat: OSType?
if let pixelFormatValue = videoOutput.videoSettings[String(kCVPixelBufferPixelFormatTypeKey)] {
if let numberValue = pixelFormatValue as? NSNumber {
actualPixelFormat = OSType(numberValue.uint32Value)
} else if let osTypeValue = pixelFormatValue as? OSType {
actualPixelFormat = osTypeValue
}
}

// Get video settings with proper codec selection
let videoSettings = try videoOutput.recommendedVideoSettings(forOptions: options,
devicePixelFormat: actualPixelFormat,
forceH264: forceH264)

VisionLogger.log(level: .info, message: "Video encoder settings: \(videoSettings)")

// Initialize video track
try recordingSession.initializeVideoTrack(withSettings: videoSettings)

// start recording session with or without audio.
// Start recording
try recordingSession.start()
self.didCancelRecording = false
self.recordingSession = recordingSession
Expand All @@ -149,7 +212,8 @@ extension CameraSession {
self.recordingSizeTimer = timer
self.recordingSizeTimer?.resume()
let end = DispatchTime.now()
VisionLogger.log(level: .info, message: "RecordingSesssion started in \(Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000)ms!")
VisionLogger.log(level: .info,
message: "Recording started in \(Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000)ms!")
} catch let error as CameraError {
onError(error)
} catch let error as NSError {
Expand All @@ -165,7 +229,8 @@ extension CameraSession {
CameraQueues.cameraQueue.async {
withPromise(promise) {
guard let recordingSession = self.recordingSession else {
throw CameraError.capture(.noRecordingInProgress)
VisionLogger.log(level: .warning, message: "stopRecording() was called but there is no active recording session.")
return nil
}
recordingSession.stop()
return nil
Expand Down
Loading