diff --git a/Moblin/Media/HaishinKit/Media/Audio/AudioUnit.swift b/Moblin/Media/HaishinKit/Media/Audio/AudioUnit.swift index d6f779355..16b90031f 100644 --- a/Moblin/Media/HaishinKit/Media/Audio/AudioUnit.swift +++ b/Moblin/Media/HaishinKit/Media/Audio/AudioUnit.swift @@ -24,6 +24,10 @@ func makeChannelMap( return channelMap.map { NSNumber(value: $0) } } +private let mixerOutputSampleRate: Double = 48000 +private let mixerOutputChannels: AVAudioChannelCount = 1 +private let mixerOutputSamplesPerBuffer: AVAudioFrameCount = 1024 + final class AudioUnit: NSObject { let encoder = AudioEncoder(lockQueue: processorPipelineQueue) private var input: AVCaptureDeviceInput? @@ -36,6 +40,13 @@ final class AudioUnit: NSObject { private var speechToTextEnabled = false private var bufferedBuiltinAudio: BufferedAudio? private var latestAudioStatusTime = 0.0 + private let builtinInputId = UUID() + private var mixer: AudioMixer? + private var mixerInputFormats: [UUID: AVAudioFormat] = [:] + private var mixerProcessTimer = SimpleTimer(queue: processorPipelineQueue) + private var mixerOutputPresentationTimeStamp: CMTime = .zero + private var mixerStarted = false + private var mixerSourceIds: Set = [] private var inputSourceFormat: AudioStreamBasicDescription? { didSet { @@ -52,6 +63,9 @@ final class AudioUnit: NSObject { func stopRunning() { session.stopRunning() + processorPipelineQueue.async { + self.stopMixer() + } } func attach(params: AudioUnitAttachParams) throws { @@ -79,6 +93,7 @@ final class AudioUnit: NSObject { encoder.stopRunning() processorPipelineQueue.async { self.inputSourceFormat = nil + self.stopMixer() } } @@ -155,6 +170,7 @@ final class AudioUnit: NSObject { private func removeBufferedAudioInternal(cameraId: UUID) { bufferedAudios.removeValue(forKey: cameraId)?.stopOutput() + removeMixerSourceInternal(sourceId: cameraId) } private func appendBufferedAudioSampleBufferInternal(cameraId: UUID, _ sampleBuffer: CMSampleBuffer) { @@ -169,6 +185,140 @@ final class AudioUnit: NSObject { bufferedAudios[cameraId]?.setTargetLatency(latency: latency) } + func addMixerSource(sourceId: UUID) { + processorPipelineQueue.async { + self.addMixerSourceInternal(sourceId: sourceId) + } + } + + func removeMixerSource(sourceId: UUID) { + processorPipelineQueue.async { + self.removeMixerSourceInternal(sourceId: sourceId) + } + } + + func addMixerBuiltinSource() { + processorPipelineQueue.async { + self.addMixerSourceInternal(sourceId: self.builtinInputId) + } + } + + func removeMixerBuiltinSource() { + processorPipelineQueue.async { + self.removeMixerSourceInternal(sourceId: self.builtinInputId) + } + } + + private func addMixerSourceInternal(sourceId: UUID) { + mixerSourceIds.insert(sourceId) + if mixerSourceIds.count > 1 { + ensureMixerStarted() + } + } + + private func removeMixerSourceInternal(sourceId: UUID) { + mixerSourceIds.remove(sourceId) + mixer?.remove(inputId: sourceId) + mixerInputFormats.removeValue(forKey: sourceId) + if mixerSourceIds.count <= 1 { + stopMixer() + } + } + + private func shouldUseMixer() -> Bool { + return mixerSourceIds.count > 1 + } + + private func ensureMixerStarted() { + guard !mixerStarted else { + return + } + mixerStarted = true + mixer = AudioMixer( + outputSampleRate: mixerOutputSampleRate, + outputChannels: mixerOutputChannels, + outputSamplesPerBuffer: mixerOutputSamplesPerBuffer + ) + mixerInputFormats = [:] + mixerOutputPresentationTimeStamp = currentPresentationTimeStamp() + let interval = Double(mixerOutputSamplesPerBuffer) / mixerOutputSampleRate + mixerProcessTimer.startPeriodic(interval: interval) { [weak self] in + self?.processMixerOutput() + } + logger.info("audio-unit: Mixer started") + } + + private func stopMixer() { + guard mixerStarted else { + return + } + mixerStarted = false + mixerProcessTimer.stop() + mixer = nil + mixerInputFormats = [:] + logger.info("audio-unit: Mixer stopped") + } + + private func ensureMixerInput(inputId: UUID, sampleBuffer: CMSampleBuffer) { + guard let mixer else { + return + } + guard let formatDescription = sampleBuffer.formatDescription, + let asbd = formatDescription.audioStreamBasicDescription + else { + return + } + let format: AVAudioFormat + if let f = AVAudioFormat( + standardFormatWithSampleRate: asbd.mSampleRate, + channels: AVAudioChannelCount(asbd.mChannelsPerFrame) + ) { + format = f + } else { + return + } + if mixerInputFormats[inputId] == nil { + mixer.add(inputId: inputId, format: format) + mixerInputFormats[inputId] = format + } + } + + private func appendToMixer(inputId: UUID, sampleBuffer: CMSampleBuffer) { + guard let mixer else { + return + } + ensureMixerInput(inputId: inputId, sampleBuffer: sampleBuffer) + try? sampleBuffer.withAudioBufferList { audioBufferList, _ in + guard let format = mixerInputFormats[inputId], + let pcmBuffer = AVAudioPCMBuffer( + pcmFormat: format, + bufferListNoCopy: audioBufferList.unsafePointer + ) + else { + return + } + mixer.append(inputId: inputId, buffer: pcmBuffer) + } + } + + private func processMixerOutput() { + guard let mixer, let processor else { + return + } + guard let outputBuffer = mixer.process() else { + return + } + let presentationTimeStamp = mixerOutputPresentationTimeStamp + mixerOutputPresentationTimeStamp = presentationTimeStamp + CMTime( + value: CMTimeValue(mixerOutputSamplesPerBuffer), + timescale: CMTimeScale(mixerOutputSampleRate) + ) + guard let sampleBuffer = outputBuffer.makeSampleBuffer(presentationTimeStamp) else { + return + } + appendNewSampleBuffer(processor, sampleBuffer, presentationTimeStamp) + } + private func appendNewSampleBuffer(_ processor: Processor, _ sampleBuffer: CMSampleBuffer, _ presentationTimeStamp: CMTime) @@ -241,38 +391,45 @@ extension AudioUnit: AVCaptureAudioDataOutputSampleBufferDelegate { if let bufferedAudio = appendBufferedBuiltinAudio(sampleBuffer, presentationTimeStamp) { sampleBuffer = bufferedAudio.getSampleBuffer(presentationTimeStamp.seconds) ?? sampleBuffer } - guard selectedBufferedAudioId == nil else { - return - } - if shouldUpdateAudioLevel(sampleBuffer) { - var audioLevel: Float - if muted { - audioLevel = .nan - } else if let channel = connection.audioChannels.first { - audioLevel = channel.averagePowerLevel - } else { - audioLevel = 0.0 + if shouldUseMixer(), mixerSourceIds.contains(builtinInputId) { + appendToMixer(inputId: builtinInputId, sampleBuffer: sampleBuffer) + } else if selectedBufferedAudioId == nil { + if shouldUpdateAudioLevel(sampleBuffer) { + var audioLevel: Float + if muted { + audioLevel = .nan + } else if let channel = connection.audioChannels.first { + audioLevel = channel.averagePowerLevel + } else { + audioLevel = 0.0 + } + updateAudioLevel(sampleBuffer: sampleBuffer, + audioLevel: audioLevel, + numberOfAudioChannels: connection.audioChannels.count) } - updateAudioLevel(sampleBuffer: sampleBuffer, - audioLevel: audioLevel, - numberOfAudioChannels: connection.audioChannels.count) + appendNewSampleBuffer(processor, sampleBuffer, presentationTimeStamp) } - appendNewSampleBuffer(processor, sampleBuffer, presentationTimeStamp) } } extension AudioUnit: BufferedAudioSampleBufferDelegate { func didOutputBufferedSampleBuffer(cameraId: UUID, sampleBuffer: CMSampleBuffer) { - guard selectedBufferedAudioId == cameraId, let processor else { + guard let processor else { return } - if shouldUpdateAudioLevel(sampleBuffer) { - let numberOfAudioChannels = Int(sampleBuffer.formatDescription?.numberOfAudioChannels() ?? 0) - updateAudioLevel(sampleBuffer: sampleBuffer, - audioLevel: .infinity, - numberOfAudioChannels: numberOfAudioChannels) + if shouldUseMixer(), mixerSourceIds.contains(cameraId) { + appendToMixer(inputId: cameraId, sampleBuffer: sampleBuffer) + } else if selectedBufferedAudioId == cameraId { + if shouldUpdateAudioLevel(sampleBuffer) { + let numberOfAudioChannels = Int( + sampleBuffer.formatDescription?.numberOfAudioChannels() ?? 0 + ) + updateAudioLevel(sampleBuffer: sampleBuffer, + audioLevel: .infinity, + numberOfAudioChannels: numberOfAudioChannels) + } + appendNewSampleBuffer(processor, sampleBuffer, sampleBuffer.presentationTimeStamp) } - appendNewSampleBuffer(processor, sampleBuffer, sampleBuffer.presentationTimeStamp) } } diff --git a/Moblin/Media/HaishinKit/Media/Processor.swift b/Moblin/Media/HaishinKit/Media/Processor.swift index 74c6cd986..0e15637b6 100644 --- a/Moblin/Media/HaishinKit/Media/Processor.swift +++ b/Moblin/Media/HaishinKit/Media/Processor.swift @@ -164,6 +164,22 @@ final class Processor { audio.setBufferedAudioTargetLatency(cameraId: cameraId, latency: latency) } + func addMixerSource(sourceId: UUID) { + audio.addMixerSource(sourceId: sourceId) + } + + func removeMixerSource(sourceId: UUID) { + audio.removeMixerSource(sourceId: sourceId) + } + + func addMixerBuiltinSource() { + audio.addMixerBuiltinSource() + } + + func removeMixerBuiltinSource() { + audio.removeMixerBuiltinSource() + } + func registerVideoEffect(_ effect: VideoEffect) { video.registerEffect(effect) } diff --git a/Moblin/Various/Media.swift b/Moblin/Various/Media.swift index 756d0f37a..851053c87 100644 --- a/Moblin/Various/Media.swift +++ b/Moblin/Various/Media.swift @@ -864,6 +864,22 @@ final class Media: NSObject { processor?.setBufferedAudioTargetLatency(cameraId: cameraId, latency) } + func addMixerSource(sourceId: UUID) { + processor?.addMixerSource(sourceId: sourceId) + } + + func removeMixerSource(sourceId: UUID) { + processor?.removeMixerSource(sourceId: sourceId) + } + + func addMixerBuiltinSource() { + processor?.addMixerBuiltinSource() + } + + func removeMixerBuiltinSource() { + processor?.removeMixerBuiltinSource() + } + func addBufferedVideo(cameraId: UUID, name: String, latency: Double) { processor?.addBufferedVideo(cameraId: cameraId, name: name, latency: latency) } diff --git a/Moblin/Various/Model/ModelAudio.swift b/Moblin/Various/Model/ModelAudio.swift index 80c61d695..337f29da9 100644 --- a/Moblin/Various/Model/ModelAudio.swift +++ b/Moblin/Various/Model/ModelAudio.swift @@ -21,8 +21,13 @@ class AudioProvider: ObservableObject { class Mic: ObservableObject { @Published var current: SettingsMicsMic = noMic + @Published var selectedMicIds: Set = [] var requested: SettingsMicsMic? var isSwitchTimerRunning: Bool = false + + func isSelected(mic: SettingsMicsMic) -> Bool { + return selectedMicIds.contains(mic.id) + } } extension Model { @@ -183,6 +188,76 @@ extension Model { } } + func manualToggleMicById(id: String) { + guard let toggledMic = getAvailableMicById(id: id) else { + return + } + if mic.isSelected(mic: toggledMic) { + mic.selectedMicIds.remove(toggledMic.id) + removeMicFromMixer(mic: toggledMic) + if mic.selectedMicIds.isEmpty { + mic.current = noMic + } else if mic.current == toggledMic, + let nextMic = database.mics.mics.first(where: { mic.isSelected(mic: $0) }) + { + mic.current = nextMic + } + } else { + mic.selectedMicIds.insert(toggledMic.id) + addMicToMixer(mic: toggledMic) + mic.current = toggledMic + } + } + + private func addMicToMixer(mic: SettingsMicsMic) { + if mic.isAudioSession() { + selectMicDefault(mic: mic) + media.addMixerBuiltinSource() + } else if isRtmpMic(mic: mic) { + if let cameraId = getRtmpStream(idString: mic.inputUid)?.id { + media.attachBufferedAudio(cameraId: cameraId) + media.addMixerSource(sourceId: cameraId) + } + } else if isSrtlaMic(mic: mic) { + if let cameraId = getSrtlaStream(idString: mic.inputUid)?.id { + media.attachBufferedAudio(cameraId: cameraId) + media.addMixerSource(sourceId: cameraId) + } + } else if isRistMic(mic: mic) { + if let cameraId = getRistStream(idString: mic.inputUid)?.id { + media.attachBufferedAudio(cameraId: cameraId) + media.addMixerSource(sourceId: cameraId) + } + } else if isMediaPlayerMic(mic: mic) { + if let cameraId = getMediaPlayer(idString: mic.inputUid)?.id { + media.attachBufferedAudio(cameraId: cameraId) + media.addMixerSource(sourceId: cameraId) + } + } + } + + private func removeMicFromMixer(mic: SettingsMicsMic) { + if mic.isAudioSession() { + media.removeMixerBuiltinSource() + } else if isRtmpMic(mic: mic) { + if let cameraId = getRtmpStream(idString: mic.inputUid)?.id { + media.removeMixerSource(sourceId: cameraId) + } + } else if isSrtlaMic(mic: mic) { + if let cameraId = getSrtlaStream(idString: mic.inputUid)?.id { + media.removeMixerSource(sourceId: cameraId) + } + } else if isRistMic(mic: mic) { + if let cameraId = getRistStream(idString: mic.inputUid)?.id { + media.removeMixerSource(sourceId: cameraId) + } + } else if isMediaPlayerMic(mic: mic) { + if let cameraId = getMediaPlayer(idString: mic.inputUid)?.id { + media.removeMixerSource(sourceId: cameraId) + } + } + } + func selectMicById(id: String) { if let mic = getAvailableMicById(id: id) { selectMic(mic: mic) @@ -587,6 +662,7 @@ extension Model { selectMicDefault(mic: mic) } self.mic.current = mic + self.mic.selectedMicIds = [mic.id] self.mic.isSwitchTimerRunning = true DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) { self.mic.isSwitchTimerRunning = false diff --git a/Moblin/View/ControlBar/QuickButton/QuickButtonMicView.swift b/Moblin/View/ControlBar/QuickButton/QuickButtonMicView.swift index 17cb268f9..e3351b34a 100644 --- a/Moblin/View/ControlBar/QuickButton/QuickButtonMicView.swift +++ b/Moblin/View/ControlBar/QuickButton/QuickButtonMicView.swift @@ -13,15 +13,15 @@ private struct QuickButtonMicMicView: View { Text(mic.name) .lineLimit(1) Spacer() - Image(systemName: "checkmark") - .foregroundStyle(mic == modelMic.current ? .blue : .clear) + Image(systemName: modelMic.isSelected(mic: mic) ? "checkmark.circle.fill" : "circle") + .foregroundStyle(modelMic.isSelected(mic: mic) ? .blue : .gray) .bold() } .contentShape(Rectangle()) .onTapGesture { model.updateMicsListAsync { if mic.connected { - model.manualSelectMicById(id: mic.id) + model.manualToggleMicById(id: mic.id) } } } @@ -39,10 +39,6 @@ struct QuickButtonMicView: View { List { ForEach(mics.mics) { mic in QuickButtonMicMicView(model: model, mic: mic, modelMic: modelMic) - .deleteDisabled(mic == modelMic.current) - } - .onDelete { offsets in - mics.mics.remove(atOffsets: offsets) } .onMove { froms, to in mics.mics.move(fromOffsets: froms, toOffset: to) @@ -50,9 +46,9 @@ struct QuickButtonMicView: View { } } footer: { VStack(alignment: .leading) { - Text("Highest priority mic at the top of the list.") + Text("Select one or more mics to mix together.") Text("") - SwipeLeftToDeleteHelpView(kind: String(localized: "a mic")) + Text("Highest priority mic at the top of the list.") } } if false { diff --git a/Moblin/View/Settings/Audio/AudioSettingsView.swift b/Moblin/View/Settings/Audio/AudioSettingsView.swift index 9f444091a..bbf73b5af 100644 --- a/Moblin/View/Settings/Audio/AudioSettingsView.swift +++ b/Moblin/View/Settings/Audio/AudioSettingsView.swift @@ -5,6 +5,16 @@ private struct MicView: View { @ObservedObject var mics: SettingsMics @ObservedObject var mic: Mic + private var selectedMicNames: String { + let names = mics.mics + .filter { mic.isSelected(mic: $0) } + .map(\.name) + if names.isEmpty { + return mic.current.name + } + return names.joined(separator: ", ") + } + var body: some View { NavigationLink { QuickButtonMicView(model: model, mics: mics, modelMic: mic) @@ -13,7 +23,7 @@ private struct MicView: View { HStack { Text("Mic") Spacer() - GrayTextView(text: mic.current.name) + GrayTextView(text: selectedMicNames) } } icon: { Image(systemName: "music.mic") diff --git a/Moblin/View/Stream/Overlay/StreamOverlayLeftView.swift b/Moblin/View/Stream/Overlay/StreamOverlayLeftView.swift index acaeeba6c..81af0b043 100644 --- a/Moblin/View/Stream/Overlay/StreamOverlayLeftView.swift +++ b/Moblin/View/Stream/Overlay/StreamOverlayLeftView.swift @@ -149,6 +149,16 @@ private struct StatusesView: View { } } + func selectedMicNames() -> String { + let names = model.database.mics.mics + .filter { mic.isSelected(mic: $0) } + .map(\.name) + if names.isEmpty { + return mic.current.name + } + return names.joined(separator: ", ") + } + var body: some View { if model.isShowingStatusStream() { StreamStatusView(status: status, textPlacement: textPlacement) @@ -163,7 +173,7 @@ private struct StatusesView: View { if model.isShowingStatusMic() { StreamOverlayIconAndTextView( icon: "music.mic", - text: mic.current.name, + text: selectedMicNames(), textPlacement: textPlacement ) } diff --git a/MoblinTests/AudioMixerSuite.swift b/MoblinTests/AudioMixerSuite.swift index 405b06df6..1a1405a3d 100644 --- a/MoblinTests/AudioMixerSuite.swift +++ b/MoblinTests/AudioMixerSuite.swift @@ -196,4 +196,95 @@ struct AudioMixerSuite { #expect(samples[1024] == 0) #expect(samples[2047] == 0) } + + @Test + func threeMonoInputsMonoOutput() async throws { + let mixer = AudioMixer(outputSampleRate: 48000, outputChannels: 1, outputSamplesPerBuffer: 1024) + let inputId1 = UUID() + let inputId2 = UUID() + let inputId3 = UUID() + let format = try #require(AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)) + mixer.add(inputId: inputId1, format: format) + mixer.add(inputId: inputId2, format: format) + mixer.add(inputId: inputId3, format: format) + #expect(mixer.numberOfInputs() == 3) + let inputBuffer1 = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer1.frameLength = 1024 + var samples = try #require(inputBuffer1.floatChannelData?.pointee) + samples[0] = 10 + mixer.append(inputId: inputId1, buffer: inputBuffer1) + let inputBuffer2 = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer2.frameLength = 1024 + samples = try #require(inputBuffer2.floatChannelData?.pointee) + samples[0] = 20 + mixer.append(inputId: inputId2, buffer: inputBuffer2) + let inputBuffer3 = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer3.frameLength = 1024 + samples = try #require(inputBuffer3.floatChannelData?.pointee) + samples[0] = 30 + mixer.append(inputId: inputId3, buffer: inputBuffer3) + try? await sleep(milliSeconds: processDelayMs) + let outputBuffer = mixer.process() + #expect(outputBuffer?.format.sampleRate == 48000) + #expect(outputBuffer?.format.channelCount == 1) + #expect(outputBuffer?.frameLength == 1024) + samples = try #require(outputBuffer?.floatChannelData?.pointee) + #expect(samples[0] == 60 / sqrt(2)) + #expect(samples[500] == 0) + #expect(samples[1023] == 0) + mixer.remove(inputId: inputId1) + #expect(mixer.numberOfInputs() == 2) + mixer.remove(inputId: inputId2) + #expect(mixer.numberOfInputs() == 1) + mixer.remove(inputId: inputId3) + #expect(mixer.numberOfInputs() == 0) + } + + @Test + func addAndRemoveInputsDynamically() async throws { + let mixer = AudioMixer(outputSampleRate: 48000, outputChannels: 1, outputSamplesPerBuffer: 1024) + let inputId1 = UUID() + let inputId2 = UUID() + let format = try #require(AVAudioFormat(standardFormatWithSampleRate: 48000, channels: 1)) + mixer.add(inputId: inputId1, format: format) + #expect(mixer.numberOfInputs() == 1) + let inputBuffer1 = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer1.frameLength = 1024 + var samples = try #require(inputBuffer1.floatChannelData?.pointee) + samples[0] = 50 + mixer.append(inputId: inputId1, buffer: inputBuffer1) + try? await sleep(milliSeconds: processDelayMs) + var outputBuffer = mixer.process() + samples = try #require(outputBuffer?.floatChannelData?.pointee) + #expect(samples[0] == 50 / sqrt(2)) + mixer.add(inputId: inputId2, format: format) + #expect(mixer.numberOfInputs() == 2) + let inputBuffer1b = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer1b.frameLength = 1024 + samples = try #require(inputBuffer1b.floatChannelData?.pointee) + samples[0] = 30 + mixer.append(inputId: inputId1, buffer: inputBuffer1b) + let inputBuffer2 = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer2.frameLength = 1024 + samples = try #require(inputBuffer2.floatChannelData?.pointee) + samples[0] = 20 + mixer.append(inputId: inputId2, buffer: inputBuffer2) + try? await sleep(milliSeconds: processDelayMs) + outputBuffer = mixer.process() + samples = try #require(outputBuffer?.floatChannelData?.pointee) + #expect(samples[0] == 50 / sqrt(2)) + mixer.remove(inputId: inputId1) + #expect(mixer.numberOfInputs() == 1) + let inputBuffer2b = try #require(AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 1024)) + inputBuffer2b.frameLength = 1024 + samples = try #require(inputBuffer2b.floatChannelData?.pointee) + samples[0] = 100 + mixer.append(inputId: inputId2, buffer: inputBuffer2b) + try? await sleep(milliSeconds: processDelayMs) + outputBuffer = mixer.process() + samples = try #require(outputBuffer?.floatChannelData?.pointee) + #expect(samples[0] == 100 / sqrt(2)) + mixer.remove(inputId: inputId2) + #expect(mixer.numberOfInputs() == 0) + } }