From d52402725fa97ccfcb0433c48a09e47f3bde02fd Mon Sep 17 00:00:00 2001 From: Kn0ax Date: Mon, 9 Feb 2026 21:55:52 +0100 Subject: [PATCH 1/2] Add WHIP WebRTC streaming support --- Common/Localizable.xcstrings | 15 + Common/Various/Validate.swift | 15 + Moblin.xcodeproj/project.pbxproj | 19 + .../xcshareddata/swiftpm/Package.resolved | 11 +- Moblin/Media/HaishinKit/Whip/WhipStream.swift | 715 ++++++++++++++++++ Moblin/Various/Media.swift | 41 + Moblin/Various/Model/ModelStream.swift | 27 + Moblin/Various/Settings/SettingsStream.swift | 13 + Moblin/Various/Utils/Utils.swift | 19 + .../Streams/Stream/StreamSettingsView.swift | 2 + .../Stream/Url/StreamUrlSettingsView.swift | 20 + MoblinTests/UtilsSuite.swift | 19 + MoblinTests/ValidateSuite.swift | 11 + 13 files changed, 926 insertions(+), 1 deletion(-) create mode 100644 Moblin/Media/HaishinKit/Whip/WhipStream.swift create mode 100644 MoblinTests/ValidateSuite.swift diff --git a/Common/Localizable.xcstrings b/Common/Localizable.xcstrings index 351763cbc..e5b993c23 100644 --- a/Common/Localizable.xcstrings +++ b/Common/Localizable.xcstrings @@ -76585,6 +76585,12 @@ } } } + }, + "Example: whip://whip.example.com/ingest/stream" : { + + }, + "Example: whips://whip.example.com/ingest/stream" : { + }, "EXB" : { "localizations" : { @@ -109404,6 +109410,9 @@ } } } + }, + "Malformed WHIP URL" : { + }, "Manage streams" : { "localizations" : { @@ -196401,6 +196410,9 @@ } } } + }, + "Template: whips://my_whip_server/whip/endpoint" : { + }, "Tennis" : { "localizations" : { @@ -223320,6 +223332,9 @@ }, "When \"Audio only\" mode is selected, no video will be rendered at all. Only audio will play." : { + }, + "WHIP" : { + }, "Whirlpool" : { "localizations" : { diff --git a/Common/Various/Validate.swift b/Common/Various/Validate.swift index 2da27d9ac..7906923ff 100644 --- a/Common/Various/Validate.swift +++ b/Common/Various/Validate.swift @@ -56,6 +56,13 @@ func isValidRistUrl(url: String) -> String? { return nil } +func isValidWhipUrl(url: String) -> String? { + guard URL(string: url) != nil else { + return String(localized: "Malformed WHIP URL") + } + return nil +} + private func isValidIrlUrl(url: String) -> String? { guard URL(string: url) != nil else { return String(localized: "Malformed IRL URL") @@ -107,6 +114,14 @@ func isValidUrl(url value: String, if let message = isValidRistUrl(url: value) { return message } + case "whip": + if let message = isValidWhipUrl(url: value) { + return message + } + case "whips": + if let message = isValidWhipUrl(url: value) { + return message + } case "irl": if let message = isValidIrlUrl(url: value) { return message diff --git a/Moblin.xcodeproj/project.pbxproj b/Moblin.xcodeproj/project.pbxproj index a6f1da9c0..8f22b150b 100644 --- a/Moblin.xcodeproj/project.pbxproj +++ b/Moblin.xcodeproj/project.pbxproj @@ -31,6 +31,8 @@ 03B7DAB52BA640C100CEFC1B /* XMLCoder in Frameworks */ = {isa = PBXBuildFile; productRef = 03B7DAB42BA640C100CEFC1B /* XMLCoder */; }; 03BC11672AE5654700C38FC4 /* SDWebImageSwiftUI in Frameworks */ = {isa = PBXBuildFile; productRef = 03BC11662AE5654700C38FC4 /* SDWebImageSwiftUI */; }; 03BC116B2AE56C2200C38FC4 /* SDWebImageWebPCoder in Frameworks */ = {isa = PBXBuildFile; productRef = 03BC116A2AE56C2200C38FC4 /* SDWebImageWebPCoder */; }; + 03C1737B2F3AF80500487806 /* Webrtc in Frameworks */ = {isa = PBXBuildFile; productRef = 03C1737A2F3AF80500487806 /* Webrtc */; }; + 03D5A1C22F5A600100A1B2C3 /* (null) in Frameworks */ = {isa = PBXBuildFile; }; 03ECDF532B8E4E6000BD920E /* Moblin.app in Embed Watch Content */ = {isa = PBXBuildFile; fileRef = 03ECDF462B8E4E5E00BD920E /* Moblin.app */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; 03ECDF5D2B8E5F0B00BD920E /* WrappingHStack in Frameworks */ = {isa = PBXBuildFile; productRef = 03ECDF5C2B8E5F0B00BD920E /* WrappingHStack */; }; 03F465EC2C441D1400630708 /* CrcSwift in Frameworks */ = {isa = PBXBuildFile; productRef = 03F465EB2C441D1400630708 /* CrcSwift */; }; @@ -212,6 +214,7 @@ files = ( 03B7DAB52BA640C100CEFC1B /* XMLCoder in Frameworks */, 033BEAC42C0FCC0B005F4E06 /* NWWebSocket in Frameworks */, + 03C1737B2F3AF80500487806 /* Webrtc in Frameworks */, 0307869A2AEC23FB0061FDE2 /* StoreKit.framework in Frameworks */, 03BC11672AE5654700C38FC4 /* SDWebImageSwiftUI in Frameworks */, 0318D36A2CF51D6900E12F3B /* SwiftProtobuf in Frameworks */, @@ -230,6 +233,7 @@ 03A08B7C2AC295620018BA95 /* AlertToast in Frameworks */, 0377239C2DE35191007D040D /* VRMSceneKit in Frameworks */, 03BC116B2AE56C2200C38FC4 /* SDWebImageWebPCoder in Frameworks */, + 03D5A1C22F5A600100A1B2C3 /* (null) in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -364,6 +368,7 @@ 035351932F1C271700428DAC /* AppAuthCore */, 035351952F1C27A500428DAC /* AppAuth */, 0360FD152F228EEB00FF8847 /* MetalPetal */, + 03C1737A2F3AF80500487806 /* Webrtc */, ); productName = Mobs; productReference = 035E9E332A9A02D6009D4F5A /* Moblin.app */; @@ -519,6 +524,7 @@ 882D0C142DF76F5B0035BFAF /* XCRemoteSwiftPackageReference "BlackSharkLib" */, 035351922F1C271700428DAC /* XCRemoteSwiftPackageReference "AppAuth-iOS" */, 0360FD142F228EEB00FF8847 /* XCRemoteSwiftPackageReference "MetalPetal" */, + 03C173792F3AF80500487806 /* XCRemoteSwiftPackageReference "Webrtc" */, ); productRefGroup = 035E9E342A9A02D6009D4F5A /* Products */; projectDirPath = ""; @@ -1335,6 +1341,14 @@ minimumVersion = 0.14.0; }; }; + 03C173792F3AF80500487806 /* XCRemoteSwiftPackageReference "Webrtc" */ = { + isa = XCRemoteSwiftPackageReference; + repositoryURL = "https://github.com/eerimoq/Webrtc/"; + requirement = { + branch = main; + kind = branch; + }; + }; 03F465EA2C441D1400630708 /* XCRemoteSwiftPackageReference "CrcSwift" */ = { isa = XCRemoteSwiftPackageReference; repositoryURL = "https://github.com/eerimoq/CrcSwift.git"; @@ -1449,6 +1463,11 @@ package = 03BC11692AE56C2200C38FC4 /* XCRemoteSwiftPackageReference "SDWebImageWebPCoder" */; productName = SDWebImageWebPCoder; }; + 03C1737A2F3AF80500487806 /* Webrtc */ = { + isa = XCSwiftPackageProductDependency; + package = 03C173792F3AF80500487806 /* XCRemoteSwiftPackageReference "Webrtc" */; + productName = Webrtc; + }; 03ECDF5C2B8E5F0B00BD920E /* WrappingHStack */ = { isa = XCSwiftPackageProductDependency; package = 0320D8742AED36860030418F /* XCRemoteSwiftPackageReference "WrappingHStack" */; diff --git a/Moblin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/Moblin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved index e65b22013..502ed5e00 100644 --- a/Moblin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved +++ b/Moblin.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "2ec89183ce973e73157e1f6178f92aabac79182102b81c7f9097c9d2d72f6022", + "originHash" : "11ced83bb44734f5ca14e8e528b3636b03ed7e62bf0469485fda090e8672e709", "pins" : [ { "identity" : "alerttoast", @@ -154,6 +154,15 @@ "revision" : "071e0bc6a0f4f0ba2385c05332b0adb172d0f7ec" } }, + { + "identity" : "webrtc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/eerimoq/Webrtc/", + "state" : { + "branch" : "main", + "revision" : "4da5d1591360d79bb7d2d069af26b5ab3e6f3e45" + } + }, { "identity" : "wrappinghstack", "kind" : "remoteSourceControl", diff --git a/Moblin/Media/HaishinKit/Whip/WhipStream.swift b/Moblin/Media/HaishinKit/Whip/WhipStream.swift new file mode 100644 index 000000000..31991ebb1 --- /dev/null +++ b/Moblin/Media/HaishinKit/Whip/WhipStream.swift @@ -0,0 +1,715 @@ +import AVFoundation +import Foundation +import libdatachannel + +private let whipQueue = DispatchQueue(label: "com.eerimoq.Moblin.whip") +private let h264PayloadType: UInt8 = 98 +private let opusPayloadType: UInt8 = 111 +private let rtpMtu = 1200 + +private func makeSsrc() -> UInt32 { + var ssrc: UInt32 = 0 + while ssrc == 0 { + ssrc = UInt32.random(in: UInt32.min ... UInt32.max) + } + return ssrc +} + +private func checkOkReturnResult(_ result: Int32) throws -> Int32 { + guard result >= 0 else { + throw "Error \(result)" + } + return result +} + +private func checkOk(_ result: Int32) throws { + _ = try checkOkReturnResult(result) +} + +private enum ConnectionState { + case new + case connecting + case connected + case disconnected + case failed + case closed + + init?(cValue: rtcState) { + switch cValue { + case RTC_NEW: + self = .new + case RTC_CONNECTING: + self = .connecting + case RTC_CONNECTED: + self = .connected + case RTC_DISCONNECTED: + self = .disconnected + case RTC_FAILED: + self = .failed + case RTC_CLOSED: + self = .closed + default: + return nil + } + } +} + +private func makeEndpointUrl(url: String) -> URL? { + guard var components = URLComponents(string: url) else { + return nil + } + switch components.scheme { + case "whip": + components.scheme = "http" + case "whips": + components.scheme = "https" + default: + return nil + } + return components.url +} + +private enum TrackState { + case connecting + case open + case closed +} + +private struct RtpPacket { + let marker: Bool + let payloadType: UInt8 + let sequenceNumber: UInt16 + let timestamp: UInt32 + let ssrc: UInt32 + let payload: Data + + func data() -> Data { + var data = Data(capacity: 12 + payload.count) + data.append(0x80) + data.append((marker ? 0x80 : 0x00) | (payloadType & 0x7F)) + data.append(contentsOf: [ + UInt8((sequenceNumber >> 8) & 0xFF), + UInt8(sequenceNumber & 0xFF), + ]) + data.append(contentsOf: [ + UInt8((timestamp >> 24) & 0xFF), + UInt8((timestamp >> 16) & 0xFF), + UInt8((timestamp >> 8) & 0xFF), + UInt8(timestamp & 0xFF), + ]) + data.append(contentsOf: [ + UInt8((ssrc >> 24) & 0xFF), + UInt8((ssrc >> 16) & 0xFF), + UInt8((ssrc >> 8) & 0xFF), + UInt8(ssrc & 0xFF), + ]) + data.append(payload) + return data + } +} + +private func convertTimestamp(_ presentationTimeStamp: CMTime) -> UInt32 { + return UInt32(UInt64(presentationTimeStamp.seconds * 90000) & 0xFFFF_FFFF) +} + +private final class H264Packetizer { + let ssrc: UInt32 + private let payloadType: UInt8 + private var sequenceNumber: UInt16 = 0 + private var sps: Data? + private var pps: Data? + + init(ssrc: UInt32, payloadType: UInt8) { + self.ssrc = ssrc + self.payloadType = payloadType + } + + func setParameterSets(sps: Data?, pps: Data?) { + self.sps = sps + self.pps = pps + } + + func packetize(sampleBuffer: CMSampleBuffer, presentationTimeStamp: CMTime) -> [Data] { + var nalUnits = extractNalUnits(sampleBuffer: sampleBuffer) + guard !nalUnits.isEmpty else { + return [] + } + if sampleBuffer.getIsSync() { + if let sps { + nalUnits.insert(sps, at: 0) + } + if let pps { + nalUnits.insert(pps, at: min(1, nalUnits.count)) + } + } + let packetTimestamp = convertTimestamp(presentationTimeStamp) + var packets: [Data] = [] + for (index, nalUnit) in nalUnits.enumerated() { + let isLastNal = index == nalUnits.count - 1 + if nalUnit.count <= rtpMtu { + let packet = RtpPacket( + marker: isLastNal, + payloadType: payloadType, + sequenceNumber: sequenceNumber, + timestamp: packetTimestamp, + ssrc: ssrc, + payload: nalUnit + ) + packets.append(packet.data()) + sequenceNumber &+= 1 + } else { + let nalHeader = nalUnit[0] + let fuIndicator = (nalHeader & 0xE0) | 28 + let nalType = nalHeader & 0x1F + var offset = 1 + var first = true + while offset < nalUnit.count { + let chunkSize = min(rtpMtu - 2, nalUnit.count - offset) + var fuHeader = nalType + if first { + fuHeader |= 0x80 + } + let isFinalFragment = offset + chunkSize >= nalUnit.count + if isFinalFragment { + fuHeader |= 0x40 + } + var payload = Data([fuIndicator, fuHeader]) + payload.append(contentsOf: nalUnit[offset ..< offset + chunkSize]) + let packet = RtpPacket( + marker: isLastNal && isFinalFragment, + payloadType: payloadType, + sequenceNumber: sequenceNumber, + timestamp: packetTimestamp, + ssrc: ssrc, + payload: payload + ) + packets.append(packet.data()) + sequenceNumber &+= 1 + offset += chunkSize + first = false + } + } + } + return packets + } + + private func extractNalUnits(sampleBuffer: CMSampleBuffer) -> [Data] { + guard let (buffer, length) = sampleBuffer.dataBuffer?.getDataPointer() else { + return [] + } + let data = Data(bytes: buffer, count: length) + var nalUnits: [Data] = [] + var offset = 0 + while offset + 4 <= data.count { + let nalLength = Int(data.getFourBytesBe(offset: offset)) + offset += 4 + guard nalLength > 0, offset + nalLength <= data.count else { + break + } + nalUnits.append(data.subdata(in: offset ..< offset + nalLength)) + offset += nalLength + } + return nalUnits + } +} + +private final class OpusPacketizer { + let ssrc: UInt32 + + init(ssrc: UInt32) { + self.ssrc = ssrc + } + + func packetize(buffer: AVAudioCompressedBuffer) -> [Data] { + guard buffer.byteLength > 0 else { + return [] + } + let allData = Data(bytes: buffer.data, count: Int(buffer.byteLength)) + guard buffer.packetCount > 0, let descriptions = buffer.packetDescriptions else { + return [allData] + } + var packets: [Data] = [] + packets.reserveCapacity(Int(buffer.packetCount)) + for index in 0 ..< Int(buffer.packetCount) { + let description = descriptions[index] + let offset = Int(description.mStartOffset) + let size = Int(description.mDataByteSize) + guard size > 0, offset >= 0, offset + size <= allData.count else { + continue + } + packets.append(allData.subdata(in: offset ..< offset + size)) + } + return packets.isEmpty ? [allData] : packets + } +} + +private final class OpusRtpPacketizer { + private let ssrc: UInt32 + private let payloadType: UInt8 + private var sequenceNumber: UInt16 = 0 + + init(ssrc: UInt32, payloadType: UInt8) { + self.ssrc = ssrc + self.payloadType = payloadType + } + + func packetize(payload: Data, presentationTimeStamp: CMTime) -> Data { + let packetTimestamp = convertTimestamp(presentationTimeStamp) + let packet = RtpPacket( + marker: false, + payloadType: payloadType, + sequenceNumber: sequenceNumber, + timestamp: packetTimestamp, + ssrc: ssrc, + payload: payload + ) + sequenceNumber &+= 1 + return packet.data() + } +} + +private func toRtcTrack(pointer: UnsafeMutableRawPointer?) -> RtcTrack? { + guard let pointer else { + return nil + } + return Unmanaged.fromOpaque(pointer).takeUnretainedValue() +} + +private final class RtcTrack { + private let trackId: Int32 + private var state: TrackState = .connecting + + init(trackId: Int32) throws { + self.trackId = trackId + do { + rtcSetUserPointer(trackId, Unmanaged.passUnretained(self).toOpaque()) + try checkOk(rtcSetOpenCallback(trackId) { _, pointer in + toRtcTrack(pointer: pointer)?.setState(state: .open) + }) + try checkOk(rtcSetClosedCallback(trackId) { _, pointer in + toRtcTrack(pointer: pointer)?.setState(state: .closed) + }) + try checkOk(rtcSetErrorCallback(trackId) { _, _, pointer in + toRtcTrack(pointer: pointer)?.setState(state: .closed) + }) + } catch { + rtcDeleteTrack(trackId) + throw error + } + } + + deinit { + rtcDeleteTrack(trackId) + } + + func send(packet: Data) -> Bool { + guard state == .open else { + return false + } + let result = packet.withUnsafeBytes { pointer in + rtcSendMessage(trackId, pointer.bindMemory(to: CChar.self).baseAddress, Int32(packet.count)) + } + return result >= 0 + } + + private func setState(state: TrackState) { + guard state != self.state else { + return + } + self.state = state + } +} + +private struct RtcTrackConfig { + let name: String + let codec: rtcCodec + let payloadType: Int32 + let ssrc: UInt32 + let mid: String + let profile: String + + static func makeAudio(ssrc: UInt32) -> Self { + return .init(name: "audio", + codec: RTC_CODEC_OPUS, + payloadType: Int32(opusPayloadType), + ssrc: ssrc, + mid: "0", + profile: "") + } + + static func makeVideo(ssrc: UInt32) -> Self { + return .init(name: "video", + codec: RTC_CODEC_H264, + payloadType: Int32(h264PayloadType), + ssrc: ssrc, + mid: "1", + profile: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f") + } +} + +private protocol PeerConnectionDelegate: AnyObject { + func peerConnectionOnConnectionStateChanged(state: ConnectionState) +} + +private func toPeerConnection(pointer: UnsafeMutableRawPointer?) -> PeerConnection? { + guard let pointer else { + return nil + } + return Unmanaged.fromOpaque(pointer).takeUnretainedValue() +} + +private final class PeerConnection { + private let peerConnectionId: Int32 + weak var delegate: PeerConnectionDelegate? + + init(delegate: PeerConnectionDelegate, iceServers: [String]) throws { + self.delegate = delegate + var config = rtcConfiguration() + peerConnectionId = iceServers.withCPointers { + config.iceServers = $0 + config.iceServersCount = Int32(iceServers.count) + return rtcCreatePeerConnection(&config) + } + try checkOk(peerConnectionId) + do { + rtcSetUserPointer(peerConnectionId, Unmanaged.passUnretained(self).toOpaque()) + try checkOk(rtcSetStateChangeCallback(peerConnectionId) { _, state, pointer in + toPeerConnection(pointer: pointer)?.handleStateChange(state: state) + }) + } catch { + rtcDeletePeerConnection(peerConnectionId) + throw error + } + } + + deinit { + rtcDeletePeerConnection(peerConnectionId) + } + + func addTrack(config: RtcTrackConfig, streamId: String) throws -> RtcTrack { + return try config.mid.withCString { mid in + try config.name.withCString { name in + try streamId.withCString { streamId in + try UUID().uuidString.withCString { trackId in + try config.profile.withCString { profile in + var trackInit = rtcTrackInit( + direction: RTC_DIRECTION_SENDONLY, + codec: config.codec, + payloadType: config.payloadType, + ssrc: config.ssrc, + mid: mid, + name: name, + msid: streamId, + trackId: trackId, + profile: profile + ) + let trackId = try checkOkReturnResult(rtcAddTrackEx(peerConnectionId, &trackInit)) + return try RtcTrack(trackId: trackId) + } + } + } + } + } + } + + func setLocalDescriptionOffer() throws { + try checkOk(rtcSetLocalDescription(peerConnectionId, "offer")) + } + + func createOffer() throws -> String { + let size = try checkOkReturnResult(rtcCreateOffer(peerConnectionId, nil, 0)) + var buffer = [CChar](repeating: 0, count: Int(size)) + try checkOk(rtcCreateOffer(peerConnectionId, &buffer, Int32(size))) + return String(cString: buffer) + } + + func setRemoteAnswer(_ sdp: String) throws { + try checkOk(rtcSetRemoteDescription(peerConnectionId, sdp, "answer")) + } + + func close() { + _ = rtcClosePeerConnection(peerConnectionId) + } + + private func handleStateChange(state: rtcState) { + guard let state = ConnectionState(cValue: state) else { + return + } + delegate?.peerConnectionOnConnectionStateChanged(state: state) + } +} + +protocol WhipStreamDelegate: AnyObject { + func whipStreamOnConnected() + func whipStreamOnDisconnected(reason: String) +} + +final class WhipStream { + private let processor: Processor + private weak var delegate: WhipStreamDelegate? + private var peerConnection: PeerConnection? + private var videoTrack: RtcTrack? + private var audioTrack: RtcTrack? + private var videoPacketizer: H264Packetizer? + private var audioPacketizer: OpusPacketizer? + private var audioRtpPacketizer: OpusRtpPacketizer? + private var totalByteCount: Int64 = 0 + private var sessionUrl: URL? + private var endpointUrl: URL? + private var encoding = false + private var connected = false + private var offerSent = false + + init(processor: Processor, delegate: WhipStreamDelegate) { + self.processor = processor + self.delegate = delegate + } + + func start(url: String, iceServers: [String]) { + whipQueue.async { + self.startInternal(url: url, iceServers: iceServers) + } + } + + func stop() { + whipQueue.async { + self.stopInternal() + } + } + + func getTotalByteCount() -> Int64 { + return whipQueue.sync { + totalByteCount + } + } + + private func startInternal(url: String, iceServers: [String]) { + stopInternal() + guard let endpointUrl = makeEndpointUrl(url: url) else { + return + } + self.endpointUrl = endpointUrl + totalByteCount = 0 + connected = false + offerSent = false + logger.info("whip: Start URL: \(endpointUrl.absoluteString)") + let audioPacketizer = OpusPacketizer(ssrc: makeSsrc()) + let audioRtpPacketizer = OpusRtpPacketizer( + ssrc: audioPacketizer.ssrc, + payloadType: opusPayloadType + ) + let videoPacketizer = H264Packetizer(ssrc: makeSsrc(), payloadType: h264PayloadType) + self.audioPacketizer = audioPacketizer + self.audioRtpPacketizer = audioRtpPacketizer + self.videoPacketizer = videoPacketizer + do { + let peerConnection = try PeerConnection(delegate: self, iceServers: iceServers) + let streamId = UUID().uuidString + audioTrack = try peerConnection.addTrack( + config: .makeAudio(ssrc: audioPacketizer.ssrc), + streamId: streamId + ) + videoTrack = try peerConnection.addTrack( + config: .makeVideo(ssrc: videoPacketizer.ssrc), + streamId: streamId + ) + self.peerConnection = peerConnection + try peerConnection.setLocalDescriptionOffer() + let offer = try peerConnection.createOffer() + sendOffer(endpointUrl: endpointUrl, offer: offer) + } catch { + stopInternal(reason: "WHIP start failed") + } + } + + private func stopInternal(reason: String? = nil) { + stopEncoding() + if let sessionUrl { + sendDeleteRequest(url: sessionUrl) + } + sessionUrl = nil + endpointUrl = nil + peerConnection?.close() + peerConnection = nil + videoTrack = nil + audioTrack = nil + videoPacketizer = nil + audioPacketizer = nil + audioRtpPacketizer = nil + connected = false + offerSent = false + if let reason { + notifyDisconnected(reason: reason) + } + } + + private func handleConnectionStateChanged(state: ConnectionState) { + logger.info("whip: Connection state \(state)") + switch state { + case .connected: + guard !connected else { + return + } + connected = true + startEncoding() + notifyConnected() + case .disconnected, .failed, .closed: + stopInternal(reason: "WHIP disconnected (\(state))") + case .new, .connecting: + break + } + } + + private func sendOffer(endpointUrl: URL, offer: String) { + var request = URLRequest(url: endpointUrl) + request.httpMethod = "POST" + request.setContentType("application/sdp") + request.httpBody = offer.utf8Data + URLSession.shared.dataTask(with: request) { [weak self] data, response, error in + whipQueue.async { + self?.handleOfferResponse(data: data, response: response, error: error) + } + } + .resume() + } + + private func handleOfferResponse(data: Data?, response: URLResponse?, error: (any Error)?) { + if let error { + logger.info("whip: Offer request failed with error: \(error)") + stopInternal(reason: "WHIP offer failed") + return + } + guard let response = response as? HTTPURLResponse else { + logger.info("whip: Offer response was not HTTP") + stopInternal(reason: "WHIP bad server response") + return + } + guard response.http?.isSuccessful == true else { + stopInternal(reason: "WHIP server returned \(response.statusCode)") + return + } + if let locationHeader = response.value(forHTTPHeaderField: "Location") { + sessionUrl = URL(string: locationHeader, relativeTo: endpointUrl) + } + guard let data, let answer = String(data: data, encoding: .utf8) else { + stopInternal(reason: "WHIP answer missing") + return + } + do { + try peerConnection?.setRemoteAnswer(answer) + } catch { + logger.info("whip: Failed to set remote answer: \(error)") + stopInternal(reason: "WHIP answer rejected") + } + } + + private func sendDeleteRequest(url: URL) { + var request = URLRequest(url: url) + request.httpMethod = "DELETE" + URLSession.shared.dataTask(with: request) { _, _, _ in }.resume() + } + + private func startEncoding() { + guard !encoding else { + return + } + encoding = true + processorControlQueue.async { + self.processor.startEncoding(self) + } + } + + private func stopEncoding() { + guard encoding else { + return + } + encoding = false + processorControlQueue.async { + self.processor.stopEncoding(self) + } + } + + private func notifyConnected() { + delegate?.whipStreamOnConnected() + } + + private func notifyDisconnected(reason: String) { + delegate?.whipStreamOnDisconnected(reason: reason) + } + + private func handleAudioEncoderOutputBuffer(_ buffer: AVAudioCompressedBuffer, + _ presentationTimeStamp: CMTime) + { + guard connected, let packets = audioPacketizer?.packetize(buffer: buffer) else { + return + } + for packet in packets { + let outgoingPacket: Data + if let rtpPacketizer = audioRtpPacketizer { + outgoingPacket = rtpPacketizer.packetize(payload: packet, + presentationTimeStamp: presentationTimeStamp) + } else { + continue + } + if audioTrack?.send(packet: outgoingPacket) == true { + totalByteCount += Int64(outgoingPacket.count) + } + } + } + + private func handleVideoEncoderOutputFormat(_ formatDescription: CMFormatDescription) { + guard let config = MpegTsVideoConfigAvc(formatDescription: formatDescription) else { + return + } + videoPacketizer?.setParameterSets(sps: config.sequenceParameterSet, pps: config.pictureParameterSet) + } + + private func handleVideoEncoderOutputSampleBuffer(_ sampleBuffer: CMSampleBuffer, + _ presentationTimeStamp: CMTime) + { + guard connected, + let packets = videoPacketizer?.packetize(sampleBuffer: sampleBuffer, + presentationTimeStamp: presentationTimeStamp) + else { + return + } + for packet in packets where videoTrack?.send(packet: packet) == true { + self.totalByteCount += Int64(packet.count) + } + } +} + +extension WhipStream: PeerConnectionDelegate { + fileprivate func peerConnectionOnConnectionStateChanged(state: ConnectionState) { + whipQueue.async { + self.handleConnectionStateChanged(state: state) + } + } +} + +extension WhipStream: AudioEncoderDelegate { + func audioEncoderOutputFormat(_: AVAudioFormat) {} + + func audioEncoderOutputBuffer(_ buffer: AVAudioCompressedBuffer, _ presentationTimeStamp: CMTime) { + whipQueue.async { + self.handleAudioEncoderOutputBuffer(buffer, presentationTimeStamp) + } + } +} + +extension WhipStream: VideoEncoderDelegate { + func videoEncoderOutputFormat(_: VideoEncoder, _ formatDescription: CMFormatDescription) { + whipQueue.async { + self.handleVideoEncoderOutputFormat(formatDescription) + } + } + + func videoEncoderOutputSampleBuffer(_: VideoEncoder, + _ sampleBuffer: CMSampleBuffer, + _ presentationTimeStamp: CMTime) + { + whipQueue.async { + self.handleVideoEncoderOutputSampleBuffer(sampleBuffer, presentationTimeStamp) + } + } +} diff --git a/Moblin/Various/Media.swift b/Moblin/Various/Media.swift index 756d0f37a..a23d20fce 100644 --- a/Moblin/Various/Media.swift +++ b/Moblin/Various/Media.swift @@ -23,6 +23,8 @@ protocol MediaDelegate: AnyObject { func mediaOnRtmpDestinationDisconnected(_ destination: String) func mediaOnRistConnected() func mediaOnRistDisconnected() + func mediaOnWhipConnected() + func mediaOnWhipDisconnected(_ reason: String) func mediaOnAudioMuteChange() func mediaOnAudioBuffer(_ sampleBuffer: CMSampleBuffer) func mediaOnLowFpsImage(_ lowFpsImage: Data?, _ frameNumber: UInt64) @@ -53,6 +55,7 @@ final class Media: NSObject { private var srtStreamNew: SrtStreamMoblin? private var srtStreamOld: SrtStreamOfficial? private var ristStream: RistStream? + private var whipStream: WhipStream? private var srtlaClient: SrtlaClient? private var processor: Processor? private var srtTotalByteCount: Int64 = 0 @@ -101,10 +104,12 @@ final class Media: NSObject { srtStopStream() rtmpStopStream() ristStopStream() + whipStopStream() rtmpStreams.removeAll() srtStreamNew = nil srtStreamOld = nil ristStream = nil + whipStream = nil processor = nil } @@ -120,6 +125,7 @@ final class Media: NSObject { srtStopStream() rtmpStopStream() ristStopStream() + whipStopStream() let processor = Processor() switch proto { case .rtmp: @@ -138,6 +144,7 @@ final class Media: NSObject { srtStreamNew = nil srtStreamOld = nil ristStream = nil + whipStream = nil case .srt: switch srtImplementation { case .moblin: @@ -157,11 +164,19 @@ final class Media: NSObject { } rtmpStreams.removeAll() ristStream = nil + whipStream = nil case .rist: ristStream = RistStream(processor: processor, timecodesEnabled: timecodesEnabled, delegate: self) srtStreamNew = nil srtStreamOld = nil rtmpStreams.removeAll() + whipStream = nil + case .whip: + whipStream = WhipStream(processor: processor, delegate: self) + srtStreamNew = nil + srtStreamOld = nil + ristStream = nil + rtmpStreams.removeAll() } self.processor = processor processor.setDelegate(delegate: self) @@ -503,6 +518,8 @@ final class Media: NSObject { return 8 * srtTransportBitrate } else if ristStream != nil { return Int64(ristStream?.getSpeed() ?? 0) + } else if whipStream != nil { + return 0 } else { return 0 } @@ -516,6 +533,9 @@ final class Media: NSObject { if isSrtStreamActive() { return srtTotalByteCount } + if let whipStream { + return whipStream.getTotalByteCount() + } return total } @@ -607,6 +627,15 @@ final class Media: NSObject { ristStream?.stop() } + func whipStartStream(url: String) { + adaptiveBitrate = nil + whipStream?.start(url: url, iceServers: ["stun:stun.l.google.com:19302"]) + } + + func whipStopStream() { + whipStream?.stop() + } + func setTorch(on: Bool) { processor?.setTorch(value: on) } @@ -1185,3 +1214,15 @@ extension Media: RtmpStreamDelegate { } } } + +extension Media: WhipStreamDelegate { + func whipStreamOnConnected() { + delegate?.mediaOnWhipConnected() + } + + func whipStreamOnDisconnected(reason: String) { + DispatchQueue.main.async { + self.delegate?.mediaOnWhipDisconnected(reason) + } + } +} diff --git a/Moblin/Various/Model/ModelStream.swift b/Moblin/Various/Model/ModelStream.swift index 9455250f6..4bbf1f56a 100644 --- a/Moblin/Various/Model/ModelStream.swift +++ b/Moblin/Various/Model/ModelStream.swift @@ -189,6 +189,8 @@ extension Model { startNetStreamSrt() case .rist: startNetStreamRist() + case .whip: + startNetStreamWhip() } updateSpeed(now: .now) streamBecameBrokenTime = nil @@ -234,12 +236,17 @@ extension Model { updateAdaptiveBitrateRistIfEnabled() } + private func startNetStreamWhip() { + media.whipStartStream(url: stream.url) + } + func stopNetStream() { moblink.streamer?.stopTunnels() reconnectTimer.stop() media.rtmpStopStream() media.srtStopStream() media.ristStopStream() + media.whipStopStream() streamStartTime = nil updateStreamUptime(now: .now) updateSpeed(now: .now) @@ -531,6 +538,18 @@ extension Model { } } + private func handleWhipConnected() { + DispatchQueue.main.async { + self.onConnected() + } + } + + private func handleWhipDisconnected(reason: String) { + DispatchQueue.main.async { + self.onDisconnected(reason: reason) + } + } + private func handleAudioBuffer(sampleBuffer: CMSampleBuffer) { DispatchQueue.main.async { self.speechToText?.append(sampleBuffer: sampleBuffer) @@ -878,6 +897,14 @@ extension Model: MediaDelegate { handleRistDisconnected() } + func mediaOnWhipConnected() { + handleWhipConnected() + } + + func mediaOnWhipDisconnected(_ reason: String) { + handleWhipDisconnected(reason: reason) + } + func mediaOnAudioMuteChange() { updateAudioLevel() } diff --git a/Moblin/Various/Settings/SettingsStream.swift b/Moblin/Various/Settings/SettingsStream.swift index 6c09e3517..c7e5e517e 100644 --- a/Moblin/Various/Settings/SettingsStream.swift +++ b/Moblin/Various/Settings/SettingsStream.swift @@ -151,6 +151,7 @@ enum SettingsStreamProtocol: String, Codable { case rtmp = "RTMP" case srt = "SRT" case rist = "RIST" + case whip = "WHIP" init(from decoder: Decoder) throws { self = try SettingsStreamProtocol(rawValue: decoder.singleValueContainer().decode(RawValue.self)) ?? @@ -164,6 +165,8 @@ enum SettingsStreamDetailedProtocol { case srt case srtla case rist + case whip + case whips } class SettingsStreamSrtConnectionPriority: Codable, Identifiable { @@ -1410,6 +1413,10 @@ class SettingsStream: Codable, Identifiable, Equatable, ObservableObject, Named return .srt case "rist": return .rist + case "whip": + return .whip + case "whips": + return .whip default: return .rtmp } @@ -1427,6 +1434,10 @@ class SettingsStream: Codable, Identifiable, Equatable, ObservableObject, Named return .srtla case "rist": return .rist + case "whip": + return .whip + case "whips": + return .whips default: return .rtmp } @@ -1437,6 +1448,8 @@ class SettingsStream: Codable, Identifiable, Equatable, ObservableObject, Named return "SRTLA" } else if getProtocol() == .rtmp && isRtmps() { return "RTMPS" + } else if getProtocol() == .whip { + return "WHIP" } else { return getProtocol().rawValue } diff --git a/Moblin/Various/Utils/Utils.swift b/Moblin/Various/Utils/Utils.swift index adee23c67..c999d3106 100644 --- a/Moblin/Various/Utils/Utils.swift +++ b/Moblin/Various/Utils/Utils.swift @@ -361,3 +361,22 @@ func clockAsMinutesAndSeconds(clock: String) -> (Int, Int) { return (0, 0) } } + +extension Array where Element == String { + func withCPointers(_ body: (UnsafeMutablePointer?>) -> T) -> T { + let pointersArray = UnsafeMutablePointer?>.allocate(capacity: count) + defer { + pointersArray.deallocate() + } + func addAt(index: Int) -> T { + if index == count { + return body(pointersArray) + } + return self[index].withCString { cstr in + pointersArray[index] = cstr + return addAt(index: index + 1) + } + } + return addAt(index: 0) + } +} diff --git a/Moblin/View/Settings/Streams/Stream/StreamSettingsView.swift b/Moblin/View/Settings/Streams/Stream/StreamSettingsView.swift index ff729a036..093792d2b 100644 --- a/Moblin/View/Settings/Streams/Stream/StreamSettingsView.swift +++ b/Moblin/View/Settings/Streams/Stream/StreamSettingsView.swift @@ -270,6 +270,8 @@ struct StreamSettingsView: View { } label: { Text("RIST") } + case .whip: + EmptyView() } } } header: { diff --git a/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift b/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift index c2af6c43c..b083dcd40 100644 --- a/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift +++ b/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift @@ -108,6 +108,20 @@ private struct SrtHelpView: View { } } +private struct WhipHelpView: View { + var body: some View { + Section { + VStack(alignment: .leading) { + Text("Template: whips://my_whip_server/whip/endpoint") + Text("Example: whips://whip.example.com/ingest/stream") + Text("Example: whip://whip.example.com/ingest/stream") + } + } header: { + Text("WHIP") + } + } +} + private struct UrlSettingsView: View { @EnvironmentObject var model: Model @Environment(\.dismiss) var dismiss @@ -115,6 +129,7 @@ private struct UrlSettingsView: View { @Binding var url: String let allowedSchemes: [String]? let showSrtHelp: Bool + let showWhipHelp: Bool @State var value: String @State var changed: Bool = false @State var submitted: Bool = false @@ -173,6 +188,9 @@ private struct UrlSettingsView: View { if showSrtHelp { SrtHelpView() } + if showWhipHelp { + WhipHelpView() + } } .navigationTitle("Help") .toolbar { @@ -199,6 +217,7 @@ struct StreamUrlSettingsView: View { url: $stream.url, allowedSchemes: nil, showSrtHelp: true, + showWhipHelp: true, value: stream.url) } } @@ -212,6 +231,7 @@ struct StreamMultiStreamingUrlView: View { url: $destination.url, allowedSchemes: ["rtmp", "rtmps"], showSrtHelp: false, + showWhipHelp: false, value: destination.url) } } diff --git a/MoblinTests/UtilsSuite.swift b/MoblinTests/UtilsSuite.swift index 98710ed20..08f16378d 100644 --- a/MoblinTests/UtilsSuite.swift +++ b/MoblinTests/UtilsSuite.swift @@ -59,4 +59,23 @@ struct UtilsSuite { let result = try #require(UUID(uuidString: "00010000-0000-0000-0000-FAFBFD204366")) #expect(original.add(data: extra) == result) } + + @Test + func arrayWithCPointers() { + let data = ["1", "22", "333"] + data.withCPointers { array in + let first = array[0]! + #expect(first[0] == 0x31) + #expect(first[1] == 0x0) + let second = array[1]! + #expect(second[0] == 0x32) + #expect(second[1] == 0x32) + #expect(second[2] == 0x0) + let third = array[2]! + #expect(third[0] == 0x33) + #expect(third[1] == 0x33) + #expect(third[2] == 0x33) + #expect(third[3] == 0x0) + } + } } diff --git a/MoblinTests/ValidateSuite.swift b/MoblinTests/ValidateSuite.swift new file mode 100644 index 000000000..bbcd54415 --- /dev/null +++ b/MoblinTests/ValidateSuite.swift @@ -0,0 +1,11 @@ +import Foundation +@testable import Moblin +import Testing + +struct ValidateSuite { + @Test + func whipUrlValidation() { + #expect(isValidUrl(url: "whips://whip.example.com/live/123") == nil) + #expect(isValidUrl(url: "whip://whip.example.com/live/123") == nil) + } +} From 862201e2b061dcd3bdd6d4df14c4e9b2b46f461d Mon Sep 17 00:00:00 2001 From: Erik Moqvist Date: Wed, 11 Feb 2026 20:06:58 +0100 Subject: [PATCH 2/2] Refactoring. --- Common/Localizable.xcstrings | 12 ------------ Moblin/Various/Settings/SettingsStream.swift | 2 -- .../Stream/Url/StreamUrlSettingsView.swift | 17 ----------------- 3 files changed, 31 deletions(-) diff --git a/Common/Localizable.xcstrings b/Common/Localizable.xcstrings index e5b993c23..a3362aaf9 100644 --- a/Common/Localizable.xcstrings +++ b/Common/Localizable.xcstrings @@ -76585,12 +76585,6 @@ } } } - }, - "Example: whip://whip.example.com/ingest/stream" : { - - }, - "Example: whips://whip.example.com/ingest/stream" : { - }, "EXB" : { "localizations" : { @@ -196410,9 +196404,6 @@ } } } - }, - "Template: whips://my_whip_server/whip/endpoint" : { - }, "Tennis" : { "localizations" : { @@ -223332,9 +223323,6 @@ }, "When \"Audio only\" mode is selected, no video will be rendered at all. Only audio will play." : { - }, - "WHIP" : { - }, "Whirlpool" : { "localizations" : { diff --git a/Moblin/Various/Settings/SettingsStream.swift b/Moblin/Various/Settings/SettingsStream.swift index c7e5e517e..ddf3db591 100644 --- a/Moblin/Various/Settings/SettingsStream.swift +++ b/Moblin/Various/Settings/SettingsStream.swift @@ -1448,8 +1448,6 @@ class SettingsStream: Codable, Identifiable, Equatable, ObservableObject, Named return "SRTLA" } else if getProtocol() == .rtmp && isRtmps() { return "RTMPS" - } else if getProtocol() == .whip { - return "WHIP" } else { return getProtocol().rawValue } diff --git a/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift b/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift index b083dcd40..d56acb487 100644 --- a/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift +++ b/Moblin/View/Settings/Streams/Stream/Url/StreamUrlSettingsView.swift @@ -108,20 +108,6 @@ private struct SrtHelpView: View { } } -private struct WhipHelpView: View { - var body: some View { - Section { - VStack(alignment: .leading) { - Text("Template: whips://my_whip_server/whip/endpoint") - Text("Example: whips://whip.example.com/ingest/stream") - Text("Example: whip://whip.example.com/ingest/stream") - } - } header: { - Text("WHIP") - } - } -} - private struct UrlSettingsView: View { @EnvironmentObject var model: Model @Environment(\.dismiss) var dismiss @@ -188,9 +174,6 @@ private struct UrlSettingsView: View { if showSrtHelp { SrtHelpView() } - if showWhipHelp { - WhipHelpView() - } } .navigationTitle("Help") .toolbar {