From 6c31894a6df7a16823a3c9bd7901a3fe4559cd13 Mon Sep 17 00:00:00 2001 From: yumoqing Date: Wed, 24 Jun 2026 20:28:31 +0800 Subject: [PATCH] Fix: record audio from source URL, not microphone --- Sources/PlayerBridge.swift | 5 +- Sources/PlayerRecorder.swift | 192 ++++++++++++++++++++++++++++++----- 2 files changed, 168 insertions(+), 29 deletions(-) diff --git a/Sources/PlayerBridge.swift b/Sources/PlayerBridge.swift index 2397d55..274f31b 100644 --- a/Sources/PlayerBridge.swift +++ b/Sources/PlayerBridge.swift @@ -120,7 +120,10 @@ final class PlayerBridge: ObservableObject { showToast("No video playing") return } - screenRecorder.startRecording(from: output) + guard !queue.isEmpty else { return } + let sourceURL = queue[currentIndex].url + let startTime = player.currentTime() + screenRecorder.startRecording(from: output, sourceURL: sourceURL, startTime: startTime) } #endif } diff --git a/Sources/PlayerRecorder.swift b/Sources/PlayerRecorder.swift index 3738ba0..bb553e4 100644 --- a/Sources/PlayerRecorder.swift +++ b/Sources/PlayerRecorder.swift @@ -2,7 +2,7 @@ import AVFoundation #if os(macOS) import AppKit -/// 视频源录制器：从 AVPlayerItemVideoOutput 抓帧，用 AVAssetWriter 写入文件 +/// 视频源录制器：从 AVPlayerItemVideoOutput 抓帧 + 源 URL 音频轨道，写入 mp4 @MainActor final class PlayerRecorder: NSObject { @@ -11,8 +11,12 @@ final class PlayerRecorder: NSObject { private var audioInput: AVAssetWriterInput? private var timer: Timer? private var startDate: Date? - private var displayLink: CVDisplayLink? private var weakOutput: AVPlayerItemVideoOutput? + private var firstFrameTime: CMTime? + private var audioReader: AVAssetReader? + private var audioReaderOutput: AVAssetReaderTrackOutput? + private var audioCaptureQueue: DispatchQueue? + private var isRunning = false @Published var isRecording = false @Published var durationText = "00:00" @@ -29,13 +33,15 @@ final class PlayerRecorder: NSObject { return dir } - /// 开始录制（从 videoOutput 抓帧） - func startRecording(from output: AVPlayerItemVideoOutput) { + /// 开始录制 + func startRecording(from output: AVPlayerItemVideoOutput, sourceURL: URL, startTime: CMTime) { guard !isRecording else { return } weakOutput = output + firstFrameTime = nil + isRunning = false - // 创建临时文件 + // 临时文件 let formatter = DateFormatter() formatter.dateFormat = "yyyy-MM-dd_HH-mm-ss" let filename = "Recording_\(formatter.string(from: Date())).mp4" @@ -49,6 +55,8 @@ final class PlayerRecorder: NSObject { return } + guard let writer = writer else { return } + // 视频输入 let videoSettings: [String: Any] = [ AVVideoCodecKey: AVVideoCodecType.h264, @@ -57,25 +65,43 @@ final class PlayerRecorder: NSObject { ] let vInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings) vInput.expectsMediaDataInRealTime = true - if writer?.canAdd(vInput) == true { - writer?.add(vInput) + if writer.canAdd(vInput) { + writer.add(vInput) videoInput = vInput } - writer?.startWriting() - writer?.startSession(atSourceTime: .zero) + // 音频输入 + let audioSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVSampleRateKey: 44100, + AVNumberOfChannelsKey: 2, + AVEncoderBitRateKey: 128000 + ] + let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings) + aInput.expectsMediaDataInRealTime = true + if writer.canAdd(aInput) { + writer.add(aInput) + audioInput = aInput + } + writer.startWriting() + + // 启动视频抓帧 isRecording = true startTimer() startCaptureLoop() + + // 启动音频读取（从源 URL） + startAudioCapture(sourceURL: sourceURL, startTime: startTime, audioWriterInput: aInput) } /// 停止录制 func stopRecording() { guard isRecording else { return } isRecording = false + isRunning = false stopTimer() - stopCaptureLoop() + stopAudioCapture() videoInput?.markAsFinished() audioInput?.markAsFinished() @@ -90,30 +116,51 @@ final class PlayerRecorder: NSObject { } } - // MARK: - 抓帧循环 + // MARK: - 视频抓帧 private func startCaptureLoop() { - let queue = DispatchQueue(label: "recorder.capture") - videoInput?.requestMediaDataWhenReady(on: queue) { [weak self] in - self?.captureFrame() - } - } - - private func captureFrame() { - guard let output = weakOutput, let input = videoInput, input.isReadyForMoreMediaData else { return } - - let time = output.itemTime(forHostTime: CACurrentMediaTime()) - if output.hasNewPixelBuffer(forItemTime: time), let pb = output.copyPixelBuffer(forItemTime: time, itemTimeForDisplay: nil) { - let sampleBuffer = createSampleBuffer(from: pb, time: time) - if let sb = sampleBuffer { - input.append(sb) + guard let vInput = videoInput else { return } + let queue = DispatchQueue(label: "recorder.video.capture") + vInput.requestMediaDataWhenReady(on: queue) { [weak self] in + guard let self = self, self.isRunning else { return } + Task { @MainActor in + self.captureVideoFrame() } } } - private func createSampleBuffer(from pixelBuffer: CVPixelBuffer, time: CMTime) -> CMSampleBuffer? { + private func captureVideoFrame() { + guard let output = weakOutput, let input = videoInput, input.isReadyForMoreMediaData else { return } + + let hostTime = CACurrentMediaTime() + let itemTime = output.itemTime(forHostTime: hostTime) + + guard output.hasNewPixelBuffer(forItemTime: itemTime), + let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) else { return } + + // 第一帧：用它的时间作为 session 起点 + if firstFrameTime == nil { + firstFrameTime = itemTime + writer?.startSession(atSourceTime: itemTime) + isRunning = true + } + + // 相对时间戳 + guard let firstTime = firstFrameTime else { return } + let relativeTime = CMTimeSubtract(itemTime, firstTime) + + guard relativeTime.seconds >= 0 else { return } + + let sampleBuffer = Self.createSampleBuffer(from: pb, time: relativeTime) + if let sb = sampleBuffer { + _ = input.append(sb) + } + } + + private static func createSampleBuffer(from pixelBuffer: CVPixelBuffer, time: CMTime) -> CMSampleBuffer? { var info = CMSampleTimingInfo() info.presentationTimeStamp = time info.duration = CMTime(value: 1, timescale: 30) + info.decodeTimeStamp = .invalid var formatDescription: CMFormatDescription? CMVideoFormatDescriptionCreateForImageBuffer(allocator: kCFAllocatorDefault, imageBuffer: pixelBuffer, formatDescriptionOut: &formatDescription) @@ -125,8 +172,97 @@ final class PlayerRecorder: NSObject { return sampleBuffer } - private func stopCaptureLoop() { - // requestMediaDataWhenReady 会在 isReadyForMoreMediaData=false 时停止 + // MARK: - 音频读取（从源 URL） + private func startAudioCapture(sourceURL: URL, startTime: CMTime, audioWriterInput: AVAssetWriterInput) { + let asset = AVURLAsset(url: sourceURL) + + Task { + do { + let audioTracks = try await asset.loadTracks(withMediaType: .audio) + guard let audioTrack = audioTracks.first else { + print("[Recorder] No audio track in source") + return + } + + let reader = try AVAssetReader(asset: asset) + let output = AVAssetReaderTrackOutput(track: audioTrack, outputSettings: [ + AVFormatIDKey: kAudioFormatLinearPCM, + AVSampleRateKey: 44100, + AVNumberOfChannelsKey: 2, + AVLinearPCMBitDepthKey: 16, + AVLinearPCMIsFloatKey: false, + AVLinearPCMIsBigEndianKey: false, + AVLinearPCMIsNonInterleaved: false + ]) + output.alwaysCopiesSampleData = false + + // 从播放位置开始读取 + reader.timeRange = CMTimeRange(start: startTime, duration: .positiveInfinity) + + if reader.canAdd(output) { + reader.add(output) + reader.startReading() + + self.audioReader = reader + self.audioReaderOutput = output + + // 后台线程读取音频 + let queue = DispatchQueue(label: "recorder.audio.capture") + self.audioCaptureQueue = queue + queue.async { [weak self] in + self?.readAudioLoop(reader: reader, output: output) + } + } + } catch { + print("[Recorder] Audio capture setup failed: \(error)") + } + } + } + + private nonisolated func readAudioLoop(reader: AVAssetReader, output: AVAssetReaderTrackOutput) { + while reader.status == .reading { + guard let sampleBuffer = output.copyNextSampleBuffer() else { + break + } + + // 在 MainActor 上调整时间戳并写入 + Task { @MainActor [weak self] in + guard let self = self, self.isRunning else { return } + guard let firstTime = self.firstFrameTime else { return } + guard let input = self.audioInput, input.isReadyForMoreMediaData else { return } + + let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + let relativeTime = CMTimeSubtract(pts, firstTime) + guard relativeTime.seconds >= 0 else { return } + + // 重建 sample buffer with new time + var count: CMItemCount = 0 + CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: 0, arrayToFill: nil, entriesNeededOut: &count) + var timingInfo = [CMSampleTimingInfo](repeating: CMSampleTimingInfo(), count: count) + CMSampleBufferGetSampleTimingInfoArray(sampleBuffer, entryCount: count, arrayToFill: &timingInfo, entriesNeededOut: nil) + + for i in 0..