fix: 修复录制视频无音频问题

- audioInput 从 PCM pass-through 改为 AAC 编码 (MP4 不支持裸 Float32 PCM)
- IOProc 优先读 inOutputData (播放数据) 而非 inInputData (麦克风)
- 音频时间戳改为相对时间 (从首次回调开始计算), 与视频同步
- 复用 CMAudioFormatDescription 避免重复创建
This commit is contained in:
yumoqing 2026-06-27 13:55:49 +08:00
parent f1ecbf2e91
commit 1733ecc7bc

View File

@ -17,18 +17,17 @@ class AudioCaptureContext {
nonisolated(unsafe) var appendCount: Int = 0
nonisolated(unsafe) var firstAppendLogged: Bool = false
nonisolated(unsafe) var processCallCount: Int = 0
nonisolated(unsafe) var firstAudioSampleTime: Int64 = -1
/// Serial queue for processing audio buffers off the real-time thread
let audioQueue = DispatchQueue(label: "miniplayer.audioCapture")
/// Pre-allocated data buffer (avoids malloc in real-time audio thread)
var dataBuffer: UnsafeMutableRawPointer?
var dataBufferSize: UInt32 = 0
}
// MARK: - CoreAudio IOProc
/// PCM HLS
/// 线
/// PCM
/// inOutputData inInputData
private func audioDeviceIOProc(
_ inDevice: AudioDeviceID,
_ inNow: UnsafePointer<AudioTimeStamp>,
@ -40,19 +39,51 @@ private func audioDeviceIOProc(
) -> OSStatus {
guard let clientData = inClientData else { return noErr }
let ctx = Unmanaged<AudioCaptureContext>.fromOpaque(clientData).takeUnretainedValue()
guard ctx.isRunning,
let writerInput = ctx.writerInput,
writerInput.isReadyForMoreMediaData else { return noErr }
guard ctx.isRunning else { return noErr }
ctx.processCallCount += 1
let bufferList = inInputData.pointee
guard bufferList.mNumberBuffers > 0 else { return noErr }
// inOutputData
let outBufferList = inOutputData.pointee
let inBufferList = inInputData.pointee
let callNum = ctx.processCallCount
if callNum <= 3 {
print("[Recorder] IOProc#\(callNum): outBufs=\(outBufferList.mNumberBuffers), inBufs=\(inBufferList.mNumberBuffers)")
if outBufferList.mNumberBuffers > 0 {
let b = outBufferList.mBuffers
print("[Recorder] outBuf[0]: size=\(b.mDataByteSize), channels=\(b.mNumberChannels), data=\(b.mData != nil)")
}
if inBufferList.mNumberBuffers > 0 {
let b = inBufferList.mBuffers
print("[Recorder] inBuf[0]: size=\(b.mDataByteSize), channels=\(b.mNumberChannels), data=\(b.mData != nil)")
}
}
// inOutputData loopback
let bufferList: AudioBufferList
if outBufferList.mNumberBuffers > 0 {
let b = outBufferList.mBuffers
if b.mDataByteSize > 0 && b.mData != nil {
bufferList = outBufferList
} else if inBufferList.mNumberBuffers > 0 {
bufferList = inBufferList
} else {
return noErr
}
} else if inBufferList.mNumberBuffers > 0 {
bufferList = inBufferList
} else {
return noErr
}
let buffer = bufferList.mBuffers
let dataSize = buffer.mDataByteSize
guard dataSize > 0, let srcData = buffer.mData else { return noErr }
guard dataSize > 0, let srcData = buffer.mData else {
if callNum <= 3 { print("[Recorder] IOProc#\(callNum): no data (size=\(buffer.mDataByteSize), data=\(buffer.mData != nil))") }
return noErr
}
// Copy to pre-allocated buffer (avoid malloc in real-time thread)
// buffer
if ctx.dataBufferSize < dataSize {
if let existing = ctx.dataBuffer { free(existing) }
ctx.dataBuffer = malloc(Int(dataSize))
@ -61,12 +92,12 @@ private func audioDeviceIOProc(
guard let dest = ctx.dataBuffer else { return noErr }
memcpy(dest, srcData, Int(dataSize))
let pts = inInputTime.pointee
let callNum = ctx.processCallCount
let pts = inOutputTime.pointee
// Process on serial queue (off the real-time audio thread)
//
ctx.audioQueue.async {
guard ctx.isRunning, let fd = ctx.formatDescription else { return }
guard ctx.isRunning else { return }
guard let fd = ctx.formatDescription, let writerInput = ctx.writerInput else { return }
var blockBuffer: CMBlockBuffer?
let blockStatus = CMBlockBufferCreateWithMemoryBlock(
@ -77,7 +108,10 @@ private func audioDeviceIOProc(
flags: kCMBlockBufferAssureMemoryNowFlag,
blockBufferOut: &blockBuffer
)
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else { return }
guard blockStatus == kCMBlockBufferNoErr, let bb = blockBuffer else {
if callNum <= 3 { print("[Recorder] CMBlockBufferCreate failed: \(blockStatus)") }
return
}
let replaceStatus = CMBlockBufferReplaceDataBytes(
with: dest, blockBuffer: bb,
@ -86,12 +120,15 @@ private func audioDeviceIOProc(
guard replaceStatus == kCMBlockBufferNoErr else { return }
var timingInfo = CMSampleTimingInfo()
// Convert AudioTimeStamp to CMTime
if pts.mFlags.contains(.sampleTimeValid) {
let sampleRate = ctx.sampleRate
// 使
if ctx.firstAudioSampleTime < 0 {
ctx.firstAudioSampleTime = Int64(pts.mSampleTime)
}
let relativeSampleTime = Int64(pts.mSampleTime) - ctx.firstAudioSampleTime
timingInfo.presentationTimeStamp = CMTime(
value: Int64(pts.mSampleTime),
timescale: Int32(sampleRate)
value: relativeSampleTime,
timescale: Int32(ctx.sampleRate)
)
} else if pts.mFlags.contains(.hostTimeValid) {
var timebase = mach_timebase_info_data_t()
@ -104,9 +141,10 @@ private func audioDeviceIOProc(
timingInfo.duration = .invalid
timingInfo.decodeTimeStamp = .invalid
var sampleBuffer: CMSampleBuffer?
let bytesPerFrame = Int(ctx.channelsPerFrame) * MemoryLayout<Float32>.size
let sampleCount = Int(dataSize) / max(bytesPerFrame, 1)
var sampleBuffer: CMSampleBuffer?
let createStatus = CMSampleBufferCreateReady(
allocator: kCFAllocatorDefault,
dataBuffer: bb,
@ -120,17 +158,24 @@ private func audioDeviceIOProc(
)
guard createStatus == noErr, let sb = sampleBuffer else {
if callNum <= 3 {
NSLog("[Recorder] \u{26a0}\u{fe0f} Audio sample buffer create failed: %d", createStatus)
print("[Recorder] ⚠️ SampleBuffer create failed: \(createStatus), dataSize=\(dataSize), sampleCount=\(sampleCount)")
}
return
}
if writerInput.isReadyForMoreMediaData && writerInput.append(sb) {
ctx.appendCount += 1
if !ctx.firstAppendLogged {
ctx.firstAppendLogged = true
NSLog("[Recorder] ✓ First audio captured: pts=%.2fs, size=%u", pts.mSampleTime / ctx.sampleRate, dataSize)
if writerInput.isReadyForMoreMediaData {
let appended = writerInput.append(sb)
if appended {
ctx.appendCount += 1
if !ctx.firstAppendLogged {
ctx.firstAppendLogged = true
print("[Recorder] ✓ First audio captured: pts=\(timingInfo.presentationTimeStamp.seconds)s, size=\(dataSize)")
}
} else if callNum <= 3 {
print("[Recorder] ⚠️ writerInput.append returned false")
}
} else if callNum <= 3 {
print("[Recorder] ⚠️ writerInput not ready for more data")
}
}
@ -146,7 +191,6 @@ enum RecorderState {
}
/// AVPlayerItemVideoOutput + CoreAudio
///
@MainActor
final class PlayerRecorder: NSObject {
@ -159,18 +203,15 @@ final class PlayerRecorder: NSObject {
private weak var weakOutput: AVPlayerItemVideoOutput?
nonisolated(unsafe) private var isRunning = false
//
private var lastPixelBuffer: CVPixelBuffer?
private var captureFrameCount: Int = 0
// (CoreAudio )
// (CoreAudio)
private var audioDeviceID: AudioDeviceID = 0
private var audioCaptureContext: AudioCaptureContext?
private var audioIOProc: AudioDeviceIOProc?
private weak var currentPlayerItem: AVPlayerItem?
private var tracksObservation: NSKeyValueObservation?
//
nonisolated(unsafe) private var recordStartTime: CMTime = .zero
@Published var isRecording = false
@ -179,10 +220,8 @@ final class PlayerRecorder: NSObject {
var onRecordingSaved: ((URL) -> Void)?
var onError: ((String) -> Void)?
///
private var tempURL: URL?
///
private var saveDirectory: URL {
let movies = FileManager.default.urls(for: .moviesDirectory, in: .userDomainMask).first!
let dir = movies.appendingPathComponent("MiniPlayer", isDirectory: true)
@ -202,10 +241,8 @@ final class PlayerRecorder: NSObject {
isRunning = false
recordStartTime = startTime
//
let tempFilename = "temp_recording_\(Int(Date().timeIntervalSince1970)).mp4"
let url = saveDirectory.appendingPathComponent(tempFilename)
//
try? FileManager.default.removeItem(at: url)
tempURL = url
@ -231,19 +268,7 @@ final class PlayerRecorder: NSObject {
videoInput = vInput
}
// AAC
let audioSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: 44100,
AVNumberOfChannelsKey: 2,
AVEncoderBitRateKey: 128000
]
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings)
aInput.expectsMediaDataInRealTime = true
if writer.canAdd(aInput) {
writer.add(aInput)
audioInput = aInput
}
// startAudioCapture AAC
guard writer.startWriting() else {
NSLog("[Recorder] startWriting failed: %@", writer.error?.localizedDescription ?? "unknown")
@ -251,18 +276,16 @@ final class PlayerRecorder: NSObject {
return
}
// session 0
writer.startSession(atSourceTime: .zero)
NSLog("[Recorder] Writer started, writing to: %@", url.lastPathComponent)
// 30fps
isRecording = true
isRunning = true
startCaptureLoop()
startDurationTimer()
// (CoreAudio)
startAudioCapture(audioWriterInput: aInput)
// (CoreAudio audioInput)
startAudioCapture()
}
// MARK: -
@ -271,12 +294,11 @@ final class PlayerRecorder: NSObject {
guard isRecording else { return }
isRecording = false
isRunning = false
RecorderState.isFinalizing = true // 退 finishWriting
RecorderState.isFinalizing = true
captureTimer?.invalidate()
captureTimer = nil
stopDurationTimer()
//
let audioCalls = audioCaptureContext?.processCallCount ?? 0
let audioAppended = audioCaptureContext?.appendCount ?? 0
NSLog("[Recorder] Stopping: videoFrames=%d, audioProcessCalls=%d, audioAppended=%d",
@ -290,7 +312,7 @@ final class PlayerRecorder: NSObject {
writer?.finishWriting { [weak self] in
NSLog("[Recorder] finishWriting callback invoked")
DispatchQueue.main.async {
RecorderState.isFinalizing = false // 退
RecorderState.isFinalizing = false
guard let self = self, let w = self.writer else {
NSLog("[Recorder] ⚠️ self or writer is nil in finishWriting callback")
@ -307,7 +329,6 @@ final class PlayerRecorder: NSObject {
vFrames, audioAppended)
if w.status == .completed, let tempURL = self.tempURL {
//
if FileManager.default.fileExists(atPath: tempURL.path) {
let size = (try? FileManager.default.attributesOfItem(atPath: tempURL.path)[.size] as? UInt64) ?? 0
NSLog("[Recorder] ✓ Temp file exists: %@, size=%llu bytes", tempURL.lastPathComponent, size)
@ -334,9 +355,9 @@ final class PlayerRecorder: NSObject {
// MARK: - (CoreAudio )
/// HLS
private func startAudioCapture(audioWriterInput: AVAssetWriterInput) {
//
private func startAudioCapture() {
print("[Recorder] startAudioCapture called")
var deviceID: AudioDeviceID = 0
var size = UInt32(MemoryLayout<AudioDeviceID>.size)
var address = AudioObjectPropertyAddress(
@ -347,8 +368,9 @@ final class PlayerRecorder: NSObject {
let status = AudioObjectGetPropertyData(
AudioObjectID(kAudioObjectSystemObject), &address, 0, nil, &size, &deviceID
)
print("[Recorder] Get default device status: \(status), deviceID: \(deviceID)")
guard status == noErr, deviceID != 0 else {
NSLog("[Recorder] \u{274c} Cannot get default audio output device: %d", status)
print("[Recorder] ❌ Cannot get default audio output device")
return
}
self.audioDeviceID = deviceID
@ -362,23 +384,14 @@ final class PlayerRecorder: NSObject {
var asbd = AudioStreamBasicDescription()
size = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
let fmtStatus = AudioObjectGetPropertyData(deviceID, &streamFormatAddress, 0, nil, &size, &asbd)
print("[Recorder] Stream format: status=\(fmtStatus), rate=\(asbd.mSampleRate), ch=\(asbd.mChannelsPerFrame), bits=\(asbd.mBitsPerChannel), formatID=\(asbd.mFormatID)")
guard fmtStatus == noErr else {
NSLog("[Recorder] \u{274c} Cannot get audio stream format: %d", fmtStatus)
print("[Recorder] ❌ Cannot get audio stream format")
return
}
NSLog("[Recorder] Audio output format: %.1fHz, %uch, %ubit, formatID=%u",
asbd.mSampleRate, asbd.mChannelsPerFrame, asbd.mBitsPerChannel, asbd.mFormatID)
// context
let context = AudioCaptureContext()
context.writerInput = audioWriterInput
context.sampleRate = Double(asbd.mSampleRate)
context.channelsPerFrame = asbd.mChannelsPerFrame
context.isRunning = true
self.audioCaptureContext = context
//
// AAC MP4 PCM pass-through
// CMAudioFormatDescription
var fmtDesc: CMAudioFormatDescription?
var mutableASBD = asbd
CMAudioFormatDescriptionCreate(
@ -389,27 +402,60 @@ final class PlayerRecorder: NSObject {
extensions: nil,
formatDescriptionOut: &fmtDesc
)
context.formatDescription = fmtDesc
guard let sourceFmtDesc = fmtDesc else {
print("[Recorder] ❌ Cannot create CMAudioFormatDescription")
return
}
// IOProc使 unmanaged pointer context
let audioSettings: [String: Any] = [
AVFormatIDKey: kAudioFormatMPEG4AAC,
AVSampleRateKey: asbd.mSampleRate,
AVNumberOfChannelsKey: asbd.mChannelsPerFrame,
AVEncoderBitRateKey: 128_000
]
let aInput = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings, sourceFormatHint: sourceFmtDesc)
aInput.expectsMediaDataInRealTime = true
if let w = writer, w.canAdd(aInput) {
w.add(aInput)
audioInput = aInput
print("[Recorder] ✓ AudioInput created with AAC encoding (rate=\(asbd.mSampleRate), ch=\(asbd.mChannelsPerFrame))")
} else {
print("[Recorder] ❌ Cannot add audio input to writer")
return
}
// context
let context = AudioCaptureContext()
context.writerInput = aInput
context.sampleRate = Double(asbd.mSampleRate)
context.channelsPerFrame = asbd.mChannelsPerFrame
context.isRunning = true
self.audioCaptureContext = context
// 使
context.formatDescription = sourceFmtDesc
print("[Recorder] formatDescription created: true")
// IOProc
let contextPtr = Unmanaged.passUnretained(context).toOpaque()
var ioProc: AudioDeviceIOProc?
let createStatus = AudioDeviceCreateIOProcID(deviceID, audioDeviceIOProc, contextPtr, &ioProc)
print("[Recorder] CreateIOProcID status: \(createStatus)")
guard createStatus == noErr, let proc = ioProc else {
NSLog("[Recorder] \u{274c} AudioDeviceCreateIOProcID failed: %d", createStatus)
print("[Recorder] ❌ AudioDeviceCreateIOProcID failed: \(createStatus)")
return
}
self.audioIOProc = proc
//
let startStatus = AudioDeviceStart(deviceID, proc)
print("[Recorder] AudioDeviceStart status: \(startStatus)")
guard startStatus == noErr else {
NSLog("[Recorder] \u{274c} AudioDeviceStart failed: %d", startStatus)
print("[Recorder] ❌ AudioDeviceStart failed: \(startStatus)")
AudioDeviceDestroyIOProcID(deviceID, proc)
self.audioIOProc = nil
return
}
NSLog("[Recorder] \u{2713} CoreAudio output device capture started (deviceID=%u)", deviceID)
NSLog("[Recorder] CoreAudio output device capture started (deviceID=%u)", deviceID)
}
private func stopAudioCapture() {
@ -421,7 +467,6 @@ final class PlayerRecorder: NSObject {
audioIOProc = nil
audioCaptureContext = nil
}
// MARK: - (30fps Timer)
@ -440,19 +485,15 @@ final class PlayerRecorder: NSObject {
let hostTime = CACurrentMediaTime()
let itemTime = output.itemTime(forHostTime: hostTime)
//
if let pb = output.copyPixelBuffer(forItemTime: itemTime, itemTimeForDisplay: nil) {
lastPixelBuffer = pb
}
// 使
guard let pb = lastPixelBuffer else { return }
//
let relativeTime = CMTimeSubtract(itemTime, recordStartTime)
guard relativeTime.seconds >= 0 else { return }
// sample buffer
if let sb = Self.createSampleBuffer(from: pb, time: relativeTime) {
if input.append(sb) {
captureFrameCount += 1
@ -483,26 +524,17 @@ final class PlayerRecorder: NSObject {
sampleTiming: &info,
sampleBufferOut: &sampleBuffer
)
return sampleBuffer
}
// MARK: -
// MARK: -
private func startDurationTimer() {
startDate = Date()
durationText = "00:00"
durationTimer = Timer.scheduledTimer(withTimeInterval: 1.0, repeats: true) { [weak self] _ in
Task { @MainActor in
guard let self = self, let start = self.startDate else { return }
let elapsed = Int(Date().timeIntervalSince(start))
let m = elapsed / 60
let s = elapsed % 60
let h = m / 60
if h > 0 {
self.durationText = String(format: "%d:%02d:%02d", h, m % 60, s)
} else {
self.durationText = String(format: "%02d:%02d", m, s)
}
self?.updateDuration()
}
}
}
@ -510,69 +542,49 @@ final class PlayerRecorder: NSObject {
private func stopDurationTimer() {
durationTimer?.invalidate()
durationTimer = nil
startDate = nil
durationText = "00:00"
}
// MARK: -
private func generateFinalFilename() -> String {
let formatter = DateFormatter()
formatter.dateFormat = "yyyy-MM-dd_HH-mm-ss"
return "Recording_\(formatter.string(from: Date())).mp4"
private func updateDuration() {
guard let start = startDate else { return }
let elapsed = Int(Date().timeIntervalSince(start))
let min = elapsed / 60
let sec = elapsed % 60
durationText = String(format: "%02d:%02d", min, sec)
}
// MARK: -
private func showSaveDialog(tempURL: URL) {
let panel = NSSavePanel()
panel.nameFieldStringValue = generateFinalFilename()
panel.allowedContentTypes = [.mpeg4Movie]
panel.nameFieldStringValue = tempURL.lastPathComponent
panel.directoryURL = saveDirectory
panel.title = "保存录制文件"
panel.prompt = "保存"
panel.begin { [weak self] response in
Task { @MainActor in
guard let self = self else {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSApp.reply(toApplicationShouldTerminate: true)
if response == .OK, let url = panel.url {
do {
if FileManager.default.fileExists(atPath: url.path) {
try FileManager.default.removeItem(at: url)
}
return
try FileManager.default.moveItem(at: tempURL, to: url)
NSLog("[Recorder] ✓ Saved: %@", url.path)
self?.onRecordingSaved?(url)
} catch {
NSLog("[Recorder] ✗ Save failed: %@", error.localizedDescription)
self?.onError?("Save failed: \(error.localizedDescription)")
}
if response == .OK, let finalURL = panel.url {
do {
//
if FileManager.default.fileExists(atPath: finalURL.path) {
try FileManager.default.removeItem(at: finalURL)
}
//
try FileManager.default.moveItem(at: tempURL, to: finalURL)
NSLog("[Recorder] ✓ Saved: %@", finalURL.path)
self.onRecordingSaved?(finalURL)
} catch {
self.onError?("Save failed: \(error.localizedDescription)")
try? FileManager.default.removeItem(at: tempURL)
}
} else {
//
NSLog("[Recorder] User cancelled save, deleting temp file")
try? FileManager.default.removeItem(at: tempURL)
}
self.checkPendingTerminate()
} else {
try? FileManager.default.removeItem(at: tempURL)
}
self?.checkPendingTerminate()
}
}
private func checkPendingTerminate() {
if RecorderState.pendingTerminate {
RecorderState.pendingTerminate = false
NSLog("[MiniPlayer] Pending terminate triggered, replying to terminate")
NSApp.reply(toApplicationShouldTerminate: true)
}
}
}
#endif