312321
This commit is contained in:
zpc
parent
ff3eb1db34
commit
08462dad0b
|
|
@ -589,7 +589,14 @@ namespace ShengShengBuXi.Hubs
|
|||
foreach (var clientId in monitoringClients)
|
||||
{
|
||||
// 添加音频数据发送任务，使用单独的客户端连接
|
||||
tasks.Add(Clients.Client(clientId).SendAsync("ReceiveAudioData", dataToSend));
|
||||
// 使用清晰的对象格式，并添加WAV头以确保格式正确
|
||||
var wavData = _audioProcessingService.AddWavHeader(dataToSend, config.SampleRate, config.Channels);
|
||||
tasks.Add(Clients.Client(clientId).SendAsync("ReceiveAudioData", new {
|
||||
format = "WAV",
|
||||
sampleRate = config.SampleRate,
|
||||
channels = config.Channels,
|
||||
data = wavData
|
||||
}));
|
||||
}
|
||||
|
||||
// 只在日志级别为Debug时输出详细信息
|
||||
|
|
|
|||
|
|
@ -115,12 +115,12 @@
|
|||
<div class="card h-100">
|
||||
<div class="card-header">
|
||||
<h5 class="mb-0">
|
||||
文本编辑 <span style="color:#6c757d;font-size:12px;">(最多输入30个文字)</span>
|
||||
文本编辑 <span style="color:#6c757d;font-size:12px;">(最多输入100个文字)</span>
|
||||
</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<textarea id="text-input" class="form-control h-100" placeholder="请输入要显示的文本..."
|
||||
maxlength="30"></textarea>
|
||||
maxlength="100"></textarea>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -269,22 +269,6 @@
|
|||
let currentVolume = 1.0; // 默认音量
|
||||
let volumeBoost = 3.0; // 音量增益倍数，提高接收到的音频音量
|
||||
|
||||
// 添加音频缓冲区变量
|
||||
let audioBufferQueue = [];
|
||||
const MAX_BUFFER_SIZE = 400; // 增加缓冲队列大小，提供更充足的数据缓冲
|
||||
let isAudioPlaying = false;
|
||||
let audioBufferTimeout = null;
|
||||
const BUFFER_PROCESS_INTERVAL = 85; // 显著增加处理间隔，确保音频帧有足够的播放时间
|
||||
let bufferStartSizeThreshold = 25; // 增加开始阈值，确保有足够数据开始播放
|
||||
let lastAudioTimestamp = 0;
|
||||
let audioJitterBuffer = 60; // 增加抖动缓冲，避免播放过快
|
||||
let sampleRate = 16000; // 采样率固定为16kHz
|
||||
let frameSize = 320; // 每帧样本数 (20ms @@ 16kHz)
|
||||
let targetFrameDuration = 20; // 目标帧时长(毫秒)
|
||||
let receivedPacketCounter = 0;
|
||||
let lastPacketTime = 0;
|
||||
let receivedPacketRate = 0;
|
||||
|
||||
// 调试日志
|
||||
function log(message) {
|
||||
const timestamp = new Date().toLocaleTimeString();
|
||||
|
|
@ -562,9 +546,6 @@
|
|||
indicator.style.backgroundColor = "red";
|
||||
statusText.textContent = "未检测到通话";
|
||||
|
||||
// 停止缓冲区处理
|
||||
stopBufferProcessing();
|
||||
|
||||
// 如果确认对话框还在显示（用户未点击接听），则自动关闭
|
||||
const confirmDialog = bootstrap.Modal.getInstance(document.getElementById('callConfirmDialog'));
|
||||
if (confirmDialog) {
|
||||
|
|
@ -1109,9 +1090,6 @@
|
|||
clearInterval(refreshDisplayInterval);
|
||||
}
|
||||
|
||||
// 停止缓冲区处理
|
||||
stopBufferProcessing();
|
||||
|
||||
// 关闭音频上下文
|
||||
if (audioContext) {
|
||||
audioContext.close().catch(e => console.log("关闭音频上下文失败: " + e));
|
||||
|
|
@ -1441,7 +1419,6 @@
|
|||
// 创建新的音频上下文
|
||||
const AudioContext = window.AudioContext || window.webkitAudioContext;
|
||||
audioContext = new AudioContext({
|
||||
sampleRate: sampleRate, // 固定使用16kHz采样率
|
||||
latencyHint: 'interactive' // 低延迟设置
|
||||
});
|
||||
|
||||
|
|
@ -1452,9 +1429,6 @@
|
|||
|
||||
log(`音频上下文已初始化: 采样率=${audioContext.sampleRate}Hz, 状态=${audioContext.state}, 音量增益=${volumeBoost}倍`);
|
||||
|
||||
// 重置音频缓冲处理
|
||||
startBufferProcessing();
|
||||
|
||||
// 恢复音频上下文
|
||||
if (audioContext.state === 'suspended') {
|
||||
const resumeAudio = function () {
|
||||
|
|
@ -1481,182 +1455,241 @@
|
|||
}
|
||||
}
|
||||
|
||||
// 启动缓冲区处理
|
||||
function startBufferProcessing() {
|
||||
// 停止现有的处理
|
||||
if (audioBufferTimeout) {
|
||||
clearInterval(audioBufferTimeout);
|
||||
}
|
||||
// 播放实时音频 - 适应新格式
|
||||
function playRealTimeAudio(audioPacket) {
|
||||
if (!audioContext || !isAudioStreamEnabled || !callInProgress) return;
|
||||
|
||||
// 重置状态
|
||||
audioBufferQueue = [];
|
||||
isAudioPlaying = false;
|
||||
lastAudioTimestamp = 0;
|
||||
receivedPacketCounter = 0;
|
||||
lastPacketTime = 0;
|
||||
|
||||
// 启动处理间隔
|
||||
audioBufferTimeout = setInterval(processAudioBuffer, BUFFER_PROCESS_INTERVAL);
|
||||
log(`音频处理已启动: 间隔=${BUFFER_PROCESS_INTERVAL}ms, 缓冲阈值=${bufferStartSizeThreshold}帧`);
|
||||
}
|
||||
|
||||
// 停止缓冲区处理
|
||||
function stopBufferProcessing() {
|
||||
if (audioBufferTimeout) {
|
||||
clearInterval(audioBufferTimeout);
|
||||
audioBufferTimeout = null;
|
||||
}
|
||||
|
||||
// 清空状态
|
||||
audioBufferQueue = [];
|
||||
isAudioPlaying = false;
|
||||
lastAudioTimestamp = 0;
|
||||
log("音频处理已停止");
|
||||
}
|
||||
|
||||
// 处理音频缓冲区
|
||||
function processAudioBuffer() {
|
||||
if (!audioContext || !isAudioStreamEnabled || !callInProgress) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 没有数据时等待
|
||||
if (audioBufferQueue.length === 0) {
|
||||
if (isAudioPlaying) {
|
||||
log("缓冲区已空，等待更多数据...");
|
||||
isAudioPlaying = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 初始播放需要达到阈值
|
||||
if (!isAudioPlaying && audioBufferQueue.length < bufferStartSizeThreshold) {
|
||||
// 只在达到特定比例时记录
|
||||
if (audioBufferQueue.length === 1 ||
|
||||
audioBufferQueue.length === Math.floor(bufferStartSizeThreshold / 2) ||
|
||||
audioBufferQueue.length === bufferStartSizeThreshold - 1) {
|
||||
log(`缓冲中: ${audioBufferQueue.length}/${bufferStartSizeThreshold}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// 当前时间和上次播放的时间间隔检查
|
||||
const now = Date.now();
|
||||
const elapsed = now - lastAudioTimestamp;
|
||||
|
||||
// 如果上次播放时间太近，等待足够的时间间隔
|
||||
if (isAudioPlaying && elapsed < audioJitterBuffer) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 从队列获取下一个音频包
|
||||
const packet = audioBufferQueue.shift();
|
||||
if (packet && !packet.processed) {
|
||||
playBufferedAudio(packet.data, packet.estimatedDuration);
|
||||
packet.processed = true;
|
||||
isAudioPlaying = true;
|
||||
lastAudioTimestamp = now;
|
||||
|
||||
// 自适应调整缓冲区处理间隔
|
||||
const bufferRatio = audioBufferQueue.length / MAX_BUFFER_SIZE;
|
||||
|
||||
if (bufferRatio > 0.7) {
|
||||
// 数据充足，可以稍微加快处理
|
||||
if (audioBufferTimeout) {
|
||||
clearInterval(audioBufferTimeout);
|
||||
const newInterval = Math.max(BUFFER_PROCESS_INTERVAL * 0.8, 60);
|
||||
audioBufferTimeout = setInterval(processAudioBuffer, newInterval);
|
||||
log(`缓冲区数据充足 (${audioBufferQueue.length}/${MAX_BUFFER_SIZE}), 调整间隔为 ${newInterval.toFixed(0)}ms`);
|
||||
}
|
||||
} else if (bufferRatio < 0.1 && audioBufferQueue.length > 0) {
|
||||
// 数据不足，需要减缓处理速度
|
||||
if (audioBufferTimeout) {
|
||||
clearInterval(audioBufferTimeout);
|
||||
const newInterval = BUFFER_PROCESS_INTERVAL * 1.5;
|
||||
audioBufferTimeout = setInterval(processAudioBuffer, newInterval);
|
||||
log(`缓冲区数据不足 (${audioBufferQueue.length}/${MAX_BUFFER_SIZE}), 调整间隔为 ${newInterval.toFixed(0)}ms`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 播放缓冲区中的音频
|
||||
async function playBufferedAudio(pcmData, estimatedDuration) {
|
||||
try {
|
||||
// 确保音频上下文正常
|
||||
if (!audioContext || audioContext.state === 'closed') {
|
||||
initAudioContext();
|
||||
if (!audioContext) return;
|
||||
// 解析音频元数据和数据
|
||||
const { format, sampleRate, channels, data } = audioPacket;
|
||||
|
||||
// 确保格式正确
|
||||
if (!format || !data) {
|
||||
log("音频格式或数据无效");
|
||||
return;
|
||||
}
|
||||
|
||||
if (audioContext.state === 'suspended') {
|
||||
try {
|
||||
await audioContext.resume();
|
||||
} catch (e) {
|
||||
log(`接收到音频数据: 格式=${format}, 采样率=${sampleRate}, 声道=${channels}, 数据长度=${Array.isArray(data) ? data.length : '未知'}`);
|
||||
|
||||
// 根据不同的音频格式处理
|
||||
if (format === "WAV") {
|
||||
// 处理WAV格式数据 - 异步解码
|
||||
processWavData(data, sampleRate, channels)
|
||||
.then(audioBuffer => {
|
||||
if (audioBuffer) {
|
||||
playAudioBuffer(audioBuffer);
|
||||
} else {
|
||||
log("无法创建WAV音频缓冲区");
|
||||
}
|
||||
})
|
||||
.catch(err => {
|
||||
log("WAV处理错误: " + err);
|
||||
});
|
||||
} else if (format === "16bit_PCM") {
|
||||
// 处理PCM格式数据 - 同步处理
|
||||
const audioBuffer = processPcmData(data, sampleRate, channels);
|
||||
if (audioBuffer) {
|
||||
playAudioBuffer(audioBuffer);
|
||||
} else {
|
||||
log("无法创建PCM音频缓冲区");
|
||||
}
|
||||
} else {
|
||||
log("不支持的音频格式: " + format);
|
||||
}
|
||||
} catch (e) {
|
||||
log("处理实时音频失败: " + e);
|
||||
}
|
||||
}
|
||||
|
||||
// 播放音频缓冲区
|
||||
function playAudioBuffer(audioBuffer) {
|
||||
// 确保音频上下文活跃
|
||||
if (audioContext.state === 'suspended') {
|
||||
try {
|
||||
audioContext.resume();
|
||||
} catch (e) {
|
||||
log("恢复音频上下文失败: " + e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 创建音频源并连接
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
|
||||
// 应用音量控制
|
||||
source.connect(audioGainNode);
|
||||
if (audioGainNode) {
|
||||
audioGainNode.gain.value = currentVolume * volumeBoost;
|
||||
}
|
||||
|
||||
// 立即播放
|
||||
source.start(0);
|
||||
log("开始播放音频");
|
||||
}
|
||||
|
||||
// 处理WAV格式的数据 - 返回Promise
|
||||
function processWavData(data, sampleRate, channels) {
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
// 确保音频上下文存在
|
||||
if (!audioContext || audioContext.state === 'closed') {
|
||||
initAudioContext();
|
||||
if (!audioContext) {
|
||||
reject(new Error("无法初始化音频上下文"));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 将数据转换为ArrayBuffer
|
||||
let arrayBuffer;
|
||||
if (data instanceof Uint8Array) {
|
||||
arrayBuffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
||||
} else if (Array.isArray(data)) {
|
||||
// 转换数组为Uint8Array
|
||||
const uint8Array = new Uint8Array(data);
|
||||
arrayBuffer = uint8Array.buffer;
|
||||
} else if (typeof data === 'string') {
|
||||
// 处理Base64编码
|
||||
try {
|
||||
const base64Str = data.trim().replace(/^data:[^;]+;base64,/, '');
|
||||
const binary = atob(base64Str);
|
||||
const uint8Array = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
uint8Array[i] = binary.charCodeAt(i);
|
||||
}
|
||||
arrayBuffer = uint8Array.buffer;
|
||||
} catch (e) {
|
||||
log("WAV数据Base64解码失败: " + e);
|
||||
reject(e);
|
||||
return;
|
||||
}
|
||||
} else if (data.buffer) {
|
||||
arrayBuffer = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
|
||||
} else {
|
||||
const error = new Error("无法处理的WAV数据类型");
|
||||
log(error.message);
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// 数据准备
|
||||
let dataView;
|
||||
try {
|
||||
dataView = new DataView(pcmData.buffer, pcmData.byteOffset, pcmData.byteLength);
|
||||
// 使用Web Audio API解码音频
|
||||
audioContext.decodeAudioData(
|
||||
arrayBuffer,
|
||||
(buffer) => {
|
||||
log("WAV数据解码成功, 时长: " + buffer.duration.toFixed(2) + "秒");
|
||||
resolve(buffer);
|
||||
},
|
||||
(err) => {
|
||||
log("解码WAV数据失败: " + err);
|
||||
reject(err);
|
||||
}
|
||||
);
|
||||
} catch (e) {
|
||||
const newBuffer = new ArrayBuffer(pcmData.length);
|
||||
const newBufferView = new Uint8Array(newBuffer);
|
||||
newBufferView.set(pcmData);
|
||||
dataView = new DataView(newBuffer);
|
||||
log("处理WAV数据失败: " + e);
|
||||
reject(e);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// 处理PCM格式的数据
|
||||
function processPcmData(data, sampleRate, channels) {
|
||||
try {
|
||||
// 确保音频上下文存在
|
||||
if (!audioContext || audioContext.state === 'closed') {
|
||||
initAudioContext();
|
||||
if (!audioContext) return null;
|
||||
}
|
||||
|
||||
// 转换为音频数据并增强音量
|
||||
const floatData = new Float32Array(pcmData.length / 2);
|
||||
for (let i = 0, j = 0; i < pcmData.length; i += 2, j++) {
|
||||
if (i + 1 < pcmData.length) {
|
||||
const int16 = dataView.getInt16(i, true);
|
||||
// 标准化16位PCM数据到-1.0到1.0，但不进行音量增益
|
||||
// 音量增益由audioGainNode处理，避免信号失真
|
||||
floatData[j] = int16 / 32768.0;
|
||||
// 转换数据为适合的格式
|
||||
let pcmData;
|
||||
if (data instanceof Uint8Array) {
|
||||
pcmData = data;
|
||||
} else if (Array.isArray(data)) {
|
||||
pcmData = new Uint8Array(data);
|
||||
} else if (typeof data === 'object' && data.buffer) {
|
||||
pcmData = new Uint8Array(data.buffer);
|
||||
} else if (typeof data === 'string') {
|
||||
try {
|
||||
// 处理Base64编码
|
||||
const base64Str = data.trim().replace(/^data:[^;]+;base64,/, '');
|
||||
const binary = atob(base64Str);
|
||||
pcmData = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
pcmData[i] = binary.charCodeAt(i);
|
||||
}
|
||||
} catch (e) {
|
||||
log("PCM数据Base64解码失败: " + e);
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
log("不支持的PCM数据类型");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 实际帧长度计算和调整
|
||||
const actualFrameDuration = (floatData.length / sampleRate) * 1000; // 毫秒
|
||||
let outputFloatData = floatData;
|
||||
|
||||
// 确保音频帧至少有目标时长
|
||||
if (actualFrameDuration < targetFrameDuration && floatData.length > 0) {
|
||||
const targetLength = Math.ceil(targetFrameDuration * sampleRate / 1000);
|
||||
outputFloatData = new Float32Array(targetLength);
|
||||
outputFloatData.set(floatData);
|
||||
// 其余部分为0，即静音填充
|
||||
// 确保有效的数据
|
||||
if (!pcmData || pcmData.length < 2) {
|
||||
log("PCM数据无效或太短");
|
||||
return null;
|
||||
}
|
||||
|
||||
// 创建音频缓冲区
|
||||
const buffer = audioContext.createBuffer(1, outputFloatData.length, sampleRate);
|
||||
const channel = buffer.getChannelData(0);
|
||||
channel.set(outputFloatData);
|
||||
|
||||
// 创建音频源
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
|
||||
// 应用音量控制
|
||||
source.connect(audioGainNode);
|
||||
if (audioGainNode) {
|
||||
audioGainNode.gain.value = currentVolume * volumeBoost; // 应用音量增益
|
||||
// 确保数据长度是偶数（16位PCM）
|
||||
const validLength = Math.floor(pcmData.length / 2) * 2;
|
||||
if (validLength < pcmData.length) {
|
||||
pcmData = pcmData.slice(0, validLength);
|
||||
}
|
||||
|
||||
// 确保音频播放完成时进行清理
|
||||
source.onended = () => {
|
||||
// 在音频片段播放完成时可以执行一些操作
|
||||
};
|
||||
try {
|
||||
// 从Uint8Array创建Int16Array视图
|
||||
let int16Data;
|
||||
|
||||
// 使用精确调度启动播放
|
||||
const startTime = audioContext.currentTime;
|
||||
source.start(startTime);
|
||||
try {
|
||||
// 创建DataView以便正确解析16位整数
|
||||
const dataView = new DataView(pcmData.buffer, pcmData.byteOffset, pcmData.byteLength);
|
||||
int16Data = new Int16Array(pcmData.length / 2);
|
||||
|
||||
// 从小端字节序读取16位整数
|
||||
for (let i = 0; i < pcmData.length; i += 2) {
|
||||
if (i + 1 < pcmData.length) {
|
||||
int16Data[i/2] = dataView.getInt16(i, true); // true表示小端字节序
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log("创建Int16Array失败: " + e);
|
||||
|
||||
// 备用方法
|
||||
const newBuffer = new ArrayBuffer(pcmData.length);
|
||||
const newView = new Uint8Array(newBuffer);
|
||||
newView.set(pcmData);
|
||||
const dataView = new DataView(newBuffer);
|
||||
|
||||
int16Data = new Int16Array(pcmData.length / 2);
|
||||
for (let i = 0; i < pcmData.length; i += 2) {
|
||||
if (i + 1 < pcmData.length) {
|
||||
int16Data[i/2] = dataView.getInt16(i, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 创建音频缓冲区，使用实际采样率
|
||||
const audioSampleRate = sampleRate || audioContext.sampleRate;
|
||||
const buffer = audioContext.createBuffer(channels || 1, int16Data.length, audioSampleRate);
|
||||
|
||||
// 将Int16数据转换为Float32数据并存入缓冲区
|
||||
const channelData = buffer.getChannelData(0);
|
||||
for (let i = 0; i < int16Data.length; i++) {
|
||||
// 将Int16转换为-1.0到1.0的Float32
|
||||
channelData[i] = Math.max(-1, Math.min(1, int16Data[i] / 32768.0));
|
||||
}
|
||||
|
||||
return buffer;
|
||||
} catch (e) {
|
||||
log("PCM数据处理失败: " + e);
|
||||
return null;
|
||||
}
|
||||
} catch (e) {
|
||||
log("播放缓冲音频失败: " + e);
|
||||
log("处理PCM数据失败: " + e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1788,110 +1821,5 @@
|
|||
updateScreenControlUI(false);
|
||||
});
|
||||
}
|
||||
|
||||
// 播放实时音频
|
||||
function playRealTimeAudio(audioData) {
|
||||
if (!audioContext || !isAudioStreamEnabled) return;
|
||||
|
||||
try {
|
||||
// 计算数据接收速率
|
||||
const now = Date.now();
|
||||
receivedPacketCounter++;
|
||||
if (now - lastPacketTime > 1000) {
|
||||
receivedPacketRate = receivedPacketCounter;
|
||||
receivedPacketCounter = 0;
|
||||
lastPacketTime = now;
|
||||
log(`音频数据接收速率: ${receivedPacketRate} 包/秒`);
|
||||
}
|
||||
|
||||
// 处理音频数据
|
||||
let pcmData;
|
||||
|
||||
// 不同类型的音频数据处理
|
||||
if (audioData instanceof Uint8Array) {
|
||||
pcmData = audioData;
|
||||
} else if (audioData instanceof ArrayBuffer) {
|
||||
pcmData = new Uint8Array(audioData);
|
||||
} else if (Array.isArray(audioData)) {
|
||||
pcmData = new Uint8Array(audioData);
|
||||
} else if (typeof audioData === 'object' && audioData !== null) {
|
||||
if (audioData.data && (audioData.data instanceof Uint8Array || audioData.data instanceof ArrayBuffer)) {
|
||||
pcmData = audioData.data instanceof Uint8Array ? audioData.data : new Uint8Array(audioData.data);
|
||||
} else if (audioData.buffer && audioData.buffer instanceof ArrayBuffer) {
|
||||
pcmData = new Uint8Array(audioData.buffer);
|
||||
} else {
|
||||
log("无法识别的对象类型数据");
|
||||
return;
|
||||
}
|
||||
} else if (typeof audioData === 'string') {
|
||||
try {
|
||||
// 移除可能的数据URL前缀
|
||||
const base64Str = audioData.trim().replace(/^data:[^;]+;base64,/, '');
|
||||
|
||||
// Base64解码
|
||||
const binary = atob(base64Str);
|
||||
pcmData = new Uint8Array(binary.length);
|
||||
|
||||
for (let i = 0; i < binary.length; i++) {
|
||||
pcmData[i] = binary.charCodeAt(i);
|
||||
}
|
||||
} catch (e) {
|
||||
try {
|
||||
pcmData = new Uint8Array(audioData.length);
|
||||
for (let i = 0; i < audioData.length; i++) {
|
||||
pcmData[i] = audioData.charCodeAt(i);
|
||||
}
|
||||
} catch (e2) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
|
||||
// 数据有效性检查
|
||||
if (!pcmData || pcmData.length < 2) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 确保数据长度是偶数（16位样本需要2个字节）
|
||||
const validLength = Math.floor(pcmData.length / 2) * 2;
|
||||
if (validLength < pcmData.length) {
|
||||
pcmData = pcmData.slice(0, validLength);
|
||||
}
|
||||
|
||||
// 估算当前数据帧的时长
|
||||
const numSamples = validLength / 2; // 16位PCM = 2字节/样本
|
||||
const frameDuration = (numSamples / sampleRate) * 1000; // 毫秒
|
||||
|
||||
// 时间戳校准功能 - 固定帧率处理
|
||||
const timestamp = Date.now();
|
||||
const packet = {
|
||||
data: pcmData,
|
||||
timestamp: timestamp,
|
||||
estimatedDuration: frameDuration,
|
||||
processed: false
|
||||
};
|
||||
|
||||
// 添加到缓冲队列
|
||||
if (audioBufferQueue.length < MAX_BUFFER_SIZE) {
|
||||
audioBufferQueue.push(packet);
|
||||
|
||||
// 只在达到重要阈值时记录
|
||||
if (audioBufferQueue.length === bufferStartSizeThreshold ||
|
||||
audioBufferQueue.length % 20 === 0) {
|
||||
log(`缓冲区状态: ${audioBufferQueue.length}/${MAX_BUFFER_SIZE}, 估计时长: ${frameDuration.toFixed(1)}ms`);
|
||||
}
|
||||
} else {
|
||||
// 缓冲区已满时的处理策略
|
||||
const keepFrames = Math.floor(MAX_BUFFER_SIZE * 0.75);
|
||||
audioBufferQueue.splice(0, MAX_BUFFER_SIZE - keepFrames);
|
||||
audioBufferQueue.push(packet);
|
||||
log(`缓冲区已满 (${MAX_BUFFER_SIZE}), 丢弃旧数据, 保留${keepFrames}帧`);
|
||||
}
|
||||
} catch (e) {
|
||||
log("处理实时音频失败: " + e);
|
||||
}
|
||||
}
|
||||
</script>
|
||||
}
|
||||
|
|
|
|||
|
|
@ -189,7 +189,7 @@ public class AudioProcessingService : IAudioProcessingService
|
|||
}
|
||||
|
||||
/// <summary>
|
||||
/// 应用降噪处理
|
||||
/// 应用噪声消除
|
||||
/// </summary>
|
||||
/// <param name="audioData">音频数据</param>
|
||||
/// <param name="sampleRate">采样率</param>
|
||||
|
|
@ -197,55 +197,116 @@ public class AudioProcessingService : IAudioProcessingService
|
|||
/// <param name="noiseThreshold">噪声门限值</param>
|
||||
/// <param name="attackSeconds">攻击时间</param>
|
||||
/// <param name="releaseSeconds">释放时间</param>
|
||||
/// <param name="highPassCutoff">高通滤波器截止频率</param>
|
||||
/// <param name="highPassCutoff">高通滤波器截止频率(Hz)</param>
|
||||
/// <param name="q">滤波器Q值</param>
|
||||
/// <returns></returns>
|
||||
public byte[] ApplyNoiseReduction(byte[] audioData, int sampleRate = 16000, int channels = 1, float noiseThreshold = 0.015f, float attackSeconds = 0.01f, float releaseSeconds = 0.1f, int highPassCutoff = 80, float q = 1.0f)
|
||||
public byte[] ApplyNoiseReduction(byte[] audioData, int sampleRate = 16000, int channels = 1, float noiseThreshold = 0.02f, float attackSeconds = 0.01f, float releaseSeconds = 0.1f, int highPassCutoff = 80, float q = 1.0f)
|
||||
{
|
||||
// 调用内部实现
|
||||
return ApplyNoiseReductionInternal(audioData, noiseThreshold, attackSeconds, releaseSeconds, highPassCutoff, q);
|
||||
// 基本的噪声消除实现（简化版）
|
||||
// 实际项目中，这里可以使用更复杂的噪声消除算法，如频域滤波、自适应滤波等
|
||||
// 为简单起见，我们这里只实现一个简单的高通滤波和噪声门限
|
||||
|
||||
if (audioData == null || audioData.Length < 4)
|
||||
{
|
||||
return audioData;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// 转换为16位的短整型数据（PCM格式）
|
||||
short[] samples = new short[audioData.Length / 2];
|
||||
for (int i = 0; i < samples.Length; i++)
|
||||
{
|
||||
// 假设数据是小端字节序
|
||||
samples[i] = (short)(audioData[i * 2] | (audioData[i * 2 + 1] << 8));
|
||||
}
|
||||
|
||||
// 应用噪声门限 - 低于门限的声音被静音
|
||||
float maxAmplitude = samples.Max(s => Math.Abs(s)) / 32768.0f; // 归一化
|
||||
if (maxAmplitude < noiseThreshold)
|
||||
{
|
||||
// 整个片段低于门限，认为是噪声，全部静音
|
||||
Array.Clear(samples, 0, samples.Length);
|
||||
}
|
||||
else
|
||||
{
|
||||
// 应用高通滤波（简单的单阶IIR滤波器）
|
||||
float alpha = (float)Math.Exp(-2 * Math.PI * highPassCutoff / sampleRate);
|
||||
float filterState = 0;
|
||||
|
||||
for (int i = 0; i < samples.Length; i++)
|
||||
{
|
||||
// 高通滤波
|
||||
float input = samples[i] / 32768.0f;
|
||||
float filtered = alpha * (filterState + input - filterState);
|
||||
filterState = input;
|
||||
|
||||
// 噪声门限
|
||||
float absFiltered = Math.Abs(filtered);
|
||||
if (absFiltered < noiseThreshold)
|
||||
{
|
||||
filtered = 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
// 应用平滑瞬态响应（攻击/释放）
|
||||
// 这里简化实现
|
||||
}
|
||||
|
||||
// 转回16位整数
|
||||
samples[i] = (short)(filtered * 32768.0f);
|
||||
}
|
||||
}
|
||||
|
||||
// 转回字节数组
|
||||
byte[] result = new byte[samples.Length * 2];
|
||||
for (int i = 0; i < samples.Length; i++)
|
||||
{
|
||||
// 小端字节序
|
||||
result[i * 2] = (byte)(samples[i] & 0xFF);
|
||||
result[i * 2 + 1] = (byte)((samples[i] >> 8) & 0xFF);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "应用噪声消除时发生错误");
|
||||
return audioData; // 失败时返回原始数据
|
||||
}
|
||||
}
|
||||
|
||||
private byte[] ApplyNoiseReductionInternal(
|
||||
byte[] audioData,
|
||||
float noiseThreshold = 0.015f, // 降低噪声门限值，使其更温和
|
||||
float attackSeconds = 0.05f, // 增加攻击时间，使开启更平滑
|
||||
float releaseSeconds = 0.15f, // 增加释放时间，使关闭更平滑
|
||||
int highPassCutoff = 60, // 降低高通滤波器截止频率，减少声音失真
|
||||
float q = 0.7071f) // 使用更平缓的Q值（巴特沃斯滤波器标准Q值）
|
||||
/// <summary>
|
||||
/// 为PCM数据添加WAV头
|
||||
/// </summary>
|
||||
/// <param name="pcmData">原始PCM数据</param>
|
||||
/// <param name="sampleRate">采样率</param>
|
||||
/// <param name="channels">声道数</param>
|
||||
/// <returns>包含WAV头的完整WAV格式数据</returns>
|
||||
public byte[] AddWavHeader(byte[] pcmData, int sampleRate = 16000, int channels = 1)
|
||||
{
|
||||
// 1. 将字节数组转换为 WaveStream
|
||||
using (var inputStream = new MemoryStream(audioData))
|
||||
using (var waveStream = new RawSourceWaveStream(inputStream, new WaveFormat(16000, 16, 1)))
|
||||
using (var stream = new MemoryStream())
|
||||
{
|
||||
// 2. 转换为浮点样本便于处理
|
||||
var sampleProvider = waveStream.ToSampleProvider();
|
||||
|
||||
// 3. 应用改进的噪声门，使用平滑过渡
|
||||
var noiseGate = new ImprovedNoiseGate(sampleProvider)
|
||||
{
|
||||
Threshold = noiseThreshold,
|
||||
AttackSeconds = attackSeconds,
|
||||
ReleaseSeconds = releaseSeconds,
|
||||
HoldSeconds = 0.1f, // 添加保持时间，防止快速开关
|
||||
SoftKneeDb = 6.0f // 添加软膝，使过渡更平滑
|
||||
};
|
||||
|
||||
// 4. 应用高通滤波器去除低频噪音，使用更温和的设置
|
||||
var highPassFilter = new BiQuadFilterSampleProvider(noiseGate);
|
||||
highPassFilter.Filter = BiQuadFilter.HighPassFilter(
|
||||
sampleProvider.WaveFormat.SampleRate,
|
||||
highPassCutoff,
|
||||
q);
|
||||
|
||||
// 5. 添加平滑处理器
|
||||
var smoothedProvider = new SmoothingSampleProvider(highPassFilter, 5);
|
||||
|
||||
// 6. 处理后的音频转回字节数组
|
||||
var outputStream = new MemoryStream();
|
||||
WaveFileWriter.WriteWavFileToStream(outputStream, smoothedProvider.ToWaveProvider16());
|
||||
|
||||
return outputStream.ToArray();
|
||||
// RIFF头
|
||||
stream.Write(System.Text.Encoding.ASCII.GetBytes("RIFF"), 0, 4);
|
||||
stream.Write(BitConverter.GetBytes(pcmData.Length + 36), 0, 4);
|
||||
stream.Write(System.Text.Encoding.ASCII.GetBytes("WAVEfmt "), 0, 8);
|
||||
|
||||
// 格式块
|
||||
stream.Write(BitConverter.GetBytes(16), 0, 4); // fmt块大小
|
||||
stream.Write(BitConverter.GetBytes((ushort)1), 0, 2); // PCM格式
|
||||
stream.Write(BitConverter.GetBytes((ushort)channels), 0, 2);
|
||||
stream.Write(BitConverter.GetBytes(sampleRate), 0, 4);
|
||||
stream.Write(BitConverter.GetBytes(sampleRate * channels * 2), 0, 4); // 字节率
|
||||
stream.Write(BitConverter.GetBytes((ushort)(channels * 2)), 0, 2); // 块对齐
|
||||
stream.Write(BitConverter.GetBytes((ushort)16), 0, 2); // 位深
|
||||
|
||||
// 数据块
|
||||
stream.Write(System.Text.Encoding.ASCII.GetBytes("data"), 0, 4);
|
||||
stream.Write(BitConverter.GetBytes(pcmData.Length), 0, 4);
|
||||
stream.Write(pcmData, 0, pcmData.Length);
|
||||
|
||||
return stream.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -91,5 +91,14 @@ namespace ShengShengBuXi.Services
|
|||
/// <param name="q">滤波器Q值</param>
|
||||
/// <returns></returns>
|
||||
byte[] ApplyNoiseReduction(byte[] audioData, int sampleRate = 16000, int channels = 1, float noiseThreshold = 0.02f, float attackSeconds = 0.01f, float releaseSeconds = 0.1f, int highPassCutoff = 80, float q = 1.0f);
|
||||
|
||||
/// <summary>
|
||||
/// 为PCM数据添加WAV头
|
||||
/// </summary>
|
||||
/// <param name="pcmData">原始PCM数据</param>
|
||||
/// <param name="sampleRate">采样率</param>
|
||||
/// <param name="channels">声道数</param>
|
||||
/// <returns>包含WAV头的完整WAV格式数据</returns>
|
||||
byte[] AddWavHeader(byte[] pcmData, int sampleRate = 16000, int channels = 1);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user