feat(flow): 添加语音输入节点功能
- 新增 MicrophoneNode 组件，支持语音识别与转文字功能 - 在节点类型配置中注册了 'MICRO' 类型及其映射关系 - 实现音频采集与 WebSocket 实时传输模块 - 提供 audioService 封装录音与语音识别逻辑 - 添加 NodeContentMicrophone 组件用于展示语音节点 UI 与控制按钮 - 集成 WebSocketConnectMethod 处理与语音服务端的通信 - 在侧边栏配置中增加“语音输入”节点选项 - 支持开始/停止录音及实时结果显示功能master
parent
fbd60bc488
commit
45fe56c543
@ -0,0 +1,43 @@
|
||||
import React from 'react';
|
||||
import { useStore } from '@xyflow/react';
|
||||
import styles from '@/components/FlowEditor/node/style/baseOther.module.less';
|
||||
import DynamicIcon from '@/components/DynamicIcon';
|
||||
import NodeContentMicrophone from '@/pages/flowEditor/components/nodeContentMicrophone';
|
||||
import NodeStatusIndicator, { NodeStatus } from '@/components/FlowEditor/NodeStatusIndicator';
|
||||
import { useStore as useFlowStore } from '@xyflow/react';
|
||||
|
||||
const setIcon = () => {
|
||||
return <DynamicIcon type="IconVoice" style={{ fontSize: '16px', marginRight: '5px' }} />;
|
||||
};
|
||||
|
||||
const MicrophoneNode = ({ data, id }: { data: any; id: string }) => {
|
||||
const title = data.title || '语音输入';
|
||||
|
||||
// 获取节点选中状态 - 适配React Flow v12 API
|
||||
const isSelected = useStore((state) =>
|
||||
state.nodeLookup.get(id)?.selected || false
|
||||
);
|
||||
|
||||
// 获取节点运行状态
|
||||
const nodeStatus: NodeStatus = useFlowStore((state) =>
|
||||
(state.nodeLookup.get(id)?.data?.status as NodeStatus) || 'waiting'
|
||||
);
|
||||
|
||||
// 获取运行状态可见性
|
||||
const isStatusVisible = useFlowStore((state) =>
|
||||
!!state.nodeLookup.get(id)?.data?.isStatusVisible
|
||||
);
|
||||
|
||||
return (
|
||||
<div className={`${styles['node-container']} ${isSelected ? styles.selected : ''}`}>
|
||||
<div className={styles['node-header']} style={{ backgroundColor: '#1890ff' }}>
|
||||
{setIcon()}
|
||||
{title}
|
||||
<NodeStatusIndicator status={nodeStatus} isVisible={isStatusVisible} />
|
||||
</div>
|
||||
<NodeContentMicrophone data={data} />
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default MicrophoneNode;
|
||||
@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights
|
||||
* Reserved. MIT License (https://opensource.org/licenses/MIT)
|
||||
*/
|
||||
|
||||
/* 2021-2023 by zhaoming,mali aihealthx.com */
|
||||
|
||||
export default function WebSocketConnectMethod(config) {
|
||||
//定义socket连接方法类
|
||||
let speechSokt;
|
||||
let connKeeperID;
|
||||
|
||||
let msgHandle = config.msgHandle;
|
||||
let stateHandle = config.stateHandle;
|
||||
|
||||
this.wsStart = function () {
|
||||
let Uri = config.url; //"wss://111.205.137.58:5821/wss/" //设置wss asr online接口地址 如 wss://X.X.X.X:port/wss/
|
||||
if (Uri.match(/wss:\S*|ws:\S*/)) {
|
||||
console.log('Uri' + Uri);
|
||||
} else {
|
||||
alert('请检查wss地址正确性');
|
||||
return 0;
|
||||
}
|
||||
|
||||
if ('WebSocket' in window) {
|
||||
speechSokt = new WebSocket(Uri); // 定义socket连接对象
|
||||
speechSokt.onopen = function (e) {
|
||||
onOpen(e);
|
||||
}; // 定义响应函数
|
||||
speechSokt.onclose = function (e) {
|
||||
console.log('onclose ws!');
|
||||
//speechSokt.close();
|
||||
onClose(e);
|
||||
};
|
||||
speechSokt.onmessage = function (e) {
|
||||
onMessage(e);
|
||||
};
|
||||
speechSokt.onerror = function (e) {
|
||||
onError(e);
|
||||
};
|
||||
return 1;
|
||||
} else {
|
||||
alert('当前浏览器不支持 WebSocket');
|
||||
return 0;
|
||||
}
|
||||
};
|
||||
|
||||
// 定义停止与发送函数
|
||||
this.wsStop = function () {
|
||||
if (speechSokt !== undefined) {
|
||||
console.log('stop ws!');
|
||||
speechSokt.close();
|
||||
}
|
||||
};
|
||||
|
||||
this.wsSend = function (oneData) {
|
||||
if (speechSokt === undefined) return;
|
||||
if (speechSokt.readyState === 1) {
|
||||
// 0:CONNECTING, 1:OPEN, 2:CLOSING, 3:CLOSED
|
||||
speechSokt.send(oneData);
|
||||
}
|
||||
};
|
||||
|
||||
// SOCEKT连接中的消息与状态响应
|
||||
function onOpen(e) {
|
||||
// 发送json
|
||||
let chunk_size = new Array(5, 10, 5);
|
||||
let request = {
|
||||
chunk_interval: 10,
|
||||
chunk_size: chunk_size,
|
||||
hotwords: '{"阿里巴巴":20,"hello world":40}',
|
||||
is_speaking: true,
|
||||
itn: false,
|
||||
mode: '2pass',
|
||||
wav_name: 'h5'
|
||||
};
|
||||
console.log(JSON.stringify(request));
|
||||
speechSokt.send(JSON.stringify(request));
|
||||
console.log('连接成功');
|
||||
stateHandle(0);
|
||||
}
|
||||
|
||||
function onClose(e) {
|
||||
console.log(e);
|
||||
stateHandle(1);
|
||||
}
|
||||
|
||||
function onMessage(e) {
|
||||
msgHandle(e);
|
||||
}
|
||||
|
||||
function onError(e) {
|
||||
console.log(e);
|
||||
stateHandle(2);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue';
|
||||
import { audioService } from '@/components/audio-recognize/audio/main';
|
||||
|
||||
const emit = defineEmits(['recordFinish']);
|
||||
|
||||
const isRecording = ref(false);
|
||||
|
||||
const resultText = ref('');
|
||||
function setResultText(value: string) {
|
||||
resultText.value = value;
|
||||
}
|
||||
|
||||
const { connect, stop } = audioService(setResultText);
|
||||
|
||||
function handleComplete() {
|
||||
if (isRecording.value) {
|
||||
console.log('用户结束语音转文字');
|
||||
stop();
|
||||
isRecording.value = false;
|
||||
emit('recordFinish', { result: resultText.value });
|
||||
}
|
||||
}
|
||||
|
||||
function handleStartRecord() {
|
||||
if (!isRecording.value) {
|
||||
console.log('用户开始语音转文字');
|
||||
connect();
|
||||
isRecording.value = true;
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div style="margin-right: 10px">
|
||||
<a-button v-if="isRecording" type="secondary" @click="handleComplete">
|
||||
<template #icon>
|
||||
<icon-pause />
|
||||
</template>
|
||||
</a-button>
|
||||
<a-button v-else type="primary" @click="handleStartRecord">
|
||||
<template #icon>
|
||||
<icon-customer-service />
|
||||
</template>
|
||||
</a-button>
|
||||
</div>
|
||||
</template>
|
||||
@ -0,0 +1,311 @@
|
||||
import React, { useEffect, useState } from 'react';
|
||||
import styles from '@/components/FlowEditor/node/style/baseOther.module.less';
|
||||
import { Handle, Position } from '@xyflow/react';
|
||||
import { formatDataType } from '@/utils/common';
|
||||
import { Button } from '@arco-design/web-react';
|
||||
import { audioService } from '@/components/audio-recognize/audio/main';
|
||||
|
||||
interface NodeContentData {
|
||||
parameters?: {
|
||||
dataIns?: any[];
|
||||
dataOuts?: any[];
|
||||
apiIns?: any[];
|
||||
apiOuts?: any[];
|
||||
};
|
||||
showFooter?: boolean;
|
||||
type?: string;
|
||||
|
||||
[key: string]: any;
|
||||
}
|
||||
|
||||
// 定义通用的句柄样式
|
||||
const handleStyles = {
|
||||
mainSource: {
|
||||
background: '#2290f6',
|
||||
width: '8px',
|
||||
height: '8px',
|
||||
border: '2px solid #fff',
|
||||
boxShadow: '0 0 4px rgba(0,0,0,0.2)'
|
||||
},
|
||||
mainTarget: {
|
||||
background: '#2290f6',
|
||||
width: '8px',
|
||||
height: '8px',
|
||||
border: '2px solid #fff',
|
||||
boxShadow: '0 0 4px rgba(0,0,0,0.2)'
|
||||
},
|
||||
data: {
|
||||
background: '#555',
|
||||
width: '6px',
|
||||
height: '6px',
|
||||
border: '1px solid #fff',
|
||||
boxShadow: '0 0 2px rgba(0,0,0,0.2)'
|
||||
}
|
||||
};
|
||||
|
||||
// 渲染特殊节点（开始/结束节点）的句柄
|
||||
const renderSpecialNodeHandles = (isStartNode: boolean, isEndNode: boolean, dataIns: any[], dataOuts: any[], apiIns: any[], apiOuts: any[]) => {
|
||||
const renderStartNodeHandles = () => {
|
||||
if (!isStartNode) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
{apiOuts.map((_, index) => (
|
||||
<Handle
|
||||
key={`start-output-handle-${index}`}
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
id={apiOuts[index].name || `start-output-${index}`}
|
||||
style={{
|
||||
...handleStyles.mainSource,
|
||||
top: `${35 + index * 20}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{dataOuts.length > 0 && dataOuts.map((_, index) => (
|
||||
<Handle
|
||||
key={`output-handle-${index}`}
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
id={dataOuts[index].name || `output-${index}`}
|
||||
style={{
|
||||
...handleStyles.data,
|
||||
top: `${70 + apiIns.length * 20 + index * 20}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const renderEndNodeHandles = () => {
|
||||
if (!isEndNode) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
{apiIns.map((_, index) => (
|
||||
<Handle
|
||||
key={`end-input-handle-${index}`}
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
id={apiIns[index].name || `end-input-${index}`}
|
||||
style={{
|
||||
...handleStyles.mainTarget,
|
||||
top: `${35 + index * 20}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{dataIns.length > 0 && dataIns.map((_, index) => (
|
||||
<Handle
|
||||
key={`input-handle-${index}`}
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
id={dataIns[index].name || `input-${index}`}
|
||||
style={{
|
||||
...handleStyles.data,
|
||||
top: `${70 + apiIns.length * 20 + index * 20}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
{renderStartNodeHandles()}
|
||||
{renderEndNodeHandles()}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
// 渲染普通节点的句柄
|
||||
const renderRegularNodeHandles = (dataIns: any[], dataOuts: any[], apiIns: any[], apiOuts: any[]) => {
|
||||
return (
|
||||
<>
|
||||
{apiOuts.map((_, index) => (
|
||||
<Handle
|
||||
key={`api-output-handle-${index}`}
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
id={apiOuts[index].name || apiOuts[index].id || `output-${index}`}
|
||||
style={{
|
||||
...handleStyles.mainSource,
|
||||
top: `${37 + index * 22}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
{apiIns.map((_, index) => (
|
||||
<Handle
|
||||
key={`api-input-handle-${index}`}
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
id={apiIns[index].name || apiIns[index].id || `input-${index}`}
|
||||
style={{
|
||||
...handleStyles.mainTarget,
|
||||
top: `${37 + index * 22}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
|
||||
{/* 输入参数连接端点 */}
|
||||
{dataIns.map((_, index) => (
|
||||
<Handle
|
||||
key={`data-input-handle-${index}`}
|
||||
type="target"
|
||||
position={Position.Left}
|
||||
id={dataIns[index].name || dataIns[index].id || `input-${index}`}
|
||||
style={{
|
||||
...handleStyles.data,
|
||||
top: `${65 + (apiIns.length + index) * 22}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
|
||||
{/* 输出参数连接端点 */}
|
||||
{dataOuts.map((_, index) => (
|
||||
<Handle
|
||||
key={`data-output-handle-${index}`}
|
||||
type="source"
|
||||
position={Position.Right}
|
||||
id={dataOuts[index].name || dataOuts[index].id || `output-${index}`}
|
||||
style={{
|
||||
...handleStyles.data,
|
||||
top: `${65 + (apiIns.length + index) * 22}px`
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
const formatTitle = (text) => {
|
||||
return text === 'start' || text === 'end' ? '' : text;
|
||||
};
|
||||
|
||||
const NodeContentMicrophone = ({ data }: { data: NodeContentData }) => {
|
||||
const [isRecording, setIsRecording] = useState(false);
|
||||
const [resultText, setResultText] = useState('');
|
||||
const { connect, stop } = audioService(setResultText);
|
||||
|
||||
const apiIns = data.parameters?.apiIns || [];
|
||||
const apiOuts = data.parameters?.apiOuts || [];
|
||||
const dataIns = data.parameters?.dataIns || [];
|
||||
const dataOuts = data.parameters?.dataOuts || [];
|
||||
|
||||
// 判断节点类型
|
||||
const isStartNode = data.type === 'start';
|
||||
const isEndNode = data.type === 'end';
|
||||
const isSpecialNode = isStartNode || isEndNode;
|
||||
|
||||
|
||||
const handleStartRecordWithCheck = () => {
|
||||
setIsRecording(true);
|
||||
handleStartRecord();
|
||||
};
|
||||
|
||||
const handleCompleteWithCheck = () => {
|
||||
setIsRecording(false);
|
||||
handleComplete();
|
||||
};
|
||||
|
||||
const handleStartRecord = () => {
|
||||
console.log('用户开始语音转文字');
|
||||
connect();
|
||||
};
|
||||
|
||||
const handleComplete = () => {
|
||||
console.log('用户结束语音转文字');
|
||||
stop();
|
||||
// TODO 调接口 等待后端出接口
|
||||
// voiceTrigger(params);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
console.log('resultText:', resultText);
|
||||
}, [resultText]);
|
||||
|
||||
return (
|
||||
<>
|
||||
{/*content栏-api部分*/}
|
||||
<div className={styles['node-api-box']}>
|
||||
<div className={styles['node-content-api']}>
|
||||
{apiIns.length > 0 && (
|
||||
<div className={styles['node-inputs']}>
|
||||
{apiIns.map((input, index) => (
|
||||
<div key={input.id || `input-${index}`} className={styles['node-input-label']}>
|
||||
{formatTitle(input.desc || input.id || input.name)}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{apiOuts.length > 0 && (
|
||||
<div className={styles['node-outputs']}>
|
||||
{apiOuts.map((output, index) => (
|
||||
<div key={output.id || `output-${index}`} className={styles['node-output-label']}>
|
||||
{output.desc}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
{(dataIns.length > 0 || dataOuts.length > 0) && (
|
||||
<>
|
||||
{/*分割*/}
|
||||
<div className={styles['node-split-line']}></div>
|
||||
|
||||
{/*content栏-data部分*/}
|
||||
<div className={styles['node-data-box']}>
|
||||
<div className={styles['node-content']}>
|
||||
<div className={styles['node-inputs']}>
|
||||
{dataIns.map((input, index) => (
|
||||
<div key={input.id || `input-${index}`} className={styles['node-input-label']}>
|
||||
<span
|
||||
className={styles['node-data-type']}
|
||||
>{input.id || `输入${index + 1}`} {formatDataType(input.dataType)}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{dataOuts.length > 0 && !isEndNode && (
|
||||
<div className={styles['node-outputs']}>
|
||||
{dataOuts.map((output, index) => (
|
||||
<div key={output.id || `output-${index}`} className={styles['node-output-label']}>
|
||||
<span
|
||||
className={styles['node-data-type']}
|
||||
>{formatDataType(output.dataType)} {output.id || `输出${index + 1}`}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{/*footer栏*/}
|
||||
<div className={styles['node-footer']}>
|
||||
{!isRecording ?
|
||||
<Button shape="round" type="primary" style={{ backgroundColor: '#1890ff' }}
|
||||
onClick={() => handleStartRecordWithCheck()}>开启语音输入</Button>
|
||||
:
|
||||
<Button shape="round" type="primary" style={{ backgroundColor: '#c05144' }}
|
||||
onClick={() => handleCompleteWithCheck()}>停止语音输入</Button>
|
||||
}
|
||||
|
||||
</div>
|
||||
|
||||
{/* 根据节点类型渲染不同的句柄 */}
|
||||
{isSpecialNode
|
||||
? renderSpecialNodeHandles(isStartNode, isEndNode, dataIns, dataOuts, apiIns, apiOuts)
|
||||
: renderRegularNodeHandles(dataIns, dataOuts, apiIns, apiOuts)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
;
|
||||
|
||||
export default NodeContentMicrophone;
|
||||
Loading…
Reference in New Issue