From 45fe56c543b3b3ddbbdc9ace7b7df9b276acb501 Mon Sep 17 00:00:00 2001 From: ZLY Date: Tue, 9 Dec 2025 14:40:44 +0800 Subject: [PATCH] =?UTF-8?q?feat(flow):=20=E6=B7=BB=E5=8A=A0=E8=AF=AD?= =?UTF-8?q?=E9=9F=B3=E8=BE=93=E5=85=A5=E8=8A=82=E7=82=B9=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 MicrophoneNode 组件，支持语音识别与转文字功能 - 在节点类型配置中注册了 'MICRO' 类型及其映射关系 - 实现音频采集与 WebSocket 实时传输模块 - 提供 audioService 封装录音与语音识别逻辑 - 添加 NodeContentMicrophone 组件用于展示语音节点 UI 与控制按钮 - 集成 WebSocketConnectMethod 处理与语音服务端的通信 - 在侧边栏配置中增加“语音输入”节点选项 - 支持开始/停止录音及实时结果显示功能 --- src/components/FlowEditor/node/index.tsx | 10 +- .../node/microphoneNode/MicrophoneNode.tsx | 43 +++ src/components/audio-recognize/audio/main.js | 140 ++++++++ .../audio-recognize/audio/wsconnecter.js | 96 ++++++ src/components/audio-recognize/index.vue | 47 +++ .../components/nodeContentMicrophone.tsx | 311 ++++++++++++++++++ .../sideBar/config/localNodeData.ts | 1 + 7 files changed, 645 insertions(+), 3 deletions(-) create mode 100644 src/components/FlowEditor/node/microphoneNode/MicrophoneNode.tsx create mode 100644 src/components/audio-recognize/audio/main.js create mode 100644 src/components/audio-recognize/audio/wsconnecter.js create mode 100644 src/components/audio-recognize/index.vue create mode 100644 src/pages/flowEditor/components/nodeContentMicrophone.tsx diff --git a/src/components/FlowEditor/node/index.tsx b/src/components/FlowEditor/node/index.tsx index 0d39d40..8e66ce7 100644 --- a/src/components/FlowEditor/node/index.tsx +++ b/src/components/FlowEditor/node/index.tsx @@ -9,6 +9,7 @@ import ImageNode from './imageNode/ImageNode'; import RestNode from './restNode/RestNode'; import SwitchNode from './switchNode/SwitchNode'; import LoopNode from './loopNode/LoopNode'; +import MicrophoneNode from './microphoneNode/MicrophoneNode'; // 定义所有可用的节点类型 export const nodeTypes: NodeTypes = { @@ -21,7 +22,8 @@ export const nodeTypes: NodeTypes = { IMAGE: ImageNode, REST: RestNode, SWITCH: SwitchNode, - LOOP: LoopNode + LOOP: LoopNode, + MICRO: MicrophoneNode }; // 节点类型映射，用于创建节点时的类型查找 @@ -35,7 +37,8 @@ export const nodeTypeMap: Record = { 'image': 'IMAGE', 'rest': 'REST', 'switch': 'SWITCH', - 'loop': 'LOOP' + 'loop': 'LOOP', + 'micro': 'MICRO' }; // 节点显示名称映射 @@ -49,7 +52,8 @@ export const nodeTypeNameMap: Record = { 'image': '图片节点', 'rest': 'REST节点', 'switch': '条件节点', - 'loop': '循环节点' + 'loop': '循环节点', + 'micro': '语音节点' }; // 注册新节点类型的函数 diff --git a/src/components/FlowEditor/node/microphoneNode/MicrophoneNode.tsx b/src/components/FlowEditor/node/microphoneNode/MicrophoneNode.tsx new file mode 100644 index 0000000..6afe236 --- /dev/null +++ b/src/components/FlowEditor/node/microphoneNode/MicrophoneNode.tsx @@ -0,0 +1,43 @@ +import React from 'react'; +import { useStore } from '@xyflow/react'; +import styles from '@/components/FlowEditor/node/style/baseOther.module.less'; +import DynamicIcon from '@/components/DynamicIcon'; +import NodeContentMicrophone from '@/pages/flowEditor/components/nodeContentMicrophone'; +import NodeStatusIndicator, { NodeStatus } from '@/components/FlowEditor/NodeStatusIndicator'; +import { useStore as useFlowStore } from '@xyflow/react'; + +const setIcon = () => { + return ; +}; + +const MicrophoneNode = ({ data, id }: { data: any; id: string }) => { + const title = data.title || '语音输入'; + + // 获取节点选中状态 - 适配React Flow v12 API + const isSelected = useStore((state) => + state.nodeLookup.get(id)?.selected || false + ); + + // 获取节点运行状态 + const nodeStatus: NodeStatus = useFlowStore((state) => + (state.nodeLookup.get(id)?.data?.status as NodeStatus) || 'waiting' + ); + + // 获取运行状态可见性 + const isStatusVisible = useFlowStore((state) => + !!state.nodeLookup.get(id)?.data?.isStatusVisible + ); + + return ( +
+
+ {setIcon()} + {title} + +
+ +
+ ); +}; + +export default MicrophoneNode; \ No newline at end of file diff --git a/src/components/audio-recognize/audio/main.js b/src/components/audio-recognize/audio/main.js new file mode 100644 index 0000000..447063f --- /dev/null +++ b/src/components/audio-recognize/audio/main.js @@ -0,0 +1,140 @@ +import Recorder from 'recorder-core'; +import 'recorder-core/src/engine/pcm'; +import 'recorder-core/src/engine/wav'; +import WebSocketConnectMethod from './wsconnecter'; + +let wsconnecter; +let rec; +export const audioService = setResult => { + // 判断当前访问协议，匹配websocket连接 + // const protocol = window.location.protocol.replace(':', ''); + // const url = protocol === 'http' ? 'ws://asr.ngsk.tech:7000/' : 'wss://asr.ngsk.tech:7000/'; + // const url = 'wss://asr.ngsk.tech:7001/'; + const url = 'wss://medicalai.ngsk.tech:7001/api/v1/ws'; + + // 连接; 定义socket连接类对象与语音对象 + let sampleBuf = new Int16Array(); + let rec_text = ''; + let offline_text = ''; + let isRec = false; + + // 语音识别结果; 对jsonMsg数据解析,将识别结果附加到编辑框中 + function getJsonMessage(jsonMsg) { + console.log(`message: ${JSON.parse(jsonMsg.data).text}`); + const rectxt = `${JSON.parse(jsonMsg.data).text}`; + const asrmodel = JSON.parse(jsonMsg.data).mode; + const {is_final} = JSON.parse(jsonMsg.data); + if (asrmodel === '2pass-offline' || asrmodel === 'offline') { + offline_text = `${offline_text + rectxt.replace(/ +/g, '')}\n`; // handleWithTimestamp(rectxt,timestamp); //rectxt; //.replace(/ +/g,""); + rec_text = offline_text; + } else { + rec_text += rectxt; // .replace(/ +/g,""); + } + if (rec_text.length > 0) { + setResult(rec_text); + } + + console.log(`offline_text: ${asrmodel},${offline_text}`); + console.log(`rec_text: ${rec_text}`); + } + + // 连接状态响应 + function getConnState(connState) { + if (connState === 2) { + stop(); + console.log('connection error'); + } + } + + // 开始录音 + function record() { + // 录音; 定义录音对象,wav格式 + rec = Recorder({ + type: 'pcm', + bitRate: 16, + sampleRate: 16000, + onProcess: recProcess + }); + rec.open(function () { + rec.start(); + console.log('开始'); + }); + } + + // 识别启动、停止、清空操作 + function connect() { + // 控件状态更新 + wsconnecter = new WebSocketConnectMethod({ + msgHandle: getJsonMessage, + stateHandle: getConnState, + url + }); + + // 启动连接 + const ret = wsconnecter.wsStart(); + // 1 is ok, 0 is error + if (ret === 1) { + isRec = true; + record(); + return 1; + } + return 0; + } + + function stop() { + rec_text = ''; + offline_text = ''; + if (sampleBuf.length > 0) { + wsconnecter.wsSend(sampleBuf); + console.log(`sampleBuf.length${sampleBuf.length}`); + sampleBuf = new Int16Array(); + } + // 控件状态更新 + isRec = false; + const request = { + chunk_interval: 10, + chunk_size: [5, 10, 5], + is_speaking: false, + mode: '2pass', + wav_name: 'h5' + }; + wsconnecter.wsSend(JSON.stringify(request)); + // wait 3s for asr result + + rec.stop( + function (blob, duration) { + console.log(blob); + }, + function (errMsg) { + console.log(`errMsg: ${errMsg}`); + } + ); + setTimeout(function () { + console.log('call stop ws!'); + wsconnecter.wsStop(); + }, 1000); + } + + function recProcess(buffer, powerLevel, bufferDuration, bufferSampleRate, newBufferIdx, asyncEnd) { + if (isRec === true) { + const data_48k = buffer[buffer.length - 1]; + + const array_48k = new Array(data_48k); + const data_16k = Recorder.SampleData(array_48k, bufferSampleRate, 16000).data; + + sampleBuf = Int16Array.from([...sampleBuf, ...data_16k]); + const chunk_size = 960; // for asr chunk_size [5, 10, 5] + while (sampleBuf.length >= chunk_size) { + const sendBuf = sampleBuf.slice(0, chunk_size); + sampleBuf = sampleBuf.slice(chunk_size, sampleBuf.length); + wsconnecter.wsSend(sendBuf); + } + } + } + + return { + stop, + record, + connect + }; +}; diff --git a/src/components/audio-recognize/audio/wsconnecter.js b/src/components/audio-recognize/audio/wsconnecter.js new file mode 100644 index 0000000..6755ef7 --- /dev/null +++ b/src/components/audio-recognize/audio/wsconnecter.js @@ -0,0 +1,96 @@ +/** + * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights + * Reserved. MIT License (https://opensource.org/licenses/MIT) + */ + +/* 2021-2023 by zhaoming,mali aihealthx.com */ + +export default function WebSocketConnectMethod(config) { + //定义socket连接方法类 + let speechSokt; + let connKeeperID; + + let msgHandle = config.msgHandle; + let stateHandle = config.stateHandle; + + this.wsStart = function () { + let Uri = config.url; //"wss://111.205.137.58:5821/wss/" //设置wss asr online接口地址 如 wss://X.X.X.X:port/wss/ + if (Uri.match(/wss:\S*|ws:\S*/)) { + console.log('Uri' + Uri); + } else { + alert('请检查wss地址正确性'); + return 0; + } + + if ('WebSocket' in window) { + speechSokt = new WebSocket(Uri); // 定义socket连接对象 + speechSokt.onopen = function (e) { + onOpen(e); + }; // 定义响应函数 + speechSokt.onclose = function (e) { + console.log('onclose ws!'); + //speechSokt.close(); + onClose(e); + }; + speechSokt.onmessage = function (e) { + onMessage(e); + }; + speechSokt.onerror = function (e) { + onError(e); + }; + return 1; + } else { + alert('当前浏览器不支持 WebSocket'); + return 0; + } + }; + + // 定义停止与发送函数 + this.wsStop = function () { + if (speechSokt !== undefined) { + console.log('stop ws!'); + speechSokt.close(); + } + }; + + this.wsSend = function (oneData) { + if (speechSokt === undefined) return; + if (speechSokt.readyState === 1) { + // 0:CONNECTING, 1:OPEN, 2:CLOSING, 3:CLOSED + speechSokt.send(oneData); + } + }; + + // SOCEKT连接中的消息与状态响应 + function onOpen(e) { + // 发送json + let chunk_size = new Array(5, 10, 5); + let request = { + chunk_interval: 10, + chunk_size: chunk_size, + hotwords: '{"阿里巴巴":20,"hello world":40}', + is_speaking: true, + itn: false, + mode: '2pass', + wav_name: 'h5' + }; + console.log(JSON.stringify(request)); + speechSokt.send(JSON.stringify(request)); + console.log('连接成功'); + stateHandle(0); + } + + function onClose(e) { + console.log(e); + stateHandle(1); + } + + function onMessage(e) { + msgHandle(e); + } + + function onError(e) { + console.log(e); + stateHandle(2); + } +} diff --git a/src/components/audio-recognize/index.vue b/src/components/audio-recognize/index.vue new file mode 100644 index 0000000..71c4461 --- /dev/null +++ b/src/components/audio-recognize/index.vue @@ -0,0 +1,47 @@ + + + diff --git a/src/pages/flowEditor/components/nodeContentMicrophone.tsx b/src/pages/flowEditor/components/nodeContentMicrophone.tsx new file mode 100644 index 0000000..b8a7f9c --- /dev/null +++ b/src/pages/flowEditor/components/nodeContentMicrophone.tsx @@ -0,0 +1,311 @@ +import React, { useEffect, useState } from 'react'; +import styles from '@/components/FlowEditor/node/style/baseOther.module.less'; +import { Handle, Position } from '@xyflow/react'; +import { formatDataType } from '@/utils/common'; +import { Button } from '@arco-design/web-react'; +import { audioService } from '@/components/audio-recognize/audio/main'; + +interface NodeContentData { + parameters?: { + dataIns?: any[]; + dataOuts?: any[]; + apiIns?: any[]; + apiOuts?: any[]; + }; + showFooter?: boolean; + type?: string; + + [key: string]: any; +} + +// 定义通用的句柄样式 +const handleStyles = { + mainSource: { + background: '#2290f6', + width: '8px', + height: '8px', + border: '2px solid #fff', + boxShadow: '0 0 4px rgba(0,0,0,0.2)' + }, + mainTarget: { + background: '#2290f6', + width: '8px', + height: '8px', + border: '2px solid #fff', + boxShadow: '0 0 4px rgba(0,0,0,0.2)' + }, + data: { + background: '#555', + width: '6px', + height: '6px', + border: '1px solid #fff', + boxShadow: '0 0 2px rgba(0,0,0,0.2)' + } +}; + +// 渲染特殊节点（开始/结束节点）的句柄 +const renderSpecialNodeHandles = (isStartNode: boolean, isEndNode: boolean, dataIns: any[], dataOuts: any[], apiIns: any[], apiOuts: any[]) => { + const renderStartNodeHandles = () => { + if (!isStartNode) return null; + + return ( + <> + {apiOuts.map((_, index) => ( + + ))} + {dataOuts.length > 0 && dataOuts.map((_, index) => ( + + ))} + + ); + }; + + const renderEndNodeHandles = () => { + if (!isEndNode) return null; + + return ( + <> + {apiIns.map((_, index) => ( + + ))} + {dataIns.length > 0 && dataIns.map((_, index) => ( + + ))} + + ); + }; + + return ( + <> + {renderStartNodeHandles()} + {renderEndNodeHandles()} + + ); +}; + +// 渲染普通节点的句柄 +const renderRegularNodeHandles = (dataIns: any[], dataOuts: any[], apiIns: any[], apiOuts: any[]) => { + return ( + <> + {apiOuts.map((_, index) => ( + + ))} + {apiIns.map((_, index) => ( + + ))} + + {/* 输入参数连接端点 */} + {dataIns.map((_, index) => ( + + ))} + + {/* 输出参数连接端点 */} + {dataOuts.map((_, index) => ( + + ))} + + ); +}; + +const formatTitle = (text) => { + return text === 'start' || text === 'end' ? '' : text; +}; + +const NodeContentMicrophone = ({ data }: { data: NodeContentData }) => { + const [isRecording, setIsRecording] = useState(false); + const [resultText, setResultText] = useState(''); + const { connect, stop } = audioService(setResultText); + + const apiIns = data.parameters?.apiIns || []; + const apiOuts = data.parameters?.apiOuts || []; + const dataIns = data.parameters?.dataIns || []; + const dataOuts = data.parameters?.dataOuts || []; + + // 判断节点类型 + const isStartNode = data.type === 'start'; + const isEndNode = data.type === 'end'; + const isSpecialNode = isStartNode || isEndNode; + + + const handleStartRecordWithCheck = () => { + setIsRecording(true); + handleStartRecord(); + }; + + const handleCompleteWithCheck = () => { + setIsRecording(false); + handleComplete(); + }; + + const handleStartRecord = () => { + console.log('用户开始语音转文字'); + connect(); + }; + + const handleComplete = () => { + console.log('用户结束语音转文字'); + stop(); + // TODO 调接口 等待后端出接口 + // voiceTrigger(params); + }; + + useEffect(() => { + console.log('resultText:', resultText); + }, [resultText]); + + return ( + <> + {/*content栏-api部分*/} +
+
+ {apiIns.length > 0 && ( +
+ {apiIns.map((input, index) => ( +
+ {formatTitle(input.desc || input.id || input.name)} +
+ ))} +
+ )} + + {apiOuts.length > 0 && ( +
+ {apiOuts.map((output, index) => ( +
+ {output.desc} +
+ ))} +
+ )} +
+
+ {(dataIns.length > 0 || dataOuts.length > 0) && ( + <> + {/*分割*/} +
+ + {/*content栏-data部分*/} +
+
+
+ {dataIns.map((input, index) => ( +
+ {input.id || `输入${index + 1}`} {formatDataType(input.dataType)} + +
+ ))} +
+ + {dataOuts.length > 0 && !isEndNode && ( +
+ {dataOuts.map((output, index) => ( +
+ {formatDataType(output.dataType)} {output.id || `输出${index + 1}`} + +
+ ))} +
+ )} +
+
+ + )} + + {/*footer栏*/} +
+ {!isRecording ? + + : + + } + +
+ + {/* 根据节点类型渲染不同的句柄 */} + {isSpecialNode + ? renderSpecialNodeHandles(isStartNode, isEndNode, dataIns, dataOuts, apiIns, apiOuts) + : renderRegularNodeHandles(dataIns, dataOuts, apiIns, apiOuts)} + + ); + } +; + +export default NodeContentMicrophone; \ No newline at end of file diff --git a/src/pages/flowEditor/sideBar/config/localNodeData.ts b/src/pages/flowEditor/sideBar/config/localNodeData.ts index d9b5f4b..4937d0a 100644 --- a/src/pages/flowEditor/sideBar/config/localNodeData.ts +++ b/src/pages/flowEditor/sideBar/config/localNodeData.ts @@ -117,6 +117,7 @@ const nodeDefinitions = [ { nodeName: '周期', nodeType: 'CYCLE', nodeGroup: 'common', icon: 'IconSchedule' }, { nodeName: '事件接收', nodeType: 'EVENTLISTENE', nodeGroup: 'common', icon: 'IconImport' }, { nodeName: '事件发送', nodeType: 'EVENTSEND', nodeGroup: 'common', icon: 'IconExport' }, + { nodeName: '语音输入', nodeType: 'MICRO', nodeGroup: 'common', icon: 'IconVoice' }, // { nodeName: 'JSON转字符串', nodeType: 'JSON2STR', nodeGroup: 'common', icon: 'IconCodeBlock' }, // { nodeName: '字符串转JSON', nodeType: 'STR2JSON', nodeGroup: 'common', icon: 'IconCodeSquare' }, // { nodeName: 'JSON封装', nodeType: 'JSONCONVERT', nodeGroup: 'common', icon: 'IconTranslate' },