feat: add screen share for vision model & fix subtitle and ui bugs & update sdk version to 4.66.1
This commit is contained in:
quemingyi.wudong
parent
33f82e8b11
commit
4f088b0e89
22
README.md
22
README.md
@ -8,11 +8,12 @@
|
||||
## 【必看】环境准备
|
||||
- **Node 版本: 16.0+**
|
||||
1. 需要准备两个 Terminal，分别启动服务端、前端页面。
|
||||
2. **根据你自定义的
|
||||
2. 开通 ASR、TTS、LLM、RTC 等服务，可通过 [无代码跑通实时对话式](https://console.volcengine.com/rtc/guide) 快速开通服务, 点击 **快速开始** 中的 **跑通 Demo** 进行服务开通。
|
||||
3. **根据你自定义的
|
||||
RoomId、UserId 以及申请的 AppID、BusinessID(如有)、Token、ASR AppID、TTS AppID，修改 `src/config/config.ts` 文件中 `ConfigFactory` 中 `BaseConfig` 的配置信息**。
|
||||
3. 使用火山引擎控制台账号的 [AK、SK](https://console.volcengine.com/iam/keymanage?s=g)、[SessionToken](https://www.volcengine.com/docs/6348/1315561#sub?s=g)(临时token, 子账号才需要), 修改 `Server/app.js` 文件中的 `ACCOUNT_INFO`。
|
||||
4. 您需要在 [火山方舟-在线推理](https://console.volcengine.com/ark/region:ark+cn-beijing/endpoint?config=%7B%7D&s=g) 中创建接入点, 并将模型对应的接入点 ID 填入 `src/config/common.ts` 文件中的 `ARK_V3_MODEL_ID`, 否则无法正常启动智能体。
|
||||
5. 如果您已经自行完成了服务端的逻辑，可以不依赖 Demo 中的 Server，直接修改前端代码文件 `src/config/index.ts` 中的 `AIGC_PROXY_HOST` 请求域名和接口，并在 `src/app/api.ts` 中修改接口的参数配置 `APIS_CONFIG`。
|
||||
4. 使用火山引擎控制台账号的 [AK、SK](https://console.volcengine.com/iam/keymanage?s=g)、[SessionToken](https://www.volcengine.com/docs/6348/1315561#sub?s=g)(临时token, 子账号才需要), 修改 `Server/app.js` 文件中的 `ACCOUNT_INFO`。
|
||||
5. 您需要在 [火山方舟-在线推理](https://console.volcengine.com/ark/region:ark+cn-beijing/endpoint?config=%7B%7D&s=g) 中创建接入点, 并将模型对应的接入点 ID 填入 `src/config/common.ts` 文件中的 `ARK_V3_MODEL_ID`, 否则无法正常启动智能体。
|
||||
6. 如果您已经自行完成了服务端的逻辑，可以不依赖 Demo 中的 Server，直接修改前端代码文件 `src/config/index.ts` 中的 `AIGC_PROXY_HOST` 请求域名和接口，并在 `src/app/api.ts` 中修改接口的参数配置 `APIS_CONFIG`。
|
||||
|
||||
## 快速开始
|
||||
请注意，服务端和 Web 端都需要启动, 启动步骤如下:
|
||||
@ -44,8 +45,8 @@ yarn dev
|
||||
| :-- | :-- |
|
||||
| **启动智能体之后, 对话无反馈，或者一直停留在 "AI 准备中, 请稍侯"** | <li>可能因为控制台中相关权限没有正常授予，请参考[流程](https://www.volcengine.com/docs/6348/1315561?s=g)再次确认下是否完成相关操作。此问题的可能性较大，建议仔细对照是否已经将相应的权限开通。</li><li>参数传递可能有问题, 例如参数大小写、类型等问题，请再次确认下这类型问题是否存在。</li><li>相关资源可能未开通或者用量不足，请再次确认。</li><li>**请检查当前使用的模型 ID 等内容都是正确且可用的。**</li> |
|
||||
| `Server/app.js` 中的 `sessionToken` 是什么，该怎么填，为什么要填 | `sessionToken` 是火山引擎子账号发起 OpenAPI 请求时所必须携带的临时 Token，获取方式可参考 [此文章末尾](https://www.volcengine.com/docs/6348/1315561?s=g)。 |
|
||||
| **浏览器报了 `Uncaught (in promise) r: token_error` 错误** | 请检查您填在项目中的 RTC Token 是否合法，检测用于生成 Token 的 UserId、RoomId 是否与项目中填写的一致。 |
|
||||
| [StartVoiceChat]Failed(Reason: The task has been started. Please do not call the startup task interface repeatedly.) 报错 | 由于目前设置的 RoomId、UserId 为固定值，重复调用 startAudioBot 会导致出错，只需先调用 stopAudioBot 后再重新 startAudioBot 即可。 |
|
||||
| **浏览器报了 `Uncaught (in promise) r: token_error` 错误** | 请检查您填在项目中的 RTC Token 是否合法，检测用于生成 Token 的 UserId、RoomId 是否与项目中填写的一致；或者 Token 可能过期, 可尝试重新生成下。 |
|
||||
| **[StartVoiceChat]Failed(Reason: The task has been started. Please do not call the startup task interface repeatedly.)** 报错 | 由于目前设置的 RoomId、UserId 为固定值，重复调用 startAudioBot 会导致出错，只需先调用 stopAudioBot 后再重新 startAudioBot 即可。 |
|
||||
| 为什么我的麦克风正常、摄像头也正常，但是设备没有正常工作? | 可能是设备权限未授予，详情可参考 [Web 排查设备权限获取失败问题](https://www.volcengine.com/docs/6348/1356355?s=g)。 |
|
||||
| 接口调用时, 返回 "Invalid 'Authorization' header, Pls check your authorization header" 错误 | `Server/app.js` 中的 AK/SK/SessionToken 不正确 |
|
||||
| 什么是 RTC | **R**eal **T**ime **C**ommunication, RTC 的概念可参考[官网文档](https://www.volcengine.com/docs/6348/66812?s=g)。 |
|
||||
@ -57,3 +58,12 @@ yarn dev
|
||||
- [场景介绍](https://www.volcengine.com/docs/6348/1310537?s=g)
|
||||
- [Demo 体验](https://www.volcengine.com/docs/6348/1310559?s=g)
|
||||
- [场景搭建方案](https://www.volcengine.com/docs/6348/1310560?s=g)
|
||||
|
||||
## 更新日志
|
||||
|
||||
### [1.5.0] - [2025-03-31]
|
||||
- 修复部分 UI 问题
|
||||
- 追加屏幕共享能力 (视觉模型可用，**读屏助手** 人设下可使用)
|
||||
- 修改字幕逻辑，避免字幕回调中标点符号、大小写不一致引起的字幕重复问题
|
||||
- 更新 RTC Web SDK 版本至 4.66.1
|
||||
- 追加设备权限未授予时的提示
|
||||
@ -1,11 +1,11 @@
|
||||
{
|
||||
"name": "aigc",
|
||||
"version": "1.4.0",
|
||||
"version": "1.5.0",
|
||||
"license": "BSD-3-Clause",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@reduxjs/toolkit": "^1.8.3",
|
||||
"@volcengine/rtc": "4.58.9",
|
||||
"@volcengine/rtc": "4.66.1",
|
||||
"@arco-design/web-react": "^2.65.0",
|
||||
"autolinker": "^4.0.0",
|
||||
"i18next": "^21.8.16",
|
||||
|
||||
@ -63,6 +63,6 @@ export const resultHandler = (res: any) => {
|
||||
const error = ResponseMetadata?.Error?.Message || Result;
|
||||
Modal.error({
|
||||
title: '接口调用错误',
|
||||
content: `[${ResponseMetadata?.Action}]Failed(Reason: ${error})`,
|
||||
content: `[${ResponseMetadata?.Action}]Failed(Reason: ${error}), 请参考 README 文档排查问题。`,
|
||||
});
|
||||
};
|
||||
|
||||
BIN
src/assets/img/SCREEN_READER.png
Normal file
BIN
src/assets/img/SCREEN_READER.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 51 KiB |
5
src/assets/img/ScreenCloseNote.svg
Normal file
5
src/assets/img/ScreenCloseNote.svg
Normal file
@ -0,0 +1,5 @@
|
||||
<svg class="icon" width="48" height="48" style="color: #A2AFC3; vertical-align: middle;fill: currentColor;overflow: hidden;" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8403">
|
||||
<rect y="1024" width="1024" height="1024" rx="4096" transform="rotate(-90 0 1024)" fill="#DEE3E9" fill-opacity="0.4" x="20"/>
|
||||
<path d="M171.677538 898.993231a39.384615 39.384615 0 0 1-27.844923-67.229539L866.776615 97.122462a39.384615 39.384615 0 0 1 55.689847 55.689846L199.522462 887.453538a39.384615 39.384615 0 0 1-27.844924 11.539693z" fill="rgb(243, 247, 255)" p-id="8404" transform="scale(0.55) translate(410 460)"/>
|
||||
<path d="M755.830154 906.043077H256.945231a35.446154 35.446154 0 0 1-20.716308-6.656l64.630154-64.630154h454.971077a35.643077 35.643077 0 0 1 0 71.286154z m-606.523077-142.454154a71.404308 71.404308 0 0 1-70.498462-71.207385V228.627692A71.325538 71.325538 0 0 1 150.055385 157.380923h605.144615L149.346462 763.155692z m713.491692 0H372.145231L930.107077 205.627077a70.892308 70.892308 0 0 1 3.938461 23.394461v463.281231a71.325538 71.325538 0 0 1-71.364923 71.010462z" fill="rgb(243, 247, 255)" p-id="8405" transform="scale(0.5) translate(530 530)"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
5
src/assets/img/ScreenOff.svg
Normal file
5
src/assets/img/ScreenOff.svg
Normal file
@ -0,0 +1,5 @@
|
||||
<svg class="icon" width="48" height="48" style="color: #666666; vertical-align: middle;fill: currentColor;overflow: hidden;" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="8403">
|
||||
<rect y="1024" width="1024" height="1024" rx="4096" transform="rotate(-90 0 1024)" fill="#DEE3E9" fill-opacity="0.4" x="20"/>
|
||||
<path d="M171.677538 898.993231a39.384615 39.384615 0 0 1-27.844923-67.229539L866.776615 97.122462a39.384615 39.384615 0 0 1 55.689847 55.689846L199.522462 887.453538a39.384615 39.384615 0 0 1-27.844924 11.539693z" fill="#EC3528" p-id="8404" transform="scale(0.55) translate(410 460)"/>
|
||||
<path d="M755.830154 906.043077H256.945231a35.446154 35.446154 0 0 1-20.716308-6.656l64.630154-64.630154h454.971077a35.643077 35.643077 0 0 1 0 71.286154z m-606.523077-142.454154a71.404308 71.404308 0 0 1-70.498462-71.207385V228.627692A71.325538 71.325538 0 0 1 150.055385 157.380923h605.144615L149.346462 763.155692z m713.491692 0H372.145231L930.107077 205.627077a70.892308 70.892308 0 0 1 3.938461 23.394461v463.281231a71.325538 71.325538 0 0 1-71.364923 71.010462z" fill="#F53F3F" p-id="8405" transform="scale(0.5) translate(530 530)"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
4
src/assets/img/ScreenOn.svg
Normal file
4
src/assets/img/ScreenOn.svg
Normal file
@ -0,0 +1,4 @@
|
||||
<svg class="icon" width="48" height="48" style="color: #666666; vertical-align: middle;fill: currentColor;overflow: hidden;" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg" p-id="11415">
|
||||
<rect y="1024" width="1024" height="1024" rx="4096" transform="rotate(-90 0 1024)" fill="#DEE3E9" fill-opacity="0.4" x="20"/>
|
||||
<path d="M191.872 728.224h640.288c58.176 0 88.384-29.248 88.384-88.736V269.216c0-59.456-30.208-88.704-88.384-88.704H191.808c-58.496 0-88.704 29.248-88.704 88.704v370.272c0 59.488 30.208 88.736 88.736 88.736z m320.448-94.112c-17.344 0-30.528-12.224-30.528-29.568V448l3.2-68.48-29.568 37.312-37.28 38.88a28.128 28.128 0 0 1-20.576 8.672c-16.064 0-27.648-11.552-27.648-27.328 0-8.32 2.24-14.432 8.032-20.544l110.272-111.872c7.68-7.712 14.784-11.232 24.096-11.232 9.632 0 16.704 3.84 24.096 11.232l110.272 111.872c5.76 6.08 8.32 12.192 8.32 20.576 0 15.744-11.84 27.296-27.936 27.296a28.16 28.16 0 0 1-20.576-8.64l-36.96-38.592-30.208-37.952 3.2 68.8v156.544c0 17.344-12.864 29.568-30.208 29.568z m163.936 206.272H347.424a29.088 29.088 0 1 1 0-58.176h328.832c16.064 0 29.248 13.184 29.248 29.248a29.28 29.28 0 0 1-29.248 28.928z" p-id="11416" transform="scale(0.55) translate(410 460)"></path>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
@ -152,60 +152,6 @@
|
||||
}
|
||||
|
||||
|
||||
|
||||
.button {
|
||||
position: relative;
|
||||
width: max-content !important;
|
||||
height: 24px !important;
|
||||
margin-top: 8px;
|
||||
border-radius: 4px !important;
|
||||
font-size: 12px !important;
|
||||
background: linear-gradient(77.86deg, rgba(229, 242, 255, 0.5) -3.23%, rgba(217, 229, 255, 0.5) 51.11%, rgba(246, 226, 255, 0.5) 98.65%);
|
||||
cursor: pointer;
|
||||
|
||||
.button-text {
|
||||
background: linear-gradient(77.86deg, #3384FF -3.23%, #014BDE 51.11%, #A945FB 98.65%);
|
||||
-webkit-background-clip: text;
|
||||
background-clip: text;
|
||||
color: transparent;
|
||||
font-weight: 500;
|
||||
line-height: 20px;
|
||||
text-align: center;
|
||||
}
|
||||
}
|
||||
|
||||
.button::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
border-radius: 3px;
|
||||
top: 0px;
|
||||
left: 0px;
|
||||
width: 100%;
|
||||
height: 22px;
|
||||
background: white;
|
||||
z-index: -1;
|
||||
}
|
||||
|
||||
.button::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
border-radius: 5px;
|
||||
top: -2px;
|
||||
left: -2px;
|
||||
width: calc(100% + 4px);
|
||||
height: 26px;
|
||||
background: linear-gradient(90deg, rgba(0, 139, 255, 0.5) 0%, rgba(0, 98, 255, 0.5) 49.5%, rgba(207, 92, 255, 0.5) 100%);
|
||||
z-index: -2;
|
||||
}
|
||||
|
||||
.button:hover {
|
||||
background: linear-gradient(77.86deg, rgba(200, 220, 255, 0.7) -3.23%, rgba(190, 210, 255, 0.7) 51.11%, rgba(230, 210, 255, 0.7) 98.65%);
|
||||
}
|
||||
|
||||
.button:active {
|
||||
background: linear-gradient(77.86deg, rgba(170, 190, 255, 0.9) -3.23%, rgba(160, 180, 255, 0.9) 51.11%, rgba(210, 180, 255, 0.9) 98.65%);
|
||||
}
|
||||
|
||||
.footer {
|
||||
width: calc(100% - 12px);
|
||||
display: flex;
|
||||
|
||||
@ -7,6 +7,7 @@ import { Button, Drawer, Input, Message } from '@arco-design/web-react';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import { IconSwap } from '@arco-design/web-react/icon';
|
||||
import { StreamIndex } from '@volcengine/rtc';
|
||||
import CheckIcon from '../CheckIcon';
|
||||
import Config, {
|
||||
Icon,
|
||||
@ -20,6 +21,7 @@ import Config, {
|
||||
ModelSourceType,
|
||||
VOICE_INFO_MAP,
|
||||
VOICE_TYPE,
|
||||
isVisionMode,
|
||||
} from '@/config';
|
||||
import TitleCard from '../TitleCard';
|
||||
import CheckBoxSelector from '@/components/CheckBoxSelector';
|
||||
@ -28,15 +30,22 @@ import { clearHistoryMsg, updateAIConfig, updateScene } from '@/store/slices/roo
|
||||
import { RootState } from '@/store';
|
||||
import utils from '@/utils/utils';
|
||||
import { useDeviceState } from '@/lib/useCommon';
|
||||
|
||||
import VoiceTypeChangeSVG from '@/assets/img/VoiceTypeChange.svg';
|
||||
import DoubaoModelSVG from '@/assets/img/DoubaoModel.svg';
|
||||
import ModelChangeSVG from '@/assets/img/ModelChange.svg';
|
||||
import styles from './index.module.less';
|
||||
|
||||
export interface IAISettingsProps {
|
||||
open: boolean;
|
||||
onOk?: () => void;
|
||||
onCancel?: () => void;
|
||||
}
|
||||
|
||||
const SCENES = [
|
||||
SCENE.INTELLIGENT_ASSISTANT,
|
||||
SCENE.SCREEN_READER,
|
||||
SCENE.VIRTUAL_GIRL_FRIEND,
|
||||
// SCENE.TEACHER,
|
||||
SCENE.TRANSLATE,
|
||||
SCENE.CHILDREN_ENCYCLOPEDIA,
|
||||
SCENE.CUSTOMER_SERVICE,
|
||||
@ -44,13 +53,13 @@ const SCENES = [
|
||||
SCENE.CUSTOM,
|
||||
];
|
||||
|
||||
function AISettings() {
|
||||
function AISettings({ open, onCancel, onOk }: IAISettingsProps) {
|
||||
const dispatch = useDispatch();
|
||||
const { isVideoPublished, switchCamera } = useDeviceState();
|
||||
const { isVideoPublished, isScreenPublished, switchScreenCapture, switchCamera } =
|
||||
useDeviceState();
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [use3Part, setUse3Part] = useState(false);
|
||||
const [open, setOpen] = useState(false);
|
||||
const [scene, setScene] = useState(room.scene);
|
||||
const [data, setData] = useState({
|
||||
prompt: Prompt[scene],
|
||||
@ -63,10 +72,6 @@ function AISettings() {
|
||||
customModelName: '',
|
||||
});
|
||||
|
||||
const handleClick = () => {
|
||||
setOpen(true);
|
||||
};
|
||||
|
||||
const handleVoiceTypeChanged = (key: string) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
@ -116,18 +121,33 @@ function AISettings() {
|
||||
Config.WelcomeSpeech = data.welcome;
|
||||
dispatch(updateAIConfig(Config.aigcConfig));
|
||||
|
||||
if (isVisionMode(data.model)) {
|
||||
switch (scene) {
|
||||
case SCENE.SCREEN_READER:
|
||||
/** 关摄像头，打开屏幕采集 */
|
||||
room.isJoined && isVideoPublished && switchCamera();
|
||||
Config.VisionSourceType = StreamIndex.STREAM_INDEX_SCREEN;
|
||||
break;
|
||||
default:
|
||||
/** 关屏幕采集，打开摄像头 */
|
||||
room.isJoined && !isVideoPublished && switchCamera();
|
||||
room.isJoined && isScreenPublished && switchScreenCapture();
|
||||
Config.VisionSourceType = StreamIndex.STREAM_INDEX_MAIN;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
/** 全关 */
|
||||
room.isJoined && isVideoPublished && switchCamera();
|
||||
room.isJoined && isScreenPublished && switchScreenCapture();
|
||||
}
|
||||
|
||||
if (RtcClient.getAudioBotEnabled()) {
|
||||
dispatch(clearHistoryMsg());
|
||||
await RtcClient.updateAudioBot();
|
||||
}
|
||||
if (data.model === AI_MODEL.VISION) {
|
||||
room.isJoined && !isVideoPublished && switchCamera(true);
|
||||
} else {
|
||||
room.isJoined && isVideoPublished && switchCamera(true);
|
||||
}
|
||||
|
||||
setLoading(false);
|
||||
setOpen(false);
|
||||
onOk?.();
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
@ -137,193 +157,193 @@ function AISettings() {
|
||||
}, [open]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Button className={styles.button} onClick={handleClick}>
|
||||
<div className={styles['button-text']}>修改 AI 设定</div>
|
||||
</Button>
|
||||
<Drawer
|
||||
width={utils.isMobile() ? '100%' : 870}
|
||||
closable={false}
|
||||
maskClosable={false}
|
||||
title={null}
|
||||
className={styles.container}
|
||||
style={{
|
||||
padding: utils.isMobile() ? '0px' : '16px 8px',
|
||||
}}
|
||||
footer={
|
||||
<div className={styles.footer}>
|
||||
<div className={styles.suffix}>AI 配置修改后，退出房间将不再保存该配置方案</div>
|
||||
<Button loading={loading} className={styles.cancel} onClick={() => setOpen(false)}>
|
||||
取消
|
||||
</Button>
|
||||
<Button loading={loading} className={styles.confirm} onClick={handleUpdateConfig}>
|
||||
确定
|
||||
</Button>
|
||||
</div>
|
||||
}
|
||||
visible={open}
|
||||
onCancel={() => setOpen(false)}
|
||||
>
|
||||
<div className={styles.title}>
|
||||
选择你所需要的
|
||||
<span className={styles['special-text']}> AI 人设</span>
|
||||
<Drawer
|
||||
width={utils.isMobile() ? '100%' : 940}
|
||||
closable={false}
|
||||
maskClosable={false}
|
||||
title={null}
|
||||
className={styles.container}
|
||||
style={{
|
||||
padding: utils.isMobile() ? '0px' : '16px 8px',
|
||||
}}
|
||||
footer={
|
||||
<div className={styles.footer}>
|
||||
<div className={styles.suffix}>AI 配置修改后，退出房间将不再保存该配置方案</div>
|
||||
<Button loading={loading} className={styles.cancel} onClick={onCancel}>
|
||||
取消
|
||||
</Button>
|
||||
<Button loading={loading} className={styles.confirm} onClick={handleUpdateConfig}>
|
||||
确定
|
||||
</Button>
|
||||
</div>
|
||||
<div className={styles['sub-title']}>
|
||||
我们已为您配置好对应人设的基本参数，您也可以根据自己的需求进行自定义设置
|
||||
</div>
|
||||
<div className={utils.isMobile() ? styles['scenes-mobile'] : styles.scenes}>
|
||||
{SCENES.map((key) => (
|
||||
}
|
||||
visible={open}
|
||||
onCancel={onCancel}
|
||||
>
|
||||
<div className={styles.title}>
|
||||
选择你所需要的
|
||||
<span className={styles['special-text']}> AI 人设</span>
|
||||
</div>
|
||||
<div className={styles['sub-title']}>
|
||||
我们已为您配置好对应人设的基本参数，您也可以根据自己的需求进行自定义设置
|
||||
</div>
|
||||
<div className={utils.isMobile() ? styles['scenes-mobile'] : styles.scenes}>
|
||||
{[...SCENES, null].map((key) =>
|
||||
key ? (
|
||||
<CheckIcon
|
||||
key={key}
|
||||
tag={key === SCENE.TEACHING_ASSISTANT ? '视觉理解模型' : ''}
|
||||
tag={
|
||||
[SCENE.TEACHING_ASSISTANT, SCENE.SCREEN_READER].includes(key) ? '视觉理解模型' : ''
|
||||
}
|
||||
icon={Icon[key as keyof typeof Icon]}
|
||||
title={Name[key as keyof typeof Name]}
|
||||
checked={key === scene}
|
||||
blur={key !== scene && key === SCENE.CUSTOM}
|
||||
onClick={() => handleChecked(key as SCENE)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<div className={styles.configuration}>
|
||||
{utils.isMobile() ? null : (
|
||||
<div
|
||||
className={styles.anchor}
|
||||
style={{
|
||||
/**
|
||||
* @note 单个场景卡片 100px, 间距 14px;
|
||||
*/
|
||||
left: `${SCENES.indexOf(scene) * 100 + 50 + SCENES.indexOf(scene) * 14}px`,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
<TitleCard title="Prompt">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.prompt}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
prompt: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入你需要的 Prompt 设定"
|
||||
/>
|
||||
</TitleCard>
|
||||
<TitleCard title="欢迎语">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.welcome}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
welcome: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入欢迎语"
|
||||
/>
|
||||
</TitleCard>
|
||||
) : utils.isMobile() ? (
|
||||
<div style={{ width: '100px', height: '100px' }} />
|
||||
) : null
|
||||
)}
|
||||
</div>
|
||||
<div className={styles.configuration}>
|
||||
{utils.isMobile() ? null : (
|
||||
<div
|
||||
className={styles['ai-settings']}
|
||||
className={styles.anchor}
|
||||
style={{
|
||||
flexWrap: utils.isMobile() ? 'wrap' : 'nowrap',
|
||||
/**
|
||||
* @note 单个场景卡片 100px, 间距 14px;
|
||||
*/
|
||||
left: `${SCENES.indexOf(scene) * 100 + 50 + SCENES.indexOf(scene) * 14}px`,
|
||||
}}
|
||||
>
|
||||
<TitleCard title="音色">
|
||||
<div className={styles['ai-settings-wrapper']}>
|
||||
<CheckBoxSelector
|
||||
label="音色选择"
|
||||
data={Object.keys(VOICE_TYPE).map((type) => {
|
||||
const info = VOICE_INFO_MAP[VOICE_TYPE[type as keyof typeof VOICE_TYPE]];
|
||||
return {
|
||||
key: VOICE_TYPE[type as keyof typeof VOICE_TYPE],
|
||||
label: type,
|
||||
icon: info.icon,
|
||||
description: info.description,
|
||||
};
|
||||
})}
|
||||
onChange={handleVoiceTypeChanged}
|
||||
value={data.voice}
|
||||
moreIcon={VoiceTypeChangeSVG}
|
||||
moreText="更换音色"
|
||||
placeHolder="请选择你需要的音色"
|
||||
/>
|
||||
</div>
|
||||
</TitleCard>
|
||||
<div className={styles['ai-settings-model']}>
|
||||
{use3Part ? (
|
||||
<>
|
||||
<TitleCard required title="第三方模型地址">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.Url}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
Url: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入第三方模型地址"
|
||||
/>
|
||||
</TitleCard>
|
||||
<TitleCard title="请求密钥">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.APIKey}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
APIKey: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入请求密钥"
|
||||
/>
|
||||
</TitleCard>
|
||||
<TitleCard title="模型名称">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.customModelName}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
customModelName: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入模型名称"
|
||||
/>
|
||||
</TitleCard>
|
||||
</>
|
||||
) : (
|
||||
<TitleCard title="官方模型">
|
||||
<CheckBoxSelector
|
||||
label="模型选择"
|
||||
data={Object.keys(AI_MODEL)
|
||||
.map((type) => ({
|
||||
key: AI_MODEL[type as keyof typeof AI_MODEL],
|
||||
label: type.replaceAll('_', ' '),
|
||||
icon: DoubaoModelSVG,
|
||||
}))}
|
||||
moreIcon={ModelChangeSVG}
|
||||
moreText="更换模型"
|
||||
placeHolder="请选择你需要的模型"
|
||||
onChange={(key) => {
|
||||
/>
|
||||
)}
|
||||
<TitleCard title="Prompt">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.prompt}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
prompt: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入你需要的 Prompt 设定"
|
||||
/>
|
||||
</TitleCard>
|
||||
<TitleCard title="欢迎语">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.welcome}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
welcome: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入欢迎语"
|
||||
/>
|
||||
</TitleCard>
|
||||
<div
|
||||
className={styles['ai-settings']}
|
||||
style={{
|
||||
flexWrap: utils.isMobile() ? 'wrap' : 'nowrap',
|
||||
}}
|
||||
>
|
||||
<TitleCard title="音色">
|
||||
<div className={styles['ai-settings-wrapper']}>
|
||||
<CheckBoxSelector
|
||||
label="音色选择"
|
||||
data={Object.keys(VOICE_TYPE).map((type) => {
|
||||
const info = VOICE_INFO_MAP[VOICE_TYPE[type as keyof typeof VOICE_TYPE]];
|
||||
return {
|
||||
key: VOICE_TYPE[type as keyof typeof VOICE_TYPE],
|
||||
label: type,
|
||||
icon: info.icon,
|
||||
description: info.description,
|
||||
};
|
||||
})}
|
||||
onChange={handleVoiceTypeChanged}
|
||||
value={data.voice}
|
||||
moreIcon={VoiceTypeChangeSVG}
|
||||
moreText="更换音色"
|
||||
placeHolder="请选择你需要的音色"
|
||||
/>
|
||||
</div>
|
||||
</TitleCard>
|
||||
<div className={styles['ai-settings-model']}>
|
||||
{use3Part ? (
|
||||
<>
|
||||
<TitleCard required title="第三方模型地址">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.Url}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
model: key as AI_MODEL,
|
||||
Url: val,
|
||||
}));
|
||||
}}
|
||||
value={data.model}
|
||||
placeholder="请输入第三方模型地址"
|
||||
/>
|
||||
</TitleCard>
|
||||
)}
|
||||
<TitleCard title="请求密钥">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.APIKey}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
APIKey: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入请求密钥"
|
||||
/>
|
||||
</TitleCard>
|
||||
<TitleCard title="模型名称">
|
||||
<Input.TextArea
|
||||
autoSize
|
||||
value={data.customModelName}
|
||||
onChange={(val) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
customModelName: val,
|
||||
}));
|
||||
}}
|
||||
placeholder="请输入模型名称"
|
||||
/>
|
||||
</TitleCard>
|
||||
</>
|
||||
) : (
|
||||
<TitleCard title="官方模型">
|
||||
<CheckBoxSelector
|
||||
label="模型选择"
|
||||
data={Object.keys(AI_MODEL).map((type) => ({
|
||||
key: AI_MODEL[type as keyof typeof AI_MODEL],
|
||||
label: type.replaceAll('_', ' '),
|
||||
icon: DoubaoModelSVG,
|
||||
}))}
|
||||
moreIcon={ModelChangeSVG}
|
||||
moreText="更换模型"
|
||||
placeHolder="请选择你需要的模型"
|
||||
onChange={(key) => {
|
||||
setData((prev) => ({
|
||||
...prev,
|
||||
model: key as AI_MODEL,
|
||||
}));
|
||||
}}
|
||||
value={data.model}
|
||||
/>
|
||||
</TitleCard>
|
||||
)}
|
||||
|
||||
<Button size="mini" type="text" onClick={handleUseThirdPart}>
|
||||
{use3Part ? '使用官方模型' : '使用第三方模型'} <IconSwap />
|
||||
</Button>
|
||||
</div>
|
||||
<Button size="mini" type="text" onClick={handleUseThirdPart}>
|
||||
{use3Part ? '使用官方模型' : '使用第三方模型'} <IconSwap />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</Drawer>
|
||||
</>
|
||||
</div>
|
||||
</Drawer>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@ -137,4 +137,57 @@
|
||||
background-color: #EAEDF1;
|
||||
transform: rotate(-90deg);
|
||||
}
|
||||
}
|
||||
|
||||
.button {
|
||||
position: relative;
|
||||
width: max-content !important;
|
||||
height: 24px !important;
|
||||
margin-top: 8px;
|
||||
border-radius: 4px !important;
|
||||
font-size: 12px !important;
|
||||
background: linear-gradient(77.86deg, rgba(229, 242, 255, 0.5) -3.23%, rgba(217, 229, 255, 0.5) 51.11%, rgba(246, 226, 255, 0.5) 98.65%);
|
||||
cursor: pointer;
|
||||
|
||||
.button-text {
|
||||
background: linear-gradient(77.86deg, #3384FF -3.23%, #014BDE 51.11%, #A945FB 98.65%);
|
||||
-webkit-background-clip: text;
|
||||
background-clip: text;
|
||||
color: transparent;
|
||||
font-weight: 500;
|
||||
line-height: 20px;
|
||||
text-align: center;
|
||||
}
|
||||
}
|
||||
|
||||
.button::after {
|
||||
content: '';
|
||||
position: absolute;
|
||||
border-radius: 3px;
|
||||
top: 0px;
|
||||
left: 0px;
|
||||
width: 100%;
|
||||
height: 22px;
|
||||
background: white;
|
||||
z-index: -1;
|
||||
}
|
||||
|
||||
.button::before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
border-radius: 5px;
|
||||
top: -2px;
|
||||
left: -2px;
|
||||
width: calc(100% + 4px);
|
||||
height: 26px;
|
||||
background: linear-gradient(90deg, rgba(0, 139, 255, 0.5) 0%, rgba(0, 98, 255, 0.5) 49.5%, rgba(207, 92, 255, 0.5) 100%);
|
||||
z-index: -2;
|
||||
}
|
||||
|
||||
.button:hover {
|
||||
background: linear-gradient(77.86deg, rgba(200, 220, 255, 0.7) -3.23%, rgba(190, 210, 255, 0.7) 51.11%, rgba(230, 210, 255, 0.7) 98.65%);
|
||||
}
|
||||
|
||||
.button:active {
|
||||
background: linear-gradient(77.86deg, rgba(170, 190, 255, 0.9) -3.23%, rgba(160, 180, 255, 0.9) 51.11%, rgba(210, 180, 255, 0.9) 98.65%);
|
||||
}
|
||||
@ -4,6 +4,8 @@
|
||||
*/
|
||||
|
||||
import { useSelector } from 'react-redux';
|
||||
import { Button } from '@arco-design/web-react';
|
||||
import { useState } from 'react';
|
||||
import AISettings from '../AISettings';
|
||||
import style from './index.module.less';
|
||||
import DouBaoAvatar from '@/assets/img/DoubaoAvatarGIF.webp';
|
||||
@ -14,16 +16,24 @@ interface IAvatarCardProps extends React.HTMLAttributes<HTMLDivElement> {
|
||||
avatar?: string;
|
||||
}
|
||||
|
||||
const ReversedVoiceType = Object.entries(VOICE_TYPE).reduce<Record<string, string>>((acc, [key, value]) => {
|
||||
acc[value] = key;
|
||||
return acc;
|
||||
}, {});
|
||||
const ReversedVoiceType = Object.entries(VOICE_TYPE).reduce<Record<string, string>>(
|
||||
(acc, [key, value]) => {
|
||||
acc[value] = key;
|
||||
return acc;
|
||||
},
|
||||
{}
|
||||
);
|
||||
|
||||
function AvatarCard(props: IAvatarCardProps) {
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
const [open, setOpen] = useState(false);
|
||||
const scene = room.scene;
|
||||
const { LLMConfig, TTSConfig } = room.aiConfig.Config || {};
|
||||
const { avatar, className, ...rest } = props;
|
||||
const voice = TTSConfig.ProviderParams.audio.voice_type;
|
||||
|
||||
const handleOpenDrawer = () => setOpen(true);
|
||||
const handleCloseDrawer = () => setOpen(false);
|
||||
|
||||
return (
|
||||
<div className={`${style.card} ${className}`} {...rest}>
|
||||
@ -40,11 +50,12 @@ function AvatarCard(props: IAvatarCardProps) {
|
||||
<div className={style['text-wrapper']}>
|
||||
<div className={style['user-info']}>
|
||||
<div className={style.title}>{Name[scene]}</div>
|
||||
<div className={style.description}>
|
||||
声源来自 {ReversedVoiceType[TTSConfig?.VoiceType || '']}
|
||||
</div>
|
||||
<div className={style.description}>声源来自 {ReversedVoiceType[voice || '']}</div>
|
||||
<div className={style.description}>模型 {LLMConfig.ModelName}</div>
|
||||
<AISettings />
|
||||
<AISettings open={open} onOk={handleCloseDrawer} onCancel={handleCloseDrawer} />
|
||||
<Button className={style.button} onClick={handleOpenDrawer}>
|
||||
<div className={style['button-text']}>修改 AI 设定</div>
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -11,6 +11,7 @@ import TRANSLATE from '@/assets/img/TRANSLATE.png';
|
||||
import CHILDREN_ENCYCLOPEDIA from '@/assets/img/CHILDREN_ENCYCLOPEDIA.png';
|
||||
import TEACHING_ASSISTANT from '@/assets/img/TEACHING_ASSISTANT.png';
|
||||
import CUSTOMER_SERVICE from '@/assets/img/CUSTOMER_SERVICE.png';
|
||||
import SCREEN_READER from '@/assets/img/SCREEN_READER.png';
|
||||
|
||||
export enum ModelSourceType {
|
||||
Custom = 'Custom',
|
||||
@ -130,9 +131,12 @@ export enum SCENE {
|
||||
CUSTOMER_SERVICE = 'CUSTOMER_SERVICE',
|
||||
CHILDREN_ENCYCLOPEDIA = 'CHILDREN_ENCYCLOPEDIA',
|
||||
TEACHING_ASSISTANT = 'TEACHING_ASSISTANT',
|
||||
SCREEN_READER = 'SCREEN_READER',
|
||||
CUSTOM = 'CUSTOM',
|
||||
}
|
||||
|
||||
export const ScreenShareScene = [SCENE.SCREEN_READER];
|
||||
|
||||
export const Icon = {
|
||||
[SCENE.INTELLIGENT_ASSISTANT]: INTELLIGENT_ASSISTANT,
|
||||
[SCENE.VIRTUAL_GIRL_FRIEND]: VIRTUAL_GIRL_FRIEND,
|
||||
@ -140,6 +144,7 @@ export const Icon = {
|
||||
[SCENE.CHILDREN_ENCYCLOPEDIA]: CHILDREN_ENCYCLOPEDIA,
|
||||
[SCENE.CUSTOMER_SERVICE]: CUSTOMER_SERVICE,
|
||||
[SCENE.TEACHING_ASSISTANT]: TEACHING_ASSISTANT,
|
||||
[SCENE.SCREEN_READER]: SCREEN_READER,
|
||||
[SCENE.CUSTOM]: INTELLIGENT_ASSISTANT,
|
||||
};
|
||||
|
||||
@ -150,6 +155,7 @@ export const Name = {
|
||||
[SCENE.CHILDREN_ENCYCLOPEDIA]: '儿童百科',
|
||||
[SCENE.CUSTOMER_SERVICE]: '售后客服',
|
||||
[SCENE.TEACHING_ASSISTANT]: '课后助教',
|
||||
[SCENE.SCREEN_READER]: '读屏助手',
|
||||
[SCENE.CUSTOM]: '自定义',
|
||||
};
|
||||
|
||||
@ -163,6 +169,7 @@ export const Welcome = {
|
||||
[SCENE.CHILDREN_ENCYCLOPEDIA]: '你好小朋友，你的小脑袋里又有什么问题啦？',
|
||||
[SCENE.CUSTOMER_SERVICE]: '感谢您在我们餐厅用餐，请问您有什么问题需要反馈吗？',
|
||||
[SCENE.TEACHING_ASSISTANT]: '你碰到什么问题啦？让我来帮帮你。',
|
||||
[SCENE.SCREEN_READER]: '欢迎使用读屏助手, 请开启屏幕采集，我会为你解说屏幕内容。',
|
||||
[SCENE.CUSTOM]: '',
|
||||
};
|
||||
|
||||
@ -173,6 +180,7 @@ export const Model = {
|
||||
[SCENE.CHILDREN_ENCYCLOPEDIA]: AI_MODEL.DOUBAO_PRO_32K,
|
||||
[SCENE.CUSTOMER_SERVICE]: AI_MODEL.DOUBAO_PRO_32K,
|
||||
[SCENE.TEACHING_ASSISTANT]: AI_MODEL.VISION,
|
||||
[SCENE.SCREEN_READER]: AI_MODEL.VISION,
|
||||
[SCENE.CUSTOM]: AI_MODEL.DOUBAO_PRO_32K,
|
||||
};
|
||||
|
||||
@ -183,6 +191,7 @@ export const Voice = {
|
||||
[SCENE.CHILDREN_ENCYCLOPEDIA]: VOICE_TYPE.通用女声,
|
||||
[SCENE.CUSTOMER_SERVICE]: VOICE_TYPE.通用女声,
|
||||
[SCENE.TEACHING_ASSISTANT]: VOICE_TYPE.通用女声,
|
||||
[SCENE.SCREEN_READER]: VOICE_TYPE.通用男声,
|
||||
[SCENE.CUSTOM]: VOICE_TYPE.通用女声,
|
||||
};
|
||||
|
||||
@ -213,6 +222,7 @@ export const Questions = {
|
||||
'你们空调开得太冷了。',
|
||||
],
|
||||
[SCENE.TEACHING_ASSISTANT]: ['这个单词是什么意思？', '这道题该怎么做？', '我的表情是什么样的？'],
|
||||
[SCENE.SCREEN_READER]: ['屏幕里这是什么?', '这道题你会做吗?', '帮我翻译解说下屏幕里的内容?'],
|
||||
[SCENE.CUSTOM]: ['你能帮我解决什么问题?', '今天北京天气怎么样?', '你喜欢哪位流行歌手?'],
|
||||
};
|
||||
|
||||
@ -299,5 +309,23 @@ export const Prompt = {
|
||||
##约束
|
||||
- 回答问题要简明扼要，避免复杂冗长的表述，尽量不超过50个字；
|
||||
- 回答中不要有“图片”、“图中”等相关字眼；`,
|
||||
[SCENE.SCREEN_READER]: `##人设
|
||||
你是人们的 AI 伙伴，可以通过 【屏幕共享实时解析】+【百科知识】来为人们提供服务。
|
||||
|
||||
##技能
|
||||
1. 实时理解屏幕中的内容，包括图片、文字、窗口焦点，自动捕捉光标轨迹；
|
||||
2. 拥有丰富的百科知识；
|
||||
3. 如果用户询问与视频和图片有关的问题，请结合【屏幕共享实时解析】的内容、你的【知识】和【用户问题】进行回答；
|
||||
|
||||
##风格
|
||||
语言风格可以随着屏幕内容和用户需求调整，可以是幽默搞笑的娱乐解说，也可以是严谨硬核的技术分析。
|
||||
- 如果屏幕内容是娱乐节目、动画、游戏等，语言风格偏幽默、活波一些，可以使用夸张的比喻、流行梗、弹幕互动式语言；
|
||||
- 如果屏幕内容是办公软件、新闻、文章等，语言风格偏专业、正经一些。
|
||||
|
||||
## 约束
|
||||
不要有任何特殊标点符号和任何 Markdown 格式输出，例如 *，# 等。
|
||||
`,
|
||||
[SCENE.CUSTOM]: '',
|
||||
};
|
||||
|
||||
export const isVisionMode = (model: AI_MODEL) => model.startsWith('Vision');
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
import { StreamIndex } from '@volcengine/rtc';
|
||||
import {
|
||||
TTS_CLUSTER,
|
||||
ARK_V3_MODEL_ID,
|
||||
@ -16,6 +17,7 @@ import {
|
||||
AI_MODEL,
|
||||
AI_MODE_MAP,
|
||||
AI_MODEL_MODE,
|
||||
isVisionMode,
|
||||
} from '.';
|
||||
|
||||
export const CONVERSATION_SIGNATURE = 'conversation';
|
||||
@ -37,6 +39,7 @@ export class ConfigFactory {
|
||||
BusinessId: undefined,
|
||||
/**
|
||||
* @brief 必填, 房间 ID, 自定义即可，例如 "Room123"。
|
||||
* @note 建议使用有特定规则、不重复的房间号名称。
|
||||
*/
|
||||
RoomId: 'Room123',
|
||||
/**
|
||||
@ -59,7 +62,7 @@ export class ConfigFactory {
|
||||
TTSAppId: 'Your TTS AppId',
|
||||
/**
|
||||
* @brief 已开通需要的语音合成服务的token。
|
||||
* 使用火山引擎双向流式语音合成服务时必填。
|
||||
* 使用火山引擎双向流式语音合成服务时 必填。
|
||||
*/
|
||||
TTSToken: undefined,
|
||||
/**
|
||||
@ -69,7 +72,7 @@ export class ConfigFactory {
|
||||
ASRAppId: 'Your ASR AppId',
|
||||
/**
|
||||
* @brief 已开通流式语音识别大模型服务 AppId 对应的 Access Token。
|
||||
* 使用流式语音识别大模型服务时该参数为必填。
|
||||
* 使用流式语音识别大模型服务时该参数为 必填。
|
||||
*/
|
||||
ASRToken: undefined,
|
||||
};
|
||||
@ -116,6 +119,11 @@ export class ConfigFactory {
|
||||
*/
|
||||
InterruptMode = true;
|
||||
|
||||
/**
|
||||
* @brief 如果使用视觉模型，用的是哪种源，有摄像头采集流/屏幕流
|
||||
*/
|
||||
VisionSourceType = StreamIndex.STREAM_INDEX_MAIN;
|
||||
|
||||
get LLMConfig() {
|
||||
const params: Record<string, unknown> = {
|
||||
Mode: AI_MODE_MAP[this.Model || ''] || AI_MODEL_MODE.CUSTOM,
|
||||
@ -134,16 +142,21 @@ export class ConfigFactory {
|
||||
Url: this.Url,
|
||||
Feature: JSON.stringify({ Http: true }),
|
||||
};
|
||||
if (this.Model === AI_MODEL.VISION) {
|
||||
if (isVisionMode(this.Model)) {
|
||||
params.VisionConfig = {
|
||||
Enable: true,
|
||||
SnapshotConfig: {
|
||||
StreamType: this.VisionSourceType,
|
||||
Height: 640,
|
||||
ImagesLimit: 1,
|
||||
},
|
||||
};
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
get ASRConfig() {
|
||||
return {
|
||||
const params: Record<string, any> = {
|
||||
Provider: 'volcano',
|
||||
ProviderParams: {
|
||||
/**
|
||||
@ -152,7 +165,6 @@ export class ConfigFactory {
|
||||
*/
|
||||
Mode: 'smallmodel',
|
||||
AppId: this.BaseConfig.ASRAppId,
|
||||
...(this.BaseConfig.ASRToken ? { AccessToken: this.BaseConfig.ASRToken } : {}),
|
||||
/**
|
||||
* @note 具体流式语音识别服务对应的 Cluster ID，可在流式语音服务控制台开通对应服务后查询。
|
||||
* 具体链接为: https://console.volcengine.com/speech/service/16?s=g
|
||||
@ -165,15 +177,18 @@ export class ConfigFactory {
|
||||
},
|
||||
VolumeGain: 0.3,
|
||||
};
|
||||
if (this.BaseConfig.ASRToken) {
|
||||
params.ProviderParams.AccessToken = this.BaseConfig.ASRToken;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
get TTSConfig() {
|
||||
return {
|
||||
const params: Record<string, any> = {
|
||||
Provider: 'volcano',
|
||||
ProviderParams: {
|
||||
app: {
|
||||
AppId: this.BaseConfig.TTSAppId,
|
||||
...(this.BaseConfig.TTSToken ? { Token: this.BaseConfig.TTSToken } : {}),
|
||||
Cluster: TTS_CLUSTER.TTS,
|
||||
},
|
||||
audio: {
|
||||
@ -183,6 +198,10 @@ export class ConfigFactory {
|
||||
},
|
||||
IgnoreBracketText: [1, 2, 3, 4, 5],
|
||||
};
|
||||
if (this.BaseConfig.TTSToken) {
|
||||
params.ProviderParams.app.Token = this.BaseConfig.TTSToken;
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
get aigcConfig() {
|
||||
|
||||
@ -3,7 +3,6 @@
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
|
||||
import VERTC, {
|
||||
MirrorType,
|
||||
StreamIndex,
|
||||
@ -23,8 +22,10 @@ import VERTC, {
|
||||
PlayerEvent,
|
||||
NetworkQuality,
|
||||
VideoRenderMode,
|
||||
ScreenEncoderConfig,
|
||||
} from '@volcengine/rtc';
|
||||
import RTCAIAnsExtension from '@volcengine/rtc/extension-ainr';
|
||||
import { Message } from '@arco-design/web-react';
|
||||
import openAPIs from '@/app/api';
|
||||
import aigcConfig from '@/config';
|
||||
import Utils from '@/utils/utils';
|
||||
@ -34,6 +35,7 @@ export interface IEventListener {
|
||||
handleError: (e: { errorCode: any }) => void;
|
||||
handleUserJoin: (e: onUserJoinedEvent) => void;
|
||||
handleUserLeave: (e: onUserLeaveEvent) => void;
|
||||
handleTrackEnded: (e: { kind: string; isScreen: boolean }) => void;
|
||||
handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void;
|
||||
handleUserUnpublishStream: (e: {
|
||||
userId: string;
|
||||
@ -45,7 +47,6 @@ export interface IEventListener {
|
||||
handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void;
|
||||
handleRemoteAudioPropertiesReport: (e: RemoteAudioPropertiesInfo[]) => void;
|
||||
handleAudioDeviceStateChanged: (e: DeviceInfo) => void;
|
||||
handleUserMessageReceived: (e: { userId: string; message: any }) => void;
|
||||
handleAutoPlayFail: (e: AutoPlayFailedEvent) => void;
|
||||
handlePlayerEvent: (e: PlayerEvent) => void;
|
||||
handleUserStartAudioCapture: (e: { userId: string }) => void;
|
||||
@ -103,7 +104,9 @@ export class RTCClient {
|
||||
await this.engine.registerExtension(AIAnsExtension);
|
||||
AIAnsExtension.enable();
|
||||
} catch (error) {
|
||||
console.error((error as any).message);
|
||||
console.warn(
|
||||
`当前环境不支持 AI 降噪, 此错误可忽略, 不影响实际使用, e: ${(error as any).message}`
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@ -111,6 +114,7 @@ export class RTCClient {
|
||||
handleError,
|
||||
handleUserJoin,
|
||||
handleUserLeave,
|
||||
handleTrackEnded,
|
||||
handleUserPublishStream,
|
||||
handleUserUnpublishStream,
|
||||
handleRemoteStreamStats,
|
||||
@ -118,7 +122,6 @@ export class RTCClient {
|
||||
handleLocalAudioPropertiesReport,
|
||||
handleRemoteAudioPropertiesReport,
|
||||
handleAudioDeviceStateChanged,
|
||||
handleUserMessageReceived,
|
||||
handleAutoPlayFail,
|
||||
handlePlayerEvent,
|
||||
handleUserStartAudioCapture,
|
||||
@ -129,6 +132,7 @@ export class RTCClient {
|
||||
this.engine.on(VERTC.events.onError, handleError);
|
||||
this.engine.on(VERTC.events.onUserJoined, handleUserJoin);
|
||||
this.engine.on(VERTC.events.onUserLeave, handleUserLeave);
|
||||
this.engine.on(VERTC.events.onTrackEnded, handleTrackEnded);
|
||||
this.engine.on(VERTC.events.onUserPublishStream, handleUserPublishStream);
|
||||
this.engine.on(VERTC.events.onUserUnpublishStream, handleUserUnpublishStream);
|
||||
this.engine.on(VERTC.events.onRemoteStreamStats, handleRemoteStreamStats);
|
||||
@ -136,7 +140,6 @@ export class RTCClient {
|
||||
this.engine.on(VERTC.events.onAudioDeviceStateChanged, handleAudioDeviceStateChanged);
|
||||
this.engine.on(VERTC.events.onLocalAudioPropertiesReport, handleLocalAudioPropertiesReport);
|
||||
this.engine.on(VERTC.events.onRemoteAudioPropertiesReport, handleRemoteAudioPropertiesReport);
|
||||
this.engine.on(VERTC.events.onUserMessageReceived, handleUserMessageReceived);
|
||||
this.engine.on(VERTC.events.onAutoplayFailed, handleAutoPlayFail);
|
||||
this.engine.on(VERTC.events.onPlayerEvent, handlePlayerEvent);
|
||||
this.engine.on(VERTC.events.onUserStartAudioCapture, handleUserStartAudioCapture);
|
||||
@ -193,21 +196,42 @@ export class RTCClient {
|
||||
audioOutputs: MediaDeviceInfo[];
|
||||
videoInputs: MediaDeviceInfo[];
|
||||
}> {
|
||||
const { video, audio = true } = props || {};
|
||||
const { video = false, audio = true } = props || {};
|
||||
let audioInputs: MediaDeviceInfo[] = [];
|
||||
let audioOutputs: MediaDeviceInfo[] = [];
|
||||
let videoInputs: MediaDeviceInfo[] = [];
|
||||
const { video: hasVideoPermission, audio: hasAudioPermission } = await VERTC.enableDevices({
|
||||
video,
|
||||
audio,
|
||||
});
|
||||
if (audio) {
|
||||
const inputs = await VERTC.enumerateAudioCaptureDevices();
|
||||
const outputs = await VERTC.enumerateAudioPlaybackDevices();
|
||||
audioInputs = inputs.filter((i) => i.deviceId && i.kind === 'audioinput');
|
||||
audioOutputs = outputs.filter((i) => i.deviceId && i.kind === 'audiooutput');
|
||||
this._audioCaptureDevice = audioInputs.filter((i) => i.deviceId)?.[0]?.deviceId;
|
||||
if (hasAudioPermission) {
|
||||
if (!audioInputs?.length) {
|
||||
Message.error('无麦克风设备, 请先确认设备情况。');
|
||||
}
|
||||
if (!audioOutputs?.length) {
|
||||
Message.error('无扬声器设备, 请先确认设备情况。');
|
||||
}
|
||||
} else {
|
||||
Message.error('暂无麦克风设备权限, 请先确认设备权限授予情况。');
|
||||
}
|
||||
}
|
||||
if (video) {
|
||||
videoInputs = await VERTC.enumerateVideoCaptureDevices();
|
||||
videoInputs = videoInputs.filter((i) => i.deviceId && i.kind === 'videoinput');
|
||||
this._videoCaptureDevice = videoInputs?.[0]?.deviceId;
|
||||
if (hasVideoPermission) {
|
||||
if (!videoInputs?.length) {
|
||||
Message.error('无摄像头设备, 请先确认设备情况。');
|
||||
}
|
||||
} else {
|
||||
Message.error('暂无摄像头设备权限, 请先确认设备权限授予情况。');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
@ -226,6 +250,16 @@ export class RTCClient {
|
||||
await this.engine.stopVideoCapture();
|
||||
};
|
||||
|
||||
startScreenCapture = async (enableAudio = false) => {
|
||||
await this.engine.startScreenCapture({
|
||||
enableAudio,
|
||||
});
|
||||
};
|
||||
|
||||
stopScreenCapture = async () => {
|
||||
await this.engine.stopScreenCapture();
|
||||
};
|
||||
|
||||
startAudioCapture = async (mic?: string) => {
|
||||
await this.engine.startAudioCapture(mic || this._audioCaptureDevice);
|
||||
};
|
||||
@ -242,6 +276,18 @@ export class RTCClient {
|
||||
this.engine.unpublishStream(mediaType);
|
||||
};
|
||||
|
||||
publishScreenStream = async (mediaType: MediaType) => {
|
||||
await this.engine.publishScreen(mediaType);
|
||||
};
|
||||
|
||||
unpublishScreenStream = async (mediaType: MediaType) => {
|
||||
await this.engine.unpublishScreen(mediaType);
|
||||
};
|
||||
|
||||
setScreenEncoderConfig = async (description: ScreenEncoderConfig) => {
|
||||
await this.engine.setScreenEncoderConfig(description);
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief 设置业务标识参数
|
||||
* @param businessId
|
||||
@ -286,12 +332,19 @@ export class RTCClient {
|
||||
return this.engine.setLocalVideoMirrorType(type);
|
||||
};
|
||||
|
||||
setLocalVideoPlayer = (userId: string, renderDom?: string | HTMLElement) => {
|
||||
return this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, {
|
||||
renderDom,
|
||||
userId,
|
||||
renderMode: VideoRenderMode.RENDER_MODE_HIDDEN,
|
||||
});
|
||||
setLocalVideoPlayer = (
|
||||
userId: string,
|
||||
renderDom?: string | HTMLElement,
|
||||
isScreenShare = false
|
||||
) => {
|
||||
return this.engine.setLocalVideoPlayer(
|
||||
isScreenShare ? StreamIndex.STREAM_INDEX_SCREEN : StreamIndex.STREAM_INDEX_MAIN,
|
||||
{
|
||||
renderDom,
|
||||
userId,
|
||||
renderMode: VideoRenderMode.RENDER_MODE_FILL,
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
/**
|
||||
@ -344,11 +397,7 @@ export class RTCClient {
|
||||
/**
|
||||
* @brief 命令 AIGC
|
||||
*/
|
||||
commandAudioBot = (
|
||||
command: COMMAND,
|
||||
interruptMode = INTERRUPT_PRIORITY.NONE,
|
||||
message = ''
|
||||
) => {
|
||||
commandAudioBot = (command: COMMAND, interruptMode = INTERRUPT_PRIORITY.NONE, message = '') => {
|
||||
if (this.audioBotEnabled) {
|
||||
this.engine.sendUserBinaryMessage(
|
||||
aigcConfig.BotName,
|
||||
|
||||
@ -30,14 +30,11 @@ import {
|
||||
addAutoPlayFail,
|
||||
removeAutoPlayFail,
|
||||
updateAITalkState,
|
||||
setHistoryMsg,
|
||||
setCurrentMsg,
|
||||
updateNetworkQuality,
|
||||
} from '@/store/slices/room';
|
||||
import RtcClient, { IEventListener } from './RtcClient';
|
||||
|
||||
import { setMicrophoneList, updateSelectedDevice } from '@/store/slices/device';
|
||||
import Utils from '@/utils/utils';
|
||||
import { useMessageHandler } from '@/utils/handler';
|
||||
|
||||
const useRtcListeners = (): IEventListener => {
|
||||
@ -45,12 +42,19 @@ const useRtcListeners = (): IEventListener => {
|
||||
const { parser } = useMessageHandler();
|
||||
const playStatus = useRef<{ [key: string]: { audio: boolean; video: boolean } }>({});
|
||||
|
||||
const debounceSetHistoryMsg = Utils.debounce((text: string, user: string) => {
|
||||
const isAudioEnable = RtcClient.getAudioBotEnabled();
|
||||
if (isAudioEnable) {
|
||||
dispatch(setHistoryMsg({ text, user }));
|
||||
const handleTrackEnded = async (event: { kind: string; isScreen: boolean }) => {
|
||||
const { kind, isScreen } = event;
|
||||
/** 浏览器自带的屏幕共享关闭触发方式，通过 onTrackEnd 事件去关闭 */
|
||||
if (isScreen && kind === 'video') {
|
||||
await RtcClient.stopScreenCapture();
|
||||
await RtcClient.unpublishScreenStream(MediaType.VIDEO);
|
||||
dispatch(
|
||||
updateLocalUser({
|
||||
publishScreen: false,
|
||||
})
|
||||
);
|
||||
}
|
||||
}, 600);
|
||||
};
|
||||
|
||||
const handleUserJoin = (e: onUserJoinedEvent) => {
|
||||
const extraInfo = JSON.parse(e.userInfo.extraInfo || '{}');
|
||||
@ -167,22 +171,6 @@ const useRtcListeners = (): IEventListener => {
|
||||
}
|
||||
};
|
||||
|
||||
const handleUserMessageReceived = (e: { userId: string; message: any }) => {
|
||||
/** debounce 记录用户输入文字 */
|
||||
if (e.message) {
|
||||
const msgObj = JSON.parse(e.message || '{}');
|
||||
if (msgObj.text) {
|
||||
const { text: msg, definite, user_id: user } = msgObj;
|
||||
if ((window as any)._debug_mode) {
|
||||
dispatch(setHistoryMsg({ msg, user }));
|
||||
} else {
|
||||
debounceSetHistoryMsg(msg, user);
|
||||
}
|
||||
dispatch(setCurrentMsg({ msg, definite, user }));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleAutoPlayFail = (event: AutoPlayFailedEvent) => {
|
||||
const { userId, kind } = event;
|
||||
let playUser = playStatus.current?.[userId] || {};
|
||||
@ -264,6 +252,7 @@ const useRtcListeners = (): IEventListener => {
|
||||
handleError,
|
||||
handleUserJoin,
|
||||
handleUserLeave,
|
||||
handleTrackEnded,
|
||||
handleUserPublishStream,
|
||||
handleUserUnpublishStream,
|
||||
handleRemoteStreamStats,
|
||||
@ -271,7 +260,6 @@ const useRtcListeners = (): IEventListener => {
|
||||
handleLocalAudioPropertiesReport,
|
||||
handleRemoteAudioPropertiesReport,
|
||||
handleAudioDeviceStateChanged,
|
||||
handleUserMessageReceived,
|
||||
handleAutoPlayFail,
|
||||
handlePlayerEvent,
|
||||
handleUserStartAudioCapture,
|
||||
|
||||
@ -27,7 +27,7 @@ import {
|
||||
setDevicePermissions,
|
||||
} from '@/store/slices/device';
|
||||
import logger from '@/utils/logger';
|
||||
import aigcConfig, { AI_MODEL } from '@/config';
|
||||
import aigcConfig, { ScreenShareScene, isVisionMode } from '@/config';
|
||||
|
||||
export interface FormProps {
|
||||
username: string;
|
||||
@ -37,148 +37,9 @@ export interface FormProps {
|
||||
|
||||
export const useVisionMode = () => {
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
return [AI_MODEL.VISION].includes(room.aiConfig?.Config?.LLMConfig.ModelName);
|
||||
};
|
||||
|
||||
export const useGetDevicePermission = () => {
|
||||
const [permission, setPermission] = useState<{
|
||||
audio: boolean;
|
||||
}>();
|
||||
|
||||
const dispatch = useDispatch();
|
||||
|
||||
useEffect(() => {
|
||||
(async () => {
|
||||
const permission = await RtcClient.checkPermission();
|
||||
dispatch(setDevicePermissions(permission));
|
||||
setPermission(permission);
|
||||
})();
|
||||
}, [dispatch]);
|
||||
return permission;
|
||||
};
|
||||
|
||||
export const useJoin = (): [
|
||||
boolean,
|
||||
(formValues: FormProps, fromRefresh: boolean) => Promise<void | boolean>
|
||||
] => {
|
||||
const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
|
||||
const dispatch = useDispatch();
|
||||
|
||||
const [joining, setJoining] = useState(false);
|
||||
const listeners = useRtcListeners();
|
||||
|
||||
const handleAIGCModeStart = async () => {
|
||||
if (room.isAIGCEnable) {
|
||||
await RtcClient.stopAudioBot();
|
||||
dispatch(clearCurrentMsg());
|
||||
await RtcClient.startAudioBot();
|
||||
} else {
|
||||
await RtcClient.startAudioBot();
|
||||
}
|
||||
dispatch(updateAIGCState({ isAIGCEnable: true }));
|
||||
};
|
||||
|
||||
async function disPatchJoin(formValues: FormProps): Promise<boolean | undefined> {
|
||||
if (joining) {
|
||||
return;
|
||||
}
|
||||
|
||||
const isSupported = await VERTC.isSupported();
|
||||
if (!isSupported) {
|
||||
Modal.error({
|
||||
title: '不支持 RTC',
|
||||
content: '您的浏览器可能不支持 RTC 功能，请尝试更换浏览器或升级浏览器后再重试。',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setJoining(true);
|
||||
const { username, roomId } = formValues;
|
||||
const isVisionMode = aigcConfig.Model === AI_MODEL.VISION;
|
||||
|
||||
const token = aigcConfig.BaseConfig.Token;
|
||||
|
||||
/** 1. Create RTC Engine */
|
||||
await RtcClient.createEngine({
|
||||
appId: aigcConfig.BaseConfig.AppId,
|
||||
roomId,
|
||||
uid: username,
|
||||
} as any);
|
||||
|
||||
/** 2.1 Set events callbacks */
|
||||
RtcClient.addEventListeners(listeners);
|
||||
|
||||
/** 2.2 RTC starting to join room */
|
||||
await RtcClient.joinRoom(token!, username);
|
||||
console.log(' ------ userJoinRoom\n', `roomId: ${roomId}\n`, `uid: ${username}`);
|
||||
/** 3. Set users' devices info */
|
||||
const mediaDevices = await RtcClient.getDevices({
|
||||
audio: true,
|
||||
video: isVisionMode,
|
||||
});
|
||||
|
||||
if (devicePermissions.audio) {
|
||||
try {
|
||||
await RtcClient.startAudioCapture();
|
||||
// RtcClient.setAudioVolume(30);
|
||||
} catch (e) {
|
||||
logger.debug('No permission for mic');
|
||||
}
|
||||
}
|
||||
|
||||
if (devicePermissions.video && isVisionMode) {
|
||||
try {
|
||||
await RtcClient.startVideoCapture();
|
||||
} catch (e) {
|
||||
logger.debug('No permission for camera');
|
||||
}
|
||||
}
|
||||
|
||||
dispatch(
|
||||
localJoinRoom({
|
||||
roomId,
|
||||
user: {
|
||||
username,
|
||||
userId: username,
|
||||
publishAudio: true,
|
||||
publishVideo: devicePermissions.video && isVisionMode,
|
||||
},
|
||||
})
|
||||
);
|
||||
dispatch(
|
||||
updateSelectedDevice({
|
||||
selectedMicrophone: mediaDevices.audioInputs[0]?.deviceId,
|
||||
selectedCamera: mediaDevices.videoInputs[0]?.deviceId,
|
||||
})
|
||||
);
|
||||
dispatch(updateMediaInputs(mediaDevices));
|
||||
|
||||
setJoining(false);
|
||||
|
||||
Utils.setSessionInfo({
|
||||
username,
|
||||
roomId,
|
||||
publishAudio: true,
|
||||
});
|
||||
|
||||
handleAIGCModeStart();
|
||||
}
|
||||
|
||||
return [joining, disPatchJoin];
|
||||
};
|
||||
|
||||
export const useLeave = () => {
|
||||
const dispatch = useDispatch();
|
||||
|
||||
return async function () {
|
||||
dispatch(localLeaveRoom());
|
||||
dispatch(updateAIGCState({ isAIGCEnable: false }));
|
||||
await Promise.all([RtcClient.stopAudioCapture]);
|
||||
RtcClient.leaveRoom();
|
||||
dispatch(clearHistoryMsg());
|
||||
dispatch(clearCurrentMsg());
|
||||
return {
|
||||
isVisionMode: isVisionMode(room.aiConfig?.Config?.LLMConfig.ModelName),
|
||||
isScreenMode: ScreenShareScene.includes(room.scene),
|
||||
};
|
||||
};
|
||||
|
||||
@ -188,7 +49,7 @@ export const useDeviceState = () => {
|
||||
const localUser = room.localUser;
|
||||
const isAudioPublished = localUser.publishAudio;
|
||||
const isVideoPublished = localUser.publishVideo;
|
||||
|
||||
const isScreenPublished = localUser.publishScreen;
|
||||
const queryDevices = async (type: MediaType) => {
|
||||
const mediaDevices = await RtcClient.getDevices({
|
||||
audio: type === MediaType.AUDIO,
|
||||
@ -220,40 +81,207 @@ export const useDeviceState = () => {
|
||||
return mediaDevices;
|
||||
};
|
||||
|
||||
const switchMic = (publish = true) => {
|
||||
if (publish) {
|
||||
!isAudioPublished
|
||||
const switchMic = async (controlPublish = true) => {
|
||||
if (controlPublish) {
|
||||
await (!isAudioPublished
|
||||
? RtcClient.publishStream(MediaType.AUDIO)
|
||||
: RtcClient.unpublishStream(MediaType.AUDIO);
|
||||
: RtcClient.unpublishStream(MediaType.AUDIO));
|
||||
}
|
||||
queryDevices(MediaType.AUDIO);
|
||||
!isAudioPublished ? RtcClient.startAudioCapture() : RtcClient.stopAudioCapture();
|
||||
await (!isAudioPublished ? RtcClient.startAudioCapture() : RtcClient.stopAudioCapture());
|
||||
dispatch(
|
||||
updateLocalUser({
|
||||
publishAudio: !localUser.publishAudio,
|
||||
publishAudio: !isAudioPublished,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const switchCamera = (publish = true) => {
|
||||
if (publish) {
|
||||
!isVideoPublished
|
||||
const switchCamera = async (controlPublish = true) => {
|
||||
if (controlPublish) {
|
||||
await (!isVideoPublished
|
||||
? RtcClient.publishStream(MediaType.VIDEO)
|
||||
: RtcClient.unpublishStream(MediaType.VIDEO);
|
||||
: RtcClient.unpublishStream(MediaType.VIDEO));
|
||||
}
|
||||
queryDevices(MediaType.VIDEO);
|
||||
!localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();
|
||||
await (!isVideoPublished ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture());
|
||||
dispatch(
|
||||
updateLocalUser({
|
||||
publishVideo: !localUser.publishVideo,
|
||||
publishVideo: !isVideoPublished,
|
||||
})
|
||||
);
|
||||
};
|
||||
|
||||
const switchScreenCapture = async (controlPublish = true) => {
|
||||
try {
|
||||
if (controlPublish) {
|
||||
await (!isScreenPublished
|
||||
? RtcClient.publishScreenStream(MediaType.VIDEO)
|
||||
: RtcClient.unpublishScreenStream(MediaType.VIDEO));
|
||||
}
|
||||
await (!isScreenPublished ? RtcClient.startScreenCapture() : RtcClient.stopScreenCapture());
|
||||
dispatch(
|
||||
updateLocalUser({
|
||||
publishScreen: !isScreenPublished,
|
||||
})
|
||||
);
|
||||
} catch {
|
||||
console.warn('Not Authorized.');
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
isAudioPublished,
|
||||
isVideoPublished,
|
||||
isScreenPublished,
|
||||
switchMic,
|
||||
switchCamera,
|
||||
switchScreenCapture,
|
||||
};
|
||||
};
|
||||
|
||||
export const useGetDevicePermission = () => {
|
||||
const [permission, setPermission] = useState<{
|
||||
audio: boolean;
|
||||
}>();
|
||||
|
||||
const dispatch = useDispatch();
|
||||
|
||||
useEffect(() => {
|
||||
(async () => {
|
||||
const permission = await RtcClient.checkPermission();
|
||||
dispatch(setDevicePermissions(permission));
|
||||
setPermission(permission);
|
||||
})();
|
||||
}, [dispatch]);
|
||||
return permission;
|
||||
};
|
||||
|
||||
export const useJoin = (): [
|
||||
boolean,
|
||||
(formValues: FormProps, fromRefresh: boolean) => Promise<void | boolean>
|
||||
] => {
|
||||
const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
|
||||
const dispatch = useDispatch();
|
||||
|
||||
const { switchCamera, switchMic } = useDeviceState();
|
||||
const [joining, setJoining] = useState(false);
|
||||
const listeners = useRtcListeners();
|
||||
|
||||
const handleAIGCModeStart = async () => {
|
||||
if (room.isAIGCEnable) {
|
||||
await RtcClient.stopAudioBot();
|
||||
dispatch(clearCurrentMsg());
|
||||
await RtcClient.startAudioBot();
|
||||
} else {
|
||||
await RtcClient.startAudioBot();
|
||||
}
|
||||
dispatch(updateAIGCState({ isAIGCEnable: true }));
|
||||
};
|
||||
|
||||
async function disPatchJoin(formValues: FormProps): Promise<boolean | undefined> {
|
||||
if (joining) {
|
||||
return;
|
||||
}
|
||||
|
||||
const isSupported = await VERTC.isSupported();
|
||||
if (!isSupported) {
|
||||
Modal.error({
|
||||
title: '不支持 RTC',
|
||||
content: '您的浏览器可能不支持 RTC 功能，请尝试更换浏览器或升级浏览器后再重试。',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
setJoining(true);
|
||||
const { username, roomId } = formValues;
|
||||
const isVision = isVisionMode(aigcConfig.Model);
|
||||
const shouldGetVideoPermission = isVision && !ScreenShareScene.includes(room.scene);
|
||||
|
||||
const token = aigcConfig.BaseConfig.Token;
|
||||
|
||||
/** 1. Create RTC Engine */
|
||||
const engineParams = {
|
||||
appId: aigcConfig.BaseConfig.AppId,
|
||||
roomId,
|
||||
uid: username,
|
||||
};
|
||||
await RtcClient.createEngine(engineParams);
|
||||
|
||||
/** 2.1 Set events callbacks */
|
||||
RtcClient.addEventListeners(listeners);
|
||||
|
||||
/** 2.2 RTC starting to join room */
|
||||
await RtcClient.joinRoom(token!, username);
|
||||
console.log(' ------ userJoinRoom\n', `roomId: ${roomId}\n`, `uid: ${username}`);
|
||||
/** 3. Set users' devices info */
|
||||
const mediaDevices = await RtcClient.getDevices({
|
||||
audio: true,
|
||||
video: shouldGetVideoPermission,
|
||||
});
|
||||
|
||||
dispatch(
|
||||
localJoinRoom({
|
||||
roomId,
|
||||
user: {
|
||||
username,
|
||||
userId: username,
|
||||
},
|
||||
})
|
||||
);
|
||||
dispatch(
|
||||
updateSelectedDevice({
|
||||
selectedMicrophone: mediaDevices.audioInputs[0]?.deviceId,
|
||||
selectedCamera: mediaDevices.videoInputs[0]?.deviceId,
|
||||
})
|
||||
);
|
||||
dispatch(updateMediaInputs(mediaDevices));
|
||||
|
||||
setJoining(false);
|
||||
|
||||
if (devicePermissions.audio) {
|
||||
try {
|
||||
await switchMic();
|
||||
// RtcClient.setAudioVolume(30);
|
||||
} catch (e) {
|
||||
logger.debug('No permission for mic');
|
||||
}
|
||||
}
|
||||
|
||||
if (devicePermissions.video && shouldGetVideoPermission) {
|
||||
try {
|
||||
await switchCamera();
|
||||
} catch (e) {
|
||||
logger.debug('No permission for camera');
|
||||
}
|
||||
}
|
||||
|
||||
Utils.setSessionInfo({
|
||||
username,
|
||||
roomId,
|
||||
publishAudio: true,
|
||||
});
|
||||
|
||||
handleAIGCModeStart();
|
||||
}
|
||||
|
||||
return [joining, disPatchJoin];
|
||||
};
|
||||
|
||||
export const useLeave = () => {
|
||||
const dispatch = useDispatch();
|
||||
|
||||
return async function () {
|
||||
await Promise.all([
|
||||
RtcClient.stopAudioCapture,
|
||||
RtcClient.stopScreenCapture,
|
||||
RtcClient.stopVideoCapture,
|
||||
]);
|
||||
await RtcClient.leaveRoom();
|
||||
dispatch(clearHistoryMsg());
|
||||
dispatch(clearCurrentMsg());
|
||||
dispatch(localLeaveRoom());
|
||||
dispatch(updateAIGCState({ isAIGCEnable: false }));
|
||||
};
|
||||
};
|
||||
|
||||
@ -3,68 +3,88 @@
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import { useSelector } from 'react-redux';
|
||||
import { useEffect } from 'react';
|
||||
import { MediaType } from '@volcengine/rtc';
|
||||
import { RootState } from '@/store';
|
||||
import { useVisionMode } from '@/lib/useCommon';
|
||||
import { useDeviceState, useVisionMode } from '@/lib/useCommon';
|
||||
import RtcClient from '@/lib/RtcClient';
|
||||
import { ScreenShareScene } from '@/config';
|
||||
|
||||
import styles from './index.module.less';
|
||||
import CameraCloseNoteSVG from '@/assets/img/CameraCloseNote.svg';
|
||||
import RtcClient from '@/lib/RtcClient';
|
||||
import { updateLocalUser } from '@/store/slices/room';
|
||||
import ScreenCloseNoteSVG from '@/assets/img/ScreenCloseNote.svg';
|
||||
|
||||
const LocalVideoID = 'local-video-player';
|
||||
const LocalScreenID = 'local-screen-player';
|
||||
|
||||
function CameraArea(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
const { className, ...rest } = props;
|
||||
const dispatch = useDispatch();
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
const isVisionMode = useVisionMode();
|
||||
const localUser = room.localUser;
|
||||
const isVideoPublished = localUser.publishVideo;
|
||||
const { isVisionMode } = useVisionMode();
|
||||
const isScreenMode = ScreenShareScene.includes(room.scene);
|
||||
const { isVideoPublished, isScreenPublished, switchCamera, switchScreenCapture } =
|
||||
useDeviceState();
|
||||
|
||||
const setVideoPlayer = () => {
|
||||
if (isVisionMode && (isVideoPublished || isScreenPublished)) {
|
||||
RtcClient.setLocalVideoPlayer(
|
||||
room.localUser.username!,
|
||||
isScreenMode ? LocalScreenID : LocalVideoID,
|
||||
isScreenPublished
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const handleOperateCamera = () => {
|
||||
!localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();
|
||||
switchCamera();
|
||||
};
|
||||
|
||||
!localUser.publishVideo
|
||||
? RtcClient.publishStream(MediaType.VIDEO)
|
||||
: RtcClient.unpublishStream(MediaType.VIDEO);
|
||||
|
||||
dispatch(
|
||||
updateLocalUser({
|
||||
publishVideo: !localUser.publishVideo,
|
||||
})
|
||||
);
|
||||
const handleOperateScreenShare = () => {
|
||||
switchScreenCapture();
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (isVisionMode && isVideoPublished) {
|
||||
RtcClient.setLocalVideoPlayer(room.localUser.username!, LocalVideoID);
|
||||
} else {
|
||||
RtcClient.setLocalVideoPlayer(room.localUser.username!);
|
||||
}
|
||||
}, [isVisionMode, isVideoPublished]);
|
||||
setVideoPlayer();
|
||||
}, [isVideoPublished, isScreenPublished, isScreenMode]);
|
||||
|
||||
return isVisionMode ? (
|
||||
<div className={`${styles['camera-wrapper']} ${className}`} {...rest}>
|
||||
{isVideoPublished ? (
|
||||
<div id={LocalVideoID} className={styles['camera-player']} />
|
||||
) : (
|
||||
<div className={styles['camera-placeholder']}>
|
||||
<img
|
||||
src={CameraCloseNoteSVG}
|
||||
alt="close"
|
||||
className={styles['camera-placeholder-close-note']}
|
||||
/>
|
||||
<div>
|
||||
请
|
||||
<div
|
||||
id={LocalVideoID}
|
||||
className={`${styles['camera-player']} ${
|
||||
isVideoPublished && !isScreenMode ? '' : styles['camera-player-hidden']
|
||||
}`}
|
||||
/>
|
||||
<div
|
||||
id={LocalScreenID}
|
||||
className={`${styles['camera-player']} ${
|
||||
isScreenPublished && isScreenMode ? '' : styles['camera-player-hidden']
|
||||
}`}
|
||||
/>
|
||||
<div
|
||||
className={`${styles['camera-placeholder']} ${
|
||||
isVideoPublished || isScreenPublished ? styles['camera-player-hidden'] : ''
|
||||
}`}
|
||||
>
|
||||
<img
|
||||
src={isScreenMode ? ScreenCloseNoteSVG : CameraCloseNoteSVG}
|
||||
alt="close"
|
||||
className={styles['camera-placeholder-close-note']}
|
||||
/>
|
||||
<div>
|
||||
请
|
||||
{isScreenMode ? (
|
||||
<span onClick={handleOperateScreenShare} className={styles['camera-open-btn']}>
|
||||
打开屏幕采集
|
||||
</span>
|
||||
) : (
|
||||
<span onClick={handleOperateCamera} className={styles['camera-open-btn']}>
|
||||
打开摄像头
|
||||
</span>
|
||||
</div>
|
||||
<div>体验豆包视觉理解模型</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
<div>体验豆包视觉理解模型</div>
|
||||
</div>
|
||||
</div>
|
||||
) : null;
|
||||
}
|
||||
|
||||
@ -21,6 +21,14 @@ function Conversation(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
const isAIReady = msgHistory.length > 0;
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
const isUserTextLoading = (owner: string) => {
|
||||
return owner === userId && isUserTalking;
|
||||
};
|
||||
|
||||
const isAITextLoading = (owner: string) => {
|
||||
return owner === Config.BotName && isAITalking;
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
if (container) {
|
||||
@ -53,7 +61,9 @@ function Conversation(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
<div className={styles.content}>
|
||||
{value}
|
||||
<div className={styles['loading-wrapper']}>
|
||||
{isAIReady && (isUserTalking || isAITalking) && index === msgHistory.length - 1 ? (
|
||||
{isAIReady &&
|
||||
(isUserTextLoading(user) || isAITextLoading(user)) &&
|
||||
index === msgHistory.length - 1 ? (
|
||||
<Loading gap={3} className={styles.loading} dotClassName={styles.dot} />
|
||||
) : (
|
||||
''
|
||||
|
||||
@ -4,11 +4,12 @@
|
||||
*/
|
||||
|
||||
import { useSelector } from 'react-redux';
|
||||
import { useState } from 'react';
|
||||
import { memo, useState } from 'react';
|
||||
import { Drawer } from '@arco-design/web-react';
|
||||
import { useDeviceState, useLeave } from '@/lib/useCommon';
|
||||
import { RootState } from '@/store';
|
||||
import { AI_MODEL } from '@/config';
|
||||
import { isVisionMode } from '@/config/common';
|
||||
import { ScreenShareScene } from '@/config';
|
||||
import utils from '@/utils/utils';
|
||||
import Menu from '../../Menu';
|
||||
|
||||
@ -19,14 +20,25 @@ import MicOpenSVG from '@/assets/img/MicOpen.svg';
|
||||
import SettingSVG from '@/assets/img/Setting.svg';
|
||||
import MicCloseSVG from '@/assets/img/MicClose.svg';
|
||||
import LeaveRoomSVG from '@/assets/img/LeaveRoom.svg';
|
||||
import ScreenOnSVG from '@/assets/img/ScreenOn.svg';
|
||||
import ScreenOffSVG from '@/assets/img/ScreenOff.svg';
|
||||
|
||||
function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
const { className, ...rest } = props;
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
const [open, setOpen] = useState(false);
|
||||
const model = room.aiConfig.Config.LLMConfig?.ModelName;
|
||||
const isScreenMode = ScreenShareScene.includes(room.scene);
|
||||
const leaveRoom = useLeave();
|
||||
const { isAudioPublished, isVideoPublished, switchMic, switchCamera } = useDeviceState();
|
||||
const {
|
||||
isAudioPublished,
|
||||
isVideoPublished,
|
||||
isScreenPublished,
|
||||
switchMic,
|
||||
switchCamera,
|
||||
switchScreenCapture,
|
||||
} = useDeviceState();
|
||||
|
||||
const handleSetting = () => {
|
||||
setOpen(true);
|
||||
};
|
||||
@ -41,13 +53,22 @@ function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
className={style.btn}
|
||||
alt="mic"
|
||||
/>
|
||||
{model === AI_MODEL.VISION ? (
|
||||
<img
|
||||
src={isVideoPublished ? CameraOpenSVG : CameraCloseSVG}
|
||||
onClick={() => switchCamera(true)}
|
||||
className={style.btn}
|
||||
alt="camera"
|
||||
/>
|
||||
{isVisionMode(model) ? (
|
||||
isScreenMode ? (
|
||||
<img
|
||||
src={isScreenPublished ? ScreenOnSVG : ScreenOffSVG}
|
||||
onClick={() => switchScreenCapture()}
|
||||
className={style.btn}
|
||||
alt="screenShare"
|
||||
/>
|
||||
) : (
|
||||
<img
|
||||
src={isVideoPublished ? CameraOpenSVG : CameraCloseSVG}
|
||||
onClick={() => switchCamera(true)}
|
||||
className={style.btn}
|
||||
alt="camera"
|
||||
/>
|
||||
)
|
||||
) : (
|
||||
''
|
||||
)}
|
||||
@ -60,6 +81,7 @@ function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
style={{
|
||||
width: 'max-content',
|
||||
}}
|
||||
footer={null}
|
||||
>
|
||||
<Menu />
|
||||
</Drawer>
|
||||
@ -67,4 +89,4 @@ function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
export default ToolBar;
|
||||
export default memo(ToolBar);
|
||||
|
||||
@ -172,6 +172,15 @@
|
||||
line-height: 22px;
|
||||
}
|
||||
|
||||
.closed {
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
color: #737A87;
|
||||
font-size: 14px;
|
||||
font-weight: 400;
|
||||
line-height: 19.6px;
|
||||
}
|
||||
|
||||
.btns {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
@ -262,6 +271,10 @@
|
||||
border-radius: 8px;
|
||||
}
|
||||
|
||||
.camera-player-hidden {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
.camera-placeholder {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
@ -273,6 +286,8 @@
|
||||
|
||||
.camera-placeholder-close-note {
|
||||
margin-bottom: 8px;
|
||||
width: 60px;
|
||||
height: 60px;
|
||||
}
|
||||
|
||||
.camera-open-btn {
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
/**
|
||||
* Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
.row {
|
||||
width: 100%;
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
cursor: pointer;
|
||||
|
||||
.firstPart {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
width: 90%;
|
||||
color: var(--text-color-text-2, var(--text-color-text-2, #42464E));
|
||||
text-align: center;
|
||||
|
||||
font-family: "PingFang SC";
|
||||
font-size: 13px;
|
||||
font-style: normal;
|
||||
font-weight: 500;
|
||||
line-height: 22px;
|
||||
letter-spacing: 0.039px;
|
||||
}
|
||||
|
||||
.finalPart {
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
align-items: center;
|
||||
width: 10%;
|
||||
justify-content: flex-end;
|
||||
|
||||
.rightOutlined {
|
||||
font-size: 12px;
|
||||
}
|
||||
}
|
||||
|
||||
.icon {
|
||||
margin-right: 4px;
|
||||
}
|
||||
}
|
||||
29
src/pages/MainPage/Menu/components/AISettingAnchor/index.tsx
Normal file
29
src/pages/MainPage/Menu/components/AISettingAnchor/index.tsx
Normal file
@ -0,0 +1,29 @@
|
||||
/**
|
||||
* Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
import { useState } from 'react';
|
||||
import { IconRight } from '@arco-design/web-react/icon';
|
||||
import AISettings from '@/components/AISettings';
|
||||
import styles from './index.module.less';
|
||||
|
||||
function AISettingAnchor() {
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
const handleOpenDrawer = () => setOpen(true);
|
||||
const handleCloseDrawer = () => setOpen(false);
|
||||
return (
|
||||
<>
|
||||
<div className={styles.row} onClick={handleOpenDrawer}>
|
||||
<div className={styles.firstPart}>AI 设置</div>
|
||||
<div className={styles.finalPart}>
|
||||
<IconRight className={styles.rightOutlined} />
|
||||
</div>
|
||||
</div>
|
||||
<AISettings open={open} onOk={handleCloseDrawer} onCancel={handleCloseDrawer} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default AISettingAnchor;
|
||||
@ -6,16 +6,18 @@
|
||||
import { MediaType } from '@volcengine/rtc';
|
||||
import DeviceDrawerButton from '../DeviceDrawerButton';
|
||||
import { useVisionMode } from '@/lib/useCommon';
|
||||
import AISettingAnchor from '../AISettingAnchor';
|
||||
import Interrupt from '../Interrupt';
|
||||
import styles from './index.module.less';
|
||||
|
||||
function Operation() {
|
||||
const isVisionMode = useVisionMode();
|
||||
const { isVisionMode, isScreenMode } = useVisionMode();
|
||||
return (
|
||||
<div className={`${styles.box} ${styles.device}`}>
|
||||
<Interrupt />
|
||||
<AISettingAnchor />
|
||||
<DeviceDrawerButton />
|
||||
{isVisionMode ? <DeviceDrawerButton type={MediaType.VIDEO} /> : ''}
|
||||
{isVisionMode && !isScreenMode ? <DeviceDrawerButton type={MediaType.VIDEO} /> : ''}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
import VERTC from '@volcengine/rtc';
|
||||
import { useEffect, useState } from 'react';
|
||||
import { Tooltip, Typography } from '@arco-design/web-react';
|
||||
import { useDispatch, useSelector } from 'react-redux';
|
||||
import { useVisionMode } from '@/lib/useCommon';
|
||||
@ -13,37 +14,39 @@ import Operation from './components/Operation';
|
||||
import { Questions } from '@/config';
|
||||
import { COMMAND, INTERRUPT_PRIORITY } from '@/utils/handler';
|
||||
import CameraArea from '../MainArea/Room/CameraArea';
|
||||
import { setCurrentMsg, setHistoryMsg } from '@/store/slices/room';
|
||||
import { setHistoryMsg, setInterruptMsg } from '@/store/slices/room';
|
||||
import utils from '@/utils/utils';
|
||||
import packageJson from '../../../../package.json';
|
||||
import styles from './index.module.less';
|
||||
|
||||
function Menu() {
|
||||
const dispatch = useDispatch();
|
||||
const [question, setQuestion] = useState('');
|
||||
const room = useSelector((state: RootState) => state.room);
|
||||
const scene = room.scene;
|
||||
const isJoined = room?.isJoined;
|
||||
const isVisionMode = useVisionMode();
|
||||
|
||||
const handleQuestion = (question: string) => {
|
||||
RtcClient.commandAudioBot(COMMAND.EXTERNAL_TEXT_TO_LLM, INTERRUPT_PRIORITY.HIGH, question);
|
||||
dispatch(
|
||||
setHistoryMsg({
|
||||
text: question,
|
||||
user: RtcClient.basicInfo.user_id,
|
||||
paragraph: true,
|
||||
definite: true,
|
||||
})
|
||||
);
|
||||
dispatch(
|
||||
setCurrentMsg({
|
||||
text: question,
|
||||
user: RtcClient.basicInfo.user_id,
|
||||
paragraph: true,
|
||||
definite: true,
|
||||
})
|
||||
);
|
||||
const handleQuestion = (que: string) => {
|
||||
RtcClient.commandAudioBot(COMMAND.EXTERNAL_TEXT_TO_LLM, INTERRUPT_PRIORITY.HIGH, que);
|
||||
setQuestion(que);
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (question && !room.isAITalking) {
|
||||
dispatch(setInterruptMsg());
|
||||
dispatch(
|
||||
setHistoryMsg({
|
||||
text: question,
|
||||
user: RtcClient.basicInfo.user_id,
|
||||
paragraph: true,
|
||||
definite: true,
|
||||
})
|
||||
);
|
||||
setQuestion('');
|
||||
}
|
||||
}, [question, room.isAITalking]);
|
||||
|
||||
return (
|
||||
<div className={styles.wrapper}>
|
||||
{isJoined && utils.isMobile() && isVisionMode ? (
|
||||
@ -52,7 +55,7 @@ function Menu() {
|
||||
</div>
|
||||
) : null}
|
||||
<div className={`${styles.box} ${styles.info}`}>
|
||||
<div className={styles.bold}>Demo Version 1.4.0</div>
|
||||
<div className={styles.bold}>Demo Version {packageJson.version}</div>
|
||||
<div className={styles.bold}>SDK Version {VERTC.getSdkVersion()}</div>
|
||||
{isJoined ? (
|
||||
<div className={styles.gray}>
|
||||
|
||||
@ -11,7 +11,6 @@ import {
|
||||
RemoteAudioStats,
|
||||
} from '@volcengine/rtc';
|
||||
import config, { SCENE } from '@/config';
|
||||
import utils from '@/utils/utils';
|
||||
|
||||
export interface IUser {
|
||||
username?: string;
|
||||
@ -33,6 +32,7 @@ export interface Msg {
|
||||
time: string;
|
||||
user: string;
|
||||
paragraph?: boolean;
|
||||
definite?: boolean;
|
||||
isInterrupted?: boolean;
|
||||
}
|
||||
|
||||
@ -59,6 +59,10 @@ export interface RoomState {
|
||||
* @brief AI 是否正在说话
|
||||
*/
|
||||
isAITalking: boolean;
|
||||
/**
|
||||
* @brief AI 思考中
|
||||
*/
|
||||
isAIThinking: boolean;
|
||||
/**
|
||||
* @brief 用户是否正在说话
|
||||
*/
|
||||
@ -99,12 +103,14 @@ const initialState: RoomState = {
|
||||
scene: SCENE.INTELLIGENT_ASSISTANT,
|
||||
remoteUsers: [],
|
||||
localUser: {
|
||||
publishAudio: true,
|
||||
publishVideo: true,
|
||||
publishAudio: false,
|
||||
publishVideo: false,
|
||||
publishScreen: false,
|
||||
},
|
||||
autoPlayFailUser: [],
|
||||
isJoined: false,
|
||||
isAIGCEnable: false,
|
||||
isAIThinking: false,
|
||||
isAITalking: false,
|
||||
isUserTalking: false,
|
||||
networkQuality: NetworkQuality.UNKNOWN,
|
||||
@ -131,15 +137,19 @@ export const roomSlice = createSlice({
|
||||
}
|
||||
) => {
|
||||
state.roomId = payload.roomId;
|
||||
state.localUser = payload.user;
|
||||
state.localUser = {
|
||||
...state.localUser,
|
||||
...payload.user,
|
||||
};
|
||||
state.isJoined = true;
|
||||
},
|
||||
localLeaveRoom: (state) => {
|
||||
state.roomId = undefined;
|
||||
state.time = -1;
|
||||
state.localUser = {
|
||||
publishAudio: true,
|
||||
publishVideo: true,
|
||||
publishAudio: false,
|
||||
publishVideo: false,
|
||||
publishScreen: false,
|
||||
};
|
||||
state.remoteUsers = [];
|
||||
state.isJoined = false;
|
||||
@ -159,7 +169,7 @@ export const roomSlice = createSlice({
|
||||
updateLocalUser: (state, { payload }: { payload: Partial<LocalUser> }) => {
|
||||
state.localUser = {
|
||||
...state.localUser,
|
||||
...payload,
|
||||
...(payload || {}),
|
||||
};
|
||||
},
|
||||
|
||||
@ -204,8 +214,14 @@ export const roomSlice = createSlice({
|
||||
state.isAIGCEnable = payload.isAIGCEnable;
|
||||
},
|
||||
updateAITalkState: (state, { payload }) => {
|
||||
state.isAIThinking = false;
|
||||
state.isUserTalking = false;
|
||||
state.isAITalking = payload.isAITalking;
|
||||
},
|
||||
updateAIThinkState: (state, { payload }) => {
|
||||
state.isAIThinking = payload.isAIThinking;
|
||||
state.isUserTalking = false;
|
||||
},
|
||||
updateAIConfig: (state, { payload }) => {
|
||||
state.aiConfig = Object.assign(state.aiConfig, payload);
|
||||
},
|
||||
@ -213,36 +229,74 @@ export const roomSlice = createSlice({
|
||||
state.msgHistory = [];
|
||||
},
|
||||
setHistoryMsg: (state, { payload }) => {
|
||||
const paragraph = payload.paragraph;
|
||||
const aiTalking = payload.user === config.BotName;
|
||||
const userTalking = payload.user === state.localUser.userId;
|
||||
if (paragraph) {
|
||||
if (state.isAITalking) {
|
||||
state.isAITalking = false;
|
||||
}
|
||||
if (state.isUserTalking) {
|
||||
state.isUserTalking = false;
|
||||
const { paragraph, definite } = payload;
|
||||
/** 是否需要再创建新句子 */
|
||||
const shouldCreateSentence = payload.definite;
|
||||
state.isUserTalking = payload.user === state.localUser.userId;
|
||||
if (state.msgHistory.length) {
|
||||
const lastMsg = state.msgHistory.at(-1)!;
|
||||
/** 当前讲话人更新字幕 */
|
||||
if (lastMsg.user === payload.user) {
|
||||
/** 如果上一句话是完整的 & 本次的话也是完整的, 则直接塞入 */
|
||||
if (lastMsg.definite) {
|
||||
state.msgHistory.push({
|
||||
value: payload.text,
|
||||
time: new Date().toString(),
|
||||
user: payload.user,
|
||||
definite,
|
||||
paragraph,
|
||||
});
|
||||
} else {
|
||||
/** 话未说完, 更新文字内容 */
|
||||
lastMsg.value = payload.text;
|
||||
lastMsg.time = new Date().toString();
|
||||
lastMsg.paragraph = paragraph;
|
||||
lastMsg.definite = definite;
|
||||
lastMsg.user = payload.user;
|
||||
}
|
||||
/** 如果本次的话已经说完了, 提前塞入空字符串做准备 */
|
||||
if (shouldCreateSentence) {
|
||||
state.msgHistory.push({
|
||||
value: '',
|
||||
time: new Date().toString(),
|
||||
user: '',
|
||||
});
|
||||
}
|
||||
} else {
|
||||
/** 换人说话了，塞入新句子 */
|
||||
state.msgHistory.push({
|
||||
value: payload.text,
|
||||
time: new Date().toString(),
|
||||
user: payload.user,
|
||||
definite,
|
||||
paragraph,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
if (state.isAITalking !== aiTalking) {
|
||||
state.isAITalking = aiTalking;
|
||||
}
|
||||
if (state.isUserTalking !== userTalking) {
|
||||
state.isUserTalking = userTalking;
|
||||
}
|
||||
/** 首句话首字不会被打断 */
|
||||
state.msgHistory.push({
|
||||
value: payload.text,
|
||||
time: new Date().toString(),
|
||||
user: payload.user,
|
||||
paragraph,
|
||||
});
|
||||
}
|
||||
utils.addMsgWithoutDuplicate(state.msgHistory, {
|
||||
user: payload.user,
|
||||
value: payload.text,
|
||||
time: new Date().toLocaleString(),
|
||||
isInterrupted: false,
|
||||
paragraph,
|
||||
});
|
||||
},
|
||||
setInterruptMsg: (state) => {
|
||||
const msg = state.msgHistory[state.msgHistory.length - 1];
|
||||
msg.isInterrupted = true;
|
||||
state.msgHistory[state.msgHistory.length - 1] = msg;
|
||||
state.isAITalking = false;
|
||||
if (!state.msgHistory.length) {
|
||||
return;
|
||||
}
|
||||
/** 找到最后一个末尾的字幕, 将其状态置换为打断 */
|
||||
for (let id = state.msgHistory.length - 1; id >= 0; id--) {
|
||||
const msg = state.msgHistory[id];
|
||||
if (msg.value) {
|
||||
if (!msg.definite) {
|
||||
state.msgHistory[id].isInterrupted = true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
},
|
||||
clearCurrentMsg: (state) => {
|
||||
state.currentConversation = {};
|
||||
@ -250,10 +304,6 @@ export const roomSlice = createSlice({
|
||||
state.isAITalking = false;
|
||||
state.isUserTalking = false;
|
||||
},
|
||||
setCurrentMsg: (state, { payload }) => {
|
||||
const { user, ...info } = payload;
|
||||
state.currentConversation[user || state.localUser.userId] = info;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@ -270,9 +320,9 @@ export const {
|
||||
clearAutoPlayFail,
|
||||
updateAIGCState,
|
||||
updateAITalkState,
|
||||
updateAIThinkState,
|
||||
updateAIConfig,
|
||||
setHistoryMsg,
|
||||
setCurrentMsg,
|
||||
clearHistoryMsg,
|
||||
clearCurrentMsg,
|
||||
setInterruptMsg,
|
||||
|
||||
@ -6,10 +6,10 @@
|
||||
import { useDispatch } from 'react-redux';
|
||||
import logger from './logger';
|
||||
import {
|
||||
setCurrentMsg,
|
||||
setHistoryMsg,
|
||||
setInterruptMsg,
|
||||
updateAITalkState,
|
||||
updateAIThinkState,
|
||||
} from '@/store/slices/room';
|
||||
import RtcClient from '@/lib/RtcClient';
|
||||
import Utils from '@/utils/utils';
|
||||
@ -89,11 +89,16 @@ export const useMessageHandler = () => {
|
||||
const { Code, Description } = Stage || {};
|
||||
logger.debug(Code, Description);
|
||||
switch (Code) {
|
||||
case AGENT_BRIEF.THINKING:
|
||||
dispatch(updateAIThinkState({ isAIThinking: true }));
|
||||
break;
|
||||
case AGENT_BRIEF.SPEAKING:
|
||||
dispatch(updateAITalkState({ isAITalking: true }));
|
||||
break;
|
||||
case AGENT_BRIEF.FINISHED:
|
||||
dispatch(updateAITalkState({ isAITalking: false }));
|
||||
break;
|
||||
case AGENT_BRIEF.INTERRUPTED:
|
||||
dispatch(updateAITalkState({ isAITalking: false }));
|
||||
dispatch(setInterruptMsg());
|
||||
break;
|
||||
default:
|
||||
@ -118,7 +123,6 @@ export const useMessageHandler = () => {
|
||||
dispatch(setHistoryMsg({ text: msg, user, paragraph, definite }));
|
||||
}
|
||||
}
|
||||
dispatch(setCurrentMsg({ msg, definite, user, paragraph }));
|
||||
}
|
||||
},
|
||||
/**
|
||||
@ -141,8 +145,8 @@ export const useMessageHandler = () => {
|
||||
ToolCallID: parsed?.tool_calls?.[0]?.id,
|
||||
Content: map[name.toLocaleLowerCase().replaceAll('_', '')],
|
||||
}),
|
||||
'func',
|
||||
),
|
||||
'func'
|
||||
)
|
||||
);
|
||||
},
|
||||
};
|
||||
|
||||
@ -3,12 +3,7 @@
|
||||
* SPDX-license-identifier: BSD-3-Clause
|
||||
*/
|
||||
|
||||
import { Msg, RoomState } from '@/store/slices/room';
|
||||
import RtcClient from '@/lib/RtcClient';
|
||||
|
||||
|
||||
class Utils {
|
||||
|
||||
formatTime = (time: number): string => {
|
||||
if (time < 0) {
|
||||
return '00:00';
|
||||
@ -46,9 +41,9 @@ class Utils {
|
||||
const query = window.location.search.substring(1);
|
||||
const pairs = query.split('&');
|
||||
return pairs.reduce<{ [key: string]: string }>((queries, pair) => {
|
||||
const [key, value] = decodeURIComponent(pair).split('=');
|
||||
const [key, value] = pair.split('=');
|
||||
if (key && value) {
|
||||
queries[key] = value;
|
||||
queries[key] = decodeURIComponent(value);
|
||||
}
|
||||
return queries;
|
||||
}, {});
|
||||
@ -58,34 +53,6 @@ class Utils {
|
||||
|
||||
isArray = Array.isArray;
|
||||
|
||||
debounce = (func: (...args: any[]) => void, wait: number) => {
|
||||
let timeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
return function (...args: any[]) {
|
||||
if (timeoutId !== null) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
|
||||
timeoutId = setTimeout(() => {
|
||||
func(...args);
|
||||
}, wait);
|
||||
};
|
||||
};
|
||||
|
||||
addMsgWithoutDuplicate = (arr: RoomState['msgHistory'], added: Msg) => {
|
||||
if (arr.length) {
|
||||
const last = arr.at(-1)!;
|
||||
const { user, value, isInterrupted } = last;
|
||||
if (
|
||||
(added.user === RtcClient.basicInfo.user_id && last.user === added.user) ||
|
||||
(user === added.user && added.value.startsWith(value) && value.trim())
|
||||
) {
|
||||
arr.pop();
|
||||
added.isInterrupted = isInterrupted;
|
||||
}
|
||||
}
|
||||
arr.push(added);
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief 将字符串包装成 TLV
|
||||
*/
|
||||
@ -119,7 +86,7 @@ class Utils {
|
||||
* @note TLV 数据格式
|
||||
* | magic number | length(big-endian) | value |
|
||||
* @param {ArrayBufferLike} tlvBuffer
|
||||
* @returns
|
||||
* @returns
|
||||
*/
|
||||
tlv2String(tlvBuffer: ArrayBufferLike) {
|
||||
const typeBuffer = new Uint8Array(tlvBuffer, 0, 4);
|
||||
|
||||
@ -2210,10 +2210,10 @@
|
||||
"@typescript-eslint/types" "5.31.0"
|
||||
eslint-visitor-keys "^3.3.0"
|
||||
|
||||
"@volcengine/rtc@4.58.9":
|
||||
version "4.58.9"
|
||||
resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.58.9.tgz#841ebaddd5d4963c71abd33037bd76d1d490d928"
|
||||
integrity sha512-nnXnNW9pVo8ynBSxVe0ikNIdxWfoSx5oOnwK7EoMCXdc2bJgHATpz/B+Kv2F1k4GjYAbo7ZcOm/g3cchvHgH5Q==
|
||||
"@volcengine/rtc@4.66.1":
|
||||
version "4.66.1"
|
||||
resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.66.1.tgz#1934c269b31216f43718ae46b169c59ac5e474f2"
|
||||
integrity sha512-APznH6eosmKJC1HYJJ8s6G3Mq3OSgw6ivv6uCiayM5QNMBj+GW6zxf+MVsk5rm6r4R92TLwQErWonJ8yzGO4xA==
|
||||
dependencies:
|
||||
eventemitter3 "^4.0.7"
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user