AI 配置修改后，退出房间将不再保存该配置方案
-
-
-
- }
- visible={open}
- onCancel={() => setOpen(false)}
- >
-
- 选择你所需要的
- AI 人设
+
+
- AI 配置修改后，退出房间将不再保存该配置方案
+
+
- 我们已为您配置好对应人设的基本参数，您也可以根据自己的需求进行自定义设置
-
-
- {SCENES.map((key) => (
+ }
+ visible={open}
+ onCancel={onCancel}
+ >
+
+ 选择你所需要的
+ AI 人设
+
+
+ 我们已为您配置好对应人设的基本参数，您也可以根据自己的需求进行自定义设置
+
+
+ {[...SCENES, null].map((key) =>
+ key ? (
handleChecked(key as SCENE)}
/>
- ))}
-
-
- {utils.isMobile() ? null : (
-
- )}
-
- {
- setData((prev) => ({
- ...prev,
- prompt: val,
- }));
- }}
- placeholder="请输入你需要的 Prompt 设定"
- />
-
-
- {
- setData((prev) => ({
- ...prev,
- welcome: val,
- }));
- }}
- placeholder="请输入欢迎语"
- />
-
+ ) : utils.isMobile() ? (
+
+ ) : null
+ )}
+
+
+ {utils.isMobile() ? null : (
{
avatar?: string;
}
-const ReversedVoiceType = Object.entries(VOICE_TYPE).reduce>((acc, [key, value]) => {
- acc[value] = key;
- return acc;
-}, {});
+const ReversedVoiceType = Object.entries(VOICE_TYPE).reduce>(
+ (acc, [key, value]) => {
+ acc[value] = key;
+ return acc;
+ },
+ {}
+);
function AvatarCard(props: IAvatarCardProps) {
const room = useSelector((state: RootState) => state.room);
+ const [open, setOpen] = useState(false);
const scene = room.scene;
const { LLMConfig, TTSConfig } = room.aiConfig.Config || {};
const { avatar, className, ...rest } = props;
+ const voice = TTSConfig.ProviderParams.audio.voice_type;
+
+ const handleOpenDrawer = () => setOpen(true);
+ const handleCloseDrawer = () => setOpen(false);
return (
= {
Mode: AI_MODE_MAP[this.Model || ''] || AI_MODEL_MODE.CUSTOM,
@@ -134,16 +142,21 @@ export class ConfigFactory {
Url: this.Url,
Feature: JSON.stringify({ Http: true }),
};
- if (this.Model === AI_MODEL.VISION) {
+ if (isVisionMode(this.Model)) {
params.VisionConfig = {
Enable: true,
+ SnapshotConfig: {
+ StreamType: this.VisionSourceType,
+ Height: 640,
+ ImagesLimit: 1,
+ },
};
}
return params;
}
get ASRConfig() {
- return {
+ const params: Record = {
Provider: 'volcano',
ProviderParams: {
/**
@@ -152,7 +165,6 @@ export class ConfigFactory {
*/
Mode: 'smallmodel',
AppId: this.BaseConfig.ASRAppId,
- ...(this.BaseConfig.ASRToken ? { AccessToken: this.BaseConfig.ASRToken } : {}),
/**
* @note 具体流式语音识别服务对应的 Cluster ID，可在流式语音服务控制台开通对应服务后查询。
* 具体链接为: https://console.volcengine.com/speech/service/16?s=g
@@ -165,15 +177,18 @@ export class ConfigFactory {
},
VolumeGain: 0.3,
};
+ if (this.BaseConfig.ASRToken) {
+ params.ProviderParams.AccessToken = this.BaseConfig.ASRToken;
+ }
+ return params;
}
get TTSConfig() {
- return {
+ const params: Record = {
Provider: 'volcano',
ProviderParams: {
app: {
AppId: this.BaseConfig.TTSAppId,
- ...(this.BaseConfig.TTSToken ? { Token: this.BaseConfig.TTSToken } : {}),
Cluster: TTS_CLUSTER.TTS,
},
audio: {
@@ -183,6 +198,10 @@ export class ConfigFactory {
},
IgnoreBracketText: [1, 2, 3, 4, 5],
};
+ if (this.BaseConfig.TTSToken) {
+ params.ProviderParams.app.Token = this.BaseConfig.TTSToken;
+ }
+ return params;
}
get aigcConfig() {
diff --git a/src/lib/RtcClient.ts b/src/lib/RtcClient.ts
index 8b2490f..b1fe23f 100644
--- a/src/lib/RtcClient.ts
+++ b/src/lib/RtcClient.ts
@@ -3,7 +3,6 @@
* SPDX-license-identifier: BSD-3-Clause
*/
-
import VERTC, {
MirrorType,
StreamIndex,
@@ -23,8 +22,10 @@ import VERTC, {
PlayerEvent,
NetworkQuality,
VideoRenderMode,
+ ScreenEncoderConfig,
} from '@volcengine/rtc';
import RTCAIAnsExtension from '@volcengine/rtc/extension-ainr';
+import { Message } from '@arco-design/web-react';
import openAPIs from '@/app/api';
import aigcConfig from '@/config';
import Utils from '@/utils/utils';
@@ -34,6 +35,7 @@ export interface IEventListener {
handleError: (e: { errorCode: any }) => void;
handleUserJoin: (e: onUserJoinedEvent) => void;
handleUserLeave: (e: onUserLeaveEvent) => void;
+ handleTrackEnded: (e: { kind: string; isScreen: boolean }) => void;
handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void;
handleUserUnpublishStream: (e: {
userId: string;
@@ -45,7 +47,6 @@ export interface IEventListener {
handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void;
handleRemoteAudioPropertiesReport: (e: RemoteAudioPropertiesInfo[]) => void;
handleAudioDeviceStateChanged: (e: DeviceInfo) => void;
- handleUserMessageReceived: (e: { userId: string; message: any }) => void;
handleAutoPlayFail: (e: AutoPlayFailedEvent) => void;
handlePlayerEvent: (e: PlayerEvent) => void;
handleUserStartAudioCapture: (e: { userId: string }) => void;
@@ -103,7 +104,9 @@ export class RTCClient {
await this.engine.registerExtension(AIAnsExtension);
AIAnsExtension.enable();
} catch (error) {
- console.error((error as any).message);
+ console.warn(
+ `当前环境不支持 AI 降噪, 此错误可忽略, 不影响实际使用, e: ${(error as any).message}`
+ );
}
};
@@ -111,6 +114,7 @@ export class RTCClient {
handleError,
handleUserJoin,
handleUserLeave,
+ handleTrackEnded,
handleUserPublishStream,
handleUserUnpublishStream,
handleRemoteStreamStats,
@@ -118,7 +122,6 @@ export class RTCClient {
handleLocalAudioPropertiesReport,
handleRemoteAudioPropertiesReport,
handleAudioDeviceStateChanged,
- handleUserMessageReceived,
handleAutoPlayFail,
handlePlayerEvent,
handleUserStartAudioCapture,
@@ -129,6 +132,7 @@ export class RTCClient {
this.engine.on(VERTC.events.onError, handleError);
this.engine.on(VERTC.events.onUserJoined, handleUserJoin);
this.engine.on(VERTC.events.onUserLeave, handleUserLeave);
+ this.engine.on(VERTC.events.onTrackEnded, handleTrackEnded);
this.engine.on(VERTC.events.onUserPublishStream, handleUserPublishStream);
this.engine.on(VERTC.events.onUserUnpublishStream, handleUserUnpublishStream);
this.engine.on(VERTC.events.onRemoteStreamStats, handleRemoteStreamStats);
@@ -136,7 +140,6 @@ export class RTCClient {
this.engine.on(VERTC.events.onAudioDeviceStateChanged, handleAudioDeviceStateChanged);
this.engine.on(VERTC.events.onLocalAudioPropertiesReport, handleLocalAudioPropertiesReport);
this.engine.on(VERTC.events.onRemoteAudioPropertiesReport, handleRemoteAudioPropertiesReport);
- this.engine.on(VERTC.events.onUserMessageReceived, handleUserMessageReceived);
this.engine.on(VERTC.events.onAutoplayFailed, handleAutoPlayFail);
this.engine.on(VERTC.events.onPlayerEvent, handlePlayerEvent);
this.engine.on(VERTC.events.onUserStartAudioCapture, handleUserStartAudioCapture);
@@ -193,21 +196,42 @@ export class RTCClient {
audioOutputs: MediaDeviceInfo[];
videoInputs: MediaDeviceInfo[];
}> {
- const { video, audio = true } = props || {};
+ const { video = false, audio = true } = props || {};
let audioInputs: MediaDeviceInfo[] = [];
let audioOutputs: MediaDeviceInfo[] = [];
let videoInputs: MediaDeviceInfo[] = [];
+ const { video: hasVideoPermission, audio: hasAudioPermission } = await VERTC.enableDevices({
+ video,
+ audio,
+ });
if (audio) {
const inputs = await VERTC.enumerateAudioCaptureDevices();
const outputs = await VERTC.enumerateAudioPlaybackDevices();
audioInputs = inputs.filter((i) => i.deviceId && i.kind === 'audioinput');
audioOutputs = outputs.filter((i) => i.deviceId && i.kind === 'audiooutput');
this._audioCaptureDevice = audioInputs.filter((i) => i.deviceId)?.[0]?.deviceId;
+ if (hasAudioPermission) {
+ if (!audioInputs?.length) {
+ Message.error('无麦克风设备, 请先确认设备情况。');
+ }
+ if (!audioOutputs?.length) {
+ Message.error('无扬声器设备, 请先确认设备情况。');
+ }
+ } else {
+ Message.error('暂无麦克风设备权限, 请先确认设备权限授予情况。');
+ }
}
if (video) {
videoInputs = await VERTC.enumerateVideoCaptureDevices();
videoInputs = videoInputs.filter((i) => i.deviceId && i.kind === 'videoinput');
this._videoCaptureDevice = videoInputs?.[0]?.deviceId;
+ if (hasVideoPermission) {
+ if (!videoInputs?.length) {
+ Message.error('无摄像头设备, 请先确认设备情况。');
+ }
+ } else {
+ Message.error('暂无摄像头设备权限, 请先确认设备权限授予情况。');
+ }
}
return {
@@ -226,6 +250,16 @@ export class RTCClient {
await this.engine.stopVideoCapture();
};
+ startScreenCapture = async (enableAudio = false) => {
+ await this.engine.startScreenCapture({
+ enableAudio,
+ });
+ };
+
+ stopScreenCapture = async () => {
+ await this.engine.stopScreenCapture();
+ };
+
startAudioCapture = async (mic?: string) => {
await this.engine.startAudioCapture(mic || this._audioCaptureDevice);
};
@@ -242,6 +276,18 @@ export class RTCClient {
this.engine.unpublishStream(mediaType);
};
+ publishScreenStream = async (mediaType: MediaType) => {
+ await this.engine.publishScreen(mediaType);
+ };
+
+ unpublishScreenStream = async (mediaType: MediaType) => {
+ await this.engine.unpublishScreen(mediaType);
+ };
+
+ setScreenEncoderConfig = async (description: ScreenEncoderConfig) => {
+ await this.engine.setScreenEncoderConfig(description);
+ };
+
/**
* @brief 设置业务标识参数
* @param businessId
@@ -286,12 +332,19 @@ export class RTCClient {
return this.engine.setLocalVideoMirrorType(type);
};
- setLocalVideoPlayer = (userId: string, renderDom?: string | HTMLElement) => {
- return this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, {
- renderDom,
- userId,
- renderMode: VideoRenderMode.RENDER_MODE_HIDDEN,
- });
+ setLocalVideoPlayer = (
+ userId: string,
+ renderDom?: string | HTMLElement,
+ isScreenShare = false
+ ) => {
+ return this.engine.setLocalVideoPlayer(
+ isScreenShare ? StreamIndex.STREAM_INDEX_SCREEN : StreamIndex.STREAM_INDEX_MAIN,
+ {
+ renderDom,
+ userId,
+ renderMode: VideoRenderMode.RENDER_MODE_FILL,
+ }
+ );
};
/**
@@ -344,11 +397,7 @@ export class RTCClient {
/**
* @brief 命令 AIGC
*/
- commandAudioBot = (
- command: COMMAND,
- interruptMode = INTERRUPT_PRIORITY.NONE,
- message = ''
- ) => {
+ commandAudioBot = (command: COMMAND, interruptMode = INTERRUPT_PRIORITY.NONE, message = '') => {
if (this.audioBotEnabled) {
this.engine.sendUserBinaryMessage(
aigcConfig.BotName,
diff --git a/src/lib/listenerHooks.ts b/src/lib/listenerHooks.ts
index ff2249c..af94c00 100644
--- a/src/lib/listenerHooks.ts
+++ b/src/lib/listenerHooks.ts
@@ -30,14 +30,11 @@ import {
addAutoPlayFail,
removeAutoPlayFail,
updateAITalkState,
- setHistoryMsg,
- setCurrentMsg,
updateNetworkQuality,
} from '@/store/slices/room';
import RtcClient, { IEventListener } from './RtcClient';
import { setMicrophoneList, updateSelectedDevice } from '@/store/slices/device';
-import Utils from '@/utils/utils';
import { useMessageHandler } from '@/utils/handler';
const useRtcListeners = (): IEventListener => {
@@ -45,12 +42,19 @@ const useRtcListeners = (): IEventListener => {
const { parser } = useMessageHandler();
const playStatus = useRef<{ [key: string]: { audio: boolean; video: boolean } }>({});
- const debounceSetHistoryMsg = Utils.debounce((text: string, user: string) => {
- const isAudioEnable = RtcClient.getAudioBotEnabled();
- if (isAudioEnable) {
- dispatch(setHistoryMsg({ text, user }));
+ const handleTrackEnded = async (event: { kind: string; isScreen: boolean }) => {
+ const { kind, isScreen } = event;
+ /** 浏览器自带的屏幕共享关闭触发方式，通过 onTrackEnd 事件去关闭 */
+ if (isScreen && kind === 'video') {
+ await RtcClient.stopScreenCapture();
+ await RtcClient.unpublishScreenStream(MediaType.VIDEO);
+ dispatch(
+ updateLocalUser({
+ publishScreen: false,
+ })
+ );
}
- }, 600);
+ };
const handleUserJoin = (e: onUserJoinedEvent) => {
const extraInfo = JSON.parse(e.userInfo.extraInfo || '{}');
@@ -167,22 +171,6 @@ const useRtcListeners = (): IEventListener => {
}
};
- const handleUserMessageReceived = (e: { userId: string; message: any }) => {
- /** debounce 记录用户输入文字 */
- if (e.message) {
- const msgObj = JSON.parse(e.message || '{}');
- if (msgObj.text) {
- const { text: msg, definite, user_id: user } = msgObj;
- if ((window as any)._debug_mode) {
- dispatch(setHistoryMsg({ msg, user }));
- } else {
- debounceSetHistoryMsg(msg, user);
- }
- dispatch(setCurrentMsg({ msg, definite, user }));
- }
- }
- };
-
const handleAutoPlayFail = (event: AutoPlayFailedEvent) => {
const { userId, kind } = event;
let playUser = playStatus.current?.[userId] || {};
@@ -264,6 +252,7 @@ const useRtcListeners = (): IEventListener => {
handleError,
handleUserJoin,
handleUserLeave,
+ handleTrackEnded,
handleUserPublishStream,
handleUserUnpublishStream,
handleRemoteStreamStats,
@@ -271,7 +260,6 @@ const useRtcListeners = (): IEventListener => {
handleLocalAudioPropertiesReport,
handleRemoteAudioPropertiesReport,
handleAudioDeviceStateChanged,
- handleUserMessageReceived,
handleAutoPlayFail,
handlePlayerEvent,
handleUserStartAudioCapture,
diff --git a/src/lib/useCommon.ts b/src/lib/useCommon.ts
index 1dc25ea..14057c9 100644
--- a/src/lib/useCommon.ts
+++ b/src/lib/useCommon.ts
@@ -27,7 +27,7 @@ import {
setDevicePermissions,
} from '@/store/slices/device';
import logger from '@/utils/logger';
-import aigcConfig, { AI_MODEL } from '@/config';
+import aigcConfig, { ScreenShareScene, isVisionMode } from '@/config';
export interface FormProps {
username: string;
@@ -37,148 +37,9 @@ export interface FormProps {
export const useVisionMode = () => {
const room = useSelector((state: RootState) => state.room);
- return [AI_MODEL.VISION].includes(room.aiConfig?.Config?.LLMConfig.ModelName);
-};
-
-export const useGetDevicePermission = () => {
- const [permission, setPermission] = useState<{
- audio: boolean;
- }>();
-
- const dispatch = useDispatch();
-
- useEffect(() => {
- (async () => {
- const permission = await RtcClient.checkPermission();
- dispatch(setDevicePermissions(permission));
- setPermission(permission);
- })();
- }, [dispatch]);
- return permission;
-};
-
-export const useJoin = (): [
- boolean,
- (formValues: FormProps, fromRefresh: boolean) => Promise
-] => {
- const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
- const room = useSelector((state: RootState) => state.room);
-
- const dispatch = useDispatch();
-
- const [joining, setJoining] = useState(false);
- const listeners = useRtcListeners();
-
- const handleAIGCModeStart = async () => {
- if (room.isAIGCEnable) {
- await RtcClient.stopAudioBot();
- dispatch(clearCurrentMsg());
- await RtcClient.startAudioBot();
- } else {
- await RtcClient.startAudioBot();
- }
- dispatch(updateAIGCState({ isAIGCEnable: true }));
- };
-
- async function disPatchJoin(formValues: FormProps): Promise {
- if (joining) {
- return;
- }
-
- const isSupported = await VERTC.isSupported();
- if (!isSupported) {
- Modal.error({
- title: '不支持 RTC',
- content: '您的浏览器可能不支持 RTC 功能，请尝试更换浏览器或升级浏览器后再重试。',
- });
- return;
- }
-
- setJoining(true);
- const { username, roomId } = formValues;
- const isVisionMode = aigcConfig.Model === AI_MODEL.VISION;
-
- const token = aigcConfig.BaseConfig.Token;
-
- /** 1. Create RTC Engine */
- await RtcClient.createEngine({
- appId: aigcConfig.BaseConfig.AppId,
- roomId,
- uid: username,
- } as any);
-
- /** 2.1 Set events callbacks */
- RtcClient.addEventListeners(listeners);
-
- /** 2.2 RTC starting to join room */
- await RtcClient.joinRoom(token!, username);
- console.log(' ------ userJoinRoom\n', `roomId: ${roomId}\n`, `uid: ${username}`);
- /** 3. Set users' devices info */
- const mediaDevices = await RtcClient.getDevices({
- audio: true,
- video: isVisionMode,
- });
-
- if (devicePermissions.audio) {
- try {
- await RtcClient.startAudioCapture();
- // RtcClient.setAudioVolume(30);
- } catch (e) {
- logger.debug('No permission for mic');
- }
- }
-
- if (devicePermissions.video && isVisionMode) {
- try {
- await RtcClient.startVideoCapture();
- } catch (e) {
- logger.debug('No permission for camera');
- }
- }
-
- dispatch(
- localJoinRoom({
- roomId,
- user: {
- username,
- userId: username,
- publishAudio: true,
- publishVideo: devicePermissions.video && isVisionMode,
- },
- })
- );
- dispatch(
- updateSelectedDevice({
- selectedMicrophone: mediaDevices.audioInputs[0]?.deviceId,
- selectedCamera: mediaDevices.videoInputs[0]?.deviceId,
- })
- );
- dispatch(updateMediaInputs(mediaDevices));
-
- setJoining(false);
-
- Utils.setSessionInfo({
- username,
- roomId,
- publishAudio: true,
- });
-
- handleAIGCModeStart();
- }
-
- return [joining, disPatchJoin];
-};
-
-export const useLeave = () => {
- const dispatch = useDispatch();
-
- return async function () {
- dispatch(localLeaveRoom());
- dispatch(updateAIGCState({ isAIGCEnable: false }));
- await Promise.all([RtcClient.stopAudioCapture]);
- RtcClient.leaveRoom();
- dispatch(clearHistoryMsg());
- dispatch(clearCurrentMsg());
+ return {
+ isVisionMode: isVisionMode(room.aiConfig?.Config?.LLMConfig.ModelName),
+ isScreenMode: ScreenShareScene.includes(room.scene),
};
};
@@ -188,7 +49,7 @@ export const useDeviceState = () => {
const localUser = room.localUser;
const isAudioPublished = localUser.publishAudio;
const isVideoPublished = localUser.publishVideo;
-
+ const isScreenPublished = localUser.publishScreen;
const queryDevices = async (type: MediaType) => {
const mediaDevices = await RtcClient.getDevices({
audio: type === MediaType.AUDIO,
@@ -220,40 +81,207 @@ export const useDeviceState = () => {
return mediaDevices;
};
- const switchMic = (publish = true) => {
- if (publish) {
- !isAudioPublished
+ const switchMic = async (controlPublish = true) => {
+ if (controlPublish) {
+ await (!isAudioPublished
? RtcClient.publishStream(MediaType.AUDIO)
- : RtcClient.unpublishStream(MediaType.AUDIO);
+ : RtcClient.unpublishStream(MediaType.AUDIO));
}
queryDevices(MediaType.AUDIO);
- !isAudioPublished ? RtcClient.startAudioCapture() : RtcClient.stopAudioCapture();
+ await (!isAudioPublished ? RtcClient.startAudioCapture() : RtcClient.stopAudioCapture());
dispatch(
updateLocalUser({
- publishAudio: !localUser.publishAudio,
+ publishAudio: !isAudioPublished,
})
);
};
- const switchCamera = (publish = true) => {
- if (publish) {
- !isVideoPublished
+ const switchCamera = async (controlPublish = true) => {
+ if (controlPublish) {
+ await (!isVideoPublished
? RtcClient.publishStream(MediaType.VIDEO)
- : RtcClient.unpublishStream(MediaType.VIDEO);
+ : RtcClient.unpublishStream(MediaType.VIDEO));
}
queryDevices(MediaType.VIDEO);
- !localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();
+ await (!isVideoPublished ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture());
dispatch(
updateLocalUser({
- publishVideo: !localUser.publishVideo,
+ publishVideo: !isVideoPublished,
})
);
};
+ const switchScreenCapture = async (controlPublish = true) => {
+ try {
+ if (controlPublish) {
+ await (!isScreenPublished
+ ? RtcClient.publishScreenStream(MediaType.VIDEO)
+ : RtcClient.unpublishScreenStream(MediaType.VIDEO));
+ }
+ await (!isScreenPublished ? RtcClient.startScreenCapture() : RtcClient.stopScreenCapture());
+ dispatch(
+ updateLocalUser({
+ publishScreen: !isScreenPublished,
+ })
+ );
+ } catch {
+ console.warn('Not Authorized.');
+ }
+ };
+
return {
isAudioPublished,
isVideoPublished,
+ isScreenPublished,
switchMic,
switchCamera,
+ switchScreenCapture,
+ };
+};
+
+export const useGetDevicePermission = () => {
+ const [permission, setPermission] = useState<{
+ audio: boolean;
+ }>();
+
+ const dispatch = useDispatch();
+
+ useEffect(() => {
+ (async () => {
+ const permission = await RtcClient.checkPermission();
+ dispatch(setDevicePermissions(permission));
+ setPermission(permission);
+ })();
+ }, [dispatch]);
+ return permission;
+};
+
+export const useJoin = (): [
+ boolean,
+ (formValues: FormProps, fromRefresh: boolean) => Promise
+] => {
+ const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
+ const room = useSelector((state: RootState) => state.room);
+
+ const dispatch = useDispatch();
+
+ const { switchCamera, switchMic } = useDeviceState();
+ const [joining, setJoining] = useState(false);
+ const listeners = useRtcListeners();
+
+ const handleAIGCModeStart = async () => {
+ if (room.isAIGCEnable) {
+ await RtcClient.stopAudioBot();
+ dispatch(clearCurrentMsg());
+ await RtcClient.startAudioBot();
+ } else {
+ await RtcClient.startAudioBot();
+ }
+ dispatch(updateAIGCState({ isAIGCEnable: true }));
+ };
+
+ async function disPatchJoin(formValues: FormProps): Promise {
+ if (joining) {
+ return;
+ }
+
+ const isSupported = await VERTC.isSupported();
+ if (!isSupported) {
+ Modal.error({
+ title: '不支持 RTC',
+ content: '您的浏览器可能不支持 RTC 功能，请尝试更换浏览器或升级浏览器后再重试。',
+ });
+ return;
+ }
+
+ setJoining(true);
+ const { username, roomId } = formValues;
+ const isVision = isVisionMode(aigcConfig.Model);
+ const shouldGetVideoPermission = isVision && !ScreenShareScene.includes(room.scene);
+
+ const token = aigcConfig.BaseConfig.Token;
+
+ /** 1. Create RTC Engine */
+ const engineParams = {
+ appId: aigcConfig.BaseConfig.AppId,
+ roomId,
+ uid: username,
+ };
+ await RtcClient.createEngine(engineParams);
+
+ /** 2.1 Set events callbacks */
+ RtcClient.addEventListeners(listeners);
+
+ /** 2.2 RTC starting to join room */
+ await RtcClient.joinRoom(token!, username);
+ console.log(' ------ userJoinRoom\n', `roomId: ${roomId}\n`, `uid: ${username}`);
+ /** 3. Set users' devices info */
+ const mediaDevices = await RtcClient.getDevices({
+ audio: true,
+ video: shouldGetVideoPermission,
+ });
+
+ dispatch(
+ localJoinRoom({
+ roomId,
+ user: {
+ username,
+ userId: username,
+ },
+ })
+ );
+ dispatch(
+ updateSelectedDevice({
+ selectedMicrophone: mediaDevices.audioInputs[0]?.deviceId,
+ selectedCamera: mediaDevices.videoInputs[0]?.deviceId,
+ })
+ );
+ dispatch(updateMediaInputs(mediaDevices));
+
+ setJoining(false);
+
+ if (devicePermissions.audio) {
+ try {
+ await switchMic();
+ // RtcClient.setAudioVolume(30);
+ } catch (e) {
+ logger.debug('No permission for mic');
+ }
+ }
+
+ if (devicePermissions.video && shouldGetVideoPermission) {
+ try {
+ await switchCamera();
+ } catch (e) {
+ logger.debug('No permission for camera');
+ }
+ }
+
+ Utils.setSessionInfo({
+ username,
+ roomId,
+ publishAudio: true,
+ });
+
+ handleAIGCModeStart();
+ }
+
+ return [joining, disPatchJoin];
+};
+
+export const useLeave = () => {
+ const dispatch = useDispatch();
+
+ return async function () {
+ await Promise.all([
+ RtcClient.stopAudioCapture,
+ RtcClient.stopScreenCapture,
+ RtcClient.stopVideoCapture,
+ ]);
+ await RtcClient.leaveRoom();
+ dispatch(clearHistoryMsg());
+ dispatch(clearCurrentMsg());
+ dispatch(localLeaveRoom());
+ dispatch(updateAIGCState({ isAIGCEnable: false }));
};
};
diff --git a/src/pages/MainPage/MainArea/Room/CameraArea.tsx b/src/pages/MainPage/MainArea/Room/CameraArea.tsx
index f855dc8..7cfe32f 100644
--- a/src/pages/MainPage/MainArea/Room/CameraArea.tsx
+++ b/src/pages/MainPage/MainArea/Room/CameraArea.tsx
@@ -3,68 +3,88 @@
* SPDX-license-identifier: BSD-3-Clause
*/
-import { useDispatch, useSelector } from 'react-redux';
+import { useSelector } from 'react-redux';
import { useEffect } from 'react';
-import { MediaType } from '@volcengine/rtc';
import { RootState } from '@/store';
-import { useVisionMode } from '@/lib/useCommon';
+import { useDeviceState, useVisionMode } from '@/lib/useCommon';
+import RtcClient from '@/lib/RtcClient';
+import { ScreenShareScene } from '@/config';
+
import styles from './index.module.less';
import CameraCloseNoteSVG from '@/assets/img/CameraCloseNote.svg';
-import RtcClient from '@/lib/RtcClient';
-import { updateLocalUser } from '@/store/slices/room';
+import ScreenCloseNoteSVG from '@/assets/img/ScreenCloseNote.svg';
const LocalVideoID = 'local-video-player';
+const LocalScreenID = 'local-screen-player';
function CameraArea(props: React.HTMLAttributes) {
const { className, ...rest } = props;
- const dispatch = useDispatch();
const room = useSelector((state: RootState) => state.room);
- const isVisionMode = useVisionMode();
- const localUser = room.localUser;
- const isVideoPublished = localUser.publishVideo;
+ const { isVisionMode } = useVisionMode();
+ const isScreenMode = ScreenShareScene.includes(room.scene);
+ const { isVideoPublished, isScreenPublished, switchCamera, switchScreenCapture } =
+ useDeviceState();
+
+ const setVideoPlayer = () => {
+ if (isVisionMode && (isVideoPublished || isScreenPublished)) {
+ RtcClient.setLocalVideoPlayer(
+ room.localUser.username!,
+ isScreenMode ? LocalScreenID : LocalVideoID,
+ isScreenPublished
+ );
+ }
+ };
const handleOperateCamera = () => {
- !localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();
+ switchCamera();
+ };
- !localUser.publishVideo
- ? RtcClient.publishStream(MediaType.VIDEO)
- : RtcClient.unpublishStream(MediaType.VIDEO);
-
- dispatch(
- updateLocalUser({
- publishVideo: !localUser.publishVideo,
- })
- );
+ const handleOperateScreenShare = () => {
+ switchScreenCapture();
};
useEffect(() => {
- if (isVisionMode && isVideoPublished) {
- RtcClient.setLocalVideoPlayer(room.localUser.username!, LocalVideoID);
- } else {
- RtcClient.setLocalVideoPlayer(room.localUser.username!);
- }
- }, [isVisionMode, isVideoPublished]);
+ setVideoPlayer();
+ }, [isVideoPublished, isScreenPublished, isScreenMode]);
return isVisionMode ? (
-
-
-
+
);
}
diff --git a/src/components/AvatarCard/index.module.less b/src/components/AvatarCard/index.module.less
index be6b768..9b8d029 100644
--- a/src/components/AvatarCard/index.module.less
+++ b/src/components/AvatarCard/index.module.less
@@ -137,4 +137,57 @@
background-color: #EAEDF1;
transform: rotate(-90deg);
}
+}
+
+.button {
+ position: relative;
+ width: max-content !important;
+ height: 24px !important;
+ margin-top: 8px;
+ border-radius: 4px !important;
+ font-size: 12px !important;
+ background: linear-gradient(77.86deg, rgba(229, 242, 255, 0.5) -3.23%, rgba(217, 229, 255, 0.5) 51.11%, rgba(246, 226, 255, 0.5) 98.65%);
+ cursor: pointer;
+
+ .button-text {
+ background: linear-gradient(77.86deg, #3384FF -3.23%, #014BDE 51.11%, #A945FB 98.65%);
+ -webkit-background-clip: text;
+ background-clip: text;
+ color: transparent;
+ font-weight: 500;
+ line-height: 20px;
+ text-align: center;
+ }
+}
+
+.button::after {
+ content: '';
+ position: absolute;
+ border-radius: 3px;
+ top: 0px;
+ left: 0px;
+ width: 100%;
+ height: 22px;
+ background: white;
+ z-index: -1;
+}
+
+.button::before {
+ content: '';
+ position: absolute;
+ border-radius: 5px;
+ top: -2px;
+ left: -2px;
+ width: calc(100% + 4px);
+ height: 26px;
+ background: linear-gradient(90deg, rgba(0, 139, 255, 0.5) 0%, rgba(0, 98, 255, 0.5) 49.5%, rgba(207, 92, 255, 0.5) 100%);
+ z-index: -2;
+}
+
+.button:hover {
+ background: linear-gradient(77.86deg, rgba(200, 220, 255, 0.7) -3.23%, rgba(190, 210, 255, 0.7) 51.11%, rgba(230, 210, 255, 0.7) 98.65%);
+}
+
+.button:active {
+ background: linear-gradient(77.86deg, rgba(170, 190, 255, 0.9) -3.23%, rgba(160, 180, 255, 0.9) 51.11%, rgba(210, 180, 255, 0.9) 98.65%);
}
\ No newline at end of file
diff --git a/src/components/AvatarCard/index.tsx b/src/components/AvatarCard/index.tsx
index 0ccccc4..f30772a 100644
--- a/src/components/AvatarCard/index.tsx
+++ b/src/components/AvatarCard/index.tsx
@@ -4,6 +4,8 @@
*/
import { useSelector } from 'react-redux';
+import { Button } from '@arco-design/web-react';
+import { useState } from 'react';
import AISettings from '../AISettings';
import style from './index.module.less';
import DouBaoAvatar from '@/assets/img/DoubaoAvatarGIF.webp';
@@ -14,16 +16,24 @@ interface IAvatarCardProps extends React.HTMLAttributes
- {
- const info = VOICE_INFO_MAP[VOICE_TYPE[type as keyof typeof VOICE_TYPE]];
- return {
- key: VOICE_TYPE[type as keyof typeof VOICE_TYPE],
- label: type,
- icon: info.icon,
- description: info.description,
- };
- })}
- onChange={handleVoiceTypeChanged}
- value={data.voice}
- moreIcon={VoiceTypeChangeSVG}
- moreText="更换音色"
- placeHolder="请选择你需要的音色"
- />
-
-
- {use3Part ? (
- <>
-
- {
- setData((prev) => ({
- ...prev,
- Url: val,
- }));
- }}
- placeholder="请输入第三方模型地址"
- />
-
-
- {
- setData((prev) => ({
- ...prev,
- APIKey: val,
- }));
- }}
- placeholder="请输入请求密钥"
- />
-
-
- {
- setData((prev) => ({
- ...prev,
- customModelName: val,
- }));
- }}
- placeholder="请输入模型名称"
- />
-
-
- ) : (
-
- ({
- key: AI_MODEL[type as keyof typeof AI_MODEL],
- label: type.replaceAll('_', ' '),
- icon: DoubaoModelSVG,
- }))}
- moreIcon={ModelChangeSVG}
- moreText="更换模型"
- placeHolder="请选择你需要的模型"
- onChange={(key) => {
+ />
+ )}
+
+ {
+ setData((prev) => ({
+ ...prev,
+ prompt: val,
+ }));
+ }}
+ placeholder="请输入你需要的 Prompt 设定"
+ />
+
+
+ {
+ setData((prev) => ({
+ ...prev,
+ welcome: val,
+ }));
+ }}
+ placeholder="请输入欢迎语"
+ />
+
+
-
-
+
+
+
+
+ {
+ const info = VOICE_INFO_MAP[VOICE_TYPE[type as keyof typeof VOICE_TYPE]];
+ return {
+ key: VOICE_TYPE[type as keyof typeof VOICE_TYPE],
+ label: type,
+ icon: info.icon,
+ description: info.description,
+ };
+ })}
+ onChange={handleVoiceTypeChanged}
+ value={data.voice}
+ moreIcon={VoiceTypeChangeSVG}
+ moreText="更换音色"
+ placeHolder="请选择你需要的音色"
+ />
+
+
+ {use3Part ? (
+ <>
+
+ {
setData((prev) => ({
...prev,
- model: key as AI_MODEL,
+ Url: val,
}));
}}
- value={data.model}
+ placeholder="请输入第三方模型地址"
/>
- )}
+
+ {
+ setData((prev) => ({
+ ...prev,
+ APIKey: val,
+ }));
+ }}
+ placeholder="请输入请求密钥"
+ />
+
+
+ {
+ setData((prev) => ({
+ ...prev,
+ customModelName: val,
+ }));
+ }}
+ placeholder="请输入模型名称"
+ />
+
+
+ ) : (
+
+ ({
+ key: AI_MODEL[type as keyof typeof AI_MODEL],
+ label: type.replaceAll('_', ' '),
+ icon: DoubaoModelSVG,
+ }))}
+ moreIcon={ModelChangeSVG}
+ moreText="更换模型"
+ placeHolder="请选择你需要的模型"
+ onChange={(key) => {
+ setData((prev) => ({
+ ...prev,
+ model: key as AI_MODEL,
+ }));
+ }}
+ value={data.model}
+ />
+
+ )}
-
-
+
@@ -40,11 +50,12 @@ function AvatarCard(props: IAvatarCardProps) {
diff --git a/src/config/common.ts b/src/config/common.ts
index 0aeb7fc..7f0c33d 100644
--- a/src/config/common.ts
+++ b/src/config/common.ts
@@ -11,6 +11,7 @@ import TRANSLATE from '@/assets/img/TRANSLATE.png';
import CHILDREN_ENCYCLOPEDIA from '@/assets/img/CHILDREN_ENCYCLOPEDIA.png';
import TEACHING_ASSISTANT from '@/assets/img/TEACHING_ASSISTANT.png';
import CUSTOMER_SERVICE from '@/assets/img/CUSTOMER_SERVICE.png';
+import SCREEN_READER from '@/assets/img/SCREEN_READER.png';
export enum ModelSourceType {
Custom = 'Custom',
@@ -130,9 +131,12 @@ export enum SCENE {
CUSTOMER_SERVICE = 'CUSTOMER_SERVICE',
CHILDREN_ENCYCLOPEDIA = 'CHILDREN_ENCYCLOPEDIA',
TEACHING_ASSISTANT = 'TEACHING_ASSISTANT',
+ SCREEN_READER = 'SCREEN_READER',
CUSTOM = 'CUSTOM',
}
+export const ScreenShareScene = [SCENE.SCREEN_READER];
+
export const Icon = {
[SCENE.INTELLIGENT_ASSISTANT]: INTELLIGENT_ASSISTANT,
[SCENE.VIRTUAL_GIRL_FRIEND]: VIRTUAL_GIRL_FRIEND,
@@ -140,6 +144,7 @@ export const Icon = {
[SCENE.CHILDREN_ENCYCLOPEDIA]: CHILDREN_ENCYCLOPEDIA,
[SCENE.CUSTOMER_SERVICE]: CUSTOMER_SERVICE,
[SCENE.TEACHING_ASSISTANT]: TEACHING_ASSISTANT,
+ [SCENE.SCREEN_READER]: SCREEN_READER,
[SCENE.CUSTOM]: INTELLIGENT_ASSISTANT,
};
@@ -150,6 +155,7 @@ export const Name = {
[SCENE.CHILDREN_ENCYCLOPEDIA]: '儿童百科',
[SCENE.CUSTOMER_SERVICE]: '售后客服',
[SCENE.TEACHING_ASSISTANT]: '课后助教',
+ [SCENE.SCREEN_READER]: '读屏助手',
[SCENE.CUSTOM]: '自定义',
};
@@ -163,6 +169,7 @@ export const Welcome = {
[SCENE.CHILDREN_ENCYCLOPEDIA]: '你好小朋友，你的小脑袋里又有什么问题啦？',
[SCENE.CUSTOMER_SERVICE]: '感谢您在我们餐厅用餐，请问您有什么问题需要反馈吗？',
[SCENE.TEACHING_ASSISTANT]: '你碰到什么问题啦？让我来帮帮你。',
+ [SCENE.SCREEN_READER]: '欢迎使用读屏助手, 请开启屏幕采集，我会为你解说屏幕内容。',
[SCENE.CUSTOM]: '',
};
@@ -173,6 +180,7 @@ export const Model = {
[SCENE.CHILDREN_ENCYCLOPEDIA]: AI_MODEL.DOUBAO_PRO_32K,
[SCENE.CUSTOMER_SERVICE]: AI_MODEL.DOUBAO_PRO_32K,
[SCENE.TEACHING_ASSISTANT]: AI_MODEL.VISION,
+ [SCENE.SCREEN_READER]: AI_MODEL.VISION,
[SCENE.CUSTOM]: AI_MODEL.DOUBAO_PRO_32K,
};
@@ -183,6 +191,7 @@ export const Voice = {
[SCENE.CHILDREN_ENCYCLOPEDIA]: VOICE_TYPE.通用女声,
[SCENE.CUSTOMER_SERVICE]: VOICE_TYPE.通用女声,
[SCENE.TEACHING_ASSISTANT]: VOICE_TYPE.通用女声,
+ [SCENE.SCREEN_READER]: VOICE_TYPE.通用男声,
[SCENE.CUSTOM]: VOICE_TYPE.通用女声,
};
@@ -213,6 +222,7 @@ export const Questions = {
'你们空调开得太冷了。',
],
[SCENE.TEACHING_ASSISTANT]: ['这个单词是什么意思？', '这道题该怎么做？', '我的表情是什么样的？'],
+ [SCENE.SCREEN_READER]: ['屏幕里这是什么?', '这道题你会做吗?', '帮我翻译解说下屏幕里的内容?'],
[SCENE.CUSTOM]: ['你能帮我解决什么问题?', '今天北京天气怎么样?', '你喜欢哪位流行歌手?'],
};
@@ -299,5 +309,23 @@ export const Prompt = {
##约束
- 回答问题要简明扼要，避免复杂冗长的表述，尽量不超过50个字；
- 回答中不要有“图片”、“图中”等相关字眼；`,
+ [SCENE.SCREEN_READER]: `##人设
+你是人们的 AI 伙伴，可以通过 【屏幕共享实时解析】+【百科知识】来为人们提供服务。
+
+##技能
+1. 实时理解屏幕中的内容，包括图片、文字、窗口焦点，自动捕捉光标轨迹；
+2. 拥有丰富的百科知识；
+3. 如果用户询问与视频和图片有关的问题，请结合【屏幕共享实时解析】的内容、你的【知识】和【用户问题】进行回答；
+
+##风格
+语言风格可以随着屏幕内容和用户需求调整，可以是幽默搞笑的娱乐解说，也可以是严谨硬核的技术分析。
+- 如果屏幕内容是娱乐节目、动画、游戏等，语言风格偏幽默、活波一些，可以使用夸张的比喻、流行梗、弹幕互动式语言；
+- 如果屏幕内容是办公软件、新闻、文章等，语言风格偏专业、正经一些。
+
+## 约束
+不要有任何特殊标点符号和任何 Markdown 格式输出，例如 *，# 等。
+`,
[SCENE.CUSTOM]: '',
};
+
+export const isVisionMode = (model: AI_MODEL) => model.startsWith('Vision');
diff --git a/src/config/config.ts b/src/config/config.ts
index cdfa7b3..24d6719 100644
--- a/src/config/config.ts
+++ b/src/config/config.ts
@@ -3,6 +3,7 @@
* SPDX-license-identifier: BSD-3-Clause
*/
+import { StreamIndex } from '@volcengine/rtc';
import {
TTS_CLUSTER,
ARK_V3_MODEL_ID,
@@ -16,6 +17,7 @@ import {
AI_MODEL,
AI_MODE_MAP,
AI_MODEL_MODE,
+ isVisionMode,
} from '.';
export const CONVERSATION_SIGNATURE = 'conversation';
@@ -37,6 +39,7 @@ export class ConfigFactory {
BusinessId: undefined,
/**
* @brief 必填, 房间 ID, 自定义即可，例如 "Room123"。
+ * @note 建议使用有特定规则、不重复的房间号名称。
*/
RoomId: 'Room123',
/**
@@ -59,7 +62,7 @@ export class ConfigFactory {
TTSAppId: 'Your TTS AppId',
/**
* @brief 已开通需要的语音合成服务的token。
- * 使用火山引擎双向流式语音合成服务时必填。
+ * 使用火山引擎双向流式语音合成服务时 必填。
*/
TTSToken: undefined,
/**
@@ -69,7 +72,7 @@ export class ConfigFactory {
ASRAppId: 'Your ASR AppId',
/**
* @brief 已开通流式语音识别大模型服务 AppId 对应的 Access Token。
- * 使用流式语音识别大模型服务时该参数为必填。
+ * 使用流式语音识别大模型服务时该参数为 必填。
*/
ASRToken: undefined,
};
@@ -116,6 +119,11 @@ export class ConfigFactory {
*/
InterruptMode = true;
+ /**
+ * @brief 如果使用视觉模型，用的是哪种源，有摄像头采集流/屏幕流
+ */
+ VisionSourceType = StreamIndex.STREAM_INDEX_MAIN;
+
get LLMConfig() {
const params: Record{Name[scene]}
-
- 声源来自 {ReversedVoiceType[TTSConfig?.VoiceType || '']}
-
+ 声源来自 {ReversedVoiceType[voice || '']}
模型 {LLMConfig.ModelName}
-
- {isVideoPublished ? (
-
- ) : (
- ) {
const isAIReady = msgHistory.length > 0;
const containerRef = useRef(null);
+ const isUserTextLoading = (owner: string) => {
+ return owner === userId && isUserTalking;
+ };
+
+ const isAITextLoading = (owner: string) => {
+ return owner === Config.BotName && isAITalking;
+ };
+
useEffect(() => {
const container = containerRef.current;
if (container) {
@@ -53,7 +61,9 @@ function Conversation(props: React.HTMLAttributes) {
- 
-
) : null;
}
diff --git a/src/pages/MainPage/MainArea/Room/Conversation.tsx b/src/pages/MainPage/MainArea/Room/Conversation.tsx
index c4f50f3..ddd0e3d 100644
--- a/src/pages/MainPage/MainArea/Room/Conversation.tsx
+++ b/src/pages/MainPage/MainArea/Room/Conversation.tsx
@@ -21,6 +21,14 @@ function Conversation(props: React.HTMLAttributes
- 请
+
+
+
+ 
+
- )}
+
+ 请
+ {isScreenMode ? (
+
+ 打开屏幕采集
+
+ ) : (
打开摄像头
-
- 体验豆包视觉理解模型
+ )}
体验豆包视觉理解模型
+
{value}
- {isAIReady && (isUserTalking || isAITalking) && index === msgHistory.length - 1 ? (
+ {isAIReady &&
+ (isUserTextLoading(user) || isAITextLoading(user)) &&
+ index === msgHistory.length - 1 ? (
) {
const { className, ...rest } = props;
const room = useSelector((state: RootState) => state.room);
const [open, setOpen] = useState(false);
const model = room.aiConfig.Config.LLMConfig?.ModelName;
+ const isScreenMode = ScreenShareScene.includes(room.scene);
const leaveRoom = useLeave();
- const { isAudioPublished, isVideoPublished, switchMic, switchCamera } = useDeviceState();
+ const {
+ isAudioPublished,
+ isVideoPublished,
+ isScreenPublished,
+ switchMic,
+ switchCamera,
+ switchScreenCapture,
+ } = useDeviceState();
+
const handleSetting = () => {
setOpen(true);
};
@@ -41,13 +53,22 @@ function ToolBar(props: React.HTMLAttributes) {
className={style.btn}
alt="mic"
/>
- {model === AI_MODEL.VISION ? (
- 
switchCamera(true)}
- className={style.btn}
- alt="camera"
- />
+ {isVisionMode(model) ? (
+ isScreenMode ? (
+ 
switchScreenCapture()}
+ className={style.btn}
+ alt="screenShare"
+ />
+ ) : (
+ switchCamera(true)}
+ className={style.btn}
+ alt="camera"
+ />
+ )
) : (
''
)}
@@ -60,6 +81,7 @@ function ToolBar(props: React.HTMLAttributes) {
style={{
width: 'max-content',
}}
+ footer={null}
>
@@ -67,4 +89,4 @@ function ToolBar(props: React.HTMLAttributes) {
);
}
-export default ToolBar;
+export default memo(ToolBar);
diff --git a/src/pages/MainPage/MainArea/Room/index.module.less b/src/pages/MainPage/MainArea/Room/index.module.less
index e5978ed..03d4d62 100644
--- a/src/pages/MainPage/MainArea/Room/index.module.less
+++ b/src/pages/MainPage/MainArea/Room/index.module.less
@@ -172,6 +172,15 @@
line-height: 22px;
}
+.closed {
+ width: 100%;
+ text-align: center;
+ color: #737A87;
+ font-size: 14px;
+ font-weight: 400;
+ line-height: 19.6px;
+}
+
.btns {
width: 100%;
display: flex;
@@ -262,6 +271,10 @@
border-radius: 8px;
}
+ .camera-player-hidden {
+ display: none !important;
+ }
+
.camera-placeholder {
width: 100%;
display: flex;
@@ -273,6 +286,8 @@
.camera-placeholder-close-note {
margin-bottom: 8px;
+ width: 60px;
+ height: 60px;
}
.camera-open-btn {
diff --git a/src/pages/MainPage/Menu/components/AISettingAnchor/index.module.less b/src/pages/MainPage/Menu/components/AISettingAnchor/index.module.less
new file mode 100644
index 0000000..1ede302
--- /dev/null
+++ b/src/pages/MainPage/Menu/components/AISettingAnchor/index.module.less
@@ -0,0 +1,44 @@
+/**
+ * Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
+ * SPDX-license-identifier: BSD-3-Clause
+ */
+
+.row {
+ width: 100%;
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ cursor: pointer;
+
+ .firstPart {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ width: 90%;
+ color: var(--text-color-text-2, var(--text-color-text-2, #42464E));
+ text-align: center;
+
+ font-family: "PingFang SC";
+ font-size: 13px;
+ font-style: normal;
+ font-weight: 500;
+ line-height: 22px;
+ letter-spacing: 0.039px;
+ }
+
+ .finalPart {
+ display: flex;
+ flex-direction: row;
+ align-items: center;
+ width: 10%;
+ justify-content: flex-end;
+
+ .rightOutlined {
+ font-size: 12px;
+ }
+ }
+
+ .icon {
+ margin-right: 4px;
+ }
+}
\ No newline at end of file
diff --git a/src/pages/MainPage/Menu/components/AISettingAnchor/index.tsx b/src/pages/MainPage/Menu/components/AISettingAnchor/index.tsx
new file mode 100644
index 0000000..d3cbaf2
--- /dev/null
+++ b/src/pages/MainPage/Menu/components/AISettingAnchor/index.tsx
@@ -0,0 +1,29 @@
+/**
+ * Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
+ * SPDX-license-identifier: BSD-3-Clause
+ */
+
+import { useState } from 'react';
+import { IconRight } from '@arco-design/web-react/icon';
+import AISettings from '@/components/AISettings';
+import styles from './index.module.less';
+
+function AISettingAnchor() {
+ const [open, setOpen] = useState(false);
+
+ const handleOpenDrawer = () => setOpen(true);
+ const handleCloseDrawer = () => setOpen(false);
+ return (
+ <>
+
+
+ AI 设置
+
+
+
{isJoined && utils.isMobile() && isVisionMode ? (
@@ -52,7 +55,7 @@ function Menu() {
) : null}
-
Demo Version 1.4.0
+ Demo Version {packageJson.version}
SDK Version {VERTC.getSdkVersion()}
{isJoined ? (
diff --git a/src/store/slices/room.ts b/src/store/slices/room.ts
index 0812fe7..89e6af4 100644
--- a/src/store/slices/room.ts
+++ b/src/store/slices/room.ts
@@ -11,7 +11,6 @@ import {
RemoteAudioStats,
} from '@volcengine/rtc';
import config, { SCENE } from '@/config';
-import utils from '@/utils/utils';
export interface IUser {
username?: string;
@@ -33,6 +32,7 @@ export interface Msg {
time: string;
user: string;
paragraph?: boolean;
+ definite?: boolean;
isInterrupted?: boolean;
}
@@ -59,6 +59,10 @@ export interface RoomState {
* @brief AI 是否正在说话
*/
isAITalking: boolean;
+ /**
+ * @brief AI 思考中
+ */
+ isAIThinking: boolean;
/**
* @brief 用户是否正在说话
*/
@@ -99,12 +103,14 @@ const initialState: RoomState = {
scene: SCENE.INTELLIGENT_ASSISTANT,
remoteUsers: [],
localUser: {
- publishAudio: true,
- publishVideo: true,
+ publishAudio: false,
+ publishVideo: false,
+ publishScreen: false,
},
autoPlayFailUser: [],
isJoined: false,
isAIGCEnable: false,
+ isAIThinking: false,
isAITalking: false,
isUserTalking: false,
networkQuality: NetworkQuality.UNKNOWN,
@@ -131,15 +137,19 @@ export const roomSlice = createSlice({
}
) => {
state.roomId = payload.roomId;
- state.localUser = payload.user;
+ state.localUser = {
+ ...state.localUser,
+ ...payload.user,
+ };
state.isJoined = true;
},
localLeaveRoom: (state) => {
state.roomId = undefined;
state.time = -1;
state.localUser = {
- publishAudio: true,
- publishVideo: true,
+ publishAudio: false,
+ publishVideo: false,
+ publishScreen: false,
};
state.remoteUsers = [];
state.isJoined = false;
@@ -159,7 +169,7 @@ export const roomSlice = createSlice({
updateLocalUser: (state, { payload }: { payload: Partial }) => {
state.localUser = {
...state.localUser,
- ...payload,
+ ...(payload || {}),
};
},
@@ -204,8 +214,14 @@ export const roomSlice = createSlice({
state.isAIGCEnable = payload.isAIGCEnable;
},
updateAITalkState: (state, { payload }) => {
+ state.isAIThinking = false;
+ state.isUserTalking = false;
state.isAITalking = payload.isAITalking;
},
+ updateAIThinkState: (state, { payload }) => {
+ state.isAIThinking = payload.isAIThinking;
+ state.isUserTalking = false;
+ },
updateAIConfig: (state, { payload }) => {
state.aiConfig = Object.assign(state.aiConfig, payload);
},
@@ -213,36 +229,74 @@ export const roomSlice = createSlice({
state.msgHistory = [];
},
setHistoryMsg: (state, { payload }) => {
- const paragraph = payload.paragraph;
- const aiTalking = payload.user === config.BotName;
- const userTalking = payload.user === state.localUser.userId;
- if (paragraph) {
- if (state.isAITalking) {
- state.isAITalking = false;
- }
- if (state.isUserTalking) {
- state.isUserTalking = false;
+ const { paragraph, definite } = payload;
+ /** 是否需要再创建新句子 */
+ const shouldCreateSentence = payload.definite;
+ state.isUserTalking = payload.user === state.localUser.userId;
+ if (state.msgHistory.length) {
+ const lastMsg = state.msgHistory.at(-1)!;
+ /** 当前讲话人更新字幕 */
+ if (lastMsg.user === payload.user) {
+ /** 如果上一句话是完整的 & 本次的话也是完整的, 则直接塞入 */
+ if (lastMsg.definite) {
+ state.msgHistory.push({
+ value: payload.text,
+ time: new Date().toString(),
+ user: payload.user,
+ definite,
+ paragraph,
+ });
+ } else {
+ /** 话未说完, 更新文字内容 */
+ lastMsg.value = payload.text;
+ lastMsg.time = new Date().toString();
+ lastMsg.paragraph = paragraph;
+ lastMsg.definite = definite;
+ lastMsg.user = payload.user;
+ }
+ /** 如果本次的话已经说完了, 提前塞入空字符串做准备 */
+ if (shouldCreateSentence) {
+ state.msgHistory.push({
+ value: '',
+ time: new Date().toString(),
+ user: '',
+ });
+ }
+ } else {
+ /** 换人说话了，塞入新句子 */
+ state.msgHistory.push({
+ value: payload.text,
+ time: new Date().toString(),
+ user: payload.user,
+ definite,
+ paragraph,
+ });
}
} else {
- if (state.isAITalking !== aiTalking) {
- state.isAITalking = aiTalking;
- }
- if (state.isUserTalking !== userTalking) {
- state.isUserTalking = userTalking;
- }
+ /** 首句话首字不会被打断 */
+ state.msgHistory.push({
+ value: payload.text,
+ time: new Date().toString(),
+ user: payload.user,
+ paragraph,
+ });
}
- utils.addMsgWithoutDuplicate(state.msgHistory, {
- user: payload.user,
- value: payload.text,
- time: new Date().toLocaleString(),
- isInterrupted: false,
- paragraph,
- });
},
setInterruptMsg: (state) => {
- const msg = state.msgHistory[state.msgHistory.length - 1];
- msg.isInterrupted = true;
- state.msgHistory[state.msgHistory.length - 1] = msg;
+ state.isAITalking = false;
+ if (!state.msgHistory.length) {
+ return;
+ }
+ /** 找到最后一个末尾的字幕, 将其状态置换为打断 */
+ for (let id = state.msgHistory.length - 1; id >= 0; id--) {
+ const msg = state.msgHistory[id];
+ if (msg.value) {
+ if (!msg.definite) {
+ state.msgHistory[id].isInterrupted = true;
+ }
+ break;
+ }
+ }
},
clearCurrentMsg: (state) => {
state.currentConversation = {};
@@ -250,10 +304,6 @@ export const roomSlice = createSlice({
state.isAITalking = false;
state.isUserTalking = false;
},
- setCurrentMsg: (state, { payload }) => {
- const { user, ...info } = payload;
- state.currentConversation[user || state.localUser.userId] = info;
- },
},
});
@@ -270,9 +320,9 @@ export const {
clearAutoPlayFail,
updateAIGCState,
updateAITalkState,
+ updateAIThinkState,
updateAIConfig,
setHistoryMsg,
- setCurrentMsg,
clearHistoryMsg,
clearCurrentMsg,
setInterruptMsg,
diff --git a/src/utils/handler.ts b/src/utils/handler.ts
index ef7fc67..b232bf1 100644
--- a/src/utils/handler.ts
+++ b/src/utils/handler.ts
@@ -6,10 +6,10 @@
import { useDispatch } from 'react-redux';
import logger from './logger';
import {
- setCurrentMsg,
setHistoryMsg,
setInterruptMsg,
updateAITalkState,
+ updateAIThinkState,
} from '@/store/slices/room';
import RtcClient from '@/lib/RtcClient';
import Utils from '@/utils/utils';
@@ -89,11 +89,16 @@ export const useMessageHandler = () => {
const { Code, Description } = Stage || {};
logger.debug(Code, Description);
switch (Code) {
+ case AGENT_BRIEF.THINKING:
+ dispatch(updateAIThinkState({ isAIThinking: true }));
+ break;
+ case AGENT_BRIEF.SPEAKING:
+ dispatch(updateAITalkState({ isAITalking: true }));
+ break;
case AGENT_BRIEF.FINISHED:
dispatch(updateAITalkState({ isAITalking: false }));
break;
case AGENT_BRIEF.INTERRUPTED:
- dispatch(updateAITalkState({ isAITalking: false }));
dispatch(setInterruptMsg());
break;
default:
@@ -118,7 +123,6 @@ export const useMessageHandler = () => {
dispatch(setHistoryMsg({ text: msg, user, paragraph, definite }));
}
}
- dispatch(setCurrentMsg({ msg, definite, user, paragraph }));
}
},
/**
@@ -141,8 +145,8 @@ export const useMessageHandler = () => {
ToolCallID: parsed?.tool_calls?.[0]?.id,
Content: map[name.toLocaleLowerCase().replaceAll('_', '')],
}),
- 'func',
- ),
+ 'func'
+ )
);
},
};
diff --git a/src/utils/utils.ts b/src/utils/utils.ts
index 2166d39..ffdd964 100644
--- a/src/utils/utils.ts
+++ b/src/utils/utils.ts
@@ -3,12 +3,7 @@
* SPDX-license-identifier: BSD-3-Clause
*/
-import { Msg, RoomState } from '@/store/slices/room';
-import RtcClient from '@/lib/RtcClient';
-
-
class Utils {
-
formatTime = (time: number): string => {
if (time < 0) {
return '00:00';
@@ -46,9 +41,9 @@ class Utils {
const query = window.location.search.substring(1);
const pairs = query.split('&');
return pairs.reduce<{ [key: string]: string }>((queries, pair) => {
- const [key, value] = decodeURIComponent(pair).split('=');
+ const [key, value] = pair.split('=');
if (key && value) {
- queries[key] = value;
+ queries[key] = decodeURIComponent(value);
}
return queries;
}, {});
@@ -58,34 +53,6 @@ class Utils {
isArray = Array.isArray;
- debounce = (func: (...args: any[]) => void, wait: number) => {
- let timeoutId: ReturnType | null = null;
- return function (...args: any[]) {
- if (timeoutId !== null) {
- clearTimeout(timeoutId);
- }
-
- timeoutId = setTimeout(() => {
- func(...args);
- }, wait);
- };
- };
-
- addMsgWithoutDuplicate = (arr: RoomState['msgHistory'], added: Msg) => {
- if (arr.length) {
- const last = arr.at(-1)!;
- const { user, value, isInterrupted } = last;
- if (
- (added.user === RtcClient.basicInfo.user_id && last.user === added.user) ||
- (user === added.user && added.value.startsWith(value) && value.trim())
- ) {
- arr.pop();
- added.isInterrupted = isInterrupted;
- }
- }
- arr.push(added);
- };
-
/**
* @brief 将字符串包装成 TLV
*/
@@ -119,7 +86,7 @@ class Utils {
* @note TLV 数据格式
* | magic number | length(big-endian) | value |
* @param {ArrayBufferLike} tlvBuffer
- * @returns
+ * @returns
*/
tlv2String(tlvBuffer: ArrayBufferLike) {
const typeBuffer = new Uint8Array(tlvBuffer, 0, 4);
diff --git a/yarn.lock b/yarn.lock
index 7d8dc66..2f1e555 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2210,10 +2210,10 @@
"@typescript-eslint/types" "5.31.0"
eslint-visitor-keys "^3.3.0"
-"@volcengine/rtc@4.58.9":
- version "4.58.9"
- resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.58.9.tgz#841ebaddd5d4963c71abd33037bd76d1d490d928"
- integrity sha512-nnXnNW9pVo8ynBSxVe0ikNIdxWfoSx5oOnwK7EoMCXdc2bJgHATpz/B+Kv2F1k4GjYAbo7ZcOm/g3cchvHgH5Q==
+"@volcengine/rtc@4.66.1":
+ version "4.66.1"
+ resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.66.1.tgz#1934c269b31216f43718ae46b169c59ac5e474f2"
+ integrity sha512-APznH6eosmKJC1HYJJ8s6G3Mq3OSgw6ivv6uCiayM5QNMBj+GW6zxf+MVsk5rm6r4R92TLwQErWonJ8yzGO4xA==
dependencies:
eventemitter3 "^4.0.7"