diff --git a/README.md b/README.md index 32faeb8..29f3a24 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ Demo 中以 `Custom` 场景为例，您可以自行新增场景。 - `VoiceChat`: 场景下的 AIGC 配置。 - 可参考 https://www.volcengine.com/docs/6348/1558163 中参数描述，完整填写参数内容。 - 可通过 [快速跑通 Demo](https://console.volcengine.com/rtc/aigc/run?s=g) 快速获取参数, 跑通后点击右上角 `接入 API` 按钮复制相关代码贴到 JSON 配置文件中即可。 - ## 快速开始 请注意，服务端和 Web 端都需要启动, 启动步骤如下: ### 服务端 @@ -87,6 +86,8 @@ yarn dev ### Demo 更新 #### [1.6.0] +- 2025-09-30 + - 更新数字人场景相关配置 - 2025-07-08 - 更新 RTC Web SDK 版本至 4.66.20 - 2025-06-26 diff --git a/Server/app.js b/Server/app.js index f612853..f5ce4ae 100644 --- a/Server/app.js +++ b/Server/app.js @@ -115,7 +115,9 @@ app.use(async ctx => { SceneConfig.botName = VoiceChat?.AgentConfig?.UserId; SceneConfig.isInterruptMode = VoiceChat?.Config?.InterruptMode === 0; SceneConfig.isVision = VoiceChat?.Config?.LLMConfig?.VisionConfig?.Enable; - SceneConfig.isScreenMode = VoiceChat?.Config?.LLMConfig?.VisionConfig?.SnapshoutConfig?.StreamType === 1; + SceneConfig.isScreenMode = VoiceChat?.Config?.LLMConfig?.VisionConfig?.SnapshotConfig?.StreamType === 1; + SceneConfig.isAvatarScene = VoiceChat?.Config?.AvatarConfig?.Enabled; + SceneConfig.avatarBgUrl = VoiceChat?.Config?.AvatarConfig?.BackgroundUrl; delete RTCConfig.AppKey; return { scene: SceneConfig || {}, diff --git a/Server/scenes/Custom.json b/Server/scenes/Custom.json index 9e07370..eb16b44 100644 --- a/Server/scenes/Custom.json +++ b/Server/scenes/Custom.json @@ -60,6 +60,15 @@ "Enable": false } }, + "AvatarConfig": { + "Enabled": false, + "AvatarType": "3min", + "AvatarRole": "250623-zhibo-linyunzhi", + "BackgroundUrl": "", + "VideoBitrate": 2000, + "AvatarAppID": "", + "AvatarToken": "" + }, "InterruptMode": 0 } } diff --git a/src/components/AIAvatarLoading/index.module.less b/src/components/AIAvatarLoading/index.module.less new file mode 100644 index 0000000..0f47246 --- /dev/null +++ b/src/components/AIAvatarLoading/index.module.less @@ -0,0 +1,26 @@ +.container { + display: flex; + justify-content: center; + align-items: center; + width: 100%; + height: 100%; +} + +.avatarContainer { + position: relative; + display: flex; + flex-direction: column; + align-items: center; +} + +.avatarSvg { + max-width: 80%; + height: auto; +} + +/* 加载文字样式 */ +.loadingText { + margin-top: 30px; + font-size: 18px; + text-align: center; +} diff --git a/src/components/AIAvatarLoading/index.tsx b/src/components/AIAvatarLoading/index.tsx new file mode 100644 index 0000000..0d1d213 --- /dev/null +++ b/src/components/AIAvatarLoading/index.tsx @@ -0,0 +1,76 @@ +/** + * Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved. + * SPDX-license-identifier: BSD-3-Clause + */ + +import styles from './index.module.less'; + +function AIAvatarReadying() { + return ( +
+
+ {/* SVG 包含人型轮廓和流光效果 */} + + {/* 原始人型轮廓 */} + + + {/* 渐变定义 */} + + {/* 原始渐变 */} + + + + + + + + {/* 添加动画效果，使渐变沿着路径运动 */} + + + + + + + + + {/* 加载文字 */} +
数字人准备中，请稍候...
+
+
+ ); +} + +export default AIAvatarReadying; diff --git a/src/components/FullScreenCard/index.module.less b/src/components/FullScreenCard/index.module.less index 5a64234..af0152a 100644 --- a/src/components/FullScreenCard/index.module.less +++ b/src/components/FullScreenCard/index.module.less @@ -10,7 +10,6 @@ left: 0; right: 0; text-align: center; - text-align: center; width: 100%; display: flex; flex-direction: column; @@ -23,4 +22,16 @@ left: 16px; top: 16px; } + + &.hidden { + display: none; + } +} + +.blur-bg { + background-position: center; + background-size: cover; + background-repeat: no-repeat; + filter: blur(20px); + transform: scale(1.1); } diff --git a/src/components/FullScreenCard/index.tsx b/src/components/FullScreenCard/index.tsx index 3a2ca08..773d7db 100644 --- a/src/components/FullScreenCard/index.tsx +++ b/src/components/FullScreenCard/index.tsx @@ -3,14 +3,35 @@ * SPDX-license-identifier: BSD-3-Clause */ +import { useSelector } from 'react-redux'; import UserTag from '../UserTag'; +import { RootState } from '@/store'; import style from './index.module.less'; +import { useScene } from '@/lib/useCommon'; + +export const LocalFullID = 'local-full-player'; +export const RemoteFullID = 'remote-full-player'; function FullScreenCard() { + const isFullScreen = useSelector((state: RootState) => state.room.isFullScreen); + const scene = useScene(); return ( -
- -
+ <> +
+ +
+
+
+
+ +
+ ); } diff --git a/src/lib/RtcClient.ts b/src/lib/RtcClient.ts index a9c715d..bace6ee 100644 --- a/src/lib/RtcClient.ts +++ b/src/lib/RtcClient.ts @@ -325,10 +325,21 @@ export class RTCClient { ); }; + setRemoteVideoPlayer = (userId: string, renderDom?: string | HTMLElement, renderMode = VideoRenderMode.RENDER_MODE_HIDDEN) => { + return this.engine.setRemoteVideoPlayer( + StreamIndex.STREAM_INDEX_MAIN, + { + renderDom, + userId, + renderMode, + } + ); + } + /** * @brief 移除播放器 */ - removeVideoPlayer = (userId: string, scope: StreamIndex | 'Both' = 'Both') => { + removeLocalVideoPlayer = (userId: string, scope: StreamIndex | 'Both' = 'Both') => { let removeScreen = scope === StreamIndex.STREAM_INDEX_SCREEN; let removeCamera = scope === StreamIndex.STREAM_INDEX_MAIN; if (scope === 'Both') { diff --git a/src/lib/listenerHooks.ts b/src/lib/listenerHooks.ts index 42eb6e1..910eaf1 100644 --- a/src/lib/listenerHooks.ts +++ b/src/lib/listenerHooks.ts @@ -35,6 +35,7 @@ import RtcClient, { IEventListener } from './RtcClient'; import { setMicrophoneList, updateSelectedDevice } from '@/store/slices/device'; import { useMessageHandler } from '@/utils/handler'; +import store from '@/store'; const useRtcListeners = (): IEventListener => { const dispatch = useDispatch(); @@ -83,9 +84,15 @@ const useRtcListeners = (): IEventListener => { const { userId, mediaType } = e; const payload: IUser = { userId }; if (mediaType === MediaType.AUDIO) { - /** 暂不需要 */ + payload.publishAudio = true; + } else if (mediaType === MediaType.VIDEO) { + payload.publishVideo = true; + } else if (mediaType === MediaType.AUDIO_AND_VIDEO) { + payload.publishAudio = true; + payload.publishVideo = true; } - payload.publishAudio = true; + const isFullScreen = store.getState().room.isFullScreen; + RtcClient.setRemoteVideoPlayer(userId, isFullScreen ? 'remote-video-player' : 'remote-full-player'); dispatch(updateRemoteUser(payload)); }; @@ -104,7 +111,7 @@ const useRtcListeners = (): IEventListener => { if (mediaType === MediaType.AUDIO_AND_VIDEO) { payload.publishAudio = false; } - + RtcClient.setRemoteVideoPlayer(userId); dispatch(updateRemoteUser(payload)); }; diff --git a/src/pages/MainPage/MainArea/Antechamber/index.tsx b/src/pages/MainPage/MainArea/Antechamber/index.tsx index a038c91..0c623ec 100644 --- a/src/pages/MainPage/MainArea/Antechamber/index.tsx +++ b/src/pages/MainPage/MainArea/Antechamber/index.tsx @@ -8,15 +8,18 @@ import { isMobile } from '@/utils/utils'; import InvokeButton from '@/pages/MainPage/MainArea/Antechamber/InvokeButton'; import { useJoin, useScene } from '@/lib/useCommon'; import AIChangeCard from '@/components/AiChangeCard'; -import { updateFullScreen } from '@/store/slices/room'; +import { updateFullScreen, updateShowSubtitle } from '@/store/slices/room'; import style from './index.module.less'; function Antechamber() { const dispatch = useDispatch(); const [joining, dispatchJoin] = useJoin(); - const { isScreenMode } = useScene(); + const { isScreenMode, isAvatarScene } = useScene(); + const handleJoinRoom = () => { - dispatch(updateFullScreen({ isFullScreen: !isMobile() && !isScreenMode })); // 初始化 + dispatch(updateFullScreen({ isFullScreen: !isMobile() && !isScreenMode && !isAvatarScene })); // 初始化 + dispatch(updateShowSubtitle({ isShowSubtitle: !isAvatarScene })); + if (!joining) { dispatchJoin(); } diff --git a/src/pages/MainPage/MainArea/Room/CameraArea.tsx b/src/pages/MainPage/MainArea/Room/CameraArea.tsx index 4535b61..68d0187 100644 --- a/src/pages/MainPage/MainArea/Room/CameraArea.tsx +++ b/src/pages/MainPage/MainArea/Room/CameraArea.tsx @@ -17,27 +17,36 @@ import AiAvatarCard from '@/components/AiAvatarCard'; import UserAvatar from '@/assets/img/userAvatar.png'; import CameraCloseNoteSVG from '@/assets/img/CameraCloseNote.svg'; import ScreenCloseNoteSVG from '@/assets/img/ScreenCloseNote.svg'; +import { LocalFullID, RemoteFullID } from '@/components/FullScreenCard'; const LocalVideoID = 'local-video-player'; const LocalScreenID = 'local-screen-player'; +const RemoteVideoID = 'remote-video-player'; function CameraArea(props: React.HTMLAttributes) { const { className, ...rest } = props; const room = useSelector((state: RootState) => state.room); const { isFullScreen, scene } = room; - const { isVision, isScreenMode } = useScene(); + const { isVision, isScreenMode, botName } = useScene(); const { isVideoPublished, isScreenPublished, switchCamera, switchScreenCapture } = useDeviceState(); + const isRemoteVideoPublished = room.remoteUsers.find(user => user.username === botName)?.publishVideo ?? false const setVideoPlayer = () => { - RtcClient.removeVideoPlayer(room.localUser.username!); + RtcClient.removeLocalVideoPlayer(room.localUser.username!); if (isVideoPublished || isScreenPublished) { RtcClient.setLocalVideoPlayer( room.localUser.username!, - isFullScreen ? 'local-full-player' : isScreenMode ? LocalScreenID : LocalVideoID, + isFullScreen ? LocalFullID : isScreenMode ? LocalScreenID : LocalVideoID, isScreenPublished, isScreenMode ? VideoRenderMode.RENDER_MODE_FILL : VideoRenderMode.RENDER_MODE_HIDDEN ); + if(isRemoteVideoPublished) { + RtcClient.setRemoteVideoPlayer( + botName, + isFullScreen ? RemoteVideoID : RemoteFullID, + ); + } } }; @@ -72,6 +81,13 @@ function CameraArea(props: React.HTMLAttributes) { isScreenPublished && isScreenMode ? '' : styles['camera-player-hidden'] }`} /> +
) { - const { className, ...rest } = props; +function Conversation(props: React.HTMLAttributes & { showSubtitle: boolean }) { + const { className, showSubtitle, ...rest } = props; const room = useSelector((state: RootState) => state.room); const { msgHistory, isFullScreen } = room; const { userId } = useSelector((state: RootState) => state.room.localUser); const { isAITalking, isUserTalking, scene } = useSelector((state: RootState) => state.room); const isAIReady = msgHistory.length > 0; const containerRef = useRef(null); - const { botName, icon } = useScene(); + const { botName, icon, isAvatarScene } = useScene(); const isUserTextLoading = (owner: string) => { return owner === userId && isUserTalking; }; const isAITextLoading = (owner: string) => { - return owner === botName && isAITalking; + return (owner === botName || owner.includes('voiceChat_')) && isAITalking; }; useEffect(() => { @@ -46,20 +47,27 @@ function Conversation(props: React.HTMLAttributes) { className={`${styles.conversation} ${className} ${isFullScreen ? styles.fullScreen : ''} ${ isMobile() ? styles.mobileConversation : '' }`} + style={isAvatarScene && !isAIReady ? { justifyContent: 'center' } : {}} {...rest} > {lines.map((line) => line)} {!isAIReady ? (
- - AI 准备中, 请稍侯 + {isAvatarScene ? ( + + ) : ( + <> + + AI 准备中, 请稍侯 + + )}
) : ( '' )} - {msgHistory?.map(({ value, user, isInterrupted }, index) => { + {(showSubtitle ? msgHistory : [])?.map(({ value, user, isInterrupted }, index) => { const isUserMsg = user === userId; - const isRobotMsg = user === botName; + const isRobotMsg = user === botName || user.includes('voiceChat_'); if (!isUserMsg && !isRobotMsg) { return ''; } diff --git a/src/pages/MainPage/MainArea/Room/ToolBar.tsx b/src/pages/MainPage/MainArea/Room/ToolBar.tsx index 1cf0be3..5952195 100644 --- a/src/pages/MainPage/MainArea/Room/ToolBar.tsx +++ b/src/pages/MainPage/MainArea/Room/ToolBar.tsx @@ -58,7 +58,7 @@ function ToolBar(props: React.HTMLAttributes) { {isScreenMode && ( switchScreenCapture()} + onClick={() => switchScreenCapture(true)} className={style.btn} alt="screenShare" /> diff --git a/src/pages/MainPage/MainArea/Room/index.module.less b/src/pages/MainPage/MainArea/Room/index.module.less index 15136ea..98bc8cd 100644 --- a/src/pages/MainPage/MainArea/Room/index.module.less +++ b/src/pages/MainPage/MainArea/Room/index.module.less @@ -310,6 +310,7 @@ width: 100%; height: 184px; border-radius: 8px; + overflow: hidden; } .camera-player-hidden { diff --git a/src/pages/MainPage/MainArea/Room/index.tsx b/src/pages/MainPage/MainArea/Room/index.tsx index 0291e1a..8093ced 100644 --- a/src/pages/MainPage/MainArea/Room/index.tsx +++ b/src/pages/MainPage/MainArea/Room/index.tsx @@ -15,10 +15,12 @@ import { RootState } from '@/store'; import UserTag from '@/components/UserTag'; import FullScreenCard from '@/components/FullScreenCard'; import MobileToolBar from '@/pages/Mobile/MobileToolBar'; +import { useScene } from '@/lib/useCommon'; function Room() { const room = useSelector((state: RootState) => state.room); const { isShowSubtitle, scene, isFullScreen } = room; + const { isAvatarScene } = useScene(); return (
{isMobile() ?
: null} @@ -26,7 +28,7 @@ function Room() { {isShowSubtitle && !isMobile() ? ( ) : null} - {isFullScreen && !isMobile() ? ( + {(isFullScreen || isAvatarScene) && !isMobile() ? ( ) : isMobile() && isShowSubtitle ? null : ( )} {isMobile() ? null : } - {isShowSubtitle && } +
AI生成内容由大模型生成，不能完全保障真实
diff --git a/src/store/slices/room.ts b/src/store/slices/room.ts index bdec53d..39c4a66 100644 --- a/src/store/slices/room.ts +++ b/src/store/slices/room.ts @@ -45,6 +45,8 @@ export interface SceneConfig { isVision: boolean; isScreenMode: boolean; isInterruptMode: boolean; + isAvatarScene: boolean; + avatarBgUrl: string; } export interface RTCConfig { @@ -283,12 +285,18 @@ export const roomSlice = createSlice({ const { paragraph, definite } = payload; const lastMsg = state.msgHistory.at(-1)! || {}; /** 是否需要再创建新句子 */ - const fromBot = payload.user === state.sceneConfigMap[state.scene].botName; + const fromBot = + payload.user === state.sceneConfigMap[state.scene].botName || + payload.user.includes('voiceChat_'); /** - * Bot 的语句以 definite 判断是否需要追加新内容 + * Bot 的语句： + * 1. 在 SubtitleMode=0 时（未启用数字人时默认值），以 definite 判断是否需要追加新内容 + * 2. 在 SubtitleMode=1 时（启用数字人时强制设定为 1），以 paragraph 判断是否需要追加新内容 * User 的语句以 paragraph 判断是否需要追加新内容 */ - const lastMsgCompleted = fromBot ? lastMsg.definite : lastMsg.paragraph; + const currentSubtitleMode = state.sceneConfigMap[state.scene].isAvatarScene ? 1 : 0; + const lastMsgCompleted = + !fromBot || currentSubtitleMode ? lastMsg.paragraph : lastMsg.definite; if (state.msgHistory.length) { /** 如果上一句话是完整的则新增语句 */ @@ -302,7 +310,11 @@ export const roomSlice = createSlice({ }); } else { /** 话未说完, 更新文字内容 */ - lastMsg.value = payload.text; + if (fromBot && currentSubtitleMode) { + lastMsg.value += payload.text; + } else { + lastMsg.value = payload.text; + } lastMsg.time = new Date().toString(); lastMsg.paragraph = paragraph; lastMsg.definite = definite;