feat: support AvatarConfig

This commit is contained in:
jinyaozzz 2025-09-30 15:21:41 +08:00
parent f25673b882
commit 22e1d32415
16 changed files with 237 additions and 31 deletions

View File

@ -33,7 +33,6 @@ Demo 中以 `Custom` 场景为例,您可以自行新增场景。
- `VoiceChat`: 场景下的 AIGC 配置。
- 可参考 https://www.volcengine.com/docs/6348/1558163 中参数描述,完整填写参数内容。
- 可通过 [快速跑通 Demo](https://console.volcengine.com/rtc/aigc/run?s=g) 快速获取参数, 跑通后点击右上角 `接入 API` 按钮复制相关代码贴到 JSON 配置文件中即可。
## 快速开始
请注意,服务端和 Web 端都需要启动, 启动步骤如下:
### 服务端
@ -87,6 +86,8 @@ yarn dev
### Demo 更新
#### [1.6.0]
- 2025-09-30
- 更新数字人场景相关配置
- 2025-07-08
- 更新 RTC Web SDK 版本至 4.66.20
- 2025-06-26

View File

@ -115,7 +115,9 @@ app.use(async ctx => {
SceneConfig.botName = VoiceChat?.AgentConfig?.UserId;
SceneConfig.isInterruptMode = VoiceChat?.Config?.InterruptMode === 0;
SceneConfig.isVision = VoiceChat?.Config?.LLMConfig?.VisionConfig?.Enable;
SceneConfig.isScreenMode = VoiceChat?.Config?.LLMConfig?.VisionConfig?.SnapshoutConfig?.StreamType === 1;
SceneConfig.isScreenMode = VoiceChat?.Config?.LLMConfig?.VisionConfig?.SnapshotConfig?.StreamType === 1;
SceneConfig.isAvatarScene = VoiceChat?.Config?.AvatarConfig?.Enabled;
SceneConfig.avatarBgUrl = VoiceChat?.Config?.AvatarConfig?.BackgroundUrl;
delete RTCConfig.AppKey;
return {
scene: SceneConfig || {},

View File

@ -60,6 +60,15 @@
"Enable": false
}
},
"AvatarConfig": {
"Enabled": false,
"AvatarType": "3min",
"AvatarRole": "250623-zhibo-linyunzhi",
"BackgroundUrl": "",
"VideoBitrate": 2000,
"AvatarAppID": "",
"AvatarToken": ""
},
"InterruptMode": 0
}
}

View File

@ -0,0 +1,26 @@
.container {
display: flex;
justify-content: center;
align-items: center;
width: 100%;
height: 100%;
}
.avatarContainer {
position: relative;
display: flex;
flex-direction: column;
align-items: center;
}
.avatarSvg {
max-width: 80%;
height: auto;
}
/* 加载文字样式 */
.loadingText {
margin-top: 30px;
font-size: 18px;
text-align: center;
}

View File

@ -0,0 +1,76 @@
/**
* Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
* SPDX-license-identifier: BSD-3-Clause
*/
import styles from './index.module.less';
function AIAvatarReadying() {
return (
<div className={styles.container}>
<div className={styles.avatarContainer}>
{/* SVG 包含人型轮廓和流光效果 */}
<svg
className={styles.avatarSvg}
width="35vh"
height="42vh"
viewBox="0 0 457 549"
fill="none"
xmlns="http://www.w3.org/2000/svg"
>
{/* 原始人型轮廓 */}
<path
d="M175.137 244.821C175.12 240.915 174.986 232.095 174.729 228.127L174.727 228.106L174.726 228.087L174.668 227.474C174.045 221.385 171.924 216.347 168.181 212.481L167.801 212.098C164.091 208.429 159.982 204.706 155.477 200.929C153.336 198.887 151.625 196.437 150.444 193.724L150.433 193.697L150.42 193.671L149.841 192.422C148.509 189.52 147.278 186.572 146.149 183.585C144.846 179.572 143.541 175.295 142.214 170.751L141.912 169.719L140.886 169.401L140.555 169.295C138.934 168.753 137.41 167.955 136.041 166.931C134.262 165.342 132.653 163.572 131.238 161.651C129.191 158.679 127.692 155.364 126.813 151.863L126.785 151.752L126.745 151.645L126.492 150.944C125.279 147.431 124.799 143.704 125.085 139.994C125.478 136.364 126.156 133.326 127.121 130.858L127.127 130.842L127.133 130.826C128.206 127.935 129.823 125.278 131.897 122.997L132.438 122.403L132.418 121.6C132.157 111.445 132.679 101.284 133.98 91.2086C135.184 81.8895 137.078 72.6727 139.647 63.6344L139.651 63.6207L139.654 63.608C142.166 54.2848 146.286 45.4712 151.827 37.5641L151.84 37.5446L151.854 37.525C156.768 30.1535 162.226 24.1949 168.194 19.6344C174.287 15.0104 180.474 11.3978 186.775 8.77893L186.779 8.77698C192.676 6.31057 198.595 4.59753 204.487 3.63049L205.665 3.4469C212.051 2.52917 218.207 2.07191 224.173 2.0719H226.075C232.924 2.20416 239.709 3.04996 246.463 4.62952L246.472 4.63147C253.613 6.26715 260.58 8.59178 267.272 11.5719V11.5709C273.733 14.4735 279.477 17.7844 284.513 21.4576C289.542 25.1405 293.399 28.7986 296.171 32.3785L296.179 32.3893L296.188 32.4C302.893 40.8262 307.772 50.094 310.875 60.2135L310.878 60.2203C313.858 69.819 316.08 79.6371 317.523 89.5836C318.827 100.26 319.489 111.077 319.489 122.036V123.048L320.306 123.648C321.902 124.82 323.183 126.369 324.034 128.157L324.062 128.216L324.095 128.273C325.021 129.93 325.815 132.135 326.456 134.989V134.99C327.041 137.657 327.136 141.06 326.627 145.227L326.624 145.255L326.621 145.285C326.108 150.808 325.015 155.025 323.453 158.076L323.449 158.082L323.446 158.089C321.83 161.302 320.045 163.696 318.165 165.395L318.148 165.41L318.133 165.425C316.207 167.244 313.9 168.612 311.381 169.429L310.377 169.755L310.079 170.768C308.749 175.302 307.465 179.574 306.145 183.578C304.848 187.009 303.364 190.366 301.697 193.633L301.694 193.639C300.082 196.825 298.307 199.225 296.446 200.897C292.063 204.433 288.379 207.515 285.397 210.127L284.158 211.221C280.486 214.496 278.291 219.798 277.187 226.703C276.335 231 276.102 240.098 276.349 244.339L276.35 244.348V244.357C276.652 248.822 277.859 253.232 279.914 257.618L279.917 257.625L279.921 257.631C282.032 262.048 285.236 266.234 289.477 270.139C293.797 274.118 299.755 277.607 307.276 280.617V280.618C313.953 283.342 321.212 285.795 329.068 287.978L329.071 287.979C331.164 288.557 334.747 289.129 338.918 289.705C343.13 290.286 348.095 290.889 353.013 291.508C357.944 292.129 362.834 292.768 366.946 293.424C371.118 294.091 374.29 294.746 375.915 295.364V295.365C377.947 296.146 381.43 296.926 385.408 297.711C389.43 298.504 394.194 299.346 398.827 300.216C403.491 301.092 408.052 302.002 411.788 302.947C413.656 303.419 415.286 303.893 416.601 304.365C417.952 304.85 418.829 305.283 419.296 305.629L419.3 305.631C424.589 309.523 428.393 314.851 430.644 321.74C445.731 382.021 453.785 439.764 454.411 481.881C454.725 502.978 453.172 520 449.79 531.429C448.096 537.155 445.996 541.291 443.602 543.85C441.281 546.33 438.717 547.314 435.77 546.913L435.755 546.911L435.741 546.91L433.611 546.654C388.296 541.315 305.942 536.993 226.451 532.245C145.556 527.413 67.7489 522.144 34.3936 514.951H34.3926C31.4052 514.293 29.0212 513.642 27.2168 513.005C25.3611 512.349 24.3001 511.77 23.8018 511.342L23.7822 511.325L23.7617 511.309L23.4365 511.037C20.0453 508.119 15.7049 502.035 11.8477 492.491C7.88642 482.689 4.49264 469.376 3.15625 452.444C0.490903 418.673 6.01322 370.536 31.4814 307.264C34.9856 304.532 39.8152 302.214 45.5293 300.212C51.4403 298.14 58.1707 296.444 65.1162 294.954C72.0624 293.463 79.164 292.19 85.8398 290.95C92.4891 289.714 98.7347 288.507 103.871 287.155C108.374 285.97 118.591 284.526 128.896 282.753C133.986 281.877 139.052 280.926 143.308 279.9C147.458 278.899 151.135 277.76 153.242 276.407L153.243 276.408C158.848 272.856 163.246 269.382 166.311 265.953C169.34 262.586 171.586 259.177 173.007 255.68C174.418 252.207 175.137 248.593 175.137 244.83V244.821Z"
stroke="url(#paint0_linear)"
strokeWidth="4"
/>
{/* 渐变定义 */}
<defs>
{/* 原始渐变 */}
<linearGradient
id="paint0_linear"
x1="142.5"
y1="83.5"
x2="299.5"
y2="401.5"
gradientUnits="userSpaceOnUse"
>
<stop stopColor="#6792FF" />
<stop offset="0.138788" stopColor="#D093FF" />
<stop offset="0.282833" stopColor="#9DFFE3" stopOpacity="0.318618" />
<stop offset="0.519953" stopColor="white" stopOpacity="0" />
<stop offset="1" stopColor="white" stopOpacity="0" />
{/* 添加动画效果,使渐变沿着路径运动 */}
<animate attributeName="x1" values="0; 457; 0" dur="4s" repeatCount="indefinite" />
<animate
attributeName="y1"
values="549; 157; 549"
dur="4s"
repeatCount="indefinite"
/>
<animate
attributeName="x2"
values="157; 614; 614"
dur="4s"
repeatCount="indefinite"
/>
<animate
attributeName="y2"
values="157; 706; 157"
dur="4s"
repeatCount="indefinite"
/>
</linearGradient>
</defs>
</svg>
{/* 加载文字 */}
<div className={styles.loadingText}>...</div>
</div>
</div>
);
}
export default AIAvatarReadying;

View File

@ -10,7 +10,6 @@
left: 0;
right: 0;
text-align: center;
text-align: center;
width: 100%;
display: flex;
flex-direction: column;
@ -23,4 +22,16 @@
left: 16px;
top: 16px;
}
&.hidden {
display: none;
}
}
.blur-bg {
background-position: center;
background-size: cover;
background-repeat: no-repeat;
filter: blur(20px);
transform: scale(1.1);
}

View File

@ -3,14 +3,35 @@
* SPDX-license-identifier: BSD-3-Clause
*/
import { useSelector } from 'react-redux';
import UserTag from '../UserTag';
import { RootState } from '@/store';
import style from './index.module.less';
import { useScene } from '@/lib/useCommon';
export const LocalFullID = 'local-full-player';
export const RemoteFullID = 'remote-full-player';
function FullScreenCard() {
const isFullScreen = useSelector((state: RootState) => state.room.isFullScreen);
const scene = useScene();
return (
<div className={`${style.card}`} id="local-full-player">
<>
<div className={`${style.card} ${!isFullScreen ? style.hidden : ''}`} id={LocalFullID}>
<UserTag name="我" className={style.tag} />
</div>
<div
className={`${style.card} ${isFullScreen ? style.hidden : ''} ${style['blur-bg']}`}
style={{ backgroundImage: `url(${scene.avatarBgUrl})` }}
/>
<div
className={`${style.card} ${isFullScreen ? style.hidden : ''}`}
style={{ background: 'unset' }}
>
<div id={RemoteFullID} style={{ width: '60%', height: '100%' }} />
<UserTag name="AI" className={style.tag} />
</div>
</>
);
}

View File

@ -325,10 +325,21 @@ export class RTCClient {
);
};
setRemoteVideoPlayer = (userId: string, renderDom?: string | HTMLElement, renderMode = VideoRenderMode.RENDER_MODE_HIDDEN) => {
return this.engine.setRemoteVideoPlayer(
StreamIndex.STREAM_INDEX_MAIN,
{
renderDom,
userId,
renderMode,
}
);
}
/**
* @brief
*/
removeVideoPlayer = (userId: string, scope: StreamIndex | 'Both' = 'Both') => {
removeLocalVideoPlayer = (userId: string, scope: StreamIndex | 'Both' = 'Both') => {
let removeScreen = scope === StreamIndex.STREAM_INDEX_SCREEN;
let removeCamera = scope === StreamIndex.STREAM_INDEX_MAIN;
if (scope === 'Both') {

View File

@ -35,6 +35,7 @@ import RtcClient, { IEventListener } from './RtcClient';
import { setMicrophoneList, updateSelectedDevice } from '@/store/slices/device';
import { useMessageHandler } from '@/utils/handler';
import store from '@/store';
const useRtcListeners = (): IEventListener => {
const dispatch = useDispatch();
@ -83,9 +84,15 @@ const useRtcListeners = (): IEventListener => {
const { userId, mediaType } = e;
const payload: IUser = { userId };
if (mediaType === MediaType.AUDIO) {
/** 暂不需要 */
}
payload.publishAudio = true;
} else if (mediaType === MediaType.VIDEO) {
payload.publishVideo = true;
} else if (mediaType === MediaType.AUDIO_AND_VIDEO) {
payload.publishAudio = true;
payload.publishVideo = true;
}
const isFullScreen = store.getState().room.isFullScreen;
RtcClient.setRemoteVideoPlayer(userId, isFullScreen ? 'remote-video-player' : 'remote-full-player');
dispatch(updateRemoteUser(payload));
};
@ -104,7 +111,7 @@ const useRtcListeners = (): IEventListener => {
if (mediaType === MediaType.AUDIO_AND_VIDEO) {
payload.publishAudio = false;
}
RtcClient.setRemoteVideoPlayer(userId);
dispatch(updateRemoteUser(payload));
};

View File

@ -8,15 +8,18 @@ import { isMobile } from '@/utils/utils';
import InvokeButton from '@/pages/MainPage/MainArea/Antechamber/InvokeButton';
import { useJoin, useScene } from '@/lib/useCommon';
import AIChangeCard from '@/components/AiChangeCard';
import { updateFullScreen } from '@/store/slices/room';
import { updateFullScreen, updateShowSubtitle } from '@/store/slices/room';
import style from './index.module.less';
function Antechamber() {
const dispatch = useDispatch();
const [joining, dispatchJoin] = useJoin();
const { isScreenMode } = useScene();
const { isScreenMode, isAvatarScene } = useScene();
const handleJoinRoom = () => {
dispatch(updateFullScreen({ isFullScreen: !isMobile() && !isScreenMode })); // 初始化
dispatch(updateFullScreen({ isFullScreen: !isMobile() && !isScreenMode && !isAvatarScene })); // 初始化
dispatch(updateShowSubtitle({ isShowSubtitle: !isAvatarScene }));
if (!joining) {
dispatchJoin();
}

View File

@ -17,27 +17,36 @@ import AiAvatarCard from '@/components/AiAvatarCard';
import UserAvatar from '@/assets/img/userAvatar.png';
import CameraCloseNoteSVG from '@/assets/img/CameraCloseNote.svg';
import ScreenCloseNoteSVG from '@/assets/img/ScreenCloseNote.svg';
import { LocalFullID, RemoteFullID } from '@/components/FullScreenCard';
const LocalVideoID = 'local-video-player';
const LocalScreenID = 'local-screen-player';
const RemoteVideoID = 'remote-video-player';
function CameraArea(props: React.HTMLAttributes<HTMLDivElement>) {
const { className, ...rest } = props;
const room = useSelector((state: RootState) => state.room);
const { isFullScreen, scene } = room;
const { isVision, isScreenMode } = useScene();
const { isVision, isScreenMode, botName } = useScene();
const { isVideoPublished, isScreenPublished, switchCamera, switchScreenCapture } =
useDeviceState();
const isRemoteVideoPublished = room.remoteUsers.find(user => user.username === botName)?.publishVideo ?? false
const setVideoPlayer = () => {
RtcClient.removeVideoPlayer(room.localUser.username!);
RtcClient.removeLocalVideoPlayer(room.localUser.username!);
if (isVideoPublished || isScreenPublished) {
RtcClient.setLocalVideoPlayer(
room.localUser.username!,
isFullScreen ? 'local-full-player' : isScreenMode ? LocalScreenID : LocalVideoID,
isFullScreen ? LocalFullID : isScreenMode ? LocalScreenID : LocalVideoID,
isScreenPublished,
isScreenMode ? VideoRenderMode.RENDER_MODE_FILL : VideoRenderMode.RENDER_MODE_HIDDEN
);
if(isRemoteVideoPublished) {
RtcClient.setRemoteVideoPlayer(
botName,
isFullScreen ? RemoteVideoID : RemoteFullID,
);
}
}
};
@ -72,6 +81,13 @@ function CameraArea(props: React.HTMLAttributes<HTMLDivElement>) {
isScreenPublished && isScreenMode ? '' : styles['camera-player-hidden']
}`}
/>
<div
id={RemoteVideoID}
className={`${styles['camera-player']} ${
isFullScreen && isRemoteVideoPublished ? '' : styles['camera-player-hidden']
}`}
style={{ position: 'absolute' }}
/>
<div
className={`${styles['camera-placeholder']} ${
isVideoPublished || isScreenPublished ? styles['camera-player-hidden'] : ''

View File

@ -12,25 +12,26 @@ import { isMobile } from '@/utils/utils';
import { useScene } from '@/lib/useCommon';
import USER_AVATAR from '@/assets/img/userAvatar.png';
import styles from './index.module.less';
import AIAvatarReadying from '@/components/AIAvatarLoading';
const lines: (string | React.ReactNode)[] = [];
function Conversation(props: React.HTMLAttributes<HTMLDivElement>) {
const { className, ...rest } = props;
function Conversation(props: React.HTMLAttributes<HTMLDivElement> & { showSubtitle: boolean }) {
const { className, showSubtitle, ...rest } = props;
const room = useSelector((state: RootState) => state.room);
const { msgHistory, isFullScreen } = room;
const { userId } = useSelector((state: RootState) => state.room.localUser);
const { isAITalking, isUserTalking, scene } = useSelector((state: RootState) => state.room);
const isAIReady = msgHistory.length > 0;
const containerRef = useRef<HTMLDivElement>(null);
const { botName, icon } = useScene();
const { botName, icon, isAvatarScene } = useScene();
const isUserTextLoading = (owner: string) => {
return owner === userId && isUserTalking;
};
const isAITextLoading = (owner: string) => {
return owner === botName && isAITalking;
return (owner === botName || owner.includes('voiceChat_')) && isAITalking;
};
useEffect(() => {
@ -46,20 +47,27 @@ function Conversation(props: React.HTMLAttributes<HTMLDivElement>) {
className={`${styles.conversation} ${className} ${isFullScreen ? styles.fullScreen : ''} ${
isMobile() ? styles.mobileConversation : ''
}`}
style={isAvatarScene && !isAIReady ? { justifyContent: 'center' } : {}}
{...rest}
>
{lines.map((line) => line)}
{!isAIReady ? (
<div className={styles.aiReadying}>
{isAvatarScene ? (
<AIAvatarReadying />
) : (
<>
<Spin size={16} className={styles['aiReading-spin']} />
AI ,
</>
)}
</div>
) : (
''
)}
{msgHistory?.map(({ value, user, isInterrupted }, index) => {
{(showSubtitle ? msgHistory : [])?.map(({ value, user, isInterrupted }, index) => {
const isUserMsg = user === userId;
const isRobotMsg = user === botName;
const isRobotMsg = user === botName || user.includes('voiceChat_');
if (!isUserMsg && !isRobotMsg) {
return '';
}

View File

@ -58,7 +58,7 @@ function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
{isScreenMode && (
<img
src={isScreenPublished ? ScreenOnSVG : ScreenOffSVG}
onClick={() => switchScreenCapture()}
onClick={() => switchScreenCapture(true)}
className={style.btn}
alt="screenShare"
/>

View File

@ -310,6 +310,7 @@
width: 100%;
height: 184px;
border-radius: 8px;
overflow: hidden;
}
.camera-player-hidden {

View File

@ -15,10 +15,12 @@ import { RootState } from '@/store';
import UserTag from '@/components/UserTag';
import FullScreenCard from '@/components/FullScreenCard';
import MobileToolBar from '@/pages/Mobile/MobileToolBar';
import { useScene } from '@/lib/useCommon';
function Room() {
const room = useSelector((state: RootState) => state.room);
const { isShowSubtitle, scene, isFullScreen } = room;
const { isAvatarScene } = useScene();
return (
<div className={`${style.wrapper} ${isMobile() ? style.mobile : ''}`}>
{isMobile() ? <div className={style.mobilePlayer} id="mobile-local-player" /> : null}
@ -26,7 +28,7 @@ function Room() {
{isShowSubtitle && !isMobile() ? (
<UserTag name={scene} className={style.subTitleUserTag} />
) : null}
{isFullScreen && !isMobile() ? (
{(isFullScreen || isAvatarScene) && !isMobile() ? (
<FullScreenCard />
) : isMobile() && isShowSubtitle ? null : (
<AiAvatarCard
@ -36,7 +38,7 @@ function Room() {
/>
)}
{isMobile() ? null : <CameraArea />}
{isShowSubtitle && <Conversation className={style.conversation} />}
<Conversation className={style.conversation} showSubtitle={isShowSubtitle} />
<ToolBar className={style.toolBar} />
<AudioController className={style.controller} />
<div className={style.declare}>AI生成内容由大模型生成</div>

View File

@ -45,6 +45,8 @@ export interface SceneConfig {
isVision: boolean;
isScreenMode: boolean;
isInterruptMode: boolean;
isAvatarScene: boolean;
avatarBgUrl: string;
}
export interface RTCConfig {
@ -283,12 +285,18 @@ export const roomSlice = createSlice({
const { paragraph, definite } = payload;
const lastMsg = state.msgHistory.at(-1)! || {};
/** 是否需要再创建新句子 */
const fromBot = payload.user === state.sceneConfigMap[state.scene].botName;
const fromBot =
payload.user === state.sceneConfigMap[state.scene].botName ||
payload.user.includes('voiceChat_');
/**
* Bot definite
* Bot
* 1. SubtitleMode=0 definite
* 2. SubtitleMode=1 1 paragraph
* User paragraph
*/
const lastMsgCompleted = fromBot ? lastMsg.definite : lastMsg.paragraph;
const currentSubtitleMode = state.sceneConfigMap[state.scene].isAvatarScene ? 1 : 0;
const lastMsgCompleted =
!fromBot || currentSubtitleMode ? lastMsg.paragraph : lastMsg.definite;
if (state.msgHistory.length) {
/** 如果上一句话是完整的则新增语句 */
@ -302,7 +310,11 @@ export const roomSlice = createSlice({
});
} else {
/** 话未说完, 更新文字内容 */
if (fromBot && currentSubtitleMode) {
lastMsg.value += payload.text;
} else {
lastMsg.value = payload.text;
}
lastMsg.time = new Date().toString();
lastMsg.paragraph = paragraph;
lastMsg.definite = definite;