From b76084fdd0c49afb45a9b4055254c0b8e59acceb Mon Sep 17 00:00:00 2001 From: gxz Date: Tue, 29 Oct 2024 15:38:05 +0800 Subject: [PATCH] chore: migrate WebDecoder --- .eslintrc | 3 + example/src/main/index.js | 1 + .../basic/VideoDecoder/VideoDecoder.tsx | 264 ++++++++++++ example/src/renderer/examples/basic/index.ts | 5 + package.json | 3 + .../agora_node_ext/agora_electron_bridge.cpp | 27 ++ .../agora_node_ext/agora_electron_bridge.h | 7 + .../agora_node_ext/node_api_header.cpp | 9 + source_code/agora_node_ext/node_api_header.h | 3 + .../node_iris_event_handler.cpp | 65 ++- .../agora_node_ext/node_iris_event_handler.h | 5 + ts/Decoder/gpu-utils.ts | 92 +++++ ts/Decoder/index.ts | 191 +++++++++ ts/Private/extension/AgoraBaseExtension.ts | 15 +- ts/Private/internal/IrisApiEngine.ts | 28 +- ts/Private/internal/RtcEngineExInternal.ts | 27 +- ts/Private/ipc/main.ts | 22 + ts/Private/ipc/renderer.ts | 21 + ts/Renderer/CapabilityManager.ts | 126 ++++++ ts/Renderer/IRenderer.ts | 60 ++- ts/Renderer/IRendererCache.ts | 94 +++++ ts/Renderer/IRendererManager.ts | 316 --------------- ts/Renderer/RendererCache.ts | 109 ++--- ts/Renderer/RendererManager.ts | 380 +++++++++++++++--- ts/Renderer/WebCodecsRenderer/index.ts | 141 +++++++ ts/Renderer/WebCodecsRendererCache.ts | 140 +++++++ ts/Renderer/WebGLRenderer/index.ts | 3 + ts/Renderer/index.ts | 1 - ts/Types.ts | 78 +++- ts/Utils.ts | 50 ++- yarn.lock | 22 + 31 files changed, 1814 insertions(+), 494 deletions(-) create mode 100644 example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx create mode 100644 ts/Decoder/gpu-utils.ts create mode 100644 ts/Decoder/index.ts create mode 100644 ts/Private/ipc/main.ts create mode 100644 ts/Private/ipc/renderer.ts create mode 100644 ts/Renderer/CapabilityManager.ts create mode 100644 ts/Renderer/IRendererCache.ts delete mode 100644 ts/Renderer/IRendererManager.ts create mode 100755 ts/Renderer/WebCodecsRenderer/index.ts create mode 100644 ts/Renderer/WebCodecsRendererCache.ts diff --git a/.eslintrc b/.eslintrc index 9ed51fd00..0dff62292 100644 --- a/.eslintrc +++ b/.eslintrc @@ -91,6 +91,9 @@ "WebGLTexture": false, "WebGLBuffer": false, "WebGLProgram": false, + "VideoDecoder": false, + "VideoFrame": false, + "EncodedVideoChunk":false, "HTMLCanvasElement": false, "ResizeObserver": false, "name": false, diff --git a/example/src/main/index.js b/example/src/main/index.js index 3497489ba..571c228af 100644 --- a/example/src/main/index.js +++ b/example/src/main/index.js @@ -1,6 +1,7 @@ import path from 'path'; import { format as formatUrl } from 'url'; +import 'agora-electron-sdk/js/Private/ipc/main.js'; import { BrowserWindow, app, ipcMain, systemPreferences } from 'electron'; const isDevelopment = process.env.NODE_ENV !== 'production'; diff --git a/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx b/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx new file mode 100644 index 000000000..00ec34e1e --- /dev/null +++ b/example/src/renderer/examples/basic/VideoDecoder/VideoDecoder.tsx @@ -0,0 +1,264 @@ +import { + AgoraEnv, + ChannelProfileType, + ClientRoleType, + IRtcEngineEventHandler, + IRtcEngineEx, + LogFilterType, + RtcConnection, + RtcStats, + VideoSourceType, + createAgoraRtcEngine, +} from 'agora-electron-sdk'; +import React, { ReactElement } from 'react'; + +import { + BaseAudioComponentState, + BaseComponent, +} from '../../../components/BaseComponent'; +import { AgoraButton, AgoraTextInput } from '../../../components/ui'; +import Config from '../../../config/agora.config'; +import { askMediaAccess } from '../../../utils/permissions'; + +interface State extends BaseAudioComponentState { + token2: string; + uid2: number; + fps: number; + joinChannelExSuccess: boolean; +} + +export default class VideoDecoder + extends BaseComponent<{}, State> + implements IRtcEngineEventHandler +{ + protected engine?: IRtcEngineEx; + + protected createState(): State { + return { + appId: Config.appId, + fps: 0, + enableVideo: true, + channelId: Config.channelId, + token: Config.token, + uid: Config.uid, + joinChannelSuccess: false, + token2: '', + uid2: 0, + remoteUsers: [], + joinChannelExSuccess: false, + }; + } + + /** + * Step 1: initRtcEngine + */ + protected async initRtcEngine() { + const { appId } = this.state; + if (!appId) { + this.error(`appId is invalid`); + } + this.engine = createAgoraRtcEngine() as IRtcEngineEx; + // need to enable WebCodecsDecoder before call engine.initialize + // if enableWebCodecsDecoder is true, the video stream will be decoded by WebCodecs + // will automatically register videoEncodedFrameObserver + // videoEncodedFrameObserver will be released when engine.release + AgoraEnv.enableWebCodecsDecoder = true; + this.engine.initialize({ + appId, + logConfig: { filePath: Config.logFilePath }, + // Should use ChannelProfileLiveBroadcasting on most of cases + channelProfile: ChannelProfileType.ChannelProfileLiveBroadcasting, + }); + this.engine.setLogFilter(LogFilterType.LogFilterDebug); + this.engine.registerEventHandler(this); + + // Need granted the microphone and camera permission + await askMediaAccess(['microphone', 'camera', 'screen']); + + // Need to enable video on this case + // If you only call `enableAudio`, only relay the audio stream to the target channel + this.engine.enableVideo(); + } + + /** + * Step 2: joinChannel + */ + protected joinChannel() { + const { channelId, token, uid } = this.state; + if (!channelId) { + this.error('channelId is invalid'); + return; + } + if (uid < 0) { + this.error('uid is invalid'); + return; + } + + // start joining channel + // 1. Users can only see each other after they join the + // same channel successfully using the same app id. + // 2. If app certificate is turned on at dashboard, token is needed + // when joining channel. The channel name and uid used to calculate + // the token has to match the ones used for channel join + this.engine?.joinChannel(token, channelId, uid, { + // Make myself as the broadcaster to send stream to remote + clientRoleType: ClientRoleType.ClientRoleBroadcaster, + }); + } + + /** + * Step 2-1(optional): joinChannelEx + */ + protected joinChannelEx = () => { + const { channelId, token2, uid2 } = this.state; + if (!channelId) { + this.error('channelId is invalid'); + return; + } + if (uid2 <= 0) { + this.error('uid2 is invalid'); + return; + } + // publish screen share stream + this.engine?.joinChannelEx( + token2, + { channelId, localUid: uid2 }, + { + autoSubscribeAudio: true, + autoSubscribeVideo: true, + publishMicrophoneTrack: true, + publishCameraTrack: true, + clientRoleType: ClientRoleType.ClientRoleBroadcaster, + } + ); + }; + + /** + * Step 2-2(optional): leaveChannelEx + */ + leaveChannelEx = () => { + const { channelId, uid2 } = this.state; + this.engine?.leaveChannelEx({ channelId, localUid: uid2 }); + }; + + /** + * Step 4: leaveChannel + */ + protected leaveChannel() { + this.engine?.leaveChannel(); + } + + /** + * Step 5: releaseRtcEngine + */ + protected releaseRtcEngine() { + this.engine?.unregisterEventHandler(this); + this.engine?.release(); + } + + onJoinChannelSuccess(connection: RtcConnection, elapsed: number) { + const { uid2 } = this.state; + if (connection.localUid === uid2) { + this.setState({ joinChannelExSuccess: true }); + } else { + this.setState({ joinChannelSuccess: true }); + } + this.info( + 'onJoinChannelSuccess', + 'connection', + connection, + 'elapsed', + elapsed + ); + } + + onLeaveChannel(connection: RtcConnection, stats: RtcStats) { + const { uid2 } = this.state; + if (connection.localUid === uid2) { + this.setState({ joinChannelExSuccess: false }); + } else { + this.setState({ joinChannelSuccess: false }); + } + this.info('onLeaveChannel', 'connection', connection, 'stats', stats); + this.setState(this.createState()); + } + + protected renderUsers(): ReactElement | undefined { + let { remoteUsers, joinChannelSuccess, joinChannelExSuccess } = this.state; + return ( + <> + {joinChannelSuccess + ? remoteUsers.map((item) => + this.renderUser({ + uid: item, + // Use WebCodecs to decode video stream + useWebCodecsDecoder: true, + enableFps: true, + sourceType: VideoSourceType.VideoSourceRemote, + }) + ) + : undefined} + {joinChannelExSuccess + ? remoteUsers.map((item) => + this.renderUser({ + uid: item, + // Use WebCodecs to decode video stream + useWebCodecsDecoder: true, + enableFps: true, + sourceType: VideoSourceType.VideoSourceRemote, + }) + ) + : undefined} + + ); + } + + protected renderChannel(): ReactElement | undefined { + const { channelId, joinChannelSuccess, joinChannelExSuccess } = this.state; + return ( + <> + { + this.setState({ channelId: text }); + }} + placeholder={`channelId`} + value={channelId} + /> + { + joinChannelSuccess ? this.leaveChannel() : this.joinChannel(); + }} + /> + + ); + } + + protected renderConfiguration(): ReactElement | undefined { + let { joinChannelExSuccess, uid2, joinChannelSuccess } = this.state; + return ( + <> + { + if (isNaN(+text)) return; + this.setState({ + uid2: text === '' ? this.createState().uid2 : +text, + }); + }} + numberKeyboard={true} + placeholder={`uid2 (must > 0)`} + value={uid2 > 0 ? uid2.toString() : ''} + /> + + + ); + } +} diff --git a/example/src/renderer/examples/basic/index.ts b/example/src/renderer/examples/basic/index.ts index 255daface..db9285a32 100644 --- a/example/src/renderer/examples/basic/index.ts +++ b/example/src/renderer/examples/basic/index.ts @@ -1,6 +1,7 @@ import JoinChannelAudio from './JoinChannelAudio/JoinChannelAudio'; import JoinChannelVideo from './JoinChannelVideo/JoinChannelVideo'; import StringUid from './StringUid/StringUid'; +import VideoDecoder from './VideoDecoder/VideoDecoder'; const Basic = { title: 'Basic', @@ -17,6 +18,10 @@ const Basic = { name: 'StringUid', component: StringUid, }, + { + name: 'VideoDecoder', + component: VideoDecoder, + }, ], }; diff --git a/package.json b/package.json index c42a2955e..a63429c87 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,8 @@ "@commitlint/config-conventional": "^17.0.2", "@evilmartians/lefthook": "^1.2.2", "@release-it/conventional-changelog": "^5.0.0", + "@types/dom-webcodecs": "^0.1.11", + "@types/node": "^22.8.2", "@types/jest": "^28.1.2", "@types/json-bigint": "^1.0.1", "@types/lodash.isequal": "^4.5.6", @@ -135,6 +137,7 @@ "jsonfile": "^6.1.0", "lodash.isequal": "^4.5.0", "minimist": "^1.2.5", + "semver": "^7.6.0", "shelljs": "^0.8.4", "ts-interface-checker": "^1.0.2", "winston": "^3.3.3", diff --git a/source_code/agora_node_ext/agora_electron_bridge.cpp b/source_code/agora_node_ext/agora_electron_bridge.cpp index 3b3a9f557..6a28be765 100644 --- a/source_code/agora_node_ext/agora_electron_bridge.cpp +++ b/source_code/agora_node_ext/agora_electron_bridge.cpp @@ -40,6 +40,7 @@ napi_value AgoraElectronBridge::Init(napi_env env, napi_value exports) { napi_property_descriptor properties[] = { DECLARE_NAPI_METHOD("CallApi", CallApi), DECLARE_NAPI_METHOD("OnEvent", OnEvent), + DECLARE_NAPI_METHOD("UnEvent", UnEvent), DECLARE_NAPI_METHOD("GetBuffer", GetBuffer), DECLARE_NAPI_METHOD("EnableVideoFrameCache", EnableVideoFrameCache), DECLARE_NAPI_METHOD("DisableVideoFrameCache", DisableVideoFrameCache), @@ -253,6 +254,30 @@ napi_value AgoraElectronBridge::OnEvent(napi_env env, napi_callback_info info) { RETURE_NAPI_OBJ(); } +napi_value AgoraElectronBridge::UnEvent(napi_env env, napi_callback_info info) { + napi_status status; + size_t argc = 2; + napi_value args[2]; + napi_value jsthis; + int ret = ERR_FAILED; + status = napi_get_cb_info(env, info, &argc, args, &jsthis, nullptr); + assert(status == napi_ok); + + AgoraElectronBridge *agoraElectronBridge; + status = + napi_unwrap(env, jsthis, reinterpret_cast(&agoraElectronBridge)); + assert(status == napi_ok); + + std::string eventName = ""; + status = napi_get_value_utf8string(env, args[0], eventName); + assert(status == napi_ok); + + agoraElectronBridge->_iris_rtc_event_handler->removeEvent(eventName); + ret = ERR_OK; + + RETURE_NAPI_OBJ(); +} + napi_value AgoraElectronBridge::SetAddonLogFile(napi_env env, napi_callback_info info) { napi_status status; @@ -480,6 +505,8 @@ napi_value AgoraElectronBridge::InitializeEnv(napi_env env, status = napi_unwrap(env, jsthis, reinterpret_cast(&agoraElectronBridge)); + napi_value obj0 = args[0]; + agoraElectronBridge->Init(); LOG_F(INFO, __FUNCTION__); napi_value retValue = nullptr; diff --git a/source_code/agora_node_ext/agora_electron_bridge.h b/source_code/agora_node_ext/agora_electron_bridge.h index 4c6bd13e2..9e445caee 100644 --- a/source_code/agora_node_ext/agora_electron_bridge.h +++ b/source_code/agora_node_ext/agora_electron_bridge.h @@ -18,6 +18,10 @@ namespace electron { class NodeIrisEventHandler; +struct AgoraEnv { + bool enable_web_codecs_decoder = false; +}; + class AgoraElectronBridge { public: explicit AgoraElectronBridge(); @@ -30,6 +34,7 @@ class AgoraElectronBridge { static napi_value CallApi(napi_env env, napi_callback_info info); static napi_value GetBuffer(napi_env env, napi_callback_info info); static napi_value OnEvent(napi_env env, napi_callback_info info); + static napi_value UnEvent(napi_env env, napi_callback_info info); static napi_value EnableVideoFrameCache(napi_env env, napi_callback_info info); static napi_value DisableVideoFrameCache(napi_env env, @@ -43,8 +48,10 @@ class AgoraElectronBridge { void OnApiError(const char *errorMessage); void Init(); void Release(); + void SetAgoraEnv(const AgoraEnv &agoraEnv) { _agoraEnv = agoraEnv; } private: + AgoraEnv _agoraEnv; static const char *_class_name; static napi_ref *_ref_construcotr_ptr; static const char *_ret_code_str; diff --git a/source_code/agora_node_ext/node_api_header.cpp b/source_code/agora_node_ext/node_api_header.cpp index cc000df82..94f6f0bba 100644 --- a/source_code/agora_node_ext/node_api_header.cpp +++ b/source_code/agora_node_ext/node_api_header.cpp @@ -127,6 +127,15 @@ napi_status napi_obj_get_property(napi_env &env, napi_value &object, return status; } +napi_status napi_obj_get_property(napi_env &env, napi_value &object, + const char *utf8name, bool &result) { + napi_status status; + napi_value retValue; + napi_get_named_property(env, object, utf8name, &retValue); + status = napi_get_value_bool(env, retValue, &result); + return status; +} + napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, uint32_t &result) { napi_status status; diff --git a/source_code/agora_node_ext/node_api_header.h b/source_code/agora_node_ext/node_api_header.h index 33ca4516b..4eeaae4b5 100644 --- a/source_code/agora_node_ext/node_api_header.h +++ b/source_code/agora_node_ext/node_api_header.h @@ -66,6 +66,9 @@ napi_status napi_obj_set_property(napi_env &env, napi_value &object, napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, int &result); +napi_status napi_obj_get_property(napi_env &env, napi_value &object, + const char *utf8name, bool &result); + napi_status napi_obj_get_property(napi_env &env, napi_value &object, const char *utf8name, uint32_t &result); diff --git a/source_code/agora_node_ext/node_iris_event_handler.cpp b/source_code/agora_node_ext/node_iris_event_handler.cpp index 3b45a45c0..0339baa64 100644 --- a/source_code/agora_node_ext/node_iris_event_handler.cpp +++ b/source_code/agora_node_ext/node_iris_event_handler.cpp @@ -8,6 +8,7 @@ #include "agora_electron_bridge.h" #include #include +#include namespace agora { namespace rtc { @@ -43,9 +44,25 @@ void NodeIrisEventHandler::addEvent(const std::string &eventName, napi_env &env, env, call_bcak, 1, &(_callbacks[eventName]->call_back_ref)); } +void NodeIrisEventHandler::removeEvent(const std::string &eventName) { + auto it = _callbacks.find(eventName); + if (it != _callbacks.end()) { + napi_delete_reference(it->second->env, it->second->call_back_ref); + delete it->second; + _callbacks.erase(it); + } +} + void NodeIrisEventHandler::OnEvent(EventParam *param) { - fireEvent(_callback_key, param->event, param->data, param->buffer, - param->length, param->buffer_count); + const char *event = + "VideoEncodedFrameObserver_onEncodedVideoFrameReceived_6922697"; + + if (strcmp(event, param->event) == 0) { + onEncodedVideoFrameReceived(param->data, param->buffer[0], param->length); + } else { + fireEvent(_callback_key, param->event, param->data, param->buffer, + param->length, param->buffer_count); + } } void NodeIrisEventHandler::fireEvent(const char *callback_name, @@ -119,6 +136,50 @@ void NodeIrisEventHandler::fireEvent(const char *callback_name, }); } +void NodeIrisEventHandler::onEncodedVideoFrameReceived(const char *data, + void *buffer, + unsigned int *length) { + std::string eventData = ""; + if (data) { eventData = data; } + std::vector buffer_data(length[0]); + memcpy(buffer_data.data(), buffer, length[0]); + std::string uid = ""; + std::regex uidRegex(R"("uid"\s*:\s*(\d+))"); + std::smatch match; + std::string jsonString(data); + if (std::regex_search(jsonString, match, uidRegex)) { + if (match.size() > 1) { uid = match[1].str(); } + } + + unsigned int buffer_length = length[0]; + + node_async_call::async_call( + [this, eventData, buffer_data, buffer_length, uid] { + auto it = _callbacks.find("call_back_with_encoded_video_frame_" + uid); + if (it != _callbacks.end()) { + size_t argc = 2; + napi_value args[2]; + napi_value result; + napi_status status; + status = napi_create_string_utf8(it->second->env, eventData.c_str(), + eventData.length(), &args[0]); + + napi_create_buffer_copy(it->second->env, buffer_length, + buffer_data.data(), nullptr, &args[1]); + + napi_value call_back_value; + status = napi_get_reference_value( + it->second->env, it->second->call_back_ref, &call_back_value); + + napi_value recv_value; + status = napi_get_undefined(it->second->env, &recv_value); + + status = napi_call_function(it->second->env, recv_value, + call_back_value, argc, args, &result); + } + }); +} + }// namespace electron }// namespace rtc }// namespace agora diff --git a/source_code/agora_node_ext/node_iris_event_handler.h b/source_code/agora_node_ext/node_iris_event_handler.h index c1e88b85f..863674ebd 100644 --- a/source_code/agora_node_ext/node_iris_event_handler.h +++ b/source_code/agora_node_ext/node_iris_event_handler.h @@ -31,9 +31,14 @@ class NodeIrisEventHandler : public iris::IrisEventHandler { void **buffer, unsigned int *length, unsigned int buffer_count); + void onEncodedVideoFrameReceived(const char *data, void *buffer, + unsigned int *length); + void addEvent(const std::string &eventName, napi_env &env, napi_value &call_bcak, napi_value &global); + void removeEvent(const std::string &eventName); + private: std::unordered_map _callbacks; const char *_callback_key = "call_back_with_buffer"; diff --git a/ts/Decoder/gpu-utils.ts b/ts/Decoder/gpu-utils.ts new file mode 100644 index 000000000..11f77837a --- /dev/null +++ b/ts/Decoder/gpu-utils.ts @@ -0,0 +1,92 @@ +//@ts-ignore +import { BrowserWindow } from 'electron'; + +/** + * @ignore + */ + +export type VideoDecodeAcceleratorSupportedProfile = { + codec: string; + minWidth: number; + maxWidth: number; + minHeight: number; + maxHeight: number; +}; + +/** + * @ignore + */ +export class GpuInfo { + videoDecodeAcceleratorSupportedProfile: VideoDecodeAcceleratorSupportedProfile[] = + []; +} + +/** + * @ignore + */ +export const getGpuInfoInternal = (callback: any): void => { + //@ts-ignore + if (process.type === 'renderer') { + console.error('getGpuInfoInternal should be called in main process'); + return; + } + const gpuPage = new BrowserWindow({ + show: false, + webPreferences: { offscreen: true }, + }); + gpuPage.loadURL('chrome://gpu'); + let executeJavaScriptText = + `` + + `let videoAccelerationInfo = [];` + + `let nodeList = document.querySelector('info-view')?.shadowRoot?.querySelector('#video-acceleration-info info-view-table')?.shadowRoot?.querySelectorAll('#info-view-table info-view-table-row') || [];` + + `for (node of nodeList) {` + + ` videoAccelerationInfo.push({` + + ` title: node.shadowRoot.querySelector('#title')?.innerText,` + + ` value: node.shadowRoot.querySelector('#value')?.innerText,` + + ` })` + + `}` + + `JSON.stringify(videoAccelerationInfo)`; + gpuPage.webContents + .executeJavaScript(executeJavaScriptText) + .then((result: string) => { + if (!result) { + console.error( + 'Failed to get GPU info, chrome://gpu is not available in this environment.' + ); + } + let filterResult: { title: string; value: string }[] = JSON.parse( + result + ).filter((item: any) => { + return item.title.indexOf('Decode') !== -1; + }); + let convertResult: VideoDecodeAcceleratorSupportedProfile[] = []; + const resolutionPattern = /(\d+)x(\d+) to (\d+)x(\d+)/; + for (const profile of filterResult) { + const match = profile.value.match(resolutionPattern); + if (!match) { + continue; + } + + const [_resolution, minWidth, minHeight, maxWidth, maxHeight] = match; + + convertResult.push({ + codec: profile.title, + minWidth: minWidth ? Number(minWidth) : 0, + maxWidth: maxWidth ? Number(maxWidth) : 0, + minHeight: minHeight ? Number(minHeight) : 0, + maxHeight: maxHeight ? Number(maxHeight) : 0, + }); + } + typeof callback === 'function' && callback(convertResult); + }) + .catch((error: any) => { + console.error( + 'Failed to get GPU info, please import agora-electron-sdk in main process', + error + ); + typeof callback === 'function' && callback(error); + }) + .finally(() => { + gpuPage.close(); + }); +}; diff --git a/ts/Decoder/index.ts b/ts/Decoder/index.ts new file mode 100644 index 000000000..fdec958f7 --- /dev/null +++ b/ts/Decoder/index.ts @@ -0,0 +1,191 @@ +import { + EncodedVideoFrameInfo, + VideoCodecType, + VideoFrameType, +} from '../Private/AgoraBase'; + +import { WebCodecsRenderer } from '../Renderer/WebCodecsRenderer/index'; +import { RendererCacheContext, RendererType } from '../Types'; +import { AgoraEnv, logDebug, logInfo } from '../Utils'; + +const frameTypeMapping = { + [VideoFrameType.VideoFrameTypeDeltaFrame]: 'delta', + [VideoFrameType.VideoFrameTypeKeyFrame]: 'key', + [VideoFrameType.VideoFrameTypeDroppableFrame]: 'delta', // this is a workaround for the issue that the frameType is not correct +}; + +export class WebCodecsDecoder { + private _decoder: VideoDecoder; + private renderers: WebCodecsRenderer[] = []; + private _cacheContext: RendererCacheContext; + private pendingFrame: VideoFrame | null = null; + private _currentCodecConfig: { + codecType: VideoCodecType | undefined; + codedWidth: number | undefined; + codedHeight: number | undefined; + } | null = null; + + private _base_ts = 0; + private _base_ts_ntp = 1; + private _last_ts_ntp = 1; + + constructor( + renders: WebCodecsRenderer[], + onError: (e: any) => void, + context: RendererCacheContext + ) { + this.renderers = renders; + this._cacheContext = context; + this._decoder = new VideoDecoder({ + // @ts-ignore + output: this._output.bind(this), + error: (e) => { + onError(e); + }, + }); + } + + _output(frame: VideoFrame) { + // Schedule the frame to be rendered. + this._renderFrame(frame); + } + + private _renderFrame(frame: VideoFrame) { + if (!this.pendingFrame) { + // Schedule rendering in the next animation frame. + // eslint-disable-next-line auto-import/auto-import + requestAnimationFrame(this.renderAnimationFrame.bind(this)); + } else { + // Close the current pending frame before replacing it. + this.pendingFrame.close(); + } + // Set or replace the pending frame. + this.pendingFrame = frame; + } + + renderAnimationFrame() { + for (let renderer of this.renderers) { + if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) { + continue; + } + renderer.drawFrame(this.pendingFrame); + this.pendingFrame = null; + } + } + + decoderConfigure(frameInfo: EncodedVideoFrameInfo) { + this.pendingFrame = null; + // @ts-ignore + let codec = + AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType!] + ?.codec; + if (!codec) { + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback( + this._cacheContext + ); + throw new Error( + 'codec is not in frameCodecMapping,failed to configure decoder, fallback to native decoder' + ); + } + this._currentCodecConfig = { + codecType: frameInfo.codecType, + codedWidth: frameInfo.width, + codedHeight: frameInfo.height, + }; + this._decoder!.configure({ + codec: codec, + codedWidth: frameInfo.width, + codedHeight: frameInfo.height, + }); + logInfo( + `configure decoder: codedWidth: ${frameInfo.width}, codedHeight: ${frameInfo.height},codec: ${codec}` + ); + } + + updateTimestamps(ts: number) { + if (this._base_ts !== 0) { + if (ts > this._base_ts) { + this._last_ts_ntp = + this._base_ts_ntp + Math.floor(((ts - this._base_ts) * 1000) / 90); + } else { + this._base_ts = ts; + this._last_ts_ntp++; + this._base_ts_ntp = this._last_ts_ntp; + } + } else { + this._base_ts = ts; + this._last_ts_ntp = 1; + } + } + + handleCodecIsChanged(frameInfo: EncodedVideoFrameInfo) { + if ( + this._currentCodecConfig?.codecType !== frameInfo.codecType || + this._currentCodecConfig?.codedWidth !== frameInfo.width || + this._currentCodecConfig?.codedHeight !== frameInfo.height + ) { + logInfo('frameInfo has changed, reconfigure decoder'); + this._decoder.reset(); + this.decoderConfigure(frameInfo); + } + } + + // @ts-ignore + decodeFrame( + imageBuffer: Uint8Array, + frameInfo: EncodedVideoFrameInfo, + ts: number + ) { + try { + this.handleCodecIsChanged(frameInfo); + } catch (error: any) { + logInfo(error); + return; + } + + if (!imageBuffer) { + logDebug('imageBuffer is empty, skip decode frame'); + return; + } + + let frameType: string | undefined; + if (frameInfo.frameType !== undefined) { + // @ts-ignore + frameType = frameTypeMapping[frameInfo.frameType]; + } + if (!frameType) { + logDebug('frameType is not in frameTypeMapping, skip decode frame'); + return; + } + + this.updateTimestamps(ts); + + this._decoder.decode( + new EncodedVideoChunk({ + data: imageBuffer, + timestamp: this._last_ts_ntp, + // @ts-ignore + type: frameType, + // @ts-ignore + transfer: [imageBuffer.buffer], + }) + ); + } + + reset() { + this._base_ts = 0; + this._base_ts_ntp = 1; + this._last_ts_ntp = 1; + this._decoder.reset(); + } + + release() { + try { + if (this.pendingFrame) { + this.pendingFrame.close(); + } + this._decoder.close(); + } catch (e) {} + this.pendingFrame = null; + } +} diff --git a/ts/Private/extension/AgoraBaseExtension.ts b/ts/Private/extension/AgoraBaseExtension.ts index cb0ff5c3b..e85cc857d 100644 --- a/ts/Private/extension/AgoraBaseExtension.ts +++ b/ts/Private/extension/AgoraBaseExtension.ts @@ -1 +1,14 @@ -export {}; +import '../AgoraBase'; + +declare module '../AgoraBase' { + interface VideoCanvas { + /** + * @ignore + */ + useWebCodecsDecoder?: boolean; + /** + * @ignore + */ + enableFps?: boolean; + } +} diff --git a/ts/Private/internal/IrisApiEngine.ts b/ts/Private/internal/IrisApiEngine.ts index c8d0af41a..465a1ce69 100644 --- a/ts/Private/internal/IrisApiEngine.ts +++ b/ts/Private/internal/IrisApiEngine.ts @@ -2,6 +2,8 @@ import EventEmitter from 'eventemitter3'; import JSONBigInt from 'json-bigint'; const JSON = JSONBigInt({ storeAsString: true }); +import createAgoraRtcEngine from '../../AgoraSdk'; +import { IAgoraElectronBridge } from '../../Types'; import { AgoraEnv, logDebug, logError, logInfo, logWarn } from '../../Utils'; import { IAudioEncodedFrameObserver } from '../AgoraBase'; import { @@ -67,8 +69,11 @@ import { RtcEngineExInternal } from './RtcEngineExInternal'; // @ts-ignore export const DeviceEventEmitter: EventEmitter = new EventEmitter(); -const AgoraRtcNg = AgoraEnv.AgoraElectronBridge; -AgoraRtcNg.OnEvent('call_back_with_buffer', (...params: any) => { +const AgoraNode = require('../../../build/Release/agora_node_ext'); +export const AgoraElectronBridge: IAgoraElectronBridge = + new AgoraNode.AgoraElectronBridge(); + +AgoraElectronBridge.OnEvent('call_back_with_buffer', (...params: any) => { try { handleEvent(...params); } catch (e) { @@ -441,7 +446,7 @@ function handleEvent(...[event, data, buffers]: any) { export function callIrisApi(funcName: string, params: any): any { try { const buffers: Uint8Array[] = []; - + const rtcEngine = createAgoraRtcEngine(); if (funcName.startsWith('MediaEngine_')) { switch (funcName) { case 'MediaEngine_pushAudioFrame_c71f4ab': @@ -485,16 +490,16 @@ export function callIrisApi(funcName: string, params: any): any { } else if (funcName.startsWith('RtcEngine_')) { switch (funcName) { case 'RtcEngine_initialize_0320339': - AgoraRtcNg.InitializeEnv(); + AgoraElectronBridge.InitializeEnv(); break; case 'RtcEngine_release': - AgoraRtcNg.CallApi( + AgoraElectronBridge.CallApi( funcName, JSON.stringify(params), buffers, buffers.length ); - AgoraRtcNg.ReleaseEnv(); + AgoraElectronBridge.ReleaseEnv(); return; case 'RtcEngine_sendMetaData': // metadata.buffer @@ -522,8 +527,17 @@ export function callIrisApi(funcName: string, params: any): any { break; } } + if (funcName.indexOf('joinChannel') != -1) { + if (AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + rtcEngine.getMediaEngine().registerVideoEncodedFrameObserver({}); + } + } else if (funcName.indexOf('leaveChannel') != -1) { + if (AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + rtcEngine.getMediaEngine().unregisterVideoEncodedFrameObserver({}); + } + } - let { callApiReturnCode, callApiResult } = AgoraRtcNg.CallApi( + let { callApiReturnCode, callApiResult } = AgoraElectronBridge.CallApi( funcName, JSON.stringify(params), buffers, diff --git a/ts/Private/internal/RtcEngineExInternal.ts b/ts/Private/internal/RtcEngineExInternal.ts index ff1db3e03..b63ca271f 100644 --- a/ts/Private/internal/RtcEngineExInternal.ts +++ b/ts/Private/internal/RtcEngineExInternal.ts @@ -1,11 +1,17 @@ import { createCheckers } from 'ts-interface-checker'; +import { AgoraElectronBridge } from '../../Private/internal/IrisApiEngine'; import { AgoraEnv, logError, parseIntPtr2Number } from '../../Utils'; let RendererManager: any; -if (typeof window !== 'undefined') { +let CapabilityManager: any; +//@ts-ignore +if (process.type === 'renderer') { RendererManager = require('../../Renderer/RendererManager').RendererManager; + CapabilityManager = + require('../../Renderer/CapabilityManager').CapabilityManager; } else { RendererManager = undefined; + CapabilityManager = undefined; } import { AudioEncodedFrameObserverConfig, @@ -92,22 +98,25 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { private _h265_transcoder: IH265Transcoder = new H265TranscoderInternal(); override initialize(context: RtcEngineContext): number { + const ret = super.initialize(context); + callIrisApi.call(this, 'RtcEngine_setAppType', { + appType: 3, + }); if (AgoraEnv.webEnvReady) { // @ts-ignore window.AgoraEnv = AgoraEnv; if (AgoraEnv.AgoraRendererManager === undefined && RendererManager) { AgoraEnv.AgoraRendererManager = new RendererManager(); } + if (AgoraEnv.CapabilityManager === undefined && CapabilityManager) { + AgoraEnv.CapabilityManager = new CapabilityManager(); + } } - const ret = super.initialize(context); - callIrisApi.call(this, 'RtcEngine_setAppType', { - appType: 3, - }); return ret; } override release(sync: boolean = false) { - AgoraEnv.AgoraElectronBridge.ReleaseRenderer(); + AgoraElectronBridge.ReleaseRenderer(); AgoraEnv.AgoraRendererManager?.release(); AgoraEnv.AgoraRendererManager = undefined; this._media_engine.release(); @@ -127,6 +136,8 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { MediaRecorderInternal._observers.clear(); this._h265_transcoder.release(); this.removeAllListeners(); + AgoraEnv.CapabilityManager?.release(); + AgoraEnv.CapabilityManager = undefined; super.release(sync); } @@ -502,7 +513,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { if (!value.thumbImage?.buffer || !value.thumbImage?.length) { value.thumbImage!.buffer = undefined; } else { - value.thumbImage!.buffer = AgoraEnv.AgoraElectronBridge.GetBuffer( + value.thumbImage!.buffer = AgoraElectronBridge.GetBuffer( value.thumbImage!.buffer as unknown as number, value.thumbImage.length! ); @@ -510,7 +521,7 @@ export class RtcEngineExInternal extends IRtcEngineExImpl { if (!value.iconImage?.buffer || !value.iconImage?.length) { value.iconImage!.buffer = undefined; } else { - value.iconImage.buffer = AgoraEnv.AgoraElectronBridge.GetBuffer( + value.iconImage.buffer = AgoraElectronBridge.GetBuffer( value.iconImage!.buffer as unknown as number, value.iconImage.length! ); diff --git a/ts/Private/ipc/main.ts b/ts/Private/ipc/main.ts new file mode 100644 index 000000000..578b0c000 --- /dev/null +++ b/ts/Private/ipc/main.ts @@ -0,0 +1,22 @@ +//@ts-ignore +import { app, ipcMain } from 'electron'; + +import { getGpuInfoInternal } from '../../Decoder/gpu-utils'; + +import { IPCMessageType } from '../../Types'; +//@ts-ignore +if (process.type !== 'renderer') { + ipcMain.handle(IPCMessageType.AGORA_IPC_GET_GPU_INFO, () => { + return new Promise((resolve) => { + getGpuInfoInternal((result: any) => { + resolve(result); + }); + }); + }); + console.log('[agora-electron] main process AgoraIPCMain handler registered'); + + app.on('quit', () => { + ipcMain.removeHandler(IPCMessageType.AGORA_IPC_GET_GPU_INFO); + console.log('[agora-electron] main process AgoraIPCMain handler removed'); + }); +} diff --git a/ts/Private/ipc/renderer.ts b/ts/Private/ipc/renderer.ts new file mode 100644 index 000000000..a6b14a026 --- /dev/null +++ b/ts/Private/ipc/renderer.ts @@ -0,0 +1,21 @@ +//@ts-ignore +import { ipcRenderer } from 'electron'; + +import { IPCMessageType } from '../../Types'; +import { logError } from '../../Utils'; + +export async function ipcSend( + channel: IPCMessageType, + ...args: any[] +): Promise { + if (!Object.values(IPCMessageType).includes(channel)) { + logError('Invalid IPCMessageType'); + return; + } + //@ts-ignore + if (process.type === 'renderer') { + return await ipcRenderer.invoke(channel, ...args); + } else { + logError('Not in renderer process, cannot send ipc message'); + } +} diff --git a/ts/Renderer/CapabilityManager.ts b/ts/Renderer/CapabilityManager.ts new file mode 100644 index 000000000..360208fd2 --- /dev/null +++ b/ts/Renderer/CapabilityManager.ts @@ -0,0 +1,126 @@ +import semver from 'semver'; + +import createAgoraRtcEngine from '../AgoraSdk'; + +import { + GpuInfo, + VideoDecodeAcceleratorSupportedProfile, +} from '../Decoder/gpu-utils'; +import { VideoCodecType } from '../Private/AgoraBase'; +import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx'; +import { ipcSend } from '../Private/ipc/renderer'; + +import { IPCMessageType, VideoFallbackStrategy, codecMapping } from '../Types'; +import { AgoraEnv, logError, logInfo } from '../Utils'; + +/** + * @ignore + */ +export class CapabilityManager { + gpuInfo: GpuInfo = new GpuInfo(); + frameCodecMapping: { + [key in VideoCodecType]?: VideoDecodeAcceleratorSupportedProfile; + } = {}; + webCodecsDecoderEnabled: boolean = AgoraEnv.enableWebCodecsDecoder; + private _engine: IRtcEngineEx; + + setWebCodecsDecoderEnabled(enabled: boolean): void { + this.webCodecsDecoderEnabled = enabled; + } + + constructor() { + this._engine = createAgoraRtcEngine(); + if (AgoraEnv.enableWebCodecsDecoder) { + this.getGpuInfo(() => { + if ( + AgoraEnv.videoFallbackStrategy === + VideoFallbackStrategy.PerformancePriority + ) { + if (!this.isSupportedH265()) { + if (this.isSupportedH264()) { + this._engine.setParameters( + JSON.stringify({ 'che.video.h265_dec_enable': false }) + ); + logInfo( + 'the videoFallbackStrategy is PerformancePriority, H265 is not supported, fallback to H264' + ); + } else { + this.webCodecsDecoderEnabled = false; + logInfo( + 'the videoFallbackStrategy is PerformancePriority, H264 and H265 are not supported, fallback to native decoder' + ); + } + } + } else if ( + AgoraEnv.videoFallbackStrategy === + VideoFallbackStrategy.BandwidthPriority + ) { + if (!this.isSupportedH265()) { + this.webCodecsDecoderEnabled = false; + logInfo( + 'the videoFallbackStrategy is BandwidthPriority, H265 is not supported, fallback to native decoder' + ); + } + } + }); + } + } + + public getGpuInfo(callback?: () => void): void { + //getGpuInfo and videoDecoder is not supported in electron version < 22.0.0 + //@ts-ignore + if (semver.lt(process.versions.electron, '22.0.0')) { + logError( + 'WebCodecsDecoder is not supported in electron version < 22.0.0, please upgrade electron to 22.0.0 or later.' + ); + return; + } + //@ts-ignore + if (process.type === 'renderer') { + ipcSend(IPCMessageType.AGORA_IPC_GET_GPU_INFO) + .then((result) => { + this.gpuInfo.videoDecodeAcceleratorSupportedProfile = result; + this.webCodecsDecoderEnabled = (AgoraEnv.enableWebCodecsDecoder && + this.gpuInfo.videoDecodeAcceleratorSupportedProfile.length > 0)!; + + result.forEach((profile: VideoDecodeAcceleratorSupportedProfile) => { + const match = codecMapping.find((item) => + profile.codec.includes(item.profile) + ); + if (match) { + //Normally, the range of compatible widths and heights should be the same under the same codec. + //there is no need to differentiate between different profiles. This could be optimized in the future. + this.frameCodecMapping[match.type] = { + codec: match.codec, + minWidth: profile.minWidth, + minHeight: profile.minHeight, + maxWidth: profile.maxWidth, + maxHeight: profile.maxHeight, + }; + } + }); + callback && callback(); + }) + .catch((error) => { + logError( + 'Failed to get GPU info, please check if you are already import agora-electron-sdk in the main process.', + error + ); + }); + } else { + logError('This function only works in renderer process'); + } + } + + public isSupportedH264(): boolean { + return this.frameCodecMapping[VideoCodecType.VideoCodecH264] !== undefined; + } + + public isSupportedH265(): boolean { + return this.frameCodecMapping[VideoCodecType.VideoCodecH265] !== undefined; + } + + release(): void { + AgoraEnv.enableWebCodecsDecoder = false; + } +} diff --git a/ts/Renderer/IRenderer.ts b/ts/Renderer/IRenderer.ts index 00f22fc17..3fa41df8a 100644 --- a/ts/Renderer/IRenderer.ts +++ b/ts/Renderer/IRenderer.ts @@ -1,16 +1,19 @@ import { VideoMirrorModeType } from '../Private/AgoraBase'; import { RenderModeType, VideoFrame } from '../Private/AgoraMediaBase'; -import { RendererContext } from '../Types'; +import { RendererContext, RendererType } from '../Types'; -type Context = Pick; +import { frameSize } from './WebCodecsRenderer'; export abstract class IRenderer { parentElement?: HTMLElement; container?: HTMLElement; canvas?: HTMLCanvasElement; - _context: Context = {}; + rendererType: RendererType | undefined; + context: RendererContext = {}; + private _frameCount = 0; + private _startTime: number | null = null; - public bind(element: HTMLElement) { + public bind(element: HTMLElement, _frameSize?: frameSize) { this.parentElement = element; this.container = document.createElement('div'); Object.assign(this.container.style, { @@ -50,20 +53,15 @@ export abstract class IRenderer { } } - public set context({ renderMode, mirrorMode }: Context) { - if (this.context.renderMode !== renderMode) { - this.context.renderMode = renderMode; + public setContext(context: RendererContext) { + if (this.context.renderMode !== context.renderMode) { this.updateRenderMode(); } - if (this.context.mirrorMode !== mirrorMode) { - this.context.mirrorMode = mirrorMode; + if (this.context.mirrorMode !== context.mirrorMode) { this.updateMirrorMode(); } - } - - public get context(): Context { - return this._context; + this.context = context; } protected updateRenderMode() { @@ -76,7 +74,6 @@ export abstract class IRenderer { const canvasAspectRatio = width / height; const widthScale = clientWidth / width; const heightScale = clientHeight / height; - const isHidden = this.context?.renderMode === RenderModeType.RenderModeHidden; @@ -120,4 +117,39 @@ export abstract class IRenderer { ); } } + + public getFps(): number { + let fps = 0; + if (!this.context.enableFps || !this.container) { + return fps; + } + if (this._startTime == null) { + this._startTime = performance.now(); + } else { + const elapsed = (performance.now() - this._startTime) / 1000; + fps = ++this._frameCount / elapsed; + } + + let span = this.container.querySelector('span'); + if (!span) { + span = document.createElement('span'); + + Object.assign(span.style, { + position: 'absolute', + bottom: '0', + left: '0', + zIndex: '10', + width: '55px', + background: '#fff', + }); + + this.container.style.position = 'relative'; + + this.container.appendChild(span); + } + + span.innerText = `fps: ${fps.toFixed(0)}`; + + return fps; + } } diff --git a/ts/Renderer/IRendererCache.ts b/ts/Renderer/IRendererCache.ts new file mode 100644 index 000000000..481d22be5 --- /dev/null +++ b/ts/Renderer/IRendererCache.ts @@ -0,0 +1,94 @@ +import { RendererCacheContext, RendererContext } from '../Types'; + +import { IRenderer } from './IRenderer'; + +/** + * @ignore + */ +export function generateRendererCacheKey({ + channelId, + uid, + sourceType, +}: RendererContext): string { + return `${channelId}_${uid}_${sourceType}`; +} + +/** + * @ignore + */ +export function isUseConnection(context: RendererCacheContext): boolean { + // if RtcConnection is not undefined, then use connection + return !!context.channelId && context.localUid !== undefined; +} + +export abstract class IRendererCache { + renderers: IRenderer[]; + cacheContext: RendererCacheContext; + + constructor({ + channelId, + uid, + useWebCodecsDecoder, + enableFps, + sourceType, + localUid, + }: RendererContext) { + this.renderers = []; + this.cacheContext = { + channelId, + uid, + useWebCodecsDecoder, + enableFps, + sourceType, + localUid, + }; + } + + public get key(): string { + return generateRendererCacheKey(this.cacheContext); + } + + public abstract draw(): void; + + public findRenderer(view: Element): IRenderer | undefined { + return this.renderers.find((renderer) => renderer.parentElement === view); + } + + public addRenderer(renderer: IRenderer): void { + this.renderers.push(renderer); + } + + /** + * Remove the specified renderer if it is specified, otherwise remove all renderers + */ + public removeRenderer(renderer?: IRenderer): void { + let start = 0; + let deleteCount = this.renderers.length; + if (renderer) { + start = this.renderers.indexOf(renderer); + if (start < 0) return; + deleteCount = 1; + } + this.renderers.splice(start, deleteCount).forEach((it) => it.unbind()); + } + + public setRendererContext(context: RendererContext): boolean { + if (context.view) { + const renderer = this.findRenderer(context.view); + if (renderer) { + renderer.context = context; + return true; + } + return false; + } else { + this.renderers.forEach((it) => { + it.context = context; + }); + return this.renderers.length > 0; + } + } + + public release(): void { + this.removeRenderer(); + } +} diff --git a/ts/Renderer/IRendererManager.ts b/ts/Renderer/IRendererManager.ts deleted file mode 100644 index c360a6618..000000000 --- a/ts/Renderer/IRendererManager.ts +++ /dev/null @@ -1,316 +0,0 @@ -import { VideoMirrorModeType, VideoViewSetupMode } from '../Private/AgoraBase'; -import { - RenderModeType, - VideoModulePosition, - VideoSourceType, -} from '../Private/AgoraMediaBase'; -import { RendererContext, RendererType } from '../Types'; -import { logDebug } from '../Utils'; - -import { IRenderer } from './IRenderer'; -import { RendererCache, generateRendererCacheKey } from './RendererCache'; - -/** - * @ignore - */ -export abstract class IRendererManager { - /** - * @ignore - */ - private _renderingFps: number; - /** - * @ignore - */ - private _currentFrameCount: number; - /** - * @ignore - */ - private _previousFirstFrameTime: number; - /** - * @ignore - */ - private _renderingTimer?: number; - /** - * @ignore - */ - private _rendererCaches: RendererCache[]; - /** - * @ignore - */ - private _context: RendererContext; - - constructor() { - this._renderingFps = 15; - this._currentFrameCount = 0; - this._previousFirstFrameTime = 0; - this._rendererCaches = []; - this._context = { - renderMode: RenderModeType.RenderModeHidden, - mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled, - }; - } - - public set renderingFps(fps: number) { - if (this._renderingFps !== fps) { - this._renderingFps = fps; - if (this._renderingTimer) { - this.stopRendering(); - this.startRendering(); - } - } - } - - public get renderingFps(): number { - return this._renderingFps; - } - - public set defaultChannelId(channelId: string) { - this._context.channelId = channelId; - } - - public get defaultChannelId(): string { - return this._context.channelId ?? ''; - } - - public get defaultRenderMode(): RenderModeType { - return this._context.renderMode!; - } - - public get defaultMirrorMode(): VideoMirrorModeType { - return this._context.mirrorMode!; - } - - public release(): void { - this.stopRendering(); - this.clearRendererCache(); - } - - private precheckRendererContext(context: RendererContext): RendererContext { - let { - sourceType, - uid, - channelId, - position, - mediaPlayerId, - renderMode = this.defaultRenderMode, - mirrorMode = this.defaultMirrorMode, - } = context; - switch (sourceType) { - case VideoSourceType.VideoSourceRemote: - if (uid === undefined) { - throw new Error('uid is required'); - } - channelId = channelId ?? this.defaultChannelId; - break; - case VideoSourceType.VideoSourceMediaPlayer: - if (mediaPlayerId === undefined) { - throw new Error('mediaPlayerId is required'); - } - channelId = ''; - uid = mediaPlayerId; - break; - case undefined: - if (uid) { - sourceType = VideoSourceType.VideoSourceRemote; - } - break; - default: - channelId = ''; - uid = 0; - break; - } - if (!position) { - position = - VideoModulePosition.PositionPreEncoder | - VideoModulePosition.PositionPreRenderer; - } - return { - ...context, - position, - sourceType, - uid, - channelId, - renderMode, - mirrorMode, - }; - } - - public addOrRemoveRenderer( - context: RendererContext - ): RendererCache | undefined { - // To be compatible with the old API - let { setupMode = VideoViewSetupMode.VideoViewSetupAdd } = context; - if (!context.view) setupMode = VideoViewSetupMode.VideoViewSetupRemove; - switch (setupMode) { - case VideoViewSetupMode.VideoViewSetupAdd: - return this.addRendererToCache(context); - case VideoViewSetupMode.VideoViewSetupRemove: - this.removeRendererFromCache(context); - return undefined; - case VideoViewSetupMode.VideoViewSetupReplace: - this.removeRendererFromCache(context); - return this.addRendererToCache(context); - } - } - - private addRendererToCache( - context: RendererContext - ): RendererCache | undefined { - const checkedContext = this.precheckRendererContext(context); - - if (!checkedContext.view) return undefined; - - if (this.findRenderer(checkedContext.view)) { - throw new Error('You have already added this view to the renderer'); - } - - let rendererCache = this.getRendererCache(checkedContext); - if (!rendererCache) { - rendererCache = new RendererCache(checkedContext); - this._rendererCaches.push(rendererCache); - } - rendererCache.addRenderer(this.createRenderer(checkedContext)); - this.startRendering(); - return rendererCache; - } - - public removeRendererFromCache(context: RendererContext): void { - const checkedContext = this.precheckRendererContext(context); - - const rendererCache = this.getRendererCache(checkedContext); - if (rendererCache) { - if (checkedContext.view) { - const renderer = rendererCache.findRenderer(checkedContext.view); - if (!renderer) return; - rendererCache.removeRenderer(renderer); - } else { - rendererCache.removeRenderer(); - } - if (rendererCache.renderers.length === 0) { - this._rendererCaches.splice( - this._rendererCaches.indexOf(rendererCache), - 1 - ); - } - } else { - this._rendererCaches = this._rendererCaches.filter((_rendererCache) => { - const renderer = _rendererCache.findRenderer(checkedContext.view); - if (renderer) { - _rendererCache.removeRenderer(renderer); - } - return _rendererCache.renderers.length > 0; - }); - } - } - - public clearRendererCache(): void { - for (const rendererCache of this._rendererCaches) { - rendererCache.removeRenderer(); - } - this._rendererCaches.splice(0); - } - - public getRendererCache(context: RendererContext): RendererCache | undefined { - return this._rendererCaches.find( - (cache) => cache.key === generateRendererCacheKey(context) - ); - } - - public getRenderers(context: RendererContext): IRenderer[] { - return this.getRendererCache(context)?.renderers || []; - } - - public findRenderer(view: Element): IRenderer | undefined { - for (const rendererCache of this._rendererCaches) { - const renderer = rendererCache.findRenderer(view); - if (renderer) return renderer; - } - return undefined; - } - - protected abstract createRenderer( - context: RendererContext, - rendererType?: RendererType - ): IRenderer; - - public startRendering(): void { - if (this._renderingTimer) return; - - const renderingLooper = () => { - if (this._previousFirstFrameTime === 0) { - // Get the current time as the time of the first frame of per second - this._previousFirstFrameTime = performance.now(); - // Reset the frame count - this._currentFrameCount = 0; - } - - // Increase the frame count - ++this._currentFrameCount; - - // Get the current time - const currentFrameTime = performance.now(); - // Calculate the time difference between the current frame and the previous frame - const deltaTime = currentFrameTime - this._previousFirstFrameTime; - // Calculate the expected time of the current frame - const expectedTime = - (this._currentFrameCount * 1000) / this._renderingFps; - logDebug( - new Date().toLocaleTimeString(), - 'currentFrameCount', - this._currentFrameCount, - 'expectedTime', - expectedTime, - 'deltaTime', - deltaTime - ); - - if (this._rendererCaches.length === 0) { - // If there is no renderer, stop rendering - this.stopRendering(); - return; - } - - // Render all renderers - for (const rendererCache of this._rendererCaches) { - this.doRendering(rendererCache); - } - - if (this._currentFrameCount >= this.renderingFps) { - this._previousFirstFrameTime = 0; - } - - if (deltaTime < expectedTime) { - // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference - this._renderingTimer = window.setTimeout( - renderingLooper, - expectedTime - deltaTime - ); - } else { - // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately - renderingLooper(); - } - }; - renderingLooper(); - } - - public abstract doRendering(rendererCache: RendererCache): void; - - public stopRendering(): void { - if (this._renderingTimer) { - window.clearTimeout(this._renderingTimer); - this._renderingTimer = undefined; - } - } - - public setRendererContext(context: RendererContext): boolean { - const checkedContext = this.precheckRendererContext(context); - - for (const rendererCache of this._rendererCaches) { - const result = rendererCache.setRendererContext(checkedContext); - if (result) { - return true; - } - } - return false; - } -} diff --git a/ts/Renderer/RendererCache.ts b/ts/Renderer/RendererCache.ts index dc5c63905..e67e0f8d2 100644 --- a/ts/Renderer/RendererCache.ts +++ b/ts/Renderer/RendererCache.ts @@ -1,26 +1,19 @@ import { VideoFrame } from '../Private/AgoraMediaBase'; -import { RendererCacheContext, RendererContext } from '../Types'; -import { AgoraEnv, logDebug } from '../Utils'; +import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine'; -import { IRenderer } from './IRenderer'; +import { RendererContext } from '../Types'; +import { logDebug } from '../Utils'; -export function generateRendererCacheKey({ - channelId, - uid, - sourceType, -}: RendererContext): string { - return `${channelId}_${uid}_${sourceType}`; -} +import { IRenderer } from './IRenderer'; +import { IRendererCache } from './IRendererCache'; -export class RendererCache { - private _renderers: IRenderer[]; - private _videoFrame: VideoFrame; - private _context: RendererCacheContext; +export class RendererCache extends IRendererCache { + private videoFrame: VideoFrame; private _enabled: boolean; - constructor({ channelId, uid, sourceType, position }: RendererContext) { - this._renderers = []; - this._videoFrame = { + constructor(context: RendererContext) { + super(context); + this.videoFrame = { yBuffer: Buffer.alloc(0), uBuffer: Buffer.alloc(0), vBuffer: Buffer.alloc(0), @@ -31,33 +24,9 @@ export class RendererCache { vStride: 0, rotation: 0, }; - this._context = { channelId, uid, sourceType, position }; this._enabled = false; } - public get key(): string { - return generateRendererCacheKey(this._context); - } - - public get renderers(): IRenderer[] { - return this._renderers; - } - - public get videoFrame(): VideoFrame { - return this._videoFrame; - } - - public get context(): RendererCacheContext { - return this._context; - } - - /** - * @deprecated Use renderers instead - */ - public get renders(): IRenderer[] { - return this.renderers; - } - /** * @deprecated Use videoFrame instead */ @@ -65,19 +34,15 @@ export class RendererCache { return this.videoFrame; } - private get bridge() { - return AgoraEnv.AgoraElectronBridge; - } - private enable() { if (this._enabled) return; - this.bridge.EnableVideoFrameCache(this._context); + AgoraElectronBridge.EnableVideoFrameCache(this.cacheContext); this._enabled = true; } private disable() { if (!this._enabled) return; - this.bridge.DisableVideoFrameCache(this._context); + AgoraElectronBridge.DisableVideoFrameCache(this.cacheContext); this._enabled = false; } @@ -89,9 +54,9 @@ export class RendererCache { } } - public draw() { - let { ret, isNewFrame } = this.bridge.GetVideoFrame( - this.context, + override draw() { + let { ret, isNewFrame } = AgoraElectronBridge.GetVideoFrame( + this.cacheContext, this.videoFrame ); @@ -105,7 +70,10 @@ export class RendererCache { this.videoFrame.uBuffer = Buffer.alloc(uStride! * height!); this.videoFrame.vBuffer = Buffer.alloc(vStride! * height!); - const result = this.bridge.GetVideoFrame(this.context, this.videoFrame); + const result = AgoraElectronBridge.GetVideoFrame( + this.cacheContext, + this.videoFrame + ); ret = result.ret; isNewFrame = result.isNewFrame; break; @@ -121,47 +89,20 @@ export class RendererCache { } } - public findRenderer(view: Element): IRenderer | undefined { - return this._renderers.find((renderer) => renderer.parentElement === view); - } - - public addRenderer(renderer: IRenderer): void { - this._renderers.push(renderer); + override addRenderer(renderer: IRenderer): void { + super.addRenderer(renderer); this.shouldEnable(); } /** * Remove the specified renderer if it is specified, otherwise remove all renderers */ - public removeRenderer(renderer?: IRenderer): void { - let start = 0; - let deleteCount = this._renderers.length; - if (renderer) { - start = this._renderers.indexOf(renderer); - if (start < 0) return; - deleteCount = 1; - } - this._renderers.splice(start, deleteCount).forEach((it) => it.unbind()); + override removeRenderer(renderer?: IRenderer): void { + super.removeRenderer(renderer); this.shouldEnable(); } - public setRendererContext({ - view, - renderMode, - mirrorMode, - }: RendererContext): boolean { - if (view) { - const renderer = this.findRenderer(view); - if (renderer) { - renderer.context = { renderMode, mirrorMode }; - return true; - } - return false; - } else { - this._renderers.forEach((it) => { - it.context = { renderMode, mirrorMode }; - }); - return this._renderers.length > 0; - } + public release(): void { + super.release(); } } diff --git a/ts/Renderer/RendererManager.ts b/ts/Renderer/RendererManager.ts index 09ed6b3f0..6e1a896ea 100644 --- a/ts/Renderer/RendererManager.ts +++ b/ts/Renderer/RendererManager.ts @@ -1,110 +1,386 @@ +import createAgoraRtcEngine from '../AgoraSdk'; +import { + VideoMirrorModeType, + VideoStreamType, + VideoViewSetupMode, +} from '../Private/AgoraBase'; +import { RenderModeType, VideoSourceType } from '../Private/AgoraMediaBase'; import { - RENDER_MODE, RendererCacheContext, + RendererCacheType, RendererContext, RendererType, } from '../Types'; -import { isSupportWebGL } from '../Utils'; +import { AgoraEnv, isSupportWebGL, logDebug } from '../Utils'; import { IRenderer } from './IRenderer'; -import { IRendererManager } from './IRendererManager'; +import { generateRendererCacheKey, isUseConnection } from './IRendererCache'; import { RendererCache } from './RendererCache'; +import { WebCodecsRenderer } from './WebCodecsRenderer'; +import { WebCodecsRendererCache } from './WebCodecsRendererCache'; import { WebGLFallback, WebGLRenderer } from './WebGLRenderer'; import { YUVCanvasRenderer } from './YUVCanvasRenderer'; /** * @ignore */ -export class RendererManager extends IRendererManager { +export class RendererManager { + /** + * @ignore + */ + private renderingFps: number; + /** + * @ignore + */ + private _currentFrameCount: number; + /** + * @ignore + */ + private _previousFirstFrameTime: number; + /** + * @ignore + */ + private _renderingTimer?: number; /** * @ignore */ - private _rendererType: RendererType; + private _rendererCaches: RendererCacheType[]; + /** + * @ignore + */ + private _context: RendererContext; + + /** + * @ignore + */ + private rendererType: RendererType; + + constructor() { + this.renderingFps = 15; + this._currentFrameCount = 0; + this._previousFirstFrameTime = 0; + this._rendererCaches = []; + this._context = { + renderMode: RenderModeType.RenderModeHidden, + mirrorMode: VideoMirrorModeType.VideoMirrorModeDisabled, + }; + this.rendererType = isSupportWebGL() + ? RendererType.WEBGL + : RendererType.SOFTWARE; + } - public set rendererType(rendererType: RendererType) { - if (this._rendererType !== rendererType) { - this._rendererType = rendererType; + public setRenderingFps(fps: number) { + this.renderingFps = fps; + if (this._renderingTimer) { + this.stopRendering(); + this.startRendering(); } } - public get rendererType(): RendererType { - return this._rendererType; + public set defaultChannelId(channelId: string) { + this._context.channelId = channelId; } - constructor() { - super(); - this._rendererType = isSupportWebGL() - ? RendererType.WEBGL - : RendererType.SOFTWARE; + public get defaultChannelId(): string { + return this._context.channelId ?? ''; } - /** - * @deprecated Use rendererType instead - */ - public setRenderMode(mode: RENDER_MODE) { - this.rendererType = mode; + public get defaultRenderMode(): RenderModeType { + return this._context.renderMode!; } - /** - * @deprecated Use renderingFps instead - */ - public setFPS(fps: number) { - this.renderingFps = fps; + public get defaultMirrorMode(): VideoMirrorModeType { + return this._context.mirrorMode!; } - /** - * @deprecated Use getRendererCache instead - */ - public getRender(context: RendererCacheContext): RendererCache | undefined { - return this.getRendererCache(context); + public release(): void { + this.stopRendering(); + this.clearRendererCache(); } - protected override createRenderer( - context: RendererContext, - rendererType?: RendererType - ): IRenderer { - if (rendererType === undefined) { - rendererType = this.rendererType; + private presetRendererContext(context: RendererContext): RendererContext { + //this is for preset default value + context.renderMode = context.renderMode || this.defaultRenderMode; + context.mirrorMode = context.mirrorMode || this.defaultMirrorMode; + context.useWebCodecsDecoder = context.useWebCodecsDecoder || false; + context.enableFps = context.enableFps || false; + + if (!AgoraEnv.CapabilityManager?.webCodecsDecoderEnabled) { + context.useWebCodecsDecoder = false; + } + + switch (context.sourceType) { + case VideoSourceType.VideoSourceRemote: + if (context.uid === undefined) { + throw new Error('uid is required'); + } + context.channelId = context.channelId ?? this.defaultChannelId; + break; + case VideoSourceType.VideoSourceMediaPlayer: + if (context.mediaPlayerId === undefined) { + throw new Error('mediaPlayerId is required'); + } + context.channelId = ''; + context.uid = context.mediaPlayerId; + break; + case undefined: + if (context.uid) { + context.sourceType = VideoSourceType.VideoSourceRemote; + } + break; + default: + context.channelId = ''; + context.uid = 0; + break; + } + return context; + } + + public addOrRemoveRenderer( + context: RendererContext + ): RendererCacheType | undefined { + // To be compatible with the old API + let { setupMode = VideoViewSetupMode.VideoViewSetupAdd } = context; + if (!context.view) setupMode = VideoViewSetupMode.VideoViewSetupRemove; + switch (setupMode) { + case VideoViewSetupMode.VideoViewSetupAdd: + return this.addRendererToCache(context); + case VideoViewSetupMode.VideoViewSetupRemove: + this.removeRendererFromCache(context); + return undefined; + case VideoViewSetupMode.VideoViewSetupReplace: + this.removeRendererFromCache(context); + return this.addRendererToCache(context); + } + } + + private addRendererToCache( + context: RendererContext + ): RendererCacheType | undefined { + const checkedContext = this.presetRendererContext(context); + + if (!checkedContext.view) return undefined; + + if (this.findRenderer(checkedContext.view)) { + throw new Error('You have already added this view to the renderer'); + } + + let rendererCache = this.getRendererCache(checkedContext); + if (!rendererCache) { + if (context.useWebCodecsDecoder) { + rendererCache = new WebCodecsRendererCache(checkedContext); + } else { + rendererCache = new RendererCache(checkedContext); + } + this._rendererCaches.push(rendererCache); + } + rendererCache.addRenderer(this.createRenderer(checkedContext)); + if (!context.useWebCodecsDecoder) { + this.startRendering(); + } + return rendererCache; + } + + public removeRendererFromCache(context: RendererContext): void { + const checkedContext = this.presetRendererContext(context); + + const rendererCache = this.getRendererCache(checkedContext); + if (!rendererCache) return; + if (checkedContext.view) { + const renderer = rendererCache.findRenderer(checkedContext.view); + if (!renderer) return; + rendererCache.removeRenderer(renderer); + } else { + rendererCache.removeRenderer(); + } + if (rendererCache.renderers.length === 0) { + rendererCache.release(); + this._rendererCaches.splice( + this._rendererCaches.indexOf(rendererCache), + 1 + ); + } + } + + public clearRendererCache(): void { + for (const rendererCache of this._rendererCaches) { + rendererCache.release(); } + this._rendererCaches.splice(0); + } + + public getRendererCache( + context: RendererContext + ): RendererCacheType | undefined { + return this._rendererCaches.find( + (cache) => cache.key === generateRendererCacheKey(context) + ); + } + + public getRenderers(context: RendererContext): IRenderer[] { + return this.getRendererCache(context)?.renderers || []; + } + public findRenderer(view: Element): IRenderer | undefined { + for (const rendererCache of this._rendererCaches) { + const renderer = rendererCache.findRenderer(view); + if (renderer) return renderer; + } + return undefined; + } + + protected createRenderer( + context: RendererContext, + rendererType: RendererType = this.rendererType + ): IRenderer { let renderer: IRenderer; switch (rendererType) { case RendererType.WEBGL: - renderer = new WebGLRenderer( - this.handleWebGLFallback(context).bind(this) - ); + if (context.useWebCodecsDecoder) { + renderer = new WebCodecsRenderer(); + } else { + renderer = new WebGLRenderer( + this.handleWebGLFallback(context).bind(this) + ); + renderer.bind(context.view); + } break; case RendererType.SOFTWARE: renderer = new YUVCanvasRenderer(); + renderer.bind(context.view); break; default: throw new Error('Unknown renderer type'); } - renderer.bind(context.view); - renderer.context = { - renderMode: context.renderMode, - mirrorMode: context.mirrorMode, - }; + renderer.setContext(context); return renderer; } - public override doRendering(rendererCache: RendererCache): void { + public startRendering(): void { + if (this._renderingTimer) return; + + const renderingLooper = () => { + if (this._previousFirstFrameTime === 0) { + // Get the current time as the time of the first frame of per second + this._previousFirstFrameTime = performance.now(); + // Reset the frame count + this._currentFrameCount = 0; + } + + // Increase the frame count + ++this._currentFrameCount; + + // Get the current time + const currentFrameTime = performance.now(); + // Calculate the time difference between the current frame and the previous frame + const deltaTime = currentFrameTime - this._previousFirstFrameTime; + // Calculate the expected time of the current frame + const expectedTime = (this._currentFrameCount * 1000) / this.renderingFps; + logDebug( + new Date().toLocaleTimeString(), + 'currentFrameCount', + this._currentFrameCount, + 'expectedTime', + expectedTime, + 'deltaTime', + deltaTime + ); + + if (this._rendererCaches.length === 0) { + // If there is no renderer, stop rendering + this.stopRendering(); + return; + } + + // Render all renderers that do not use WebCodecs + for (const rendererCache of this._rendererCaches.filter( + (cache) => cache instanceof RendererCache + )) { + this.doRendering(rendererCache); + } + + if (this._currentFrameCount >= this.renderingFps) { + this._previousFirstFrameTime = 0; + } + + if (deltaTime < expectedTime) { + // If the time difference between the current frame and the previous frame is less than the expected time, then wait for the difference + this._renderingTimer = window.setTimeout( + renderingLooper, + expectedTime - deltaTime + ); + } else { + // If the time difference between the current frame and the previous frame is greater than the expected time, then render immediately + renderingLooper(); + } + }; + renderingLooper(); + } + + public doRendering(rendererCache: RendererCacheType): void { rendererCache.draw(); } private handleWebGLFallback(context: RendererContext): WebGLFallback { return (renderer: WebGLRenderer) => { - const { - context: { renderMode, mirrorMode }, - } = renderer; const renderers = this.getRenderers(context); renderer.unbind(); - const newRenderer = this.createRenderer( - { ...context, renderMode, mirrorMode }, - RendererType.SOFTWARE - ); + const newRenderer = this.createRenderer(context, RendererType.SOFTWARE); renderers.splice(renderers.indexOf(renderer), 1, newRenderer); }; } + + public handleWebCodecsFallback(context: RendererCacheContext): void { + let engine = createAgoraRtcEngine(); + engine.getMediaEngine().unregisterVideoEncodedFrameObserver({}); + if (context.uid) { + if (isUseConnection(context)) { + engine.setRemoteVideoSubscriptionOptionsEx( + context.uid, + { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: false, + }, + { + channelId: context.channelId, + localUid: context.localUid, + } + ); + } else { + engine.setRemoteVideoSubscriptionOptions(context.uid, { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: false, + }); + } + } + AgoraEnv.enableWebCodecsDecoder = false; + AgoraEnv.CapabilityManager?.setWebCodecsDecoderEnabled(false); + let renderers = this.getRenderers(context); + for (let renderer of renderers) { + this.addOrRemoveRenderer({ + ...renderer.context, + setupMode: VideoViewSetupMode.VideoViewSetupReplace, + }); + } + } + + public stopRendering(): void { + if (this._renderingTimer) { + window.clearTimeout(this._renderingTimer); + this._renderingTimer = undefined; + } + } + + public setRendererContext(context: RendererContext): boolean { + const checkedContext = this.presetRendererContext(context); + + for (const rendererCache of this._rendererCaches) { + const result = rendererCache.setRendererContext(checkedContext); + if (result) { + return true; + } + } + return false; + } } diff --git a/ts/Renderer/WebCodecsRenderer/index.ts b/ts/Renderer/WebCodecsRenderer/index.ts new file mode 100755 index 000000000..06aaacc4c --- /dev/null +++ b/ts/Renderer/WebCodecsRenderer/index.ts @@ -0,0 +1,141 @@ +import { RendererType } from '../../Types'; +import { getContextByCanvas } from '../../Utils'; +import { IRenderer } from '../IRenderer'; + +export type frameSize = { + width: number; + height: number; +}; + +export class WebCodecsRenderer extends IRenderer { + gl?: WebGLRenderingContext | WebGL2RenderingContext | null; + // eslint-disable-next-line auto-import/auto-import + offscreenCanvas: OffscreenCanvas | undefined; + + constructor() { + super(); + this.rendererType = RendererType.WEBCODECSRENDERER; + } + + static vertexShaderSource = ` + attribute vec2 xy; + varying highp vec2 uv; + void main(void) { + gl_Position = vec4(xy, 0.0, 1.0); + // Map vertex coordinates (-1 to +1) to UV coordinates (0 to 1). + // UV coordinates are Y-flipped relative to vertex coordinates. + uv = vec2((1.0 + xy.x) / 2.0, (1.0 - xy.y) / 2.0); + } + `; + static fragmentShaderSource = ` + varying highp vec2 uv; + uniform sampler2D texture; + void main(void) { + gl_FragColor = texture2D(texture, uv); + } + `; + + bind(element: HTMLElement, frameSize: frameSize) { + super.bind(element); + if (!this.canvas) return; + this.canvas.width = frameSize.width; + this.canvas.height = frameSize.height; + this.offscreenCanvas = this.canvas.transferControlToOffscreen(); + this.gl = getContextByCanvas(this.offscreenCanvas); + if (!this.gl) return; + const vertexShader = this.gl.createShader(this.gl.VERTEX_SHADER); + if (!vertexShader) return; + this.gl.shaderSource(vertexShader, WebCodecsRenderer.vertexShaderSource); + this.gl.compileShader(vertexShader); + if (!this.gl.getShaderParameter(vertexShader, this.gl.COMPILE_STATUS)) { + throw this.gl.getShaderInfoLog(vertexShader); + } + const fragmentShader = this.gl.createShader(this.gl.FRAGMENT_SHADER); + if (!fragmentShader) return; + this.gl.shaderSource( + fragmentShader, + WebCodecsRenderer.fragmentShaderSource + ); + this.gl.compileShader(fragmentShader); + if (!this.gl.getShaderParameter(fragmentShader, this.gl.COMPILE_STATUS)) { + throw this.gl.getShaderInfoLog(fragmentShader); + } + const shaderProgram = this.gl.createProgram(); + if (!shaderProgram) return; + this.gl.attachShader(shaderProgram, vertexShader); + this.gl.attachShader(shaderProgram, fragmentShader); + this.gl.linkProgram(shaderProgram); + if (!this.gl.getProgramParameter(shaderProgram, this.gl.LINK_STATUS)) { + throw this.gl.getProgramInfoLog(shaderProgram); + } + this.gl.useProgram(shaderProgram); + // Vertex coordinates, clockwise from bottom-left. + const vertexBuffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, vertexBuffer); + this.gl.bufferData( + this.gl.ARRAY_BUFFER, + new Float32Array([-1.0, -1.0, -1.0, +1.0, +1.0, +1.0, +1.0, -1.0]), + this.gl.STATIC_DRAW + ); + const xyLocation = this.gl.getAttribLocation(shaderProgram, 'xy'); + this.gl.vertexAttribPointer(xyLocation, 2, this.gl.FLOAT, false, 0, 0); + this.gl.enableVertexAttribArray(xyLocation); + // Create one texture to upload frames to. + const texture = this.gl.createTexture(); + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MAG_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_MIN_FILTER, + this.gl.NEAREST + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_S, + this.gl.CLAMP_TO_EDGE + ); + this.gl.texParameteri( + this.gl.TEXTURE_2D, + this.gl.TEXTURE_WRAP_T, + this.gl.CLAMP_TO_EDGE + ); + } + + drawFrame(frame: any) { + if (!this.offscreenCanvas || !frame) return; + this.offscreenCanvas.width = frame.displayWidth; + this.offscreenCanvas.height = frame.displayHeight; + this.updateRenderMode(); + if (!this.gl) return; + + if (this.gl) { + // Upload the frame. + this.gl.texImage2D( + this.gl.TEXTURE_2D, + 0, + this.gl.RGBA, + this.gl.RGBA, + this.gl.UNSIGNED_BYTE, + frame + ); + frame.close(); + // Configure and clear the drawing area. + this.gl.viewport( + 0, + 0, + this.gl.drawingBufferWidth, + this.gl.drawingBufferHeight + ); + this.gl.clearColor(1.0, 0.0, 0.0, 1.0); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + // Draw the frame. + this.gl.drawArrays(this.gl.TRIANGLE_FAN, 0, 4); + } + super.drawFrame(); + this.getFps(); + } +} diff --git a/ts/Renderer/WebCodecsRendererCache.ts b/ts/Renderer/WebCodecsRendererCache.ts new file mode 100644 index 000000000..993f9f8d7 --- /dev/null +++ b/ts/Renderer/WebCodecsRendererCache.ts @@ -0,0 +1,140 @@ +import createAgoraRtcEngine from '../AgoraSdk'; +import { WebCodecsDecoder } from '../Decoder/index'; +import { EncodedVideoFrameInfo, VideoStreamType } from '../Private/AgoraBase'; +import { IRtcEngineEx } from '../Private/IAgoraRtcEngineEx'; +import { AgoraElectronBridge } from '../Private/internal/IrisApiEngine'; + +import { RendererContext, RendererType } from '../Types'; +import { AgoraEnv, logInfo } from '../Utils'; + +import { IRendererCache, isUseConnection } from './IRendererCache'; +import { WebCodecsRenderer } from './WebCodecsRenderer/index'; + +export class WebCodecsRendererCache extends IRendererCache { + private _decoder?: WebCodecsDecoder | null; + private _engine?: IRtcEngineEx; + private _firstFrame = true; + + constructor(context: RendererContext) { + super(context); + this._engine = createAgoraRtcEngine(); + this._decoder = new WebCodecsDecoder( + this.renderers as WebCodecsRenderer[], + this.onDecoderError.bind(this), + context + ); + this.draw(); + } + + onDecoderError(e: any) { + logInfo('webCodecsDecoder decode failed, fallback to native decoder', e); + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext); + } + + onEncodedVideoFrameReceived(...[data, buffer]: any) { + let _data: any; + try { + _data = JSON.parse(data) ?? {}; + } catch (e) { + _data = {}; + } + if ( + Object.keys(_data).length === 0 || + !this._decoder || + this.cacheContext.uid !== _data.uid + ) + return; + if (this._firstFrame) { + for (let renderer of this.renderers) { + if (renderer.rendererType !== RendererType.WEBCODECSRENDERER) { + continue; + } + renderer.bind(renderer.context.view, { + width: _data.videoEncodedFrameInfo.width!, + height: _data.videoEncodedFrameInfo.height!, + }); + } + + try { + this._decoder.decoderConfigure(_data.videoEncodedFrameInfo); + } catch (error: any) { + logInfo(error); + return; + } + this._firstFrame = false; + } + if (this.shouldFallback(_data.videoEncodedFrameInfo)) { + AgoraEnv.AgoraRendererManager?.handleWebCodecsFallback(this.cacheContext); + } else { + this._decoder.decodeFrame( + buffer, + _data.videoEncodedFrameInfo, + new Date().getTime() + ); + } + } + + public draw() { + if (isUseConnection(this.cacheContext)) { + this._engine?.setRemoteVideoSubscriptionOptionsEx( + this.cacheContext.uid!, + { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: true, + }, + { + channelId: this.cacheContext.channelId, + localUid: this.cacheContext.localUid, + } + ); + } else { + this._engine?.setRemoteVideoSubscriptionOptions(this.cacheContext.uid!, { + type: VideoStreamType.VideoStreamHigh, + encodedFrameOnly: true, + }); + } + AgoraElectronBridge.OnEvent( + `call_back_with_encoded_video_frame_${this.cacheContext.uid}`, + (...params: any) => { + try { + this.onEncodedVideoFrameReceived(...params); + } catch (e) { + console.error(e); + } + } + ); + } + + public shouldFallback(frameInfo: EncodedVideoFrameInfo): boolean { + let shouldFallback = false; + if (!frameInfo.codecType) { + shouldFallback = true; + logInfo('codecType is not supported, fallback to native decoder'); + } else { + const mapping = + AgoraEnv.CapabilityManager?.frameCodecMapping[frameInfo.codecType]; + if (mapping === undefined) { + shouldFallback = true; + logInfo('codecType is not supported, fallback to native decoder'); + } else if ( + mapping.minWidth >= frameInfo.width! && + mapping.minHeight >= frameInfo.height! && + mapping.maxWidth <= frameInfo.width! && + mapping.maxHeight <= frameInfo.height! + ) { + shouldFallback = true; + logInfo('frame size is not supported, fallback to native decoder'); + } + } + return shouldFallback; + } + + public release(): void { + AgoraElectronBridge.UnEvent( + `call_back_with_encoded_video_frame_${this.cacheContext.uid}` + ); + this._decoder?.release(); + this._decoder = null; + super.release(); + } +} diff --git a/ts/Renderer/WebGLRenderer/index.ts b/ts/Renderer/WebGLRenderer/index.ts index a6bd0f999..0c0f32c3f 100644 --- a/ts/Renderer/WebGLRenderer/index.ts +++ b/ts/Renderer/WebGLRenderer/index.ts @@ -1,4 +1,5 @@ import { VideoFrame } from '../../Private/AgoraMediaBase'; +import { RendererType } from '../../Types'; import { logWarn } from '../../Utils'; import { IRenderer } from '../IRenderer'; @@ -57,6 +58,7 @@ export class WebGLRenderer extends IRenderer { constructor(fallback?: WebGLFallback) { super(); this.gl = undefined; + this.rendererType = RendererType.WEBGL; this.yTexture = null; this.uTexture = null; this.vTexture = null; @@ -251,6 +253,7 @@ export class WebGLRenderer extends IRenderer { this.gl.drawArrays(this.gl.TRIANGLES, 0, 6); super.drawFrame(); + this.getFps(); } protected override rotateCanvas({ width, height, rotation }: VideoFrame) { diff --git a/ts/Renderer/index.ts b/ts/Renderer/index.ts index 26cd4c4db..0fa58f585 100644 --- a/ts/Renderer/index.ts +++ b/ts/Renderer/index.ts @@ -1,2 +1 @@ export * from './IRenderer'; -export * from './IRendererManager'; diff --git a/ts/Types.ts b/ts/Types.ts index c0db0bae3..13de5ea58 100644 --- a/ts/Types.ts +++ b/ts/Types.ts @@ -1,7 +1,21 @@ -import { VideoCanvas } from './Private/AgoraBase'; +import { VideoCanvas, VideoCodecType } from './Private/AgoraBase'; import { VideoFrame } from './Private/AgoraMediaBase'; import { RtcConnection } from './Private/IAgoraRtcEngineEx'; -import { IRendererManager } from './Renderer'; +import { CapabilityManager } from './Renderer/CapabilityManager'; +import { RendererCache } from './Renderer/RendererCache'; +import { RendererManager } from './Renderer/RendererManager'; +import { WebCodecsRendererCache } from './Renderer/WebCodecsRendererCache'; + +export enum VideoFallbackStrategy { + /** + * @ignore + */ + PerformancePriority = 0, + /** + * @ignore + */ + BandwidthPriority = 1, +} /** * @ignore @@ -19,6 +33,14 @@ export interface AgoraEnvOptions { * @ignore */ webEnvReady?: boolean; + /** + * @ignore + */ + enableWebCodecsDecoder: boolean; + /** + * @ignore + */ + videoFallbackStrategy: VideoFallbackStrategy; } /** @@ -28,11 +50,11 @@ export interface AgoraEnvType extends AgoraEnvOptions { /** * @ignore */ - AgoraElectronBridge: AgoraElectronBridge; + AgoraRendererManager?: RendererManager; /** * @ignore */ - AgoraRendererManager?: IRendererManager; + CapabilityManager?: CapabilityManager; } /** @@ -47,15 +69,26 @@ export enum RendererType { * @ignore */ SOFTWARE = 2, + /** + * @ignore + */ + WEBCODECSRENDERER = 3, } export type RENDER_MODE = RendererType; export type RendererContext = VideoCanvas & RtcConnection; +export type RendererCacheType = RendererCache | WebCodecsRendererCache; export type RendererCacheContext = Pick< RendererContext, - 'channelId' | 'uid' | 'sourceType' | 'position' + | 'channelId' + | 'localUid' + | 'uid' + | 'sourceType' + | 'useWebCodecsDecoder' + | 'enableFps' + | 'position' >; /** @@ -75,7 +108,7 @@ export interface Result { /** * @ignore */ -export interface AgoraElectronBridge { +export interface IAgoraElectronBridge { /** * @ignore */ @@ -90,6 +123,8 @@ export interface AgoraElectronBridge { ) => void ): void; + UnEvent(callbackName: string): void; + CallApi( funcName: string, params: any, @@ -124,3 +159,34 @@ export interface AgoraElectronBridge { bufferCount?: number ) => Result; } + +/** + * @ignore + */ +export enum IPCMessageType { + AGORA_IPC_GET_GPU_INFO = 'AGORA_IPC_GET_GPU_INFO', +} + +interface CodecMappingItem { + codec: string; + type: VideoCodecType; + profile: string; +} + +/** + * @ignore + */ +export const codecMapping: CodecMappingItem[] = [ + { + codec: 'avc1.64e01f', + type: VideoCodecType.VideoCodecH264, + profile: 'h264', + }, + { + codec: 'hvc1.1.6.L5.90', + type: VideoCodecType.VideoCodecH265, + profile: 'hevc', + }, + { codec: 'vp8', type: VideoCodecType.VideoCodecVp8, profile: 'vp8' }, + { codec: 'vp9', type: VideoCodecType.VideoCodecVp9, profile: 'vp9' }, +]; diff --git a/ts/Utils.ts b/ts/Utils.ts index bc3d6be4a..d25deb942 100644 --- a/ts/Utils.ts +++ b/ts/Utils.ts @@ -44,6 +44,21 @@ export const logError = (msg: string, ...optParams: any[]) => { console.error(`${TAG} ${msg}`, ...optParams); }; +const getCurrentTime = () => { + const date = new Date(); + + const year = date.getFullYear().toString().slice(-2); + const month = (date.getMonth() + 1).toString().padStart(2, '0'); + const day = date.getDate().toString().padStart(2, '0'); + + const hours = date.getHours().toString().padStart(2, '0'); + const minutes = date.getMinutes().toString().padStart(2, '0'); + const seconds = date.getSeconds().toString().padStart(2, '0'); + const milliseconds = date.getMilliseconds().toString().padStart(3, '0'); + + return `${month}/${day}/${year} ${hours}:${minutes}:${seconds}:${milliseconds}`; +}; + /** * @ignore */ @@ -51,7 +66,7 @@ export const logInfo = (msg: string, ...optParams: any[]) => { if (!AgoraEnv.enableLogging) { return; } - console.info(`${TAG} ${msg}`, ...optParams); + console.info(`[${getCurrentTime()}]${TAG} ${msg}`, ...optParams); }; /** @@ -138,7 +153,35 @@ export function isSupportWebGL(): boolean { return flag; } -const AgoraNode = require('../build/Release/agora_node_ext'); +/** + * @ignore + */ +export function getContextByCanvas( + // eslint-disable-next-line auto-import/auto-import + canvas: OffscreenCanvas +): WebGLRenderingContext | WebGL2RenderingContext | null { + const contextNames = ['webgl2', 'webgl', 'experimental-webgl']; + + for (const contextName of contextNames) { + //@ts-ignore + const context = canvas.getContext(contextName, { + depth: true, + stencil: true, + alpha: false, + antialias: false, + premultipliedAlpha: true, + preserveDrawingBuffer: true, + powerPreference: 'default', + failIfMajorPerformanceCaveat: false, + }) as WebGLRenderingContext | WebGL2RenderingContext | null; + + if (context) { + return context; + } + } + + return null; +} /** * @ignore @@ -147,5 +190,6 @@ export const AgoraEnv: AgoraEnvType = { enableLogging: true, enableDebugLogging: false, webEnvReady: true, - AgoraElectronBridge: new AgoraNode.AgoraElectronBridge(), + enableWebCodecsDecoder: false, + videoFallbackStrategy: 0, }; diff --git a/yarn.lock b/yarn.lock index 1583ad924..170225c3b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1733,6 +1733,11 @@ dependencies: "@babel/types" "^7.3.0" +"@types/dom-webcodecs@^0.1.11": + version "0.1.13" + resolved "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz#d8be5da4f01b20721307b08ad2cca903ccf4f47f" + integrity sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ== + "@types/graceful-fs@^4.1.3": version "4.1.6" resolved "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.6.tgz#e14b2576a1c25026b7f02ede1de3b84c3a1efeae" @@ -1814,6 +1819,13 @@ resolved "https://registry.npmjs.org/@types/node/-/node-20.0.0.tgz#081d9afd28421be956c1a47ced1c9a0034b467e2" integrity sha512-cD2uPTDnQQCVpmRefonO98/PPijuOnnEy5oytWJFPY1N9aJCz2wJ5kSGWO+zJoed2cY2JxQh6yBuUq4vIn61hw== +"@types/node@^22.8.2": + version "22.8.2" + resolved "https://registry.npmjs.org/@types/node/-/node-22.8.2.tgz#8e82bb8201c0caf751dcdc61b0a262d2002d438b" + integrity sha512-NzaRNFV+FZkvK/KLCsNdTvID0SThyrs5SHB6tsD/lajr22FGC73N2QeDPM2wHtVde8mgcXuSsHQkH5cX1pbPLw== + dependencies: + undici-types "~6.19.8" + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -8669,6 +8681,11 @@ semver@^6.0.0, semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: resolved "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz#ee0a64c8af5e8ceea67687b133761e1becbd1d3d" integrity sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw== +semver@^7.6.0: + version "7.6.3" + resolved "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz#980f7b5550bc175fb4dc09403085627f9eb33143" + integrity sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A== + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" @@ -9646,6 +9663,11 @@ undertaker@^1.2.1: object.reduce "^1.0.0" undertaker-registry "^1.0.0" +undici-types@~6.19.8: + version "6.19.8" + resolved "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw== + unicode-canonical-property-names-ecmascript@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/unicode-canonical-property-names-ecmascript/-/unicode-canonical-property-names-ecmascript-2.0.0.tgz#301acdc525631670d39f6146e0e77ff6bbdebddc"