/* * Copyright (C) 2025 Amlogic Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include "mesonvideocompositor_plugin.h" #include "mesonvideocompositor_impl.h" #include "Logger.h" #include "Times.h" #include "video_tunnel.h" #include "videotunnel.h" using namespace ATools; #define TAG "rlib:mesonvideocompositor_impl" #define UNDER_FLOW_EXPIRED_TIME_MS (83) /*if there are two output screen,display id will be from 0~1,0: first output screen 1:second output screen*/ #define DEFAULT_DISPLAY_ID (0) #define SECOND_DISPLAY_ID (1) #define VIDEO_TUNNEL_LOCAL_MODE (0) #define VIDEO_TUNNEL_REMOTE_MODE (1) #define FRAME_FREEZE_THRESHOLD(x) (x*1.67) static meson_callback_t buffer_cb = { MesonVideoCompositorImpl::bufferDisplayed, MesonVideoCompositorImpl::bufferDropped, }; MesonVideoCompositorImpl::MesonVideoCompositorImpl(MesonVideoCompositorPlugin *plugin, int logcategory) : mPlugin(plugin), mLogCategory(logcategory) { mFd = -1; mInstanceId = -1; mIsVideoTunnelConnected = false; mQueueFrameCnt = 0; mTotalQueueFrameCnt = 0; mStarted = false; mRequestStop = false; mVideotunnelLib = NULL; mDisplayAdapterLib = NULL; mWindowChanged = false; mDisplayId = DEFAULT_DISPLAY_ID; // Used to specify on which screen to display, the default setting is 0 mLayerId = -1; // Obtained from the compositor to distinguish video path mFrameWidth = 1920; mFrameHeight = 1080; mPip = 0; mZorder = 0; mImmediately = false; mHideVideo = false; mKeepLastFrame = true; mUnderFlowDetect = false; mMesonDisplayLib = NULL; mPoll = NULL; mWindow.x = 0; mWindow.y = 0; mWindow.w = 0; mWindow.h = 0; mResolutionWidth = 0; mResolutionHeight = 0; mPixelAspectRatio = 1.0; mDisplayedFrameWidth = 0; mDisplayedFrameHeight = 0; mLastDisplayFramePts = -1; mLastDisplayFrameTimeUs = -1; mFrameRateFractionNum = 0; mFrameRateFractionDenom = 1; mFrameDurationUs = 0; } MesonVideoCompositorImpl::~MesonVideoCompositorImpl() { } bool MesonVideoCompositorImpl::init() { mVideotunnelLib = videotunnelLoadLib(mLogCategory); if (!mVideotunnelLib) { ERROR(mLogCategory,"videotunnelLoadLib load symbol fail"); return false; } mDisplayAdapterLib = displayAdapterLoadLib(mLogCategory); if (!mDisplayAdapterLib) { ERROR(mLogCategory,"videotunnelLoadLib load symbol fail"); return false; } mMesonDisplayLib = mesonDisplayLoadLib(mLogCategory); if (mMesonDisplayLib) { int ret = 0; int32_t resolutionWidth = 0; int32_t resolutionHeight = 0; if (mDisplayAdapterLib->dADispAdptGetUiSize(mDisplayId, &resolutionWidth, &resolutionHeight) == 0) { INFO(mLogCategory,"displayId:%d, full screen resolution:%dx%d", mDisplayId, resolutionWidth, resolutionHeight); mResolutionWidth = resolutionWidth; mResolutionHeight = resolutionHeight; } else { ERROR(mLogCategory,"failed get screen resolution, Set to 1920x1080"); mResolutionWidth = 1920; mResolutionHeight = 1080; } mWindow.w = mResolutionWidth; mWindow.h = mResolutionHeight; } else { ERROR(mLogCategory,"mesonDisplayLoadLib load symbol fail"); return false; } return true; } bool MesonVideoCompositorImpl::release() { DEBUG(mLogCategory,"in"); if (mVideotunnelLib) { videotunnelUnloadLib(mLogCategory, mVideotunnelLib); mVideotunnelLib = NULL; } if (mDisplayAdapterLib) { displayAdapterUnloadLib(mLogCategory, mDisplayAdapterLib); mDisplayAdapterLib = NULL; } if (mMesonDisplayLib) { mesonDisplayUnLoadLib(mLogCategory, mMesonDisplayLib); mMesonDisplayLib = NULL; } DEBUG(mLogCategory,"out"); return true; } bool MesonVideoCompositorImpl::connect() { int ret = 0; int displayMode = -1; mDisplayedFrameWidth = 0; mDisplayedFrameHeight = 0; DEBUG(mLogCategory,"in"); if (!mPoll) { mPoll = new ATools::Poll(true); } mRequestStop = false; mSignalFirstFrameDiplayed = false; if (mDisplayAdapterLib && mDisplayAdapterLib->dADispAdptGetVideoTunnelMode) { displayMode = mDisplayAdapterLib->dADispAdptGetVideoTunnelMode(); INFO(mLogCategory,"display mode: %d (local:0,remote:1)",displayMode); } if (mVideotunnelLib && mVideotunnelLib->vtOpenByMode) { mFd = mVideotunnelLib->vtOpenByMode(displayMode); // open vt, mode=0: localmode mode=1: remotemode } if (mVideotunnelLib && mVideotunnelLib->vtAllocId) { ret = mVideotunnelLib->vtAllocId(mFd, &mInstanceId); } if (mVideotunnelLib && mVideotunnelLib->vtConnect) { ret = mVideotunnelLib->vtConnect(mFd, mInstanceId, VT_ROLE_PRODUCER); // connect to tunnel id, role: 0: producer, 1: consumer, gst and dtv need set to 0 } if (mDisplayAdapterLib && mDisplayAdapterLib->dACreateLayer) { ret = mDisplayAdapterLib->dACreateLayer(mDisplayId, &mLayerId); } mIsVideoTunnelConnected = true; if (mDisplayAdapterLib && mDisplayAdapterLib->dASetTunnelId) { mDisplayAdapterLib->dASetTunnelId(mDisplayId, mLayerId, mInstanceId); } if (mDisplayAdapterLib && mDisplayAdapterLib->dARegisterCallback) { mDisplayAdapterLib->dARegisterCallback(mDisplayId, mLayerId, this, &buffer_cb); } INFO(mLogCategory,"vt fd:%d, instance id:%d, layer id:%d, display id:%d",mFd, mInstanceId, mLayerId, mDisplayId); DEBUG(mLogCategory,"out"); return true; } bool MesonVideoCompositorImpl::disconnect() { mRequestStop = true; DEBUG(mLogCategory,"in"); if (isRunning()) { if (mPoll) { mPoll->setFlushing(true); } requestExitAndWait(); mStarted = false; } if (mKeepLastFrame) { //set mKeepLastFrame to compositor if (mVideotunnelLib && mVideotunnelLib->vtSendCmd) { mVideotunnelLib->vtSendCmd(mFd, mInstanceId, VT_CMD_SET_VIDEO_STATUS, VT_VIDEO_STATUS_HOLD_FRAME); } } if (mFd >= 0) { if (mIsVideoTunnelConnected) { INFO(mLogCategory,"instance id:%d, layer id:%d, display id:%d",mInstanceId, mLayerId, mDisplayId); if (mLayerId >= 0 && mDisplayAdapterLib && mDisplayAdapterLib->dADestroyLayer) { mDisplayAdapterLib->dADestroyLayer(mDisplayId, mLayerId); } if (mInstanceId >= 0 && mVideotunnelLib && mVideotunnelLib->vtDisconnect) { mVideotunnelLib->vtDisconnect(mFd, mInstanceId, VT_ROLE_PRODUCER); } mIsVideoTunnelConnected = false; } if (mInstanceId >= 0 && mVideotunnelLib && mVideotunnelLib->vtFreeId) { INFO(mLogCategory,"free instance id:%d",mInstanceId); mVideotunnelLib->vtFreeId(mFd, mInstanceId); } INFO(mLogCategory,"close vt fd:%d",mFd); if (mVideotunnelLib && mVideotunnelLib->vtClose) { mVideotunnelLib->vtClose(mFd); } mFd = -1; } //flush all buffer those do not displayed DEBUG(mLogCategory,"release all posted to videotunnel buffers"); ATools::Mutex::Autolock _l(mMutex); for (auto item = mQueueRenderBufferMap.begin(); item != mQueueRenderBufferMap.end(); ) { RenderBuffer *renderbuffer = (RenderBuffer*) item->second; mQueueRenderBufferMap.erase(item++); renderbuffer->reserved[1] = FRAME_DROP_DUE_FLUSH; mPlugin->handleFrameDropped(renderbuffer); mPlugin->handleBufferRelease(renderbuffer); } if (mPoll) { delete mPoll; mPoll = NULL; } mInstanceId = -1; mIsVideoTunnelConnected = false; mQueueFrameCnt = 0; mTotalQueueFrameCnt = 0; mStarted = false; mRequestStop = false; mVideotunnelLib = NULL; mDisplayAdapterLib = NULL; mWindowChanged = false; mDisplayId = DEFAULT_DISPLAY_ID; // Used to specify on which screen to display, the default setting is 0 mLayerId = -1; // Obtained from the compositor to distinguish video path mFrameWidth = 1920; mFrameHeight = 1080; mPip = 0; mZorder = 0; mImmediately = false; mHideVideo = false; mKeepLastFrame = true; mUnderFlowDetect = false; mMesonDisplayLib = NULL; mPoll = NULL; mWindow.x = 0; mWindow.y = 0; mWindow.w = 0; mWindow.h = 0; mResolutionWidth = 0; mResolutionHeight = 0; mPixelAspectRatio = 1.0; mDisplayedFrameWidth = 0; mDisplayedFrameHeight = 0; mLastDisplayFramePts = -1; mLastDisplayFrameTimeUs = -1; mFrameRateFractionNum = 0; mFrameRateFractionDenom = 1; mFrameDurationUs = 0; DEBUG(mLogCategory,"out"); return true; } bool MesonVideoCompositorImpl::displayFrame(RenderBuffer *buf, int64_t displayTime) { int ret = 0; if (mStarted == false) { DEBUG(mLogCategory,"to run VideoTunnelImpl"); run("VideoTunnelImpl"); mStarted = true; mSignalFirstFrameDiplayed = true; } if ((mDisplayedFrameWidth == 0 || mDisplayedFrameHeight == 0) && buf->dma.width > 0) { mDisplayedFrameWidth = buf->dma.width; mDisplayedFrameHeight = buf->dma.height; mWindowChanged = true; } if (buf->dma.width != mDisplayedFrameWidth || buf->dma.height != mDisplayedFrameHeight) { mWindowChanged = true; } if (mWindowChanged) { struct rect src_win; struct rect dst = {mWindow.x, mWindow.y, mWindow.w, mWindow.h}; src_win.x = 0; src_win.y = 0; src_win.w = mFrameWidth; src_win.h = mFrameHeight; /*if not set zorder, we set zorder to pip index*/ if (mZorder == 0 && mPip > 0) { mZorder = mPip; } if (mDisplayAdapterLib && mDisplayAdapterLib->dASetLayerSourceCrop && mDisplayAdapterLib->dASetLayerDispFrame && mDisplayAdapterLib->dASetLayerZorder) { mDisplayAdapterLib->dASetLayerSourceCrop(mDisplayId, mLayerId, src_win); mDisplayAdapterLib->dASetLayerDispFrame(mDisplayId, mLayerId, dst); mDisplayAdapterLib->dASetLayerZorder(mDisplayId, mLayerId, mZorder); } mWindowChanged = false; DEBUG(mLogCategory, "win src:x:%d, y:%d, w:%d, h:%d,dst:x:%d,y:%d,w:%d,h:%d", src_win.x, src_win.y, src_win.w, src_win.h, mWindow.x, mWindow.y, mWindow.w, mWindow.h); } int fd0 = buf->dma.fd[0]; if (mVideotunnelLib && mVideotunnelLib->vtQueueBuffer) { ret = mVideotunnelLib->vtQueueBuffer(mFd, mInstanceId, fd0, -1 /*fence_fd*/, displayTime); if (mUnderFlowDetect) { mUnderFlowDetect = false; } } ATools::Mutex::Autolock _l(mMutex); std::pair item(fd0, buf); mQueueRenderBufferMap.insert(item); ++mQueueFrameCnt; ++mTotalQueueFrameCnt; TRACE(mLogCategory,"***fd:%d,w:%d,h:%d,displaytime:%lld,inVideoTunnelCnt:%d,totalCommitCnt:%d", buf->dma.fd[0],buf->dma.width,buf->dma.height,displayTime,mQueueFrameCnt,mTotalQueueFrameCnt); mPlugin->handleFrameDisplayed(buf); mLastDisplayTime = ATools::Times::getSystemTimeMs(); return true; } void MesonVideoCompositorImpl::flush() { DEBUG(mLogCategory,"flush"); ATools::Mutex::Autolock _l(mMutex); if (mVideotunnelLib && mVideotunnelLib->vtCancelBuffer) { mVideotunnelLib->vtCancelBuffer(mFd, mInstanceId); } for (auto item = mQueueRenderBufferMap.begin(); item != mQueueRenderBufferMap.end(); ) { RenderBuffer *renderbuffer = (RenderBuffer*) item->second; mQueueRenderBufferMap.erase(item++); renderbuffer->reserved[1] = FRAME_DROP_DUE_FLUSH; mPlugin->handleFrameDropped(renderbuffer); mPlugin->handleBufferRelease(renderbuffer); } mQueueFrameCnt = 0; mTotalQueueFrameCnt=0; DEBUG(mLogCategory,"flush done"); } void MesonVideoCompositorImpl::setFrameSize(int width, int height) { if (width > 0 && height > 0 && (width != mFrameWidth || height != mFrameHeight)) { mFrameWidth = width; mFrameHeight = height; } DEBUG(mLogCategory, "set frame size, width:%d, height:%d",mFrameWidth,mFrameHeight); } void MesonVideoCompositorImpl::setWindow(RenderRect win) { if (win.x == 0 && win.y == 0 && win.w == 0 && win.h == 0) { WARNING(mLogCategory, "set full screen? %dx%dx%dx%d",win.x,win.y,win.w,win.h); mWindow.x = 0; mWindow.y = 0; mWindow.w = mResolutionWidth; mWindow.h = mResolutionHeight; mWindowChanged = true; } else if (win.x != mWindow.x || win.y != mWindow.y || win.w != mWindow.w || win.h != mWindow.h) { mWindow.x = win.x; mWindow.y = win.y; mWindow.w = win.w; mWindow.h = win.h; mWindowChanged = true; } DEBUG(mLogCategory, "set window size, x:%d, y:%d, w:%d, h:%d",mWindow.x, mWindow.y, mWindow.w, mWindow.h); } void MesonVideoCompositorImpl::setImmediatelyOutout(bool immediately) { mImmediately = immediately; DEBUG(mLogCategory,"set immediate output: %d", mImmediately); } void MesonVideoCompositorImpl::setHideVideo(bool hide) { mHideVideo = hide; //set mHideVideo to compositor if (mVideotunnelLib && mVideotunnelLib->vtSendCmd) { if (mHideVideo) { mVideotunnelLib->vtSendCmd(mFd, mInstanceId, VT_CMD_SET_VIDEO_STATUS, VT_VIDEO_STATUS_HIDE); } else { mVideotunnelLib->vtSendCmd(mFd, mInstanceId, VT_CMD_SET_VIDEO_STATUS, VT_VIDEO_STATUS_SHOW); } } DEBUG(mLogCategory,"set hide video: %d", mHideVideo); } void MesonVideoCompositorImpl::setKeepLastFrame(bool keep) { mKeepLastFrame = keep; DEBUG(mLogCategory,"set keep last frame: %d", mKeepLastFrame); } void MesonVideoCompositorImpl::setPip(int pip) { mPip = pip; DEBUG(mLogCategory,"set pip: %d", mPip); } void MesonVideoCompositorImpl::setFrameRate(int frameRateNum, int frameRateDenom) { mFrameRateFractionNum = frameRateNum; mFrameRateFractionDenom = frameRateDenom; if (mFrameRateFractionDenom == 0) { mFrameRateFractionDenom = 1; } if (mFrameRateFractionNum > 0) { mFrameDurationUs = 1000000 * (int64_t)mFrameRateFractionDenom/(int64_t)mFrameRateFractionNum; } INFO(mLogCategory,"num:%d,denom:%d,fps:%d/100,frame dur:%lld us", mFrameRateFractionNum, mFrameRateFractionDenom,mFrameRateFractionNum*100/mFrameRateFractionDenom,mFrameDurationUs); } void MesonVideoCompositorImpl::waitFence(int fence) { if (fence > 0) { mPoll->addFd(fence); mPoll->setFdReadable(fence, true); for ( ; ; ) { int rc = mPoll->wait(3000); //3 sec if ((rc == -1) && ((errno == EINTR) || (errno == EAGAIN))) { continue; } else if (rc <= 0) { if (rc == 0) errno = ETIME; } break; } mPoll->removeFd(fence); close(fence); fence = -1; } } void MesonVideoCompositorImpl::bufferDropped(void *context, int32_t fd) { MesonVideoCompositorImpl* impl = static_cast (context); RenderBuffer *buffer = NULL; ATools::Mutex::Autolock _l(impl->mMutex); auto item = impl->mQueueRenderBufferMap.find(fd); if (item == impl->mQueueRenderBufferMap.end()) { ERROR(impl->mLogCategory,"Not found in mQueueRenderBufferMap buffer fd:%d",fd); return; } buffer = (RenderBuffer*) item->second; impl->mPlugin->handleFrameDropped(buffer); } void MesonVideoCompositorImpl::bufferDisplayed(void *context, int32_t fd, long long time) { MesonVideoCompositorImpl* impl = static_cast (context); RenderBuffer *buffer = NULL; int64_t frameDisplayTimeUs = time; ATools::Mutex::Autolock _l(impl->mMutex); auto item = impl->mQueueRenderBufferMap.find(fd); if (item == impl->mQueueRenderBufferMap.end()) { ERROR(impl->mLogCategory,"Not found in mQueueRenderBufferMap buffer fd:%d",fd); return; } buffer = (RenderBuffer*) item->second; //send first frame displayed msg if (impl->mSignalFirstFrameDiplayed) { impl->mSignalFirstFrameDiplayed = false; INFO(impl->mLogCategory,"send first frame displayed msg"); impl->mPlugin->handleMsgNotify(MSG_FIRST_FRAME,(void*)&buffer->pts); } impl->mPlugin->handleFrameDisplayed(buffer); // calc display duration if (impl->mLastDisplayFramePts == -1) { impl->mLastDisplayFramePts = buffer->pts; impl->mLastDisplayFrameTimeUs = frameDisplayTimeUs; } else { //calculate last frame displayed duration int64_t duration = frameDisplayTimeUs - impl->mLastDisplayFrameTimeUs; TRACE(impl->mLogCategory,"now pts:%lld us,last dur:%lld us,time:%u us", buffer->pts/1000,duration,frameDisplayTimeUs); if (impl->mFrameDurationUs > 0 && duration > FRAME_FREEZE_THRESHOLD(impl->mFrameDurationUs)) { FrameDisplayInfo frameDisplayInfo = {-1, -1}; frameDisplayInfo.pts = impl->mLastDisplayFramePts; frameDisplayInfo.duration = duration; INFO(impl->mLogCategory,"reportFreezeFrame,pts:%lld us,dur:%lld us",frameDisplayInfo.pts/1000,frameDisplayInfo.duration); impl->mPlugin->handleMsgNotify(MSG_FRAME_FREEZE, &frameDisplayInfo); } impl->mLastDisplayFramePts = buffer->pts; impl->mLastDisplayFrameTimeUs = frameDisplayTimeUs; } } void MesonVideoCompositorImpl::readyToRun() { struct vt_rect rect; if (mFrameWidth > 0 && mFrameHeight > 0) { rect.left = 0; rect.top = 0; rect.right = mFrameWidth; rect.bottom = mFrameHeight; if (mVideotunnelLib && mVideotunnelLib->vtSetSourceCrop) { mVideotunnelLib->vtSetSourceCrop(mFd, mInstanceId, rect); } } } bool MesonVideoCompositorImpl::threadLoop() { int ret = 0; int bufferId = -1; int fenceId = -1; RenderBuffer *buffer = NULL; if (mRequestStop) { DEBUG(mLogCategory,"request stop"); return false; } //if last frame expired,and not send frame to compositor for under flow expired time //we need to send under flow msg if (mQueueFrameCnt < 1 && ATools::Times::getSystemTimeMs() - mLastDisplayTime > UNDER_FLOW_EXPIRED_TIME_MS && !mUnderFlowDetect) { mUnderFlowDetect = true; mPlugin->handleMsgNotify(MSG_UNDER_FLOW, NULL); } if (mVideotunnelLib && mVideotunnelLib->vtDequeueBuffer) { ret = mVideotunnelLib->vtDequeueBuffer(mFd, mInstanceId, &bufferId, &fenceId); } /*return -EAGAIN if videotunnel can't dequeue buffer*/ if (ret != 0) { if (mRequestStop) { DEBUG(mLogCategory,"request stop"); return false; } usleep(5000); return true; } if (mRequestStop) { DEBUG(mLogCategory,"request stop"); return false; } //invalid fenceId and invalid bufferId if (fenceId < 0 && bufferId < 0) { usleep(5000); return true; } //send uvm fd to driver after getted fence waitFence(fenceId); { ATools::Mutex::Autolock _l(mMutex); auto item = mQueueRenderBufferMap.find(bufferId); if (item == mQueueRenderBufferMap.end()) { ERROR(mLogCategory,"Not found in mQueueRenderBufferMap bufferId:%d",bufferId); usleep(5000); return true; } // try locking when removing item from mQueueRenderBufferMap buffer = (RenderBuffer*) item->second; mQueueRenderBufferMap.erase(bufferId); --mQueueFrameCnt; TRACE(mLogCategory,"***dq buffer fd:%d,commitCnt:%d",bufferId,mQueueFrameCnt); } mPlugin->handleBufferRelease(buffer); return true; } void MesonVideoCompositorImpl::setDisplayOutput(int output) { /*output 0 index map display id 0, output 1 index map display id 1 it means 0 for first screen display output, 1 for second screen display output*/ if (output == 0) { mDisplayId = DEFAULT_DISPLAY_ID; } else if (output == 1) { mDisplayId = SECOND_DISPLAY_ID; } INFO(mLogCategory,"select output: %d, id:%d", output, mDisplayId); } void MesonVideoCompositorImpl::setZorder(int zorder) { mZorder = zorder; INFO(mLogCategory,"set pip %d to zorder:%d", mPip, mZorder); } #if 0 void MesonVideoCompositorImpl::videoCenterRect(struct rect src, struct rect dst, struct rect *result) { double src_ratio = 1.0, dst_ratio = 1.0; src_ratio = (double) (src.w * mPixelAspectRatio) / src.h; dst_ratio = (double) dst.w / dst.h; if (src_ratio > dst_ratio) { result->w = dst.w; result->h = dst.w / src_ratio; result->x = dst.x; result->y = dst.y + (dst.h - result->h) / 2; } else if (src_ratio < dst_ratio) { result->w = dst.h * src_ratio; result->h = dst.h; result->x = dst.x + (dst.w - result->w) / 2; result->y = dst.y; } else { result->x = dst.x; result->y = dst.y; result->w = dst.w; result->h = dst.h; } DEBUG(mLogCategory,"source is %dx%d dest is %dx%d, result is %d,%d,%d,%d", src.w, src.h, dst.w, dst.h, result->x, result->y,result->w, result->h); } void MesonVideoCompositorImpl::setPixelAspectRatio(double ratio) { //TRACE(mLogCategory,"pixel aspect ratio:%f",ratio); if (fabs(mPixelAspectRatio - ratio) > 0.001) { mWindowChanged = true; } mPixelAspectRatio = ratio; } #endif