diff --git a/common/include/VkVideoCore/VkVideoCoreProfile.h b/common/include/VkVideoCore/VkVideoCoreProfile.h index 611ccac4..e9537f5f 100644 --- a/common/include/VkVideoCore/VkVideoCoreProfile.h +++ b/common/include/VkVideoCore/VkVideoCoreProfile.h @@ -193,7 +193,12 @@ class VkVideoCoreProfile VkVideoComponentBitDepthFlagsKHR lumaBitDepth = VK_VIDEO_COMPONENT_BIT_DEPTH_INVALID_KHR, VkVideoComponentBitDepthFlagsKHR chromaBitDepth = VK_VIDEO_COMPONENT_BIT_DEPTH_INVALID_KHR, uint32_t videoH26xProfileIdc = 0, - VkVideoEncodeTuningModeKHR tuningMode = VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR) +#if (_TRANSCODING) + VkVideoEncodeUsageInfoKHR InEncodeUsage = {} +#else + VkVideoEncodeTuningModeKHR tuningMode = VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR +#endif // _TRANSCODING + ) : m_profile({VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR, NULL, videoCodecOperation, chromaSubsampling, lumaBitDepth, chromaBitDepth}), m_profileList({VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR, NULL, 1, &m_profile}) @@ -210,8 +215,12 @@ class VkVideoCoreProfile VkVideoEncodeAV1ProfileInfoKHR encodeAV1ProfilesRequest; VkBaseInStructure* pVideoProfileExt = NULL; +#if (_TRANSCODING) + VkVideoEncodeUsageInfoKHR encodeUsageInfoRequest = InEncodeUsage; +#else VkVideoEncodeUsageInfoKHR encodeUsageInfoRequest{VK_STRUCTURE_TYPE_VIDEO_ENCODE_USAGE_INFO_KHR, NULL, 0, 0, tuningMode}; +#endif // _TRANSCODING VkBaseInStructure* pEncodeUsageInfo = (VkBaseInStructure*)&encodeUsageInfoRequest; if (videoCodecOperation == VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR) { diff --git a/common/libs/VkCodecUtils/FrameProcessor.h b/common/libs/VkCodecUtils/FrameProcessor.h index ceeee654..ddd42f53 100644 --- a/common/libs/VkCodecUtils/FrameProcessor.h +++ b/common/libs/VkCodecUtils/FrameProcessor.h @@ -25,6 +25,12 @@ #include "VkCodecUtils/VkVideoRefCountBase.h" #include "VkCodecUtils/ProgramConfig.h" +#if (_TRANSCODING) +#include "VkCodecUtils/VulkanDecodedFrame.h" +#include "VkCodecUtils/VulkanEncoderFrameProcessor.h" +#include "VkCodecUtils/ProgramConfig.h" +#include "VkVideoEncoder/VkEncoderConfig.h" +#endif // _TRANSCODING class Shell; @@ -65,6 +71,16 @@ class FrameProcessor : public VkVideoRefCountBase { const VkSemaphore* pWaitSemaphores = nullptr, uint32_t signalSemaphoreCount = 0, const VkSemaphore* pSignalSemaphores = nullptr) = 0; +#if (_TRANSCODING) + virtual bool OnFrameTranscoding( int32_t renderIndex, + ProgramConfig* programConfig, + VkSharedBaseObj& encoderConfig, + uint32_t waitSemaphoreCount = 0, + const VkSemaphore* pWaitSemaphores = nullptr, + uint32_t signalSemaphoreCount = 0, + const VkSemaphore* pSignalSemaphores = nullptr, + VulkanDecodedFrame* pLastFrameDecoded = nullptr) = 0; +#endif // _TRANSCODING uint64_t GetTimeDiffNanoseconds(bool updateStartTime = true) { diff --git a/common/libs/VkCodecUtils/HelpersDispatchTable.cpp b/common/libs/VkCodecUtils/HelpersDispatchTable.cpp index 3d678118..e482731c 100644 --- a/common/libs/VkCodecUtils/HelpersDispatchTable.cpp +++ b/common/libs/VkCodecUtils/HelpersDispatchTable.cpp @@ -98,6 +98,9 @@ void InitDispatchTableMiddle(VkInstance instance, bool include_bottom, VkInterfa #ifdef VK_USE_VIDEO_QUEUE pVkFunctions->GetPhysicalDeviceVideoCapabilitiesKHR = reinterpret_cast(getInstanceProcAddrFunc(instance, "vkGetPhysicalDeviceVideoCapabilitiesKHR")); #endif +#ifdef VK_USE_VIDEO_QUEUE + pVkFunctions->GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR = reinterpret_cast(getInstanceProcAddrFunc(instance, "vkGetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR")); +#endif if (!include_bottom) return; diff --git a/common/libs/VkCodecUtils/HelpersDispatchTable.h b/common/libs/VkCodecUtils/HelpersDispatchTable.h index 171b8315..0525a0c5 100644 --- a/common/libs/VkCodecUtils/HelpersDispatchTable.h +++ b/common/libs/VkCodecUtils/HelpersDispatchTable.h @@ -277,6 +277,7 @@ struct VkInterfaceFunctions { // VK_KHR_video_queue PFN_vkGetPhysicalDeviceVideoFormatPropertiesKHR GetPhysicalDeviceVideoFormatPropertiesKHR; PFN_vkGetPhysicalDeviceVideoCapabilitiesKHR GetPhysicalDeviceVideoCapabilitiesKHR; + PFN_vkGetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR; PFN_vkCreateVideoSessionKHR CreateVideoSessionKHR; PFN_vkDestroyVideoSessionKHR DestroyVideoSessionKHR; PFN_vkCreateVideoSessionParametersKHR CreateVideoSessionParametersKHR; diff --git a/common/libs/VkCodecUtils/ProgramConfig.h b/common/libs/VkCodecUtils/ProgramConfig.h index 04e0e573..b9626959 100644 --- a/common/libs/VkCodecUtils/ProgramConfig.h +++ b/common/libs/VkCodecUtils/ProgramConfig.h @@ -109,7 +109,9 @@ struct ProgramConfig { {"--help", nullptr, 0, "Show this help", [argv](const char **, const ProgramArgs &a) { int rtn = showHelp(argv, a); +#if (!_TRANSCODING) // transcoding: should print encode info as well exit(EXIT_SUCCESS); +#endif // !_TRANSCODING return rtn; }}, {"--enableStrDemux", nullptr, 0, "Enable stream demuxing", @@ -122,7 +124,12 @@ struct ProgramConfig { enableStreamDemuxing = false; return true; }}, - {"--codec", nullptr, 1, "Codec to decode", + { +#if (!_TRANSCODING) // transcoding: to prevent overlap with encoder's codec option + "--codec", nullptr, 1, "Codec to decode", +#else + "--codec-input", nullptr, 1, "Codec to decode", +#endif // !_TRANSCODING [this](const char **args, const ProgramArgs &a) { if ((strcmp(args[0], "hevc") == 0) || (strcmp(args[0], "h265") == 0)) { @@ -330,10 +337,15 @@ struct ProgramConfig { (a.short_flag != nullptr && strcmp(argv[i], a.short_flag) == 0); }); if (flag == spec.end()) { +#if (!_TRANSCODING) // transcoding: should parse encode info after decode info as well std::cerr << "Unknown argument \"" << argv[i] << "\"" << std::endl; std::cout << std::endl; + continue; showHelp(argv, spec); exit(EXIT_FAILURE); +#else + continue; +#endif // !_TRANSCODING } if (i + flag->numArgs >= argc) { diff --git a/common/libs/VkCodecUtils/VkImageResource.h b/common/libs/VkCodecUtils/VkImageResource.h index 314a2f01..365a6c15 100644 --- a/common/libs/VkCodecUtils/VkImageResource.h +++ b/common/libs/VkCodecUtils/VkImageResource.h @@ -141,7 +141,7 @@ class VkImageResourceView : public VkVideoRefCountBase } operator VkImageView() const { return m_imageViews[0]; } - VkImageView GetImageView() const { return m_imageViews[0]; } + VkImageView GetImageView(int i = 0) const { return m_imageViews[i]; } uint32_t GetNumberOfPlanes() const { return m_numPlanes; } VkImageView GetPlaneImageView(uint32_t planeIndex = 0) const { assert(planeIndex < m_numPlanes); return m_imageViews[planeIndex + 1]; } VkDevice GetDevice() const { return *m_vkDevCtx; } diff --git a/common/libs/VkCodecUtils/VkVideoQueue.h b/common/libs/VkCodecUtils/VkVideoQueue.h index d374d40a..4b7ed7eb 100644 --- a/common/libs/VkCodecUtils/VkVideoQueue.h +++ b/common/libs/VkCodecUtils/VkVideoQueue.h @@ -18,6 +18,10 @@ #define _VKCODECUTILS_VKVIDEOQUEUE_H_ #include "VkCodecUtils/VkVideoRefCountBase.h" +#if (_TRANSCODING) +#include "VkCodecUtils/ProgramConfig.h" +#include "VkVideoEncoder/VkEncoderConfig.h" +#endif // _TRANSCODING template class VkVideoQueue : public VkVideoRefCountBase { @@ -30,7 +34,11 @@ class VkVideoQueue : public VkVideoRefCountBase { virtual VkFormat GetFrameImageFormat(int32_t* pWidth = nullptr, int32_t* pHeight = nullptr, int32_t* pBitDepth = nullptr) const = 0; - virtual int32_t GetNextFrame(FrameDataType* pFrame, bool* endOfStream) = 0; + virtual int32_t GetNextFrame(FrameDataType* pFrame, bool* endOfStream +#if (_TRANSCODING) + , ProgramConfig* programConfig = nullptr, VkSharedBaseObj* encoderConfig = nullptr +#endif // _TRANSCODING + ) = 0; virtual int32_t ReleaseFrame(FrameDataType* pDisplayedFrame) = 0; public: virtual ~VkVideoQueue() {}; diff --git a/common/libs/VkCodecUtils/VulkanFrame.cpp b/common/libs/VkCodecUtils/VulkanFrame.cpp index 6b0c568b..3e1dab70 100644 --- a/common/libs/VkCodecUtils/VulkanFrame.cpp +++ b/common/libs/VkCodecUtils/VulkanFrame.cpp @@ -26,6 +26,9 @@ #include "VulkanFrame.h" #include "vk_enum_string_helper.h" #include "VkVideoCore/DecodeFrameBufferIf.h" +#if (_TRANSCODING) +#include "VkVideoEncoder/VkVideoEncoder.h" +#endif // _TRANSCODING template VulkanFrame::VulkanFrame(const VulkanDeviceContext* vkDevCtx, @@ -270,6 +273,37 @@ bool VulkanFrame::OnKey(Key key) return true; } +#if (_TRANSCODING) +static int InitPreset(VkSharedBaseObj& encoderConfig) +{ + if (encoderConfig->codec == VkVideoCodecOperationFlagBitsKHR::VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR) encoderConfig->qualityLevel = 0; + + if (encoderConfig->encodingProfile == EncoderConfig::LOW_LATENCY_STREAMING) + { + encoderConfig->rateControlMode = VkVideoEncodeRateControlModeFlagBitsKHR::VK_VIDEO_ENCODE_RATE_CONTROL_MODE_CBR_BIT_KHR; + encoderConfig->gopStructure.SetConsecutiveBFrameCount(0); + encoderConfig->gopStructure.SetGopFrameCount(-1); + encoderConfig->gopStructure.SetIdrPeriod(0); + encoderConfig->gopStructure.SetLastFrameType(VkVideoGopStructure::FrameType::FRAME_TYPE_P); // ? set right value + encoderConfig->encodeUsageHints = VK_VIDEO_ENCODE_USAGE_STREAMING_BIT_KHR; + encoderConfig->encodeContentHints = VK_VIDEO_ENCODE_CONTENT_RENDERED_BIT_KHR; + // encoderConfig->gopStructure.SetTemporalLayerCount(3); + } + else if (encoderConfig->encodingProfile == EncoderConfig::ARCHIVING) + { + encoderConfig->rateControlMode = VkVideoEncodeRateControlModeFlagBitsKHR::VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR; + encoderConfig->maxBitrate = (uint32_t)(1.2f * encoderConfig->averageBitrate); // This is used in Variable Bit Rate (VBR) mode and is ignored for Constant Bit Rate (CBR) mode. + encoderConfig->gopStructure.SetConsecutiveBFrameCount(3); + encoderConfig->gopStructure.SetIdrPeriod(MAX_GOP_SIZE); + encoderConfig->gopStructure.SetGopFrameCount(MAX_GOP_SIZE); + encoderConfig->gopStructure.SetTemporalLayerCount(1); + encoderConfig->encodeUsageHints = VK_VIDEO_ENCODE_USAGE_RECORDING_BIT_KHR; + encoderConfig->encodeContentHints = VK_VIDEO_ENCODE_CONTENT_RENDERED_BIT_KHR; + }; + return 0; +} +#endif // _TRANSCODING + template bool VulkanFrame::OnFrame( int32_t renderIndex, uint32_t waitSemaphoreCount, @@ -394,6 +428,143 @@ bool VulkanFrame::OnFrame( int32_t renderIndex, pSignalSemaphores, pLastDecodedFrame); + // VkImageLastDecodedFrame() + + if (VK_SUCCESS != result) { + return false; + } + + return continueLoop; +} + +#if (_TRANSCODING) +template +bool VulkanFrame::OnFrameTranscoding( int32_t renderIndex, + ProgramConfig* programConfig, + VkSharedBaseObj& encoderConfig, + uint32_t waitSemaphoreCount, + const VkSemaphore* pWaitSemaphores, + uint32_t signalSemaphoreCount, + const VkSemaphore* pSignalSemaphores, + VulkanDecodedFrame* pLastDecodedFrameRet) +{ + // must not be used by encoder + int isFunctionUsed = 0; + isFunctionUsed++; + assert(isFunctionUsed == 0);//, "must not be used by encoder"); + return false; +} + +template<> +bool VulkanFrame::OnFrameTranscoding( int32_t renderIndex, + ProgramConfig* programConfig, + VkSharedBaseObj& encoderConfig, + uint32_t waitSemaphoreCount, + const VkSemaphore* pWaitSemaphores, + uint32_t signalSemaphoreCount, + const VkSemaphore* pSignalSemaphores, + VulkanDecodedFrame* pLastDecodedFrameRet) +{ + InitPreset(encoderConfig); + bool continueLoop = true; + const bool dumpDebug = false; + const bool trainFrame = (renderIndex < 0); + const bool gfxRendererIsEnabled = (m_videoRenderer != nullptr); + m_frameCount++; + + if (dumpDebug == false) { + bool displayTimeNow = false; + float fps = GetFrameRateFps(displayTimeNow); + if (displayTimeNow) { + std::cout << "\t\tFrame " << m_frameCount << ", FPS: " << fps << std::endl; + } + } else { + uint64_t timeDiffNanoSec = GetTimeDiffNanoseconds(); + std::cout << "\t\t Time nanoseconds: " << timeDiffNanoSec << + " milliseconds: " << timeDiffNanoSec / 1000 << + " rate: " << 1000000000.0 / timeDiffNanoSec << std::endl; + } + + VulkanDecodedFrame* pLastDecodedFrame = nullptr; + // FrameDataType* pLastDecodedFrame = nullptr; + + if (m_videoQueue->IsValid() && !trainFrame) { + + pLastDecodedFrame = &m_frameData[m_frameDataIndex]; + + // Graphics and present stages are not enabled. + // Make sure the frame complete query or fence are signaled (video frame is processed) before returning the frame. + if (false && (gfxRendererIsEnabled == false) && (pLastDecodedFrame != nullptr)) { + + if (pLastDecodedFrame->queryPool != VK_NULL_HANDLE) { + auto startTime = std::chrono::steady_clock::now(); + VkQueryResultStatusKHR decodeStatus; + VkResult result = m_vkDevCtx->GetQueryPoolResults(*m_vkDevCtx, + pLastDecodedFrame->queryPool, + pLastDecodedFrame->startQueryId, + 1, + sizeof(decodeStatus), + &decodeStatus, + sizeof(decodeStatus), + VK_QUERY_RESULT_WITH_STATUS_BIT_KHR | VK_QUERY_RESULT_WAIT_BIT); + + assert(result == VK_SUCCESS); + assert(decodeStatus == VK_QUERY_RESULT_STATUS_COMPLETE_KHR); + auto deltaTime = std::chrono::steady_clock::now() - startTime; + auto diffMilliseconds = std::chrono::duration_cast(deltaTime); + auto diffMicroseconds = std::chrono::duration_cast(deltaTime); + if (dumpDebug) { + std::cout << pLastDecodedFrame->pictureIndex << ": frameWaitTime: " << + diffMilliseconds.count() << "." << diffMicroseconds.count() << " mSec" << std::endl; + } + } else if (pLastDecodedFrame->frameCompleteFence != VkFence()) { + VkResult result = m_vkDevCtx->WaitForFences(*m_vkDevCtx, 1, &pLastDecodedFrame->frameCompleteFence, true, 100 * 1000 * 1000 /* 100 mSec */); + assert(result == VK_SUCCESS); + if (result != VK_SUCCESS) { + fprintf(stderr, "\nERROR: WaitForFences() result: 0x%x\n", result); + } + result = m_vkDevCtx->GetFenceStatus(*m_vkDevCtx, pLastDecodedFrame->frameCompleteFence); + assert(result == VK_SUCCESS); + if (result != VK_SUCCESS) { + fprintf(stderr, "\nERROR: GetFenceStatus() result: 0x%x\n", result); + } + } + } + + m_videoQueue->ReleaseFrame(pLastDecodedFrame); + + pLastDecodedFrame->Reset(); + + bool endOfStream = false; + int32_t numVideoFrames = 0; + + if (pLastDecodedFrame) { + numVideoFrames = m_videoQueue->GetNextFrame(pLastDecodedFrame, &endOfStream, programConfig, &encoderConfig); + } + if (endOfStream && (numVideoFrames < 0)) { + continueLoop = false; + bool displayTimeNow = true; + float fps = GetFrameRateFps(displayTimeNow); + if (displayTimeNow) { + std::cout << "\t\tFrame " << m_frameCount << ", FPS: " << fps << std::endl; + } + } + } + + if (gfxRendererIsEnabled == false) { + m_frameDataIndex = (m_frameDataIndex + 1) % m_frameData.size(); + return continueLoop; + } + + VkResult result = DrawFrame(renderIndex, + waitSemaphoreCount, + pWaitSemaphores, + signalSemaphoreCount, + pSignalSemaphores, + pLastDecodedFrame); + + // VkImageLastDecodedFrame() + if (VK_SUCCESS != result) { return false; @@ -401,6 +572,7 @@ bool VulkanFrame::OnFrame( int32_t renderIndex, return continueLoop; } +#endif // _TRANSCODING template VkResult VulkanFrame::DrawFrame( int32_t renderIndex, diff --git a/common/libs/VkCodecUtils/VulkanFrame.h b/common/libs/VkCodecUtils/VulkanFrame.h index 9a671103..21bd2ee3 100644 --- a/common/libs/VkCodecUtils/VulkanFrame.h +++ b/common/libs/VkCodecUtils/VulkanFrame.h @@ -63,6 +63,16 @@ class VulkanFrame : public FrameProcessor { uint32_t signalSemaphoreCount = 0, const VkSemaphore* pSignalSemaphores = nullptr); +#if (_TRANSCODING) + virtual bool OnFrameTranscoding(int32_t renderIndex, + ProgramConfig* programConfig, + VkSharedBaseObj& encoderConfig, + uint32_t waitSemaphoreCount = 0, + const VkSemaphore* pWaitSemaphores = nullptr, + uint32_t signalSemaphoreCount = 0, + const VkSemaphore* pSignalSemaphores = nullptr, + VulkanDecodedFrame* pLastFrameDecoded = nullptr); +#endif // _TRANSCODING VkResult DrawFrame( int32_t renderIndex, uint32_t waitSemaphoreCount, diff --git a/common/libs/VkCodecUtils/VulkanVideoDisplayQueue.h b/common/libs/VkCodecUtils/VulkanVideoDisplayQueue.h index 144eacee..06eb8c44 100644 --- a/common/libs/VkCodecUtils/VulkanVideoDisplayQueue.h +++ b/common/libs/VkCodecUtils/VulkanVideoDisplayQueue.h @@ -22,6 +22,10 @@ #include #include "VkCodecUtils/VkVideoQueue.h" #include "VkCodecUtils/VkThreadSafeQueue.h" +#if (_TRANSCODING) +#include "VkCodecUtils/ProgramConfig.h" +#include "VkVideoEncoder/VkEncoderConfig.h" +#endif // _TRANSCODING template class VulkanVideoDisplayQueue : public VkVideoQueue { @@ -33,7 +37,11 @@ class VulkanVideoDisplayQueue : public VkVideoQueue { virtual int32_t GetBitDepth() const { return m_defaultBitDepth; } virtual VkFormat GetFrameImageFormat(int32_t* pWidth = NULL, int32_t* pHeight = NULL, int32_t* pBitDepth = NULL) const; - virtual int32_t GetNextFrame(FrameDataType* pFrame, bool* endOfStream); + virtual int32_t GetNextFrame(FrameDataType* pFrame, bool* endOfStream +#if (_TRANSCODING) + , ProgramConfig* programConfig, VkSharedBaseObj* encoderConfig +#endif // _TRANSCODING +); virtual int32_t ReleaseFrame(FrameDataType* pDisplayedFrame); static VkSharedBaseObj& invalidVulkanVideoDisplayQueue; @@ -173,7 +181,11 @@ int32_t VulkanVideoDisplayQueue::EnqueueFrame(FrameDataType* pFra } template -int32_t VulkanVideoDisplayQueue::GetNextFrame(FrameDataType* pFrame, bool* endOfStream) +int32_t VulkanVideoDisplayQueue::GetNextFrame(FrameDataType* pFrame, bool* endOfStream +#if (_TRANSCODING) + , ProgramConfig* programConfig, VkSharedBaseObj* encoderConfig +#endif // _TRANSCODING +) { if (m_exitQueueRequested) { m_queue.SetFlushAndExit(); diff --git a/common/libs/VkCodecUtils/VulkanVideoProcessor.cpp b/common/libs/VkCodecUtils/VulkanVideoProcessor.cpp index 5b3f60c7..faee07ce 100644 --- a/common/libs/VkCodecUtils/VulkanVideoProcessor.cpp +++ b/common/libs/VkCodecUtils/VulkanVideoProcessor.cpp @@ -109,16 +109,21 @@ int32_t VulkanVideoProcessor::Initialize(const VulkanDeviceContext* vkDevCtx, fprintf(stderr, "\nERROR: Create VulkanVideoFrameBuffer result: 0x%x\n", result); } + uint32_t enableDecoderFeatures = 0; + +#if (!_TRANSCODING) FILE* outFile = m_frameToFile.AttachFile(outputFileName); if ((outputFileName != nullptr) && (outFile == nullptr)) { fprintf( stderr, "Error opening the output file %s", outputFileName); return -1; } - uint32_t enableDecoderFeatures = 0; if (outFile != nullptr) { + // Transcoding does not need a linear output (but this line alone causes output's freeze on Ampere, fixed in VkVideoDecoder.cpp L246 .. + // .. by force setup if (!(videoCapabilities.flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR) to if(0) alongside enableDecoderFeatures |= VkVideoDecoder::ENABLE_LINEAR_OUTPUT; } +#endif // !_TRANSCODING if (enableHwLoadBalancing) { enableDecoderFeatures |= VkVideoDecoder::ENABLE_HW_LOAD_BALANCING; @@ -247,6 +252,29 @@ int32_t VulkanVideoProcessor::GetBitDepth() const return m_videoStreamDemuxer->GetBitDepth(); } +#if (_TRANSCODING) + +int32_t VulkanVideoProcessor::GetCodedHeight() const +{ + return m_vkVideoDecoder->GetVideoFormatInfo()->coded_height; +} + +int32_t VulkanVideoProcessor::GetCodedWidth() const +{ + return m_vkVideoDecoder->GetVideoFormatInfo()->coded_width; +} + +uint32_t VulkanVideoProcessor::GetFrameRate() const +{ + return (m_vkVideoDecoder->GetVideoFormatInfo()->frame_rate.denominator) | (m_vkVideoDecoder->GetVideoFormatInfo()->frame_rate.numerator << 14); +} + +uint32_t VulkanVideoProcessor::GetFramesCount() const +{ + return m_maxFrameCount; +} +#endif //_TRANSCODING + void VulkanVideoProcessor::Deinit() { @@ -635,7 +663,31 @@ int32_t VulkanVideoProcessor::ParserProcessNextDataChunk() return retValue; } -int32_t VulkanVideoProcessor::GetNextFrame(VulkanDecodedFrame* pFrame, bool* endOfStream) +#if (_TRANSCODING) +static void setGeneratedOutputFileName(ProgramConfig* programConfig, VkSharedBaseObj& encoderConfig) +{ + std::string filename = std::string{programConfig->videoFileName}; + size_t filePath = filename.rfind("/"); + if (filePath == std::string::npos) filePath = filename.rfind("\\"); + if (filePath == std::string::npos) filePath = 0; else filePath += 1; + filename = filename.substr(filePath, filename.size() - 1); + size_t fileExtension = filename.rfind("."); + filename = filename.substr(0, fileExtension); + encoderConfig->outputFileHandler.SetFileName((filename + + "_" + std::to_string(encoderConfig->encodeHeight) + "p" + + "_p" + std::to_string(encoderConfig->qualityLevel + 1) + + "_vbv" + std::to_string(encoderConfig->vbvbufratio) + "_" + + std::to_string(encoderConfig->averageBitrate>>20) + "Mbps" + + (encoderConfig->codec == VkVideoCodecOperationFlagBitsKHR::VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR ? ".264" : ".265") + ).c_str()); +} +#endif + +int32_t VulkanVideoProcessor::GetNextFrame(VulkanDecodedFrame* pFrame, bool* endOfStream +#if (_TRANSCODING) + , ProgramConfig* programConfig, VkSharedBaseObj* encoderConfig +#endif +) { // The below call to DequeueDecodedPicture allows returning the next frame without parsing of the stream. // Parsing is only done when there are no more frames in the queue. @@ -653,11 +705,31 @@ int32_t VulkanVideoProcessor::GetNextFrame(VulkanDecodedFrame* pFrame, bool* end if (m_videoFrameNum == 0) { DumpVideoFormat(m_vkVideoDecoder->GetVideoFormatInfo(), true); +#if (_TRANSCODING) + { + (*encoderConfig)->encodeWidth = (*encoderConfig)->encodeMaxWidth = GetWidth(); + (*encoderConfig)->encodeHeight = (*encoderConfig)->encodeMaxHeight = GetHeight(); + (*encoderConfig)->input.width = GetCodedWidth(); + (*encoderConfig)->input.height = GetCodedHeight(); + setGeneratedOutputFileName(programConfig, *encoderConfig); + uint32_t frameRate = GetFrameRate(); + (*encoderConfig)->numFrames = GetFramesCount(); + (*encoderConfig)->frameRateDenominator = frameRate & ((1 << 14) - 1); + (*encoderConfig)->frameRateNumerator = frameRate >> 14; + (*encoderConfig)->InitializeParameters(); + VkResult result = VkVideoEncoder::CreateVideoEncoder(m_vkDevCtx, *encoderConfig, m_vkVideoEncoder); + assert(result == VK_SUCCESS); + } +#endif //_TRANSCODING } +#if (!_TRANSCODING) if (m_frameToFile) { OutputFrameToFile(pFrame); } +#else + m_vkVideoEncoder->LoadNextFrameDecoded(pFrame); +#endif // !_TRANSCODING m_videoFrameNum++; } diff --git a/common/libs/VkCodecUtils/VulkanVideoProcessor.h b/common/libs/VkCodecUtils/VulkanVideoProcessor.h index ab046224..c3e29425 100644 --- a/common/libs/VkCodecUtils/VulkanVideoProcessor.h +++ b/common/libs/VkCodecUtils/VulkanVideoProcessor.h @@ -23,6 +23,10 @@ #include "VkCodecUtils/ProgramConfig.h" #include "VkCodecUtils/VkVideoQueue.h" +#if (_TRANSCODING) +#include "VkVideoEncoder/VkVideoEncoder.h" +#endif //_TRANSCODING + class VulkanVideoProcessor : public VkVideoQueue { public: @@ -30,8 +34,18 @@ class VulkanVideoProcessor : public VkVideoQueue { virtual int32_t GetWidth() const; virtual int32_t GetHeight() const; virtual int32_t GetBitDepth() const; +#if (_TRANSCODING) + virtual int32_t GetCodedWidth() const; + virtual int32_t GetCodedHeight() const; + virtual uint32_t GetFrameRate() const; + virtual uint32_t GetFramesCount() const; +#endif // _TRANSCODING virtual VkFormat GetFrameImageFormat(int32_t* pWidth = NULL, int32_t* pHeight = NULL, int32_t* pBitDepth = NULL) const; - virtual int32_t GetNextFrame(VulkanDecodedFrame* pFrame, bool* endOfStream); + virtual int32_t GetNextFrame(VulkanDecodedFrame* pFrame, bool* endOfStream +#if (_TRANSCODING) + , ProgramConfig* programConfig = nullptr, VkSharedBaseObj* encoderConfig = nullptr +#endif // _TRANSCODING +); virtual int32_t ReleaseFrame(VulkanDecodedFrame* pDisplayedFrame); static VkSharedBaseObj& invalidVulkanVideoProcessor; @@ -73,6 +87,9 @@ class VulkanVideoProcessor : public VkVideoQueue { m_videoStreamDemuxer() , m_vkVideoFrameBuffer() , m_vkVideoDecoder() +#if (_TRANSCODING) + , m_vkVideoEncoder() +#endif // _TRANSCODING , m_vkParser() , m_currentBitstreamOffset(0) , m_videoFrameNum(0) @@ -102,6 +119,13 @@ class VulkanVideoProcessor : public VkVideoQueue { bool StreamCompleted(); +#if (_TRANSCODING) +public: + VkSharedBaseObj getEncoder() const { + return m_vkVideoEncoder; + }; +#endif // _TRANSCODING + private: void WaitForFrameCompletion(VulkanDecodedFrame* pFrame, VkSharedBaseObj& imageResource); @@ -112,6 +136,9 @@ class VulkanVideoProcessor : public VkVideoQueue { VkSharedBaseObj m_videoStreamDemuxer; VkSharedBaseObj m_vkVideoFrameBuffer; VkSharedBaseObj m_vkVideoDecoder; +#if (_TRANSCODING) + VkSharedBaseObj m_vkVideoEncoder; +#endif // _TRANSCODING VkSharedBaseObj m_vkParser; int64_t m_currentBitstreamOffset; uint32_t m_videoFrameNum; diff --git a/scripts/generate-dispatch-table.py b/scripts/generate-dispatch-table.py index df148ed8..c3b10ee2 100644 --- a/scripts/generate-dispatch-table.py +++ b/scripts/generate-dispatch-table.py @@ -363,6 +363,7 @@ def __repr__(self): vk_khr_video_queue = Extension(name='VK_KHR_video_queue', version=1, guard='VK_USE_VIDEO_QUEUE', commands=[ Command(name='GetPhysicalDeviceVideoFormatPropertiesKHR', dispatch='VkPhysicalDevice'), Command(name='GetPhysicalDeviceVideoCapabilitiesKHR', dispatch='VkPhysicalDevice'), + Command(name='GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR', dispatch='VkPhysicalDevice'), Command(name='CreateVideoSessionKHR', dispatch='VkDevice'), Command(name='DestroyVideoSessionKHR', dispatch='VkDevice'), Command(name='CreateVideoSessionParametersKHR', dispatch='VkDevice'), diff --git a/vk_video_decoder/demos/CMakeLists.txt b/vk_video_decoder/demos/CMakeLists.txt index 482630e9..fdb3e61a 100644 --- a/vk_video_decoder/demos/CMakeLists.txt +++ b/vk_video_decoder/demos/CMakeLists.txt @@ -5,6 +5,10 @@ set(DEMO_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/.. ) +IF (${_TRANSCODING} MATCHES 1) + list (APPEND DEMO_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/../../vk_video_encoder/libs) +ENDIF() + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") add_definitions(-DVK_USE_PLATFORM_WIN32_KHR -DWIN32_LEAN_AND_MEAN) set(DisplayServer Win32) diff --git a/vk_video_decoder/demos/vk-video-dec/CMakeLists.txt b/vk_video_decoder/demos/vk-video-dec/CMakeLists.txt index 0a61f859..3a70c2d1 100644 --- a/vk_video_decoder/demos/vk-video-dec/CMakeLists.txt +++ b/vk_video_decoder/demos/vk-video-dec/CMakeLists.txt @@ -24,8 +24,19 @@ endmacro() generate_dispatch_table(${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/HelpersDispatchTable.h) generate_dispatch_table(${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/HelpersDispatchTable.cpp) -set(sources - Main.cpp +set(VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT + ${CMAKE_CURRENT_SOURCE_DIR}/../../../vk_video_encoder/libs +) + +if (${_TRANSCODING} MATCHES 1) + set(TARGET_EXECUTABLE_NAME vk-video-trans-test) + set(sources ${CMAKE_CURRENT_SOURCE_DIR}/../../../vk_video_transcoder/demos/vk-video-trans/Main.cpp) +else() + set(TARGET_EXECUTABLE_NAME vk-video-dec-test) + set(sources Main.cpp) +endif() + +list(APPEND sources ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkShell/Shell.cpp ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkShell/ShellDirect.cpp ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkShell/Shell.h @@ -87,6 +98,46 @@ set(sources ${VK_VIDEO_DECODER_LIBS_SOURCE_ROOT}/VulkanVideoFrameBuffer/VulkanVideoFrameBuffer.cpp ) +IF (${_TRANSCODING} MATCHES 1) +list(APPEND sources + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigH264.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigH264.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbH264.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbH264.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderH264.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderH264.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigH265.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigH265.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbH265.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbH265.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderH265.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderH265.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigAV1.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfigAV1.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbAV1.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderDpbAV1.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderAV1.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoderAV1.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkEncoderConfig.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoder.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoGopStructure.cpp + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoGopStructure.h + ${VK_VIDEO_ENCODER_LIBS_SOURCE_ROOT}/VkVideoEncoder/VkVideoEncoder.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/YCbCrConvUtilsCpu.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/YCbCrConvUtilsCpu.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoImagePool.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoImagePool.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanCommandBufferPool.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanCommandBufferPool.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanQueryPoolSet.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanQueryPoolSet.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSessionParameters.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoSessionParameters.h + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoDisplayQueue.cpp + ${VK_VIDEO_COMMON_LIBS_SOURCE_ROOT}/VkCodecUtils/VulkanVideoDisplayQueue.h) + add_compile_definitions(_TRANSCODING) +ENDIF() + set(definitions PRIVATE -DVK_NO_PROTOTYPES PRIVATE -DGLM_FORCE_RADIANS) @@ -156,10 +207,10 @@ list(APPEND includes PRIVATE ${VULKAN_VIDEO_APIS_INCLUDE}/vulkan) list(APPEND includes PRIVATE ${VULKAN_VIDEO_APIS_INCLUDE}/nvidia_utils/vulkan) list(APPEND includes PRIVATE ${SHADERC_ROOT_PATH}/install/include) -add_executable(vk-video-dec-test ${generate_helper_files} ${sources}) -target_compile_definitions(vk-video-dec-test ${definitions}) -target_include_directories(vk-video-dec-test ${includes}) -target_link_libraries(vk-video-dec-test ${libraries}) -add_dependencies(vk-video-dec-test generate_helper_files) +add_executable(${TARGET_EXECUTABLE_NAME} ${generate_helper_files} ${sources}) +target_compile_definitions(${TARGET_EXECUTABLE_NAME} ${definitions}) +target_include_directories(${TARGET_EXECUTABLE_NAME} ${includes}) +target_link_libraries(${TARGET_EXECUTABLE_NAME} ${libraries}) +add_dependencies(${TARGET_EXECUTABLE_NAME} generate_helper_files) -install(TARGETS vk-video-dec-test RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) +install(TARGETS ${TARGET_EXECUTABLE_NAME} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/vk_video_decoder/libs/VkVideoDecoder/VkVideoDecoder.cpp b/vk_video_decoder/libs/VkVideoDecoder/VkVideoDecoder.cpp index ffb072ac..82a5fa5c 100644 --- a/vk_video_decoder/libs/VkVideoDecoder/VkVideoDecoder.cpp +++ b/vk_video_decoder/libs/VkVideoDecoder/VkVideoDecoder.cpp @@ -270,11 +270,13 @@ int32_t VkVideoDecoder::StartVideoSequence(VkParserDetectedVideoFormat* pVideoFo } m_numImageTypesEnabled |= DecodeFrameBufferIf::IMAGE_TYPE_MASK_DECODE_OUT; +#if (!_TRANSCODING) if(!(videoCapabilities.flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) { // The implementation does not support individual images for DPB and so must use arrays m_useImageArray = VK_TRUE; m_useImageViewArray = VK_FALSE; } +#endif //_TRANSCODING if (m_enableDecodeComputeFilter) { diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp index c1e7baee..152bde29 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.cpp @@ -66,6 +66,17 @@ void printHelp(VkVideoCodecOperationFlagBitsKHR codec) --deviceID : deviceID to be used, \n\ --deviceUuid : deviceUuid to be used \n\ --testOutOfOrderRecording Testing only: enable testing for out-of-order-recording\n"); +#if (_TRANSCODING) + fprintf(stderr, + "--videoContentHints \n\ + default(0), camera (1), desktop(2), recorded(3) \n\ + --videoUsageHints \n\ + default(0), transcoding (1), streaming(2), recording(3), conferencing(4) \n\ + --preset : [1, 4] corresponds to p1..p4 \n\ + --profile : 0 - low latency streaming, 1 - archiving, 2 - svc \n\ + --bitrate : Target bitrate of the encoded file in Mbps (Megabit per second)\n\ + --vbvbuf-ratio : [1, 4] multiplier to vbv buffer size: CBR's vbvbuf = bitrate/framerate * vbvbuf-ratio, VBR's vbvbuf = bitrate * vbvbuf-ratio \n"); +#endif //_TRANSCODING if ((codec == VK_VIDEO_CODEC_OPERATION_NONE_KHR) || (codec == VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR)) { fprintf(stderr, "\nH264 specific arguments: None\n"); @@ -301,6 +312,48 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) } gopStructure.SetGopFrameCount(gopFrameCount); printf("Selected gopFrameCount: %d\n", gopFrameCount); +#if (_TRANSCODING) + } else if (strcmp(argv[i], "--profile") == 0) { + if ((++i >= argc) !=1) { + int encodeProfileVal; + int sscanf_result = sscanf(argv[i], "%d", &encodeProfileVal); + if (sscanf_result != 1 || encodeProfileVal < 0 || encodeProfileVal > 2) { + fprintf(stderr, "invalid parameter for %s\n", argv[i - 1]); + return -1; + } else { + encodingProfile = static_cast(encodeProfileVal); + } + } + } else if (strcmp(argv[i], "--preset") == 0) { + if ((++i >= argc) !=1) { + int pn; + int sscanf_result = sscanf(argv[i], "%d", &pn); + if (sscanf_result != 1 || pn < 1) { + fprintf(stderr, "invalid parameter for %s\n", argv[i - 1]); + return -1; + } else { + qualityLevel = std::min(pn, 4) - 1; + } + } + } else if (strcmp(argv[i], "--bitrate") == 0) { + if ((++i >= argc) !=1) { + int targetBitrate; + int sscanf_result = sscanf(argv[i], "%d", &targetBitrate); + if (sscanf_result != 1 || targetBitrate <= 0 || targetBitrate >= 200) { + fprintf(stderr, "invalid parameter for %s\n", argv[i - 1]); + return -1; + } + averageBitrate = targetBitrate << 20; + } + } else if (strcmp(argv[i], "--vbvbuf-ratio") == 0) { + int vbvbuf_ratio; + int sscanf_result = sscanf(argv[i+1], "%d", &vbvbuf_ratio); + if (sscanf_result != 1 || vbvbuf_ratio <= 0 || vbvbuf_ratio > 8) { + fprintf(stderr, "invalid parameter for %s\n", argv[i - 1]); + return VK_ERROR_VIDEO_PROFILE_CODEC_NOT_SUPPORTED_KHR; + } + vbvbufratio = vbvbuf_ratio; +#endif //_TRANSCODING } else if (args[i] == "--idrPeriod") { int32_t idrPeriod = EncoderConfig::DEFAULT_GOP_IDR_PERIOD; if (++i >= argc || sscanf(args[i].c_str(), "%d", &idrPeriod) != 1) { @@ -369,6 +422,46 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) fprintf(stderr, "Invalid tuningMode: %s\n", tuningModeStr.c_str()); return -1; } +#if (_TRANSCODING) + } else if (args[i] == "--encodeUsage") { + if (++i >= argc) { + fprintf(stderr, "Invalid parameter for %s\n", args[i - 1].c_str()); + return -1; + } + std::string encodeUsageStr = argv[i]; + if (encodeUsageStr == "0" || encodeUsageStr == "default") { + encodeUsageHints = VK_VIDEO_ENCODE_USAGE_DEFAULT_KHR; + } else if (encodeUsageStr == "1" || encodeUsageStr == "transcoding") { + encodeUsageHints = VK_VIDEO_ENCODE_USAGE_TRANSCODING_BIT_KHR; + } else if (encodeUsageStr == "2" || encodeUsageStr == "streaming") { + encodeUsageHints = VK_VIDEO_ENCODE_USAGE_STREAMING_BIT_KHR; + } else if (encodeUsageStr == "3" || encodeUsageStr == "recording") { + encodeUsageHints = VK_VIDEO_ENCODE_USAGE_RECORDING_BIT_KHR; + } else if (encodeUsageStr == "4" || encodeUsageStr == "conferencing") { + encodeUsageHints = VK_VIDEO_ENCODE_USAGE_CONFERENCING_BIT_KHR; + } else { + fprintf(stderr, "Invalid encodeUsage: %s\n", encodeUsageStr.c_str()); + return -1; + } + } else if (args[i] == "--encodeContent") { + if (++i >= argc) { + fprintf(stderr, "Invalid parameter for %s\n", args[i - 1].c_str()); + return -1; + } + std::string encodeContentStr = argv[i]; + if (encodeContentStr == "0" || encodeContentStr == "default") { + encodeContentHints = VK_VIDEO_ENCODE_CONTENT_DEFAULT_KHR; + } else if (encodeContentStr == "1" || encodeContentStr == "camera") { + encodeContentHints = VK_VIDEO_ENCODE_CONTENT_DESKTOP_BIT_KHR; + } else if (encodeContentStr == "2" || encodeContentStr == "desktop") { + encodeContentHints = VK_VIDEO_ENCODE_CONTENT_DESKTOP_BIT_KHR; + } else if (encodeContentStr == "3" || encodeContentStr == "rendered") { + encodeContentHints = VK_VIDEO_ENCODE_CONTENT_RENDERED_BIT_KHR; + } else { + fprintf(stderr, "Invalid encodeContent: %s\n", encodeContentStr.c_str()); + return -1; + } +#endif // _TRANSCODING } else if (args[i] == "--rateControlMode") { if (++i >= argc) { fprintf(stderr, "invalid parameter for %s\n", args[i-1].c_str()); @@ -469,6 +562,7 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) return -1; } +#if (!_TRANSCODING) // transcoding: the resolution will be read later when 1st frame is decoded if (input.width == 0) { fprintf(stderr, "The width was not specified\n"); return -1; @@ -478,6 +572,7 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) fprintf(stderr, "The height was not specified\n"); return -1; } +#endif // !_TRANSCODING if (!outputFileHandler.HasFileName()) { const char* defaultOutName = (codec == VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR) ? "out.264" : @@ -525,6 +620,7 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) return -1; } +#if (!_TRANSCODING) frameCount = inputFileHandler.GetFrameCount(input.width, input.height, input.bpp, input.chromaSubsampling); if (numFrames == 0 || numFrames > frameCount) { @@ -537,6 +633,7 @@ int EncoderConfig::ParseArguments(int argc, char *argv[]) return -1; } } +#endif //!_TRANSCODING return DoParseArguments(argcount, arglist.data()); } @@ -581,8 +678,10 @@ VkResult EncoderConfig::CreateCodecConfig(int argc, char *argv[], VkResult result = vkEncoderConfigh264->InitializeParameters(); if (result != VK_SUCCESS) { +#if (!_TRANSCODING) assert(!"InitializeParameters failed"); return result; +#endif // !_TRANSCODING } encoderConfig = vkEncoderConfigh264; @@ -599,8 +698,10 @@ VkResult EncoderConfig::CreateCodecConfig(int argc, char *argv[], VkResult result = vkEncoderConfigh265->InitializeParameters(); if (result != VK_SUCCESS) { +#if (!_TRANSCODING) assert(!"InitializeParameters failed"); return result; +#endif // !_TRANSCODING } encoderConfig = vkEncoderConfigh265; @@ -644,12 +745,26 @@ void EncoderConfig::InitVideoProfile() } // update the video profile +#if (_TRANSCODING) + VkVideoEncodeUsageInfoKHR encodeUsage; + encodeUsage.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_USAGE_INFO_KHR; + encodeUsage.pNext = NULL; + encodeUsage.tuningMode = tuningMode; + encodeUsage.videoContentHints = encodeContentHints; + encodeUsage.videoUsageHints = encodeUsageHints; +#endif // _TRANSCODING + videoCoreProfile = VkVideoCoreProfile(codec, encodeChromaSubsampling, GetComponentBitDepthFlagBits(encodeBitDepthLuma), GetComponentBitDepthFlagBits(encodeBitDepthChroma), (videoProfileIdc != (uint32_t)-1) ? videoProfileIdc : GetDefaultVideoProfileIdc(), - tuningMode); +#if (_TRANSCODING) + encodeUsage +#else + tuningMode +#endif // _TRANSCODING + ); } bool EncoderConfig::InitRateControl() diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h index 602058ab..53a7b8dd 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfig.h @@ -673,6 +673,10 @@ struct EncoderConfig : public VkVideoRefCountBase { uint32_t codecBlockAlignment; uint32_t qualityLevel; VkVideoEncodeTuningModeKHR tuningMode; +#if (_TRANSCODING) + VkVideoEncodeUsageFlagBitsKHR encodeUsageHints; + VkVideoEncodeContentFlagBitsKHR encodeContentHints; +#endif // _TRANSCODING VkVideoCoreProfile videoCoreProfile; VkVideoCapabilitiesKHR videoCapabilities; VkVideoEncodeCapabilitiesKHR videoEncodeCapabilities; @@ -735,6 +739,12 @@ struct EncoderConfig : public VkVideoRefCountBase { uint32_t selectVideoWithComputeQueue : 1; uint32_t enableOutOfOrderRecording : 1; // Testing only - don't use for production! +#if (_TRANSCODING) + int vbvbufratio; + enum ENCODING_PROFILE { LOW_LATENCY_STREAMING = 0, ARCHIVING, SVC, ENUM_MAXVAL_NOTSET }; + ENCODING_PROFILE encodingProfile; +#endif // _TRANSCODING + EncoderConfig() : refCount(0) , appName() @@ -761,6 +771,10 @@ struct EncoderConfig : public VkVideoRefCountBase { , codecBlockAlignment(16) , qualityLevel(0) , tuningMode(VK_VIDEO_ENCODE_TUNING_MODE_DEFAULT_KHR) +#if (_TRANSCODING) + , encodeUsageHints(VK_VIDEO_ENCODE_USAGE_DEFAULT_KHR) + , encodeContentHints(VK_VIDEO_ENCODE_CONTENT_DEFAULT_KHR) +#endif // _TRANSCODING , videoCoreProfile(codec, encodeChromaSubsampling, encodeBitDepthLuma, encodeBitDepthChroma) , videoCapabilities() , videoEncodeCapabilities() @@ -818,6 +832,10 @@ struct EncoderConfig : public VkVideoRefCountBase { , enableHwLoadBalancing(false) , selectVideoWithComputeQueue(false) , enableOutOfOrderRecording(false) +#if (_TRANSCODING) + , vbvbufratio(1) + , encodingProfile(ENUM_MAXVAL_NOTSET) +#endif // _TRANSCODING { } virtual ~EncoderConfig() {} diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp index 3d76c7c8..99bcc028 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkEncoderConfigH264.cpp @@ -380,7 +380,15 @@ VkResult EncoderConfigH264::InitDeviceCapabilities(const VulkanDeviceContext* vk int8_t EncoderConfigH264::InitDpbCount() { +#if (!_TRANSCODING) dpbCount = (gopStructure.GetConsecutiveBFrameCount() > 0) ? gopStructure.GetConsecutiveBFrameCount() : 1; +#else + dpbCount = gopStructure.GetConsecutiveBFrameCount(); + if (dpbCount == 0) { + dpbCount = 1; + return dpbCount + 1; + } +#endif // !_TRANSCODING // spsInfo->level represents the smallest level that we require for the // given stream. This level constrains the maximum size (in terms of // number of frames) that the DPB can have. levelDpbSize is this maximum @@ -458,6 +466,16 @@ bool EncoderConfigH264::InitRateControl() } // Use the level limit for the max VBV buffer size, and no more than 8 seconds at peak rate +#if (_TRANSCODING) + if (encodingProfile == EncoderConfig::LOW_LATENCY_STREAMING) + { + vbvBufferSize = averageBitrate / frameRate * vbvbufratio; // averageBitrate is a uint32_t, no overflow + } + else if (encodingProfile == EncoderConfig::ARCHIVING) + { + vbvBufferSize = averageBitrate * vbvbufratio; + } else // Tymur: temporarily disable the following feature if encodingProfile is specified (the above code will be used instead) +#endif // _TRANSCODING if (vbvBufferSize == 0) { vbvBufferSize = std::min(levelLimits[levelIdc].maxCPB * 1000, 120000000U); if (rateControlMode != VK_VIDEO_ENCODE_RATE_CONTROL_MODE_DISABLED_BIT_KHR) diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp index bab27757..4c01a191 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.cpp @@ -394,6 +394,140 @@ VkResult VkVideoEncoder::SubmitStagedInputFrame(VkSharedBaseObj encodeFrameInfo; + GetAvailablePoolNode(encodeFrameInfo); + assert(encodeFrameInfo); + encodeFrameInfo->frameInputOrderNum = m_inputFrameNum++; + encodeFrameInfo->lastFrame = !(encodeFrameInfo->frameInputOrderNum < (m_encoderConfig->numFrames - 1)); + + if (m_encoderConfig->enableQpMap) { + + if (encodeFrameInfo->srcQpMapStagingImageView == nullptr) { + bool success = m_linearQpMapImagePool->GetAvailableImage(encodeFrameInfo->srcQpMapStagingImageView, + VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + assert(success); + if (!success) { + return VK_ERROR_INITIALIZATION_FAILED; + } + assert(encodeFrameInfo->srcQpMapStagingImageView != nullptr); + } + + VkSharedBaseObj linearQpMapImageView; + encodeFrameInfo->srcQpMapStagingImageView->GetImageView(linearQpMapImageView); + + const VkSharedBaseObj& dstQpMapImageResource = linearQpMapImageView->GetImageResource(); + VkSharedBaseObj srcQpMapImageDeviceMemory(dstQpMapImageResource->GetMemory()); + + // Map the image and read the image data. + VkDeviceSize qpMapImageOffset = dstQpMapImageResource->GetImageDeviceMemoryOffset(); + VkDeviceSize qpMapMaxSize = 0; + uint8_t* writeQpMapImagePtr = srcQpMapImageDeviceMemory->GetDataPtr(qpMapImageOffset, qpMapMaxSize); + assert(writeQpMapImagePtr != nullptr); + + size_t formatSize = getFormatTexelSize(m_imageQpMapFormat); + uint32_t inputQpMapWidth = (m_encoderConfig->input.width + m_qpMapTexelSize.width - 1) / m_qpMapTexelSize.width; + uint32_t qpMapWidth = (m_encoderConfig->encodeWidth + m_qpMapTexelSize.width - 1) / m_qpMapTexelSize.width; + uint32_t qpMapHeight = (m_encoderConfig->encodeHeight + m_qpMapTexelSize.height - 1) / m_qpMapTexelSize.height; + uint64_t qpMapFileOffset = qpMapWidth * qpMapHeight * encodeFrameInfo->frameInputOrderNum * formatSize; + const uint8_t* pQpMapData = m_encoderConfig->qpMapFileHandler.GetMappedPtr(qpMapFileOffset); + + const VkSubresourceLayout* dstQpMapSubresourceLayout = dstQpMapImageResource->GetSubresourceLayout(); + + for (uint32_t j = 0; j < qpMapHeight; j++) { + memcpy(writeQpMapImagePtr + (dstQpMapSubresourceLayout[0].offset + j * dstQpMapSubresourceLayout[0].rowPitch), + pQpMapData + j * inputQpMapWidth * formatSize, qpMapWidth * formatSize); + } + } + + { + // On success, stage the input frame for the encoder video input + StageInputFrameDecoded(encodeFrameInfo, vkLastFrameDecoded); + return VK_SUCCESS; + } + + return VK_ERROR_INITIALIZATION_FAILED; +} + +VkResult VkVideoEncoder::StageInputFrameDecoded(VkSharedBaseObj& encodeFrameInfo, VulkanDecodedFrame* vkLastFrameDecoded) +{ + assert(encodeFrameInfo); + + VkResult result; + + if (m_encoderConfig->enableQpMap) { + if (encodeFrameInfo->srcQpMapImageResource == nullptr) { + bool success = m_qpMapImagePool->GetAvailableImage(encodeFrameInfo->srcQpMapImageResource, + VK_IMAGE_LAYOUT_VIDEO_ENCODE_QUANTIZATION_MAP_KHR); + assert(success); + assert(encodeFrameInfo->srcQpMapImageResource != nullptr); + if (!success || encodeFrameInfo->srcQpMapImageResource == nullptr) { + return VK_ERROR_INITIALIZATION_FAILED; + } + } + + m_inputCommandBufferPool->GetAvailablePoolNode(encodeFrameInfo->qpMapCmdBuffer); + assert(encodeFrameInfo->qpMapCmdBuffer != nullptr); + + // Make sure command buffer is not in use anymore and reset + encodeFrameInfo->qpMapCmdBuffer->ResetCommandBuffer(true, "encoderStagedInputFence"); + + // Begin command buffer + VkCommandBufferBeginInfo beginInfo = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO, nullptr }; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + VkCommandBuffer cmdBuf = encodeFrameInfo->qpMapCmdBuffer->BeginCommandBufferRecording(beginInfo); + + VkSharedBaseObj linearQpMapImageView; + encodeFrameInfo->srcQpMapStagingImageView->GetImageView(linearQpMapImageView); + + VkSharedBaseObj srcQpMapImageView; + encodeFrameInfo->srcQpMapImageResource->GetImageView(srcQpMapImageView); + + VkImageLayout linearQpMapImgNewLayout = TransitionImageLayout(cmdBuf, linearQpMapImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + VkImageLayout srcQpMapImgNewLayout = TransitionImageLayout(cmdBuf, srcQpMapImageView, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); + (void)linearQpMapImgNewLayout; + (void)srcQpMapImgNewLayout; + + VkExtent2D copyImageExtent { + (std::min(m_encoderConfig->encodeWidth, m_encoderConfig->input.width) + m_qpMapTexelSize.width - 1) / m_qpMapTexelSize.width, + (std::min(m_encoderConfig->encodeHeight, m_encoderConfig->input.height) + m_qpMapTexelSize.height - 1) / m_qpMapTexelSize.height + }; + + CopyLinearToLinearImage(cmdBuf, linearQpMapImageView, srcQpMapImageView, copyImageExtent); + + result = encodeFrameInfo->qpMapCmdBuffer->EndCommandBufferRecording(cmdBuf); + + // Now submit the staged input to the queue + SubmitStagedQpMap(encodeFrameInfo); + } + + if (encodeFrameInfo->srcEncodeImageResource == nullptr) { + bool success = m_inputImagePool->GetAvailableImage(encodeFrameInfo->srcEncodeImageResource, + VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR); + assert(success); + assert(encodeFrameInfo->srcEncodeImageResource != nullptr); + } + encodeFrameInfo->setLastDecodedFrame(vkLastFrameDecoded); + + encodeFrameInfo->constQp = m_encoderConfig->constQp; + + // and encode the input frame with the encoder next + auto timeStart = std::chrono::steady_clock::now(); + EncodeFrame(encodeFrameInfo); + auto timeEnd = std::chrono::steady_clock::now(); + m_encodeTimeMicroSec += std::chrono::duration_cast(timeEnd - timeStart).count(); + + return VK_SUCCESS; +} + +VkResult VkVideoEncoder::SubmitStagedInputFrameDecoded(VkSharedBaseObj& encodeFrameInfo, VulkanDecodedFrame& vkLastFrameDecoded) +{ + return VK_SUCCESS; // not used +} +#endif //_TRANSCODING + VkResult VkVideoEncoder::AssembleBitstreamData(VkSharedBaseObj& encodeFrameInfo, uint32_t frameIdx, uint32_t ofTotalFrames) { @@ -528,7 +662,13 @@ VkResult VkVideoEncoder::InitEncoder(VkSharedBaseObj& encoderConf std::cout << ", Consecutive B frames: " << (uint32_t)m_encoderConfig->gopStructure.GetConsecutiveBFrameCount(); std::cout << std::endl; const uint64_t maxFramesToDump = std::min(m_encoderConfig->numFrames, m_encoderConfig->gopStructure.GetGopFrameCount() + 19); - m_encoderConfig->gopStructure.PrintGopStructure(maxFramesToDump); + +#if (_TRANSCODING) + if (maxFramesToDump < 256) +#endif // _TRANSCODING + { + m_encoderConfig->gopStructure.PrintGopStructure(maxFramesToDump); + } if (m_encoderConfig->verboseFrameStruct) { m_encoderConfig->gopStructure.DumpFramesGopStructure(0, maxFramesToDump); @@ -1016,6 +1156,15 @@ VkImageLayout VkVideoEncoder::TransitionImageLayout(VkCommandBuffer cmdBuf, imageBarrier.dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR; imageBarrier.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR; imageBarrier.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR; +#if (_TRANSCODING) + } else if ((oldLayout == VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR) && (newLayout == VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR || newLayout == VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL)) { + imageBarrier.srcAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR; + imageBarrier.dstAccessMask = VK_ACCESS_2_VIDEO_ENCODE_READ_BIT_KHR; + imageBarrier.srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR; + imageBarrier.dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR; + imageBarrier.srcQueueFamilyIndex = (uint32_t)m_vkDevCtx->GetVideoDecodeQueueFamilyIdx(); + imageBarrier.dstQueueFamilyIndex = (uint32_t)m_vkDevCtx->GetVideoEncodeQueueFamilyIdx(); +#endif // _TRANSCODING } else { throw std::invalid_argument("unsupported layout transition!"); } @@ -1263,6 +1412,23 @@ VkResult VkVideoEncoder::RecordVideoCodingCmd(VkSharedBaseObjvideoSession; encodeBeginInfo.videoSessionParameters = *encodeFrameInfo->videoSessionParameters; +#if (_TRANSCODING) + VkSharedBaseObj srcEncodeImageView; + encodeFrameInfo->srcEncodeImageResource->GetImageView(srcEncodeImageView); + VkSharedBaseObj lastDecodedImageView; + VulkanDecodedFrame* vkLast = encodeFrameInfo->getLastDecodedFrame(); + bool getResourceView = vkLast->imageViews[VulkanDecodedFrame::IMAGE_VIEW_TYPE_OPTIMAL_DISPLAY].GetImageResourceView(lastDecodedImageView); + assert(getResourceView); + VkImageLayout srcImgNewLayout = TransitionImageLayout(cmdBuf, srcEncodeImageView, VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, VK_IMAGE_LAYOUT_VIDEO_ENCODE_SRC_KHR); + VkImageLayout linearImgNewLayout = TransitionImageLayout(cmdBuf, lastDecodedImageView, VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL); + (void)srcImgNewLayout; + (void)linearImgNewLayout; + VkVideoPictureResourceInfoKHR* pSrcPictureResource = encodeFrameInfo->srcEncodeImageResource->GetPictureResourceInfo(); + pSrcPictureResource->imageViewBinding = lastDecodedImageView->GetImageView(VulkanDecodedFrame::IMAGE_VIEW_TYPE_OPTIMAL_DISPLAY); + encodeFrameInfo->encodeInfo.srcPictureResource.imageViewBinding = pSrcPictureResource->imageViewBinding; + encodeFrameInfo->setDecodeCompleteSemaphore(&vkLast->frameCompleteSemaphore); +#endif // _TRANSCODING + assert((encodeFrameInfo->encodeInfo.referenceSlotCount) <= ARRAYSIZE(encodeFrameInfo->dpbImageResources)); // TODO: Calculate the number of DPB slots for begin against the multiple frames. encodeBeginInfo.referenceSlotCount = encodeFrameInfo->encodeInfo.referenceSlotCount + 1; @@ -1359,9 +1525,16 @@ VkResult VkVideoEncoder::SubmitVideoCodingCmds(VkSharedBaseObjinputCmdBuffer) { inputWaitSemaphore[waitSemaphoreCount++] = encodeFrameInfo->inputCmdBuffer->GetSemaphore(); } +#else + if (encodeFrameInfo->inputCmdBuffer == nullptr) { + inputWaitSemaphore[waitSemaphoreCount++] = *encodeFrameInfo->getDecodeCompleteSemaphore(); // transcoding: inputCmdBuff is nullptr + } +#endif //(!_TRANSCODING) if (encodeFrameInfo->qpMapCmdBuffer) { inputWaitSemaphore[waitSemaphoreCount++] = encodeFrameInfo->qpMapCmdBuffer->GetSemaphore(); } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h index cdc77383..8a697840 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoder.h @@ -40,6 +40,9 @@ #include "VkShell/Shell.h" #endif // ENCODER_DISPLAY_QUEUE_SUPPORT #include "mio/mio.hpp" +#if (_TRANSCODING) +#include "VkCodecUtils/VulkanDecodedFrame.h" +#endif // _TRANSCODING class VkVideoEncoderH264; class VkVideoEncoderH265; @@ -102,6 +105,10 @@ class VkVideoEncoder : public VkVideoRefCountBase { , m_refCount(0) , m_parent() , m_parentIndex(-1) +#if (_TRANSCODING) + , m_decodeCompeteSemaphore(nullptr) + , m_lastDecodedFrame{nullptr} +#endif // _TRANSCODING { assert(ARRAYSIZE(referenceSlotsInfo) == MAX_IMAGE_REF_RESOURCES); for (uint32_t i = 0; i < MAX_IMAGE_REF_RESOURCES; i++) { @@ -346,10 +353,31 @@ class VkVideoEncoder : public VkVideoRefCountBase { return VK_SUCCESS; } +#if (_TRANSCODING) + VkSemaphore* getDecodeCompleteSemaphore() const { + return m_decodeCompeteSemaphore; + } + + void setDecodeCompleteSemaphore(VkSemaphore* inputSemaphore) { + m_decodeCompeteSemaphore = inputSemaphore; + } +#endif // _TRANSCODING + private: std::atomic m_refCount; VkSharedBaseObj m_parent; int32_t m_parentIndex; +#if (_TRANSCODING) + VkSemaphore* m_decodeCompeteSemaphore; + VulkanDecodedFrame* m_lastDecodedFrame; + public: + void setLastDecodedFrame(VulkanDecodedFrame* lastDecodedFrame) { + m_lastDecodedFrame = lastDecodedFrame; + } + VulkanDecodedFrame* getLastDecodedFrame() const { + return m_lastDecodedFrame; + } +#endif // _TRANSCODING }; #ifdef ENCODER_DISPLAY_QUEUE_SUPPORT class DisplayQueue { @@ -472,6 +500,9 @@ class VkVideoEncoder : public VkVideoRefCountBase { , m_qpMapTiling() , m_linearQpMapImagePool() , m_qpMapImagePool() +#if (_TRANSCODING) + , m_encodeTimeMicroSec(0) +#endif // _TRANSCODING { } // Factory Function @@ -517,6 +548,11 @@ class VkVideoEncoder : public VkVideoRefCountBase { VkResult LoadNextFrame(VkSharedBaseObj& encodeFrameInfo); VkResult StageInputFrame(VkSharedBaseObj& encodeFrameInfo); VkResult SubmitStagedInputFrame(VkSharedBaseObj& encodeFrameInfo); +#if (_TRANSCODING) + VkResult LoadNextFrameDecoded(VulkanDecodedFrame* vkLastFrameDecoded); + VkResult StageInputFrameDecoded(VkSharedBaseObj& encodeFrameInfo, VulkanDecodedFrame* vkLastFrameDecoded); + VkResult SubmitStagedInputFrameDecoded(VkSharedBaseObj& encodeFrameInfo, VulkanDecodedFrame& vkLastFrameDecoded); +#endif // _TRANSCODING VkResult SubmitStagedQpMap(VkSharedBaseObj& encodeFrameInfo); virtual VkResult EncodeFrame(VkSharedBaseObj& encodeFrameInfo) = 0; // Must be implemented by the codec virtual VkResult HandleCtrlCmd(VkSharedBaseObj& encodeFrameInfo); @@ -711,6 +747,13 @@ class VkVideoEncoder : public VkVideoRefCountBase { VkImageTiling m_qpMapTiling; VkSharedBaseObj m_linearQpMapImagePool; VkSharedBaseObj m_qpMapImagePool; +#if _TRANSCODING + uint64_t m_encodeTimeMicroSec; +public: + uint64_t getFps() const { + return m_inputFrameNum * 1000000 / m_encodeTimeMicroSec; + } +#endif }; VkResult CreateVideoEncoderH264(const VulkanDeviceContext* vkDevCtx, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.cpp index fbbf1fb8..9db9a8e9 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH264.cpp @@ -51,6 +51,27 @@ VkResult VkVideoEncoderH264::InitEncoderCodec(VkSharedBaseObj& en return result; } +#if (_TRANSCODING) + const VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR deviceQualityLevelInfo { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR, nullptr, encoderConfig->videoCoreProfile.GetProfile(), encoderConfig->videoEncodeCapabilities.maxQualityLevels - 1} ; // should be less than maxQualityLevels; + VkVideoEncodeQualityLevelPropertiesKHR qualityLevelProperties { VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_PROPERTIES_KHR, nullptr, VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR, 1 }; + + if (encoderConfig->videoCoreProfile.GetProfile()->videoCodecOperation == VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR) + { + // DRAFT: I'm just filling the random data to check that the app works (doesn't crash), the results are unused + VkVideoEncodeH264QualityLevelPropertiesKHR encodeH264QualityLevelProperties; // this structure is used as pNext in for return value from getPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR + encodeH264QualityLevelProperties.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H264_QUALITY_LEVEL_PROPERTIES_KHR; + encodeH264QualityLevelProperties.pNext = NULL; + encodeH264QualityLevelProperties.preferredRateControlFlags = VK_VIDEO_ENCODE_H264_RATE_CONTROL_REGULAR_GOP_BIT_KHR; + qualityLevelProperties.pNext = &encodeH264QualityLevelProperties; + result = m_vkDevCtx->GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR(m_vkDevCtx->getPhysicalDevice(), &deviceQualityLevelInfo, &qualityLevelProperties); // result is unused currently + + if(result != VK_SUCCESS) { + fprintf(stderr, "\nInitEncoder Error: Failed to GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR.\n"); + return result; + } + } +#endif //(_TRANSCODING) + // Initialize DPB m_dpb264 = VkEncDpbH264::CreateInstance(); assert(m_dpb264); @@ -72,6 +93,17 @@ VkResult VkVideoEncoderH264::InitEncoderCodec(VkSharedBaseObj& en VkVideoSessionParametersCreateInfoKHR* encodeSessionParametersCreateInfo = videoSessionParametersInfo.getVideoSessionParametersInfo(); encodeSessionParametersCreateInfo->flags = 0; VkVideoSessionParametersKHR sessionParameters; + +#if (_TRANSCODING) + VkVideoEncodeQualityLevelInfoKHR qualityLevel; + qualityLevel.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR; + qualityLevel.pNext = nullptr; + qualityLevel.qualityLevel = encoderConfig->qualityLevel; + + VkVideoEncodeH264SessionParametersCreateInfoKHR* encodeH264SessionParametersCreateInfo = (VkVideoEncodeH264SessionParametersCreateInfoKHR*)encodeSessionParametersCreateInfo->pNext; + encodeH264SessionParametersCreateInfo->pNext = &qualityLevel; +#endif //_TRANSCODING + result = m_vkDevCtx->CreateVideoSessionParametersKHR(*m_vkDevCtx, encodeSessionParametersCreateInfo, nullptr, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.cpp index d773930e..de23d27c 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoEncoderH265.cpp @@ -51,6 +51,41 @@ VkResult VkVideoEncoderH265::InitEncoderCodec(VkSharedBaseObj& en return result; } +#if (_TRANSCODING) + const VkPhysicalDeviceVideoEncodeQualityLevelInfoKHR deviceQualityLevelInfo { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR, nullptr, encoderConfig->videoCoreProfile.GetProfile(), encoderConfig->videoEncodeCapabilities.maxQualityLevels - 1} ; // should be less than maxQualityLevels; + VkVideoEncodeQualityLevelPropertiesKHR qualityLevelProperties { VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_PROPERTIES_KHR, nullptr, VK_VIDEO_ENCODE_RATE_CONTROL_MODE_VBR_BIT_KHR, 1 }; + + if (encoderConfig->videoCoreProfile.GetProfile()->videoCodecOperation == VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR) + { + // DRAFT: I'm just filling the random data to check that the app works (doesn't crash) + VkVideoEncodeH265QualityLevelPropertiesKHR encodeH265QualityLevelProperties; // this structure is used as pNext in for return value from getPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR + encodeH265QualityLevelProperties.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_H265_QUALITY_LEVEL_PROPERTIES_KHR; + encodeH265QualityLevelProperties.pNext = NULL; + encodeH265QualityLevelProperties.preferredRateControlFlags = VK_VIDEO_ENCODE_H265_RATE_CONTROL_REGULAR_GOP_BIT_KHR; + qualityLevelProperties.pNext = &encodeH265QualityLevelProperties; + result = m_vkDevCtx->GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR(m_vkDevCtx->getPhysicalDevice(), &deviceQualityLevelInfo, &qualityLevelProperties); // result is unused currently + + if(result != VK_SUCCESS) { + fprintf(stderr, "\nInitEncoder Error: Failed to GetPhysicalDeviceVideoEncodeQualityLevelPropertiesKHR.\n"); + return result; + } + + double frameRate = ((encoderConfig->frameRateNumerator > 0) && (encoderConfig->frameRateDenominator > 0)) + ? (double)encoderConfig->frameRateNumerator / encoderConfig->frameRateDenominator + : 60.f; + + if (encoderConfig->encodingProfile == EncoderConfig::LOW_LATENCY_STREAMING) + { + m_encoderConfig->vbvBufferSize = encoderConfig->averageBitrate / frameRate * encoderConfig->vbvbufratio; // averageBitrate is a uint32_t, no overflow + printf("vbv size %u\n", m_encoderConfig->vbvBufferSize); + } + else if (encoderConfig->encodingProfile == EncoderConfig::ARCHIVING) + { + m_encoderConfig->vbvBufferSize = encoderConfig->averageBitrate * encoderConfig->vbvbufratio; // avgBitrate * 4 * (kbits-to-bits) + } + } +#endif // _TRANSCODING + // Initialize DPB m_dpb.DpbSequenceStart(m_maxDpbPicturesCount, (m_encoderConfig->numRefL0 > 0)); @@ -87,6 +122,17 @@ VkResult VkVideoEncoderH265::InitEncoderCodec(VkSharedBaseObj& en encodeSessionParametersCreateInfo.flags = 0; VkVideoSessionParametersKHR sessionParameters; + +#if (_TRANSCODING) + VkVideoEncodeQualityLevelInfoKHR qualityLevel; + qualityLevel.sType = VK_STRUCTURE_TYPE_VIDEO_ENCODE_QUALITY_LEVEL_INFO_KHR; + qualityLevel.pNext = nullptr; + qualityLevel.qualityLevel = encoderConfig->qualityLevel; + + VkVideoEncodeH265SessionParametersCreateInfoKHR* p_encodeH265SessionParametersCreateInfo = (VkVideoEncodeH265SessionParametersCreateInfoKHR*)&encodeSessionParametersCreateInfo.pNext; + p_encodeH265SessionParametersCreateInfo->pNext = &qualityLevel; +#endif //_TRANSCODING + result = m_vkDevCtx->CreateVideoSessionParametersKHR(*m_vkDevCtx, &encodeSessionParametersCreateInfo, nullptr, diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.cpp b/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.cpp index 72caddb2..9d88d5bf 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.cpp +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.cpp @@ -16,7 +16,7 @@ #include "VkVideoGopStructure.h" -VkVideoGopStructure::VkVideoGopStructure(uint8_t gopFrameCount, +VkVideoGopStructure::VkVideoGopStructure(uint16_t gopFrameCount, int32_t idrPeriod, uint8_t consecutiveBFrameCount, uint8_t temporalLayerCount, @@ -38,7 +38,7 @@ VkVideoGopStructure::VkVideoGopStructure(uint8_t gopFrameCount, bool VkVideoGopStructure::Init(uint64_t maxNumFrames) { m_gopFrameCycle = (uint8_t)(m_consecutiveBFrameCount + 1); - m_gopFrameCount = (uint8_t)std::min(m_gopFrameCount, maxNumFrames); + m_gopFrameCount = (uint16_t)std::min(m_gopFrameCount, maxNumFrames); if (m_idrPeriod > 0) { m_idrPeriod = (uint32_t)std::min(m_idrPeriod, maxNumFrames); } diff --git a/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.h b/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.h index f181e6f2..3937b3a9 100644 --- a/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.h +++ b/vk_video_encoder/libs/VkVideoEncoder/VkVideoGopStructure.h @@ -26,7 +26,7 @@ #include #include -static const uint32_t MAX_GOP_SIZE = 64; +static const uint32_t MAX_GOP_SIZE = UINT16_MAX; class VkVideoGopStructure { @@ -54,7 +54,7 @@ class VkVideoGopStructure { struct GopPosition { uint32_t inputOrder; // input order in the IDR sequence uint32_t encodeOrder; // encode order in the Gop - uint8_t inGop; // The position in Gop in input order + uint16_t inGop; // The position in Gop in input order int8_t numBFrames; // Number of B frames in this part of the Gop, -1 if not a B frame int8_t bFramePos; // The B position in Gop, -1 if not a B frame FrameType pictureType; // The type of the picture @@ -71,7 +71,7 @@ class VkVideoGopStructure { {} }; - VkVideoGopStructure(uint8_t gopFrameCount = 8, + VkVideoGopStructure(uint16_t gopFrameCount = 8, int32_t idrPeriod = 60, uint8_t consecutiveBFrameCount = 2, uint8_t temporalLayerCount = 1, @@ -106,8 +106,8 @@ class VkVideoGopStructure { // If it is set to 0, the rate control algorithm may assume an // implementation-dependent GOP length. If it is set to UINT32_MAX, // the GOP length is treated as infinite. - void SetGopFrameCount(uint8_t gopFrameCount) { m_gopFrameCount = gopFrameCount; } - uint8_t GetGopFrameCount() const { return m_gopFrameCount; } + void SetGopFrameCount(uint16_t gopFrameCount) { m_gopFrameCount = gopFrameCount; } + uint16_t GetGopFrameCount() const { return m_gopFrameCount; } // idrPeriod is the interval, in terms of number of frames, between two IDR frames (see IDR period). // If it is set to 0, the rate control algorithm may assume an implementation-dependent IDR period. @@ -173,7 +173,7 @@ class VkVideoGopStructure { // consecutiveBFrameCount can be modified before the IDR sequence uint8_t consecutiveBFrameCount = m_consecutiveBFrameCount; - gopPos.inGop = (uint8_t)(gopState.positionInInputOrder % m_gopFrameCount); + gopPos.inGop = (uint16_t)(gopState.positionInInputOrder % m_gopFrameCount); if (gopPos.inGop == 0) { // This is the start of a new (open or close) GOP. @@ -269,7 +269,7 @@ class VkVideoGopStructure { } private: - uint8_t m_gopFrameCount; + uint16_t m_gopFrameCount; uint8_t m_consecutiveBFrameCount; uint8_t m_gopFrameCycle; uint8_t m_temporalLayerCount; diff --git a/vk_video_transcoder/BUILD.md b/vk_video_transcoder/BUILD.md new file mode 100644 index 00000000..e9428fcc --- /dev/null +++ b/vk_video_transcoder/BUILD.md @@ -0,0 +1,216 @@ +# Build Instructions + +Instructions for building this repository on Linux, Windows, L4. + +## Index + +1. [Repository Set-Up](#repository-set-up) +2. [Windows Build](#building-on-windows) +3. [Linux Build](#building-on-linux) +4. [Linux for Tegra Build](#building-on-linux-for-tegra) +5. [Android Build](#building-on-android) + +## Contributing to the Repository + +If you intend to contribute, the preferred work flow is for you to develop +your contribution in a fork of this repository in your GitHub account and +then submit a pull request. +Please see the [CONTRIBUTING.md](CONTRIBUTING.md) file in this repository for more details. + +## Repository Set-Up + + Please make sure you have installed the latest NVIDIA BETA drivers from https://developer.nvidia.com/vulkan-driver. + The minimum supported BETA driver versions by this application are 527.86 (Windows) / 525.47.04 (Linux) that + must support Vulkan API version 1.3.230 or later. + The Windows and Linux BETA drivers are available for download at https://developer.nvidia.com/vulkan-beta-51769-windows + and https://developer.nvidia.com/vulkan-beta-5154924-linux, respectively. + +### Download the Repository + + Vulkan Video Test application Gerrit repository: + VULKAN_VIDEO_GIT_REPO="https://github.com/nvpro-samples/vk_video_samples.git" + + $ git clone $VULKAN_VIDEO_GIT_REPO + + APP_INSTALLED_LOC=$(pwd)/"vk_video_samples/vk_video_transcoder" + +## Building On Windows + +### Windows Build Requirements + +Windows 10 or Windows 11 with the following software packages: + +- Microsoft Visual Studio VS2017 or later (any version). +- [CMake](http://www.cmake.org/download/) + - Tell the installer to "Add CMake to the system PATH" environment variable. +- [Python 3](https://www.python.org/downloads) + - Select to install the optional sub-package to add Python to the system PATH + environment variable. + - Ensure the `pip` module is installed (it should be by default) + - Python3.3 or later is necessary for the Windows py.exe launcher that is used to select python3 + rather than python2 if both are installed +- [Git](http://git-scm.com/download/win) + - Tell the installer to allow it to be used for "Developer Prompt" as well as "Git Bash". + - Tell the installer to treat line endings "as is" (i.e. both DOS and Unix-style line endings). + - Install both the 32-bit and 64-bit versions, as the 64-bit installer does not install the + 32-bit libraries and tools. +- [Vulkan SDK](https://vulkan.lunarg.com) + - install current Vulkan SDK (i.e. VulkanSDK-1.3.*-Installer.exe) from https://vulkan.lunarg.com/ +- [FFMPEG libraries for Windows] + Download the latest version of the FFMPEG shared libraries archive from https://github.com/BtbN/FFmpeg-Builds/releases. + The archive must have the following pattern in the name ffmpeg-*-win64-*-shared.zip + For example: + https://github.com/BtbN/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-win64-gpl-shared.zip + Then extract the archive to \bin\libs\ffmpeg and add the path of the \bin\libs\ffmpeg\win64\ of + the application. Please make sure that \bin\libs\ffmpeg\win64\bin location contains + avformat-59.dll, avutil-59.dll and avcodec-59.dll shared libraries and \bin\libs\ffmpeg\win64\lib contains the + corresponding lib files. +- Notes for using [Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/install-win10) + - Vulkan Video currently does not support [Windows Subsystem for Linux]. + +### Windows Build - Microsoft Visual Studio + +1. Open a Developer Command Prompt for VS201x +2. Change directory to `/../vk_video_decoder` -- the decoder folder in the cloned git repository +3. Run `update_external_sources.bat` -- this will download and build external components +4. Change directory to `` -- the vk_video_transcoder folder in the cloned git repository +5. Create a `build` directory, change into that directory, and run cmake and build as follows: + +### Windows Build for debug - using shell + + cmake .. -DCMAKE_GENERATOR_PLATFORM=x64 -DCMAKE_INSTALL_PREFIX="$(pwd)/install/Debug" -DCMAKE_BUILD_TYPE=Debug + cmake --build . --parallel 16 --config Debug + cmake --build . --config Debug --target INSTALL + +### Windows Build for release - using shell + + cmake .. -DCMAKE_GENERATOR_PLATFORM=x64 -DCMAKE_INSTALL_PREFIX="$(pwd)/install/Release" -DCMAKE_BUILD_TYPE=Release + cmake --build . --parallel 16 --config Release + cmake --build . --config Release --target INSTALL + +### Windows Notes + +#### The Vulkan Loader Library + +Vulkan programs must be able to find and use the vulkan-1.dll library. +While several of the test and demo projects in the Windows solution set this up automatically, doing so manually may be necessary for custom projects or solutions. +Make sure the library is either installed in the C:\Windows\System folder, or that the PATH environment variable includes the folder where the library resides. + +## Building On Linux + +### Linux Build Requirements + +This repository has been built and tested on the two most recent Ubuntu LTS versions with +Vulkan SDK vulkansdk-linux-x86_64-1.2.189.0.tar.gz from https://vulkan.lunarg.com/sdk/home#linux. +Currently, the oldest supported version is Ubuntu 18.04.5 LTS, meaning that the minimum supported +compiler versions are GCC 7.5.0 and Clang 6.0.0, although earlier versions may work. +It should be straightforward to adapt this repository to other Linux distributions. + +**Required Package List:** + +1. sudo apt-get install git cmake build-essential libx11-xcb-dev libxkbcommon-dev libmirclient-dev libwayland-dev libxrandr-dev libavcodec-dev libavformat-dev libavutil-dev ninja-build + +### Linux Vulkan Video Transcoder Demo Build + +2. In a Linux terminal, `cd /vk_video_decoder` -- the root of the cloned git repository + cd $APP_INSTALLED_LOC +3. Execute `./update_external_sources.sh` -- this will download and build external components + $ ./update_external_sources.sh +4. Create a `build` directory, change into that directory: + + mkdir build + cd build + +5. Run cmake to configure the build: + + `cd /vk_video_transcoder` + + # Transcoder build configuration (please select Debug or Release build type): + For example, for Debug build: + cmake -DCMAKE_BUILD_TYPE=Debug .. + + OR for Debug build: + cmake -DCMAKE_BUILD_TYPE=Release .. + +6. Run `make -j16` to build the sample application. + +7. Usage example: + + `./demos/vk-video-dec-test -i inputfile.mkv --codec hevc --profile 0 --bitrate 5 --preset 1 --vbvbuf-ratio 2` + * `profile 0` - only p-frames, `profile 1` - 3 b-frames, `bitrate` is a target bitrate in MB/s, please see `--help` for more information + +### WSI Support Build Options + +By default, the Vulkan video demo is built with support for all 3 Vulkan-defined WSI display servers: Xcb, Xlib and Wayland, as well as direct-to-display. +It is recommended to build the repository components with support for these display servers to maximize their usability across Linux platforms. +If it is necessary to build these modules without support for one of the display servers, the appropriate CMake option of the form `BUILD_WSI_xxx_SUPPORT` can be set to `OFF`. +See the top-level CMakeLists.txt file for more info. + +### Linux Decoder Tests + +Before you begin, check if your driver has Vulkan Video extensions enabled: + + $ vulkaninfo | grep VK_KHR_video + +The output should be: + VK_KHR_video_decode_queue : extension revision 1 + VK_KHR_video_encode_queue : extension revision 1 + VK_KHR_video_queue : extension revision 1 + +To run the Vulkan Video Decode sample from the build dir (for Windows change to build\install\Debug\bin\ or build\install\Release\bin\): + + $ ./demos/vk-video-dec-test -i '