Skip to content

Commit

Permalink
Bug fix: Videos now playable on all players written using mp4Writer
Browse files Browse the repository at this point in the history
  • Loading branch information
@mohammedzakikochargi
mohammedzakikochargi committed Jun 28, 2023
1 parent 1078260 commit 5ff94b8
Show file tree
Hide file tree
Showing 5 changed files with 160 additions and 32 deletions.
11 changes: 9 additions & 2 deletions base/include/Mp4WriterSink.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@ class DetailH264;
class Mp4WriterSinkProps : public ModuleProps
{
public:
Mp4WriterSinkProps(uint32_t _chunkTime, uint32_t _syncTimeInSecs, uint16_t _fps, std::string _baseFolder, bool _recordedTSBasedDTS = true) : ModuleProps()
Mp4WriterSinkProps(uint32_t _chunkTime, uint32_t _syncTimeInSecs, uint16_t _fps, std::string _baseFolder, bool _recordedTSBasedDTS = true, bool _enableMetadata = true) : ModuleProps()
{
baseFolder = _baseFolder;
fps = _fps;
recordedTSBasedDTS = _recordedTSBasedDTS;
enableMetadata = _enableMetadata;
if ((_chunkTime >= 1 && _chunkTime <= 60) || (_chunkTime == UINT32_MAX))
{
chunkTime = _chunkTime;
Expand All @@ -39,6 +40,7 @@ class Mp4WriterSinkProps : public ModuleProps
syncTimeInSecs = 1;
fps = 30;
recordedTSBasedDTS = true;
enableMetadata = true;
}

size_t getSerializeSize()
Expand All @@ -48,14 +50,16 @@ class Mp4WriterSinkProps : public ModuleProps
sizeof(baseFolder) +
sizeof(chunkTime) +
sizeof(syncTimeInSecs) +
sizeof(fps);
sizeof(fps) +
sizeof(enableMetadata);;
}

std::string baseFolder;
uint32_t chunkTime = 1;
uint32_t syncTimeInSecs = 1;
uint16_t fps = 30;
bool recordedTSBasedDTS = true;
bool enableMetadata = true;
private:
friend class boost::serialization::access;

Expand All @@ -68,6 +72,7 @@ class Mp4WriterSinkProps : public ModuleProps
ar &chunkTime;
ar &syncTimeInSecs;
ar &fps;
ar &enableMetadata;
}
};

Expand All @@ -89,6 +94,8 @@ class Mp4WriterSink : public Module
bool setMetadata(framemetadata_sp &inputMetadata);
bool handlePropsChange(frame_sp &frame);
bool shouldTriggerSOS();
void addInputPin(framemetadata_sp& metadata, string& pinId);
bool enableMp4Metadata(framemetadata_sp &inputMetadata);
boost::shared_ptr<DetailAbs> mDetail;
Mp4WriterSinkProps mProp;

Expand Down
2 changes: 1 addition & 1 deletion base/include/Mp4WriterSinkUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ class Mp4WriterSinkUtils
bool customNamedFileDirCheck(std::string baseFolder, uint32_t chunkTimeInMinutes, boost::filesystem::path relPath, std::string& nextFrameFileName);
std::string format_hrs(int &hr);
std::string format_2(int &min);
std::string filePath(boost::filesystem::path relPath, std::string mp4FileName, std::string baseFolder);
std::string filePath(boost::filesystem::path relPath, std::string mp4FileName, std::string baseFolder, uint64_t chunkTimeInMins);
~Mp4WriterSinkUtils();
private:
int lastVideoMinute=0;
Expand Down
147 changes: 131 additions & 16 deletions base/src/Mp4WriterSink.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,8 +77,7 @@ class DetailAbs
setProps(_props);
mNextFrameFileName = "";
mux = nullptr;
mMetadataEnabled = false;

/* DTS should be based on recorded timestamps of frames or on the fps prop entirely */
if (_props.recordedTSBasedDTS)
{
mDTSCalc.reset(new DTSPassThroughStrategy);
Expand All @@ -91,7 +90,7 @@ class DetailAbs

void setProps(Mp4WriterSinkProps& _props)
{
mProps.reset(new Mp4WriterSinkProps(_props.chunkTime, _props.syncTimeInSecs, _props.fps, _props.baseFolder));
mProps.reset(new Mp4WriterSinkProps(_props.chunkTime, _props.syncTimeInSecs, _props.fps, _props.baseFolder, _props.recordedTSBasedDTS, _props.enableMetadata));
}

~DetailAbs()
Expand Down Expand Up @@ -280,6 +279,8 @@ class DetailH264 : public DetailAbs
{
}
bool write(frame_container& frames);
uint8_t* prependSizeBeforeNaluSeparatorIFrame(short naluType, frame_sp frame, size_t& frameSize);


bool set_video_decoder_config()
{
Expand Down Expand Up @@ -323,7 +324,8 @@ bool DetailJpeg::write(frame_container& frames)

if (syncFlag)
{
mp4_mux_sync(mux);
LOG_TRACE << "attempting to sync <" << mNextFrameFileName << ">";
auto ret = mp4_mux_sync(mux);
syncFlag = false;
}

Expand All @@ -348,15 +350,70 @@ bool DetailJpeg::write(frame_container& frames)

mp4_mux_track_add_sample(mux, videotrack, &mux_sample);

if (metatrack != -1 && mMetadataEnabled && inMp4MetaFrame.get())
{
mux_sample.buffer = static_cast<uint8_t*>(inMp4MetaFrame->data());
mux_sample.len = inMp4MetaFrame->size();
mp4_mux_track_add_sample(mux, metatrack, &mux_sample);
}
if (metatrack != -1 && mMetadataEnabled)
{
if (inMp4MetaFrame.get() && inMp4MetaFrame->fIndex % 3 == 0)
{
mux_sample.buffer = static_cast<uint8_t *>(inMp4MetaFrame->data());
mux_sample.len = inMp4MetaFrame->size();
}
else
{
mux_sample.buffer = nullptr;
mux_sample.len = 0;
}

mp4_mux_track_add_sample(mux, metatrack, &mux_sample);
}
return true;
}

uint8_t* DetailH264::prependSizeBeforeNaluSeparatorIFrame(short naluType, frame_sp inH264ImageFrame, size_t& frameSize)
{
char NaluSeprator[3] = { 00 ,00, 00 };
char nalusepratorIframe[2] = { 00, 00 };
auto nalu = reinterpret_cast<uint8_t*>(NaluSeprator);
uint spsPpsSize = spsBuffer.size() + ppsBuffer.size() + 8;
if (naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM)
{
frameSize = inH264ImageFrame->size();
}
else if (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE)
{
frameSize = inH264ImageFrame->size() + spsPpsSize;
}
uint8_t* newBuffer = new uint8_t[frameSize];
memcpy(newBuffer, nalu, 3);
newBuffer += 3;
newBuffer[0] = spsBuffer.size();
newBuffer += 1;
memcpy(newBuffer, spsBuffer.data(), spsBuffer.size());
newBuffer += spsBuffer.size();
memcpy(newBuffer, nalu, 3);
newBuffer += 3;
newBuffer[0] = ppsBuffer.size();
newBuffer += 1;
memcpy(newBuffer, ppsBuffer.data(), ppsBuffer.size());
newBuffer += ppsBuffer.size();
memcpy(newBuffer, nalusepratorIframe, 2);
newBuffer += 2;
newBuffer[0] = (frameSize - spsPpsSize - 4 >> 8) & 0xFF;
newBuffer[1] = frameSize - spsPpsSize - 4 & 0xFF;
newBuffer += 2;
uint8_t* tempBuffer = reinterpret_cast<uint8_t*>(inH264ImageFrame->data());
if (naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM)
{
tempBuffer = tempBuffer + spsPpsSize + 4;
}
else if (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE)
{
tempBuffer = tempBuffer + 4;
}
memcpy(newBuffer, tempBuffer, frameSize - spsPpsSize - 4);
newBuffer -= spsPpsSize + 4;
return newBuffer;
}

bool DetailH264::write(frame_container& frames)
{
auto inH264ImageFrame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA);
Expand Down Expand Up @@ -388,15 +445,40 @@ bool DetailH264::write(frame_container& frames)
return false;
}

uint8_t* frameData = reinterpret_cast<uint8_t*>(inH264ImageFrame->data());
// assign size of the frame to the last two bytes of the NALU seperator for playability in default players
frameData[2] = (inH264ImageFrame->size() - 4 >> 8) & 0xFF;
frameData[3] = inH264ImageFrame->size() - 4 & 0xFF;

mux_sample.buffer = frameData;
mux_sample.len = inH264ImageFrame->size();
auto naluType = H264Utils::getNALUType((char*)mFrameBuffer.data());
size_t frameSize;;
if (mNextFrameFileName != _nextFrameFileName)
{
mNextFrameFileName = _nextFrameFileName;
initNewMp4File(mNextFrameFileName);
if (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE)
{
auto newBuffer = prependSizeBeforeNaluSeparatorIFrame(naluType, inH264ImageFrame, frameSize);
mux_sample.buffer = newBuffer;
mux_sample.len = frameSize;
}
}

if (naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM)
{

auto newBuffer = prependSizeBeforeNaluSeparatorIFrame(naluType, inH264ImageFrame, frameSize);
mux_sample.buffer = newBuffer;
mux_sample.len = frameSize;
}


if (syncFlag)
{
mp4_mux_sync(mux);
LOG_TRACE << "attempting to sync <" << mNextFrameFileName << ">";
auto ret = mp4_mux_sync(mux);
syncFlag = false;
}

Expand All @@ -411,8 +493,6 @@ bool DetailH264::write(frame_container& frames)

addMetadataInVideoHeader(inH264ImageFrame);

mux_sample.buffer = static_cast<uint8_t*>(inH264ImageFrame->data());
mux_sample.len = inH264ImageFrame->size();
mux_sample.sync = isKeyFrame ? 1 : 0;
int64_t diffInMsecs = 0;

Expand All @@ -432,8 +512,16 @@ bool DetailH264::write(frame_container& frames)

if (metatrack != -1 && mMetadataEnabled && inMp4MetaFrame.get())
{
mux_sample.buffer = static_cast<uint8_t*>(inMp4MetaFrame->data());
mux_sample.len = inMp4MetaFrame->size();
if (inMp4MetaFrame.get())
{
mux_sample.buffer = static_cast<uint8_t*>(inMp4MetaFrame->data());
mux_sample.len = inMp4MetaFrame->size();
}
else
{
mux_sample.buffer = nullptr;
mux_sample.len = 0;
}
mp4_mux_track_add_sample(mux, metatrack, &mux_sample);
}
return true;
Expand All @@ -456,7 +544,7 @@ bool Mp4WriterSink::init()

for (auto const& element : inputPinIdMetadataMap)
{
auto& metadata = element.second;
auto metadata = element.second;
auto mFrameType = metadata->getFrameType();
if (mFrameType == FrameMetadata::FrameType::ENCODED_IMAGE)
{
Expand All @@ -468,6 +556,15 @@ bool Mp4WriterSink::init()
mDetail.reset(new DetailH264(mProp));
}
}
for (auto const& element : inputPinIdMetadataMap)
{
auto metadata = element.second;
auto mFrameType = metadata->getFrameType();
if (mFrameType == FrameMetadata::FrameType::MP4_VIDEO_METADATA && mProp.enableMetadata)
{
enableMp4Metadata(metadata);
}
}
return Module::init();
}

Expand Down Expand Up @@ -516,6 +613,24 @@ bool Mp4WriterSink::setMetadata(framemetadata_sp& inputMetadata)
return true;
}

bool Mp4WriterSink::enableMp4Metadata(framemetadata_sp &inputMetadata)
{
auto mp4VideoMetadata = FrameMetadataFactory::downcast<Mp4VideoMetadata>(inputMetadata);
std::string formatVersion = mp4VideoMetadata->getVersion();
if (formatVersion.empty())
{
LOG_ERROR << "Serialization Format Information missing from the metadata. Metadata writing will be disabled";
return false;
}
mDetail->enableMetadata(formatVersion);
return true;
}

void Mp4WriterSink::addInputPin(framemetadata_sp& metadata, string& pinId)
{
Module::addInputPin(metadata, pinId);
}

bool Mp4WriterSink::processSOS(frame_sp& frame)
{
auto inputMetadata = frame->getMetadata();
Expand Down
22 changes: 14 additions & 8 deletions base/src/Mp4WriterSinkUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,15 @@ std::string Mp4WriterSinkUtils::format_2(int &num)
}
}

std::string Mp4WriterSinkUtils::filePath(boost::filesystem::path relPath, std::string mp4FileName, std::string baseFolder)
std::string Mp4WriterSinkUtils::filePath(boost::filesystem::path relPath, std::string mp4FileName, std::string baseFolder, uint64_t chunkTimeInMinutes)
{
boost::filesystem::path finalPath;
std::string mp4VideoPath;
if (customNamedFileDirCheck(baseFolder, chunkTimeInMinutes, relPath, mp4VideoPath))
{
return mp4VideoPath;
}

auto folderPath = boost::filesystem::path(baseFolder) / relPath;
if (boost::filesystem::is_directory(folderPath))
{
Expand Down Expand Up @@ -121,11 +127,11 @@ void Mp4WriterSinkUtils::parseTSJpeg(uint64_t &ts, uint32_t &chunkTimeInMinutes,

// used cached values if the difference in ts is less than chunkTime
uint32_t chunkTimeInSecs = 60 * chunkTimeInMinutes;
if ((t - lastVideoTS) < chunkTimeInSecs && currentFolder == baseFolder && chunkTimeInMinutes != UINT32_MAX)
if ((t - lastVideoTS) < chunkTimeInSecs && currentFolder == baseFolder)
{
relPath = lastVideoFolderPath;
mp4FileName = lastVideoName;
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder);
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder, chunkTimeInMinutes);
return;
}

Expand All @@ -144,7 +150,7 @@ void Mp4WriterSinkUtils::parseTSJpeg(uint64_t &ts, uint32_t &chunkTimeInMinutes,

lastVideoName = mp4FileName;

nextFrameFileName = filePath(relPath, mp4FileName, baseFolder);
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder, chunkTimeInMinutes);
}
void Mp4WriterSinkUtils::parseTSH264(uint64_t& ts, uint32_t& chunkTimeInMinutes, uint32_t& syncTimeInSeconds,boost::filesystem::path& relPath,
std::string& mp4FileName, bool& syncFlag, short frameType, short naluType, std::string baseFolder, std::string& nextFrameFileName)
Expand All @@ -169,15 +175,15 @@ void Mp4WriterSinkUtils::parseTSH264(uint64_t& ts, uint32_t& chunkTimeInMinutes,
}
// used cached values if the difference in ts is less than chunkTime
uint32_t chunkTimeInSecs = 60 * chunkTimeInMinutes;
if ((t - lastVideoTS) < chunkTimeInSecs && currentFolder == baseFolder && chunkTimeInMinutes != UINT32_MAX)
if ((t - lastVideoTS) < chunkTimeInSecs && currentFolder == baseFolder)// && chunkTimeInMinutes != UINT32_MAX
{
relPath = lastVideoFolderPath;
mp4FileName = lastVideoName;
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder);
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder, chunkTimeInMinutes);
return;
}
// cannot be merged with if condition above.
if (naluType != H264Utils::H264_NAL_TYPE::H264_NAL_TYPE_IDR_SLICE && chunkTimeInMinutes != UINT32_MAX)
if (naluType != H264Utils::H264_NAL_TYPE::H264_NAL_TYPE_IDR_SLICE && naluType != H264Utils::H264_NAL_TYPE_SEQ_PARAM)
{
relPath = lastVideoFolderPath;
mp4FileName = lastVideoName;
Expand All @@ -200,7 +206,7 @@ void Mp4WriterSinkUtils::parseTSH264(uint64_t& ts, uint32_t& chunkTimeInMinutes,
lastVideoName = mp4FileName;
lastVideoFolderPath = relPath;

nextFrameFileName = filePath(relPath, mp4FileName, baseFolder);
nextFrameFileName = filePath(relPath, mp4FileName, baseFolder, chunkTimeInMinutes);
tempNextFrameFileName = nextFrameFileName;
}

Expand Down
Loading

0 comments on commit 5ff94b8

Please sign in to comment.