From d23cf4156e874187c18e01dfab6a8ffd52ded779 Mon Sep 17 00:00:00 2001 From: burak-58 Date: Wed, 4 Dec 2024 17:51:14 +0300 Subject: [PATCH 01/15] add segment file name format setting --- src/main/java/io/antmedia/AppSettings.java | 16 ++++++++++++ src/main/java/io/antmedia/muxer/HLSMuxer.java | 25 +++++++++---------- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/antmedia/AppSettings.java b/src/main/java/io/antmedia/AppSettings.java index a5cac1ea7..b67ee89d7 100644 --- a/src/main/java/io/antmedia/AppSettings.java +++ b/src/main/java/io/antmedia/AppSettings.java @@ -1144,6 +1144,14 @@ public class AppSettings implements Serializable{ @Value( "${hlsSegmentType:mpegts}" ) private String hlsSegmentType = "mpegts"; + + /** + * HLS segment file name format. It can be like file-%Y%m%d-%s.ts + * + */ + @Value( "${hlsSegmentFileNameFormat:}" ) + private String hlsSegmentFileNameFormat = ""; + /** * The path for manually saved used VoDs * Determines the directory to store VOD files. @@ -3923,6 +3931,14 @@ public void setHlsSegmentType(String hlsSegmentType) { this.hlsSegmentType = hlsSegmentType; } + public String getHlsSegmentFileNameFormat() { + return hlsSegmentFileNameFormat; + } + + public void setHlsSegmentFileNameFormat(String hlsSegmentFileNameFormat) { + this.hlsSegmentFileNameFormat = hlsSegmentFileNameFormat; + } + public String getRecordingSubfolder() { return recordingSubfolder; } diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 2c6db0c6d..2be01ebb4 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -147,36 +147,35 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold logger.info("hls time:{}, hls list size:{} hls playlist type:{} for stream:{}", hlsTime, hlsListSize, this.hlsPlayListType, streamId); - if (StringUtils.isNotBlank(httpEndpoint)) - { + if (StringUtils.isNotBlank(httpEndpoint)) { segmentFilename = httpEndpoint; segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; segmentFilename += (this.subFolder != null ? subFolder : ""); segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; - segmentFilename += initialResourceNameWithoutExtension; - } - else - { + segmentFilename += initialResourceNameWithoutExtension; + } else { segmentFilename = file.getParentFile().toString(); segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; segmentFilename += initialResourceNameWithoutExtension; + if(!StringUtils.isBlank(getAppSettings().getHlsSegmentFileNameFormat())) { + options.put("strftime", "1"); + segmentFilename += getAppSettings().getHlsSegmentFileNameFormat(); + } } - + //remove double slashes with single slash because it may cause problems segmentFilename = replaceDoubleSlashesWithSingleSlash(segmentFilename); - + options.put("hls_segment_type", hlsSegmentType); if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - + segmentInitFilename = initialResourceNameWithoutExtension + "_init.mp4"; options.put("hls_fmp4_init_filename", segmentInitFilename); segmentFilename += SEGMENT_SUFFIX_FMP4; - } - else { //if it's mpegts + } else { //if it's mpegts segmentFilename += SEGMENT_SUFFIX_TS; } - - + options.put("hls_segment_filename", segmentFilename); if (hlsPlayListType != null && (hlsPlayListType.equals("event") || hlsPlayListType.equals("vod"))) From 586b8ce73318240092dca8d6cacba9a5156725f5 Mon Sep 17 00:00:00 2001 From: burak Date: Sun, 8 Dec 2024 23:00:16 +0300 Subject: [PATCH 02/15] Merge remote-tracking branch 'origin/master' into hlsSegmentFileNameFormat --- .../io/antmedia/integration/MuxingTest.java | 58 +++++++++++++++++++ .../java/io/antmedia/test/MuxerUnitTest.java | 6 ++ 2 files changed, 64 insertions(+) diff --git a/src/test/java/io/antmedia/integration/MuxingTest.java b/src/test/java/io/antmedia/integration/MuxingTest.java index 83d6f58da..0c809b8a8 100644 --- a/src/test/java/io/antmedia/integration/MuxingTest.java +++ b/src/test/java/io/antmedia/integration/MuxingTest.java @@ -808,6 +808,64 @@ public static byte[] getByteArray(String address) { } return null; } + + @Test + public void testHLSSegmentFileName() { + + try { + ConsoleAppRestServiceTest.resetCookieStore(); + Result result = ConsoleAppRestServiceTest.callisFirstLogin(); + if (result.isSuccess()) { + Result createInitialUser = ConsoleAppRestServiceTest.createDefaultInitialUser(); + assertTrue(createInitialUser.isSuccess()); + } + + result = ConsoleAppRestServiceTest.authenticateDefaultUser(); + assertTrue(result.isSuccess()); + AppSettings appSettings = ConsoleAppRestServiceTest.callGetAppSettings("LiveApp"); + boolean hlsEnabled = appSettings.isHlsMuxingEnabled(); + appSettings.setHlsMuxingEnabled(true); + String hlsSegmentFileNameFormat = appSettings.getHlsSegmentFileNameFormat(); + appSettings.setHlsSegmentFileNameFormat("-%Y%m%d-%s-"); + result = ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings); + assertTrue(result.isSuccess()); + + // send rtmp stream with ffmpeg to red5 + String streamName = "live_test" + (int)(Math.random() * 999999); + + // make sure that ffmpeg is installed and in path + Process rtmpSendingProcess = execute( + ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://" + + SERVER_ADDR + "/LiveApp/" + streamName); + + try { + Process finalProcess = rtmpSendingProcess; + Awaitility.await().pollDelay(5, TimeUnit.SECONDS).atMost(10, TimeUnit.SECONDS).until(()-> { + return finalProcess.isAlive(); + }); + } + catch (Exception e) { + //try one more time because it may give high resource usage + rtmpSendingProcess = execute( + ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://" + + SERVER_ADDR + "/LiveApp/" + streamName); + } + + + + Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + return MuxingTest.testFile("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); + }); + + + appSettings.setHlsMuxingEnabled(hlsEnabled); + appSettings.setHlsSegmentFileNameFormat(hlsSegmentFileNameFormat); + ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } diff --git a/src/test/java/io/antmedia/test/MuxerUnitTest.java b/src/test/java/io/antmedia/test/MuxerUnitTest.java index 65cc334fc..d72a8f5ab 100644 --- a/src/test/java/io/antmedia/test/MuxerUnitTest.java +++ b/src/test/java/io/antmedia/test/MuxerUnitTest.java @@ -3955,6 +3955,12 @@ public void testHLSNaming() { hlsMuxer.init(appScope, "test", 300, "", 400000); assertEquals("./webapps/junit/streams/test_300p400kbps%09d.ts", hlsMuxer.getSegmentFilename()); + getAppSettings().setHlsSegmentFileNameFormat("-"); + hlsMuxer = new HLSMuxer(vertx, Mockito.mock(StorageClient.class), "", 7, null, false); + hlsMuxer.init(appScope, "test", 0, "", 0); + assertEquals("./webapps/junit/streams/test-%Y%m%d-%s-%09d.ts", hlsMuxer.getSegmentFilename()); + + } public void testHLSMuxing(String name) { From 5102dd0787734fbd9c79911e9ac67e23b9db97ae Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 10 Dec 2024 07:17:11 +0300 Subject: [PATCH 03/15] make segment file name suffix configurable --- src/main/java/io/antmedia/AppSettings.java | 16 +++++----- src/main/java/io/antmedia/muxer/HLSMuxer.java | 31 ++++++++++--------- .../io/antmedia/integration/MuxingTest.java | 4 +-- .../java/io/antmedia/test/MuxerUnitTest.java | 4 +-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/main/java/io/antmedia/AppSettings.java b/src/main/java/io/antmedia/AppSettings.java index dd929ea09..18ccc5cd7 100644 --- a/src/main/java/io/antmedia/AppSettings.java +++ b/src/main/java/io/antmedia/AppSettings.java @@ -1148,11 +1148,13 @@ public class AppSettings implements Serializable{ /** - * HLS segment file name format. It can be like file-%Y%m%d-%s.ts + * HLS segment file suffix format. + * By default: %09d which means 9 digit incremental + * To add time: It can be like %Y%m%d-%s * */ - @Value( "${hlsSegmentFileNameFormat:}" ) - private String hlsSegmentFileNameFormat = ""; + @Value( "${hlsSegmentFileSuffixFormat:%09d}" ) + private String hlsSegmentFileSuffixFormat = "%09d"; /** * The path for manually saved used VoDs @@ -3936,12 +3938,12 @@ public void setHlsSegmentType(String hlsSegmentType) { this.hlsSegmentType = hlsSegmentType; } - public String getHlsSegmentFileNameFormat() { - return hlsSegmentFileNameFormat; + public String getHlsSegmentFileSuffixFormat() { + return hlsSegmentFileSuffixFormat; } - public void setHlsSegmentFileNameFormat(String hlsSegmentFileNameFormat) { - this.hlsSegmentFileNameFormat = hlsSegmentFileNameFormat; + public void setHlsSegmentFileNameFormat(String hlsSegmentFileSuffixFormat) { + this.hlsSegmentFileSuffixFormat = hlsSegmentFileSuffixFormat; } public String getRecordingSubfolder() { diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 2be01ebb4..2810ec12e 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -29,13 +29,9 @@ public class HLSMuxer extends Muxer { public static final String SEI_USER_DATA = "sei_user_data"; - private static final String TS_EXTENSION = "ts"; - private static final String FMP4_EXTENSION = "fmp4"; + private static final String TS_EXTENSION = ".ts"; + private static final String FMP4_EXTENSION = ".fmp4"; - private static final String SEGMENT_SUFFIX_TS = "%0"+SEGMENT_INDEX_LENGTH+"d." + TS_EXTENSION; - //DASH also has m4s and ChunkTransferServlet is responsbile for streaming m4s files, so it's better to use fmp4 here - private static final String SEGMENT_SUFFIX_FMP4 = "%0"+SEGMENT_INDEX_LENGTH+"d."+ FMP4_EXTENSION; - private static final String HLS_SEGMENT_TYPE_MPEGTS = "mpegts"; private static final String HLS_SEGMENT_TYPE_FMP4 = "fmp4"; @@ -80,6 +76,8 @@ public class HLSMuxer extends Muxer { private AVPacket tmpPacketForSEI; + private String segmentFileNameSuffix; + public HLSMuxer(Vertx vertx, StorageClient storageClient, String s3StreamsFolderPath, int uploadExtensionsToS3, String httpEndpoint, boolean addDateTimeToResourceName) { super(vertx); this.storageClient = storageClient; @@ -157,11 +155,15 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold segmentFilename = file.getParentFile().toString(); segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; segmentFilename += initialResourceNameWithoutExtension; - if(!StringUtils.isBlank(getAppSettings().getHlsSegmentFileNameFormat())) { - options.put("strftime", "1"); - segmentFilename += getAppSettings().getHlsSegmentFileNameFormat(); - } } + + segmentFileNameSuffix = getAppSettings().getHlsSegmentFileSuffixFormat(); + + if(segmentFileNameSuffix.contains("%s") || segmentFileNameSuffix.contains("%Y") || segmentFileNameSuffix.contains("%m")) { + options.put("strftime", "1"); + } + + segmentFilename += getAppSettings().getHlsSegmentFileSuffixFormat(); //remove double slashes with single slash because it may cause problems segmentFilename = replaceDoubleSlashesWithSingleSlash(segmentFilename); @@ -171,10 +173,11 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold segmentInitFilename = initialResourceNameWithoutExtension + "_init.mp4"; options.put("hls_fmp4_init_filename", segmentInitFilename); - segmentFilename += SEGMENT_SUFFIX_FMP4; + segmentFilename += FMP4_EXTENSION; } else { //if it's mpegts - segmentFilename += SEGMENT_SUFFIX_TS; + segmentFilename += TS_EXTENSION; } + options.put("hls_segment_filename", segmentFilename); @@ -371,10 +374,10 @@ public synchronized void writeTrailer() { int indexOfSuffix = 0; if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_FMP4); + indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); } else { - indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_TS); + indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); } String segmentFileWithoutSuffix = segmentFilename.substring(segmentFilename.lastIndexOf("/")+1, indexOfSuffix); diff --git a/src/test/java/io/antmedia/integration/MuxingTest.java b/src/test/java/io/antmedia/integration/MuxingTest.java index 0c809b8a8..1c9a1a1a6 100644 --- a/src/test/java/io/antmedia/integration/MuxingTest.java +++ b/src/test/java/io/antmedia/integration/MuxingTest.java @@ -825,8 +825,8 @@ public void testHLSSegmentFileName() { AppSettings appSettings = ConsoleAppRestServiceTest.callGetAppSettings("LiveApp"); boolean hlsEnabled = appSettings.isHlsMuxingEnabled(); appSettings.setHlsMuxingEnabled(true); - String hlsSegmentFileNameFormat = appSettings.getHlsSegmentFileNameFormat(); - appSettings.setHlsSegmentFileNameFormat("-%Y%m%d-%s-"); + String hlsSegmentFileNameFormat = appSettings.getHlsSegmentFileSuffixFormat(); + appSettings.setHlsSegmentFileNameFormat("-%Y%m%d-%s"); result = ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings); assertTrue(result.isSuccess()); diff --git a/src/test/java/io/antmedia/test/MuxerUnitTest.java b/src/test/java/io/antmedia/test/MuxerUnitTest.java index d72a8f5ab..68d57edb0 100644 --- a/src/test/java/io/antmedia/test/MuxerUnitTest.java +++ b/src/test/java/io/antmedia/test/MuxerUnitTest.java @@ -3955,10 +3955,10 @@ public void testHLSNaming() { hlsMuxer.init(appScope, "test", 300, "", 400000); assertEquals("./webapps/junit/streams/test_300p400kbps%09d.ts", hlsMuxer.getSegmentFilename()); - getAppSettings().setHlsSegmentFileNameFormat("-"); + getAppSettings().setHlsSegmentFileNameFormat("-%Y%m%d-%s"); hlsMuxer = new HLSMuxer(vertx, Mockito.mock(StorageClient.class), "", 7, null, false); hlsMuxer.init(appScope, "test", 0, "", 0); - assertEquals("./webapps/junit/streams/test-%Y%m%d-%s-%09d.ts", hlsMuxer.getSegmentFilename()); + assertEquals("./webapps/junit/streams/test-%Y%m%d-%s.ts", hlsMuxer.getSegmentFilename()); } From 6017583fff1fc46fbcba395865ecf3296fb53af6 Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 10 Dec 2024 13:04:02 +0300 Subject: [PATCH 04/15] fix dot from extension --- src/main/java/io/antmedia/muxer/HLSMuxer.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 2810ec12e..72e400e7c 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -29,8 +29,9 @@ public class HLSMuxer extends Muxer { public static final String SEI_USER_DATA = "sei_user_data"; - private static final String TS_EXTENSION = ".ts"; - private static final String FMP4_EXTENSION = ".fmp4"; + private static final String LETTER_DOT = "."; + private static final String TS_EXTENSION = "ts"; + private static final String FMP4_EXTENSION = "fmp4"; private static final String HLS_SEGMENT_TYPE_MPEGTS = "mpegts"; private static final String HLS_SEGMENT_TYPE_FMP4 = "fmp4"; @@ -168,6 +169,7 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold //remove double slashes with single slash because it may cause problems segmentFilename = replaceDoubleSlashesWithSingleSlash(segmentFilename); + segmentFilename += LETTER_DOT; options.put("hls_segment_type", hlsSegmentType); if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { @@ -381,7 +383,7 @@ public synchronized void writeTrailer() { } String segmentFileWithoutSuffix = segmentFilename.substring(segmentFilename.lastIndexOf("/")+1, indexOfSuffix); - String regularExpression = segmentFileWithoutSuffix + "[0-9]*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$"; + String regularExpression = segmentFileWithoutSuffix + ".*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$"; File[] files = getHLSFilesInDirectory(regularExpression); if (files != null) From cf37eac172bb75a39f359530967a9a9ea57590b8 Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 10 Dec 2024 15:34:28 +0300 Subject: [PATCH 05/15] fix test case --- src/test/java/io/antmedia/test/AppSettingsUnitTest.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/antmedia/test/AppSettingsUnitTest.java b/src/test/java/io/antmedia/test/AppSettingsUnitTest.java index 9c74ce83a..c255666e0 100644 --- a/src/test/java/io/antmedia/test/AppSettingsUnitTest.java +++ b/src/test/java/io/antmedia/test/AppSettingsUnitTest.java @@ -643,6 +643,10 @@ public void testUnsetAppSettings(AppSettings appSettings) { assertEquals("", appSettings.getSubFolder()); appSettings.setSubFolder("test/folder"); assertEquals("test/folder", appSettings.getSubFolder()); + + assertEquals("%09d", appSettings.getHlsSegmentFileSuffixFormat()); + appSettings.setHlsSegmentFileNameFormat("%s"); + assertEquals("%s", appSettings.getHlsSegmentFileSuffixFormat()); //if we add a new field, we just need to check its default value in this test @@ -650,7 +654,7 @@ public void testUnsetAppSettings(AppSettings appSettings) { //by also checking its default value. assertEquals("New field is added to settings. PAY ATTENTION: Please CHECK ITS DEFAULT VALUE and fix the number of fields.", - 193, numberOfFields); + 194, numberOfFields); } From 2aad09616aa8c9f6e84dbd1c50829046fa207494 Mon Sep 17 00:00:00 2001 From: burak Date: Tue, 10 Dec 2024 22:21:26 +0300 Subject: [PATCH 06/15] fix test case --- src/test/java/io/antmedia/integration/MuxingTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/test/java/io/antmedia/integration/MuxingTest.java b/src/test/java/io/antmedia/integration/MuxingTest.java index 1c9a1a1a6..c2a90825f 100644 --- a/src/test/java/io/antmedia/integration/MuxingTest.java +++ b/src/test/java/io/antmedia/integration/MuxingTest.java @@ -857,6 +857,11 @@ public void testHLSSegmentFileName() { return MuxingTest.testFile("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); }); + rtmpSendingProcess.destroyForcibly(); + + Awaitility.await().atMost(15, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + return null == RestServiceV2Test.getBroadcast(streamName); + }); appSettings.setHlsMuxingEnabled(hlsEnabled); appSettings.setHlsSegmentFileNameFormat(hlsSegmentFileNameFormat); From 6061d0660a8d95a58c1c67e9fd6fb54c0e4f799d Mon Sep 17 00:00:00 2001 From: burak Date: Wed, 11 Dec 2024 09:18:11 +0300 Subject: [PATCH 07/15] refactor datastore factories --- .../console/datastore/ConsoleDataStoreFactory.java | 8 ++++---- .../java/io/antmedia/datastore/db/DataStoreFactory.java | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java index 2c1aa4de8..e5a88decc 100644 --- a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java +++ b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java @@ -6,6 +6,7 @@ import org.springframework.context.ApplicationContextAware; import io.antmedia.AppSettings; +import io.antmedia.datastore.db.DataStoreFactory; import io.antmedia.muxer.IAntMediaStreamHandler; import io.vertx.core.Vertx; @@ -97,16 +98,15 @@ public void setApplicationContext(ApplicationContext applicationContext) throws public AbstractConsoleDataStore getDataStore() { if (dataStore == null) { - if(dbType.contentEquals("mongodb")) + if(DataStoreFactory.DB_TYPE_MONGODB.contentEquals(dbType)) { - dataStore = new MongoStore(dbHost, dbUser, dbPassword); } - else if(dbType.contentEquals("mapdb")) + else if(DataStoreFactory.DB_TYPE_MAPDB.contentEquals(dbType)) { dataStore = new MapDBStore(vertx); } - else if(dbType.contentEquals("redisdb")) + else if(DataStoreFactory.DB_TYPE_REDISDB.contentEquals(dbType)) { dataStore = new RedisStore(dbHost); } diff --git a/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java b/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java index f11f4cf0d..08d2150bf 100644 --- a/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java +++ b/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java @@ -97,19 +97,19 @@ public void setDbPassword(String dbPassword) { public void init() { - if(dbType.contentEquals(DB_TYPE_MONGODB)) + if(DB_TYPE_MONGODB.contentEquals(dbType)) { dataStore = new MongoStore(dbHost, dbUser, dbPassword, dbName); } - else if(dbType .contentEquals(DB_TYPE_MAPDB)) + else if(DB_TYPE_MAPDB .contentEquals(dbType)) { dataStore = new MapDBStore(dbName+".db", vertx); } - else if(dbType .contentEquals(DB_TYPE_REDISDB)) + else if(DB_TYPE_REDISDB .contentEquals(dbType)) { dataStore = new RedisStore(dbHost, dbName); } - else if(dbType .contentEquals(DB_TYPE_MEMORYDB)) + else if(DB_TYPE_MEMORYDB .contentEquals(dbType)) { dataStore = new InMemoryDataStore(dbName); } From 5ca5ebee8afd1f99733b1e83f78c850ea2159bd9 Mon Sep 17 00:00:00 2001 From: burak Date: Wed, 18 Dec 2024 16:22:25 +0300 Subject: [PATCH 08/15] update test and comment --- src/main/java/io/antmedia/AppSettings.java | 3 ++ .../io/antmedia/integration/MuxingTest.java | 46 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/main/java/io/antmedia/AppSettings.java b/src/main/java/io/antmedia/AppSettings.java index 2bdfa4506..f419d2d46 100644 --- a/src/main/java/io/antmedia/AppSettings.java +++ b/src/main/java/io/antmedia/AppSettings.java @@ -1150,6 +1150,9 @@ public class AppSettings implements Serializable{ * HLS segment file suffix format. * By default: %09d which means 9 digit incremental * To add time: It can be like %Y%m%d-%s + * If you want to use both incrementing numbers and date together + * - Please use double % for the incrementing number suffix like: %s-%%09d + * - +second_level_segment_index to HLS flags * */ @Value( "${hlsSegmentFileSuffixFormat:%09d}" ) diff --git a/src/test/java/io/antmedia/integration/MuxingTest.java b/src/test/java/io/antmedia/integration/MuxingTest.java index c2a90825f..2cf751fad 100644 --- a/src/test/java/io/antmedia/integration/MuxingTest.java +++ b/src/test/java/io/antmedia/integration/MuxingTest.java @@ -13,15 +13,22 @@ import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q; import static org.junit.Assert.*; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ProcessHandle.Info; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import io.antmedia.AntMediaApplicationAdapter; import io.antmedia.AppSettings; @@ -857,6 +864,22 @@ public void testHLSSegmentFileName() { return MuxingTest.testFile("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); }); + + String content = getM3U8Content("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); + + + long now = System.currentTimeMillis(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd"); + String formattedTime = formatter.format(new Date(now)); + + //(now/10000) we can not guarantee we will have a ts created just now so use regex like live_test873835-20241218-173452XX.ts + String regex = streamName+"-"+formattedTime+"-"+(now/100000) + "\\d{2}\\.ts"; + System.out.println("regex for ts name:"+regex); + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(content); + assertTrue (matcher.find()); + rtmpSendingProcess.destroyForcibly(); Awaitility.await().atMost(15, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { @@ -872,6 +895,29 @@ public void testHLSSegmentFileName() { } } + private String getM3U8Content(String urlString) throws Exception { + URL url = new URL(urlString); + + // Open a connection and create a BufferedReader + BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); + + // Read the URL content into a StringBuilder + StringBuilder content = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + + // Close the reader + reader.close(); + + // Print the content + System.out.println("URL Content:"); + System.out.println(content.toString()); + + return content.toString(); + } + } From ce23dbec266f3565a1d21c7d41a6f80f8548733c Mon Sep 17 00:00:00 2001 From: burak Date: Thu, 19 Dec 2024 10:18:03 +0300 Subject: [PATCH 09/15] add unit tests --- .../datastore/ConsoleDataStoreFactory.java | 7 ++ .../db/ConsoleDataStoreFactoryUnitTest.java | 96 +++++++++++++++++++ 2 files changed, 103 insertions(+) create mode 100644 src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java diff --git a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java index e5a88decc..851396ed7 100644 --- a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java +++ b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java @@ -1,5 +1,7 @@ package io.antmedia.console.datastore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; @@ -44,6 +46,8 @@ public class ConsoleDataStoreFactory implements ApplicationContextAware { private Vertx vertx; + private static Logger logger = LoggerFactory.getLogger(ConsoleDataStoreFactory.class); + public String getAppName() { return appName; } @@ -110,6 +114,9 @@ else if(DataStoreFactory.DB_TYPE_REDISDB.contentEquals(dbType)) { dataStore = new RedisStore(dbHost); } + else { + logger.error("Undefined Console Datastore:{} db name:{}", dbType, dbName); + } } return dataStore; } diff --git a/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java new file mode 100644 index 000000000..60c984c6f --- /dev/null +++ b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java @@ -0,0 +1,96 @@ +package io.antmedia.test.db; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.context.ApplicationContext; + +import io.antmedia.console.datastore.AbstractConsoleDataStore; +import io.antmedia.console.datastore.ConsoleDataStoreFactory; +import io.antmedia.console.datastore.MapDBStore; +import io.antmedia.console.datastore.MongoStore; +import io.antmedia.console.datastore.RedisStore; +import io.antmedia.datastore.db.DataStoreFactory; +import io.antmedia.muxer.IAntMediaStreamHandler; +import io.vertx.core.Vertx; + +public class ConsoleDataStoreFactoryUnitTest { + + private ConsoleDataStoreFactory consoleDataStoreFactory; + + @Mock + private ApplicationContext applicationContext; + + @Mock + private Vertx vertx; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + consoleDataStoreFactory = new ConsoleDataStoreFactory(); + when(applicationContext.getBean(IAntMediaStreamHandler.VERTX_BEAN_NAME)).thenReturn(vertx); + } + + @Test + public void testSetAndGetDbName() { + String dbName = "testDbName"; + consoleDataStoreFactory.setDbName(dbName); + assertEquals("DB name should be set correctly", dbName, consoleDataStoreFactory.getDbName()); + } + + @Test + public void testSetAndGetDbType() { + String dbType = DataStoreFactory.DB_TYPE_MONGODB; + consoleDataStoreFactory.setDbType(dbType); + assertEquals("DB type should be set correctly", dbType, consoleDataStoreFactory.getDbType()); + } + + @Test + public void testGetDataStoreMongoDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_MONGODB); + consoleDataStoreFactory.setDbHost("127.0.0.1"); + consoleDataStoreFactory.setDbUser(null); + consoleDataStoreFactory.setDbPassword("password"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type MongoStore", dataStore instanceof MongoStore); + } + + @Test + public void testGetDataStoreMapDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_MAPDB); + + consoleDataStoreFactory.setApplicationContext(applicationContext); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type MapDBStore", dataStore instanceof MapDBStore); + } + + @Test + public void testGetDataStoreRedisDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_REDISDB); + consoleDataStoreFactory.setDbHost("127.0.0.1"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type RedisStore", dataStore instanceof RedisStore); + } + + @Test + public void testGetDataStoreUndefined() { + consoleDataStoreFactory.setDbType("undefined_db_type"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNull("DataStore should be null for undefined DB type", dataStore); + } +} From 5a5c609014b90d2af2fa6adc9297c82d6bfd7bf1 Mon Sep 17 00:00:00 2001 From: burak Date: Thu, 19 Dec 2024 14:21:47 +0300 Subject: [PATCH 10/15] update tests --- .../io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java index 60c984c6f..fc67f228c 100644 --- a/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java +++ b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java @@ -74,12 +74,14 @@ public void testGetDataStoreMapDB() { AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); assertNotNull("DataStore should not be null", dataStore); assertTrue("DataStore should be of type MapDBStore", dataStore instanceof MapDBStore); + dataStore.clear(); + dataStore.close(); } @Test public void testGetDataStoreRedisDB() { consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_REDISDB); - consoleDataStoreFactory.setDbHost("127.0.0.1"); + consoleDataStoreFactory.setDbHost("redis://127.0.0.1:6379"); AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); assertNotNull("DataStore should not be null", dataStore); From 3799870429113b094eaf2a25292fbebdf032e4f5 Mon Sep 17 00:00:00 2001 From: burak Date: Thu, 19 Dec 2024 19:53:09 +0300 Subject: [PATCH 11/15] fix sonar issues --- src/main/java/io/antmedia/muxer/HLSMuxer.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 72e400e7c..30f53f753 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -375,12 +375,7 @@ public synchronized void writeTrailer() { //convert segmentFileName to regular expression int indexOfSuffix = 0; - if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); - } - else { - indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); - } + indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); String segmentFileWithoutSuffix = segmentFilename.substring(segmentFilename.lastIndexOf("/")+1, indexOfSuffix); String regularExpression = segmentFileWithoutSuffix + ".*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$"; From afcabdc0190ff80a23928e6d9fab836ef20566f1 Mon Sep 17 00:00:00 2001 From: burak Date: Sat, 21 Dec 2024 23:13:52 +0300 Subject: [PATCH 12/15] add unick name to init mp4 --- src/main/java/io/antmedia/muxer/HLSMuxer.java | 2 +- src/test/java/io/antmedia/test/MuxerUnitTest.java | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 30f53f753..05c37c34d 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -173,7 +173,7 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold options.put("hls_segment_type", hlsSegmentType); if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - segmentInitFilename = initialResourceNameWithoutExtension + "_init.mp4"; + segmentInitFilename = initialResourceNameWithoutExtension + "_" + System.currentTimeMillis() + "_init.mp4"; options.put("hls_fmp4_init_filename", segmentInitFilename); segmentFilename += FMP4_EXTENSION; } else { //if it's mpegts diff --git a/src/test/java/io/antmedia/test/MuxerUnitTest.java b/src/test/java/io/antmedia/test/MuxerUnitTest.java index 68d57edb0..01241377c 100644 --- a/src/test/java/io/antmedia/test/MuxerUnitTest.java +++ b/src/test/java/io/antmedia/test/MuxerUnitTest.java @@ -625,10 +625,16 @@ public void testHEVCHLSMuxingInFMP4() { //check the init file and m4s files there assertTrue(hlsMuxer.getFile().exists()); - assertTrue(new File(hlsMuxer.getFile().getParentFile()+ "/" + streamId + "_init.mp4").exists()); - assertTrue(new File(hlsMuxer.getFile().getParentFile()+ "/" + streamId + "000000003.fmp4").exists()); - - + String[] filesInStreams = hlsMuxer.getFile().getParentFile().list(); + boolean initFileFound = false; + String regex = streamId + "_" + System.currentTimeMillis()/10000 + "\\d{4}_init.mp4"; + System.out.println("regex:"+regex); + + for (int i = 0; i < filesInStreams.length; i++) { + System.out.println("files:"+filesInStreams[i]); + initFileFound |= filesInStreams[i].matches(regex); + } + assertTrue(initFileFound); assertTrue(MuxingTest.testFile(hlsMuxer.getFile().getAbsolutePath(), 107000)); assertEquals(0, hlsMuxer.getAudioNotWrittenCount()); From 57df156c07a0b96b0fe0fb21578900a9cda352e6 Mon Sep 17 00:00:00 2001 From: burak Date: Mon, 23 Dec 2024 09:57:54 +0300 Subject: [PATCH 13/15] fix test case --- .../io/antmedia/test/StreamFetcherUnitTest.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java index ba6a9a264..011ebc3f5 100644 --- a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java +++ b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java @@ -971,8 +971,17 @@ public void testHLSSourceFmp4() { //test HLS Source String streamId = testFetchStreamSources("src/test/resources/test.m3u8", false, false, true, "fmp4"); - File f = new File("webapps/junit/streams/"+streamId +"_init.mp4"); - assertTrue(f.exists()); + String[] filesInStreams = new File("webapps/junit/streams").list(); + boolean initFileFound = false; + String regex = streamId + "_" + System.currentTimeMillis()/10000 + "\\d{4}_init.mp4"; + System.out.println("regex:"+regex); + + for (int i = 0; i < filesInStreams.length; i++) { + System.out.println("files:"+filesInStreams[i]); + initFileFound |= filesInStreams[i].matches(regex); + } + assertTrue(initFileFound); + logger.info("leaving testHLSSource"); } From a05a7cba0ccd458e3cf12f081503909e5ad1207e Mon Sep 17 00:00:00 2001 From: burak Date: Wed, 25 Dec 2024 10:29:19 +0300 Subject: [PATCH 14/15] update owasp surppression --- owasp-suppressions.xml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/owasp-suppressions.xml b/owasp-suppressions.xml index a790b9854..6d6568a2d 100644 --- a/owasp-suppressions.xml +++ b/owasp-suppressions.xml @@ -123,4 +123,14 @@ ^pkg:maven/org\.apache\.tomcat/tomcat-coyote@10\.1\.19$ CVE-2024-34750 + + + ^pkg:maven/org\.apache\.tomcat/tomcat-catalina@10\.1\.34$ + CVE-2024-56337 + + + From bb3ff4e348e6b56afcf9dabfef7343602e031113 Mon Sep 17 00:00:00 2001 From: burak Date: Wed, 25 Dec 2024 21:25:23 +0300 Subject: [PATCH 15/15] fix test case --- src/test/java/io/antmedia/test/StreamFetcherUnitTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java index 011ebc3f5..37ba9e3b7 100644 --- a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java +++ b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java @@ -973,7 +973,7 @@ public void testHLSSourceFmp4() { String[] filesInStreams = new File("webapps/junit/streams").list(); boolean initFileFound = false; - String regex = streamId + "_" + System.currentTimeMillis()/10000 + "\\d{4}_init.mp4"; + String regex = streamId + "_" + System.currentTimeMillis()/100000 + "\\d{5}_init.mp4"; System.out.println("regex:"+regex); for (int i = 0; i < filesInStreams.length; i++) {