diff --git a/owasp-suppressions.xml b/owasp-suppressions.xml index a790b9854..6d6568a2d 100644 --- a/owasp-suppressions.xml +++ b/owasp-suppressions.xml @@ -123,4 +123,14 @@ ^pkg:maven/org\.apache\.tomcat/tomcat-coyote@10\.1\.19$ CVE-2024-34750 + + + ^pkg:maven/org\.apache\.tomcat/tomcat-catalina@10\.1\.34$ + CVE-2024-56337 + + + diff --git a/src/main/java/io/antmedia/AppSettings.java b/src/main/java/io/antmedia/AppSettings.java index 1b39c89a4..26403eed8 100644 --- a/src/main/java/io/antmedia/AppSettings.java +++ b/src/main/java/io/antmedia/AppSettings.java @@ -1145,6 +1145,19 @@ public class AppSettings implements Serializable{ @Value( "${hlsSegmentType:mpegts}" ) private String hlsSegmentType = "mpegts"; + + /** + * HLS segment file suffix format. + * By default: %09d which means 9 digit incremental + * To add time: It can be like %Y%m%d-%s + * If you want to use both incrementing numbers and date together + * - Please use double % for the incrementing number suffix like: %s-%%09d + * - +second_level_segment_index to HLS flags + * + */ + @Value( "${hlsSegmentFileSuffixFormat:%09d}" ) + private String hlsSegmentFileSuffixFormat = "%09d"; + /** * The path for manually saved used VoDs * Determines the directory to store VOD files. @@ -3984,6 +3997,14 @@ public void setHlsSegmentType(String hlsSegmentType) { this.hlsSegmentType = hlsSegmentType; } + public String getHlsSegmentFileSuffixFormat() { + return hlsSegmentFileSuffixFormat; + } + + public void setHlsSegmentFileNameFormat(String hlsSegmentFileSuffixFormat) { + this.hlsSegmentFileSuffixFormat = hlsSegmentFileSuffixFormat; + } + public String getRecordingSubfolder() { return recordingSubfolder; } diff --git a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java index 2c1aa4de8..851396ed7 100644 --- a/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java +++ b/src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java @@ -1,11 +1,14 @@ package io.antmedia.console.datastore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.BeansException; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; import io.antmedia.AppSettings; +import io.antmedia.datastore.db.DataStoreFactory; import io.antmedia.muxer.IAntMediaStreamHandler; import io.vertx.core.Vertx; @@ -43,6 +46,8 @@ public class ConsoleDataStoreFactory implements ApplicationContextAware { private Vertx vertx; + private static Logger logger = LoggerFactory.getLogger(ConsoleDataStoreFactory.class); + public String getAppName() { return appName; } @@ -97,19 +102,21 @@ public void setApplicationContext(ApplicationContext applicationContext) throws public AbstractConsoleDataStore getDataStore() { if (dataStore == null) { - if(dbType.contentEquals("mongodb")) + if(DataStoreFactory.DB_TYPE_MONGODB.contentEquals(dbType)) { - dataStore = new MongoStore(dbHost, dbUser, dbPassword); } - else if(dbType.contentEquals("mapdb")) + else if(DataStoreFactory.DB_TYPE_MAPDB.contentEquals(dbType)) { dataStore = new MapDBStore(vertx); } - else if(dbType.contentEquals("redisdb")) + else if(DataStoreFactory.DB_TYPE_REDISDB.contentEquals(dbType)) { dataStore = new RedisStore(dbHost); } + else { + logger.error("Undefined Console Datastore:{} db name:{}", dbType, dbName); + } } return dataStore; } diff --git a/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java b/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java index f11f4cf0d..08d2150bf 100644 --- a/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java +++ b/src/main/java/io/antmedia/datastore/db/DataStoreFactory.java @@ -97,19 +97,19 @@ public void setDbPassword(String dbPassword) { public void init() { - if(dbType.contentEquals(DB_TYPE_MONGODB)) + if(DB_TYPE_MONGODB.contentEquals(dbType)) { dataStore = new MongoStore(dbHost, dbUser, dbPassword, dbName); } - else if(dbType .contentEquals(DB_TYPE_MAPDB)) + else if(DB_TYPE_MAPDB .contentEquals(dbType)) { dataStore = new MapDBStore(dbName+".db", vertx); } - else if(dbType .contentEquals(DB_TYPE_REDISDB)) + else if(DB_TYPE_REDISDB .contentEquals(dbType)) { dataStore = new RedisStore(dbHost, dbName); } - else if(dbType .contentEquals(DB_TYPE_MEMORYDB)) + else if(DB_TYPE_MEMORYDB .contentEquals(dbType)) { dataStore = new InMemoryDataStore(dbName); } diff --git a/src/main/java/io/antmedia/muxer/HLSMuxer.java b/src/main/java/io/antmedia/muxer/HLSMuxer.java index 2c6db0c6d..05c37c34d 100644 --- a/src/main/java/io/antmedia/muxer/HLSMuxer.java +++ b/src/main/java/io/antmedia/muxer/HLSMuxer.java @@ -29,13 +29,10 @@ public class HLSMuxer extends Muxer { public static final String SEI_USER_DATA = "sei_user_data"; + private static final String LETTER_DOT = "."; private static final String TS_EXTENSION = "ts"; private static final String FMP4_EXTENSION = "fmp4"; - private static final String SEGMENT_SUFFIX_TS = "%0"+SEGMENT_INDEX_LENGTH+"d." + TS_EXTENSION; - //DASH also has m4s and ChunkTransferServlet is responsbile for streaming m4s files, so it's better to use fmp4 here - private static final String SEGMENT_SUFFIX_FMP4 = "%0"+SEGMENT_INDEX_LENGTH+"d."+ FMP4_EXTENSION; - private static final String HLS_SEGMENT_TYPE_MPEGTS = "mpegts"; private static final String HLS_SEGMENT_TYPE_FMP4 = "fmp4"; @@ -80,6 +77,8 @@ public class HLSMuxer extends Muxer { private AVPacket tmpPacketForSEI; + private String segmentFileNameSuffix; + public HLSMuxer(Vertx vertx, StorageClient storageClient, String s3StreamsFolderPath, int uploadExtensionsToS3, String httpEndpoint, boolean addDateTimeToResourceName) { super(vertx); this.storageClient = storageClient; @@ -147,36 +146,41 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold logger.info("hls time:{}, hls list size:{} hls playlist type:{} for stream:{}", hlsTime, hlsListSize, this.hlsPlayListType, streamId); - if (StringUtils.isNotBlank(httpEndpoint)) - { + if (StringUtils.isNotBlank(httpEndpoint)) { segmentFilename = httpEndpoint; segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; segmentFilename += (this.subFolder != null ? subFolder : ""); segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; - segmentFilename += initialResourceNameWithoutExtension; - } - else - { + segmentFilename += initialResourceNameWithoutExtension; + } else { segmentFilename = file.getParentFile().toString(); segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : ""; segmentFilename += initialResourceNameWithoutExtension; } + segmentFileNameSuffix = getAppSettings().getHlsSegmentFileSuffixFormat(); + + if(segmentFileNameSuffix.contains("%s") || segmentFileNameSuffix.contains("%Y") || segmentFileNameSuffix.contains("%m")) { + options.put("strftime", "1"); + } + + segmentFilename += getAppSettings().getHlsSegmentFileSuffixFormat(); + //remove double slashes with single slash because it may cause problems segmentFilename = replaceDoubleSlashesWithSingleSlash(segmentFilename); - + + segmentFilename += LETTER_DOT; options.put("hls_segment_type", hlsSegmentType); if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - - segmentInitFilename = initialResourceNameWithoutExtension + "_init.mp4"; + + segmentInitFilename = initialResourceNameWithoutExtension + "_" + System.currentTimeMillis() + "_init.mp4"; options.put("hls_fmp4_init_filename", segmentInitFilename); - segmentFilename += SEGMENT_SUFFIX_FMP4; - } - else { //if it's mpegts - segmentFilename += SEGMENT_SUFFIX_TS; + segmentFilename += FMP4_EXTENSION; + } else { //if it's mpegts + segmentFilename += TS_EXTENSION; } - + options.put("hls_segment_filename", segmentFilename); if (hlsPlayListType != null && (hlsPlayListType.equals("event") || hlsPlayListType.equals("vod"))) @@ -371,15 +375,10 @@ public synchronized void writeTrailer() { //convert segmentFileName to regular expression int indexOfSuffix = 0; - if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) { - indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_FMP4); - } - else { - indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_TS); - } + indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix); String segmentFileWithoutSuffix = segmentFilename.substring(segmentFilename.lastIndexOf("/")+1, indexOfSuffix); - String regularExpression = segmentFileWithoutSuffix + "[0-9]*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$"; + String regularExpression = segmentFileWithoutSuffix + ".*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$"; File[] files = getHLSFilesInDirectory(regularExpression); if (files != null) diff --git a/src/test/java/io/antmedia/integration/MuxingTest.java b/src/test/java/io/antmedia/integration/MuxingTest.java index 83d6f58da..2cf751fad 100644 --- a/src/test/java/io/antmedia/integration/MuxingTest.java +++ b/src/test/java/io/antmedia/integration/MuxingTest.java @@ -13,15 +13,22 @@ import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q; import static org.junit.Assert.*; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.lang.ProcessHandle.Info; import java.net.HttpURLConnection; +import java.net.MalformedURLException; import java.net.URL; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.Iterator; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import io.antmedia.AntMediaApplicationAdapter; import io.antmedia.AppSettings; @@ -808,6 +815,108 @@ public static byte[] getByteArray(String address) { } return null; } + + @Test + public void testHLSSegmentFileName() { + + try { + ConsoleAppRestServiceTest.resetCookieStore(); + Result result = ConsoleAppRestServiceTest.callisFirstLogin(); + if (result.isSuccess()) { + Result createInitialUser = ConsoleAppRestServiceTest.createDefaultInitialUser(); + assertTrue(createInitialUser.isSuccess()); + } + + result = ConsoleAppRestServiceTest.authenticateDefaultUser(); + assertTrue(result.isSuccess()); + AppSettings appSettings = ConsoleAppRestServiceTest.callGetAppSettings("LiveApp"); + boolean hlsEnabled = appSettings.isHlsMuxingEnabled(); + appSettings.setHlsMuxingEnabled(true); + String hlsSegmentFileNameFormat = appSettings.getHlsSegmentFileSuffixFormat(); + appSettings.setHlsSegmentFileNameFormat("-%Y%m%d-%s"); + result = ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings); + assertTrue(result.isSuccess()); + + // send rtmp stream with ffmpeg to red5 + String streamName = "live_test" + (int)(Math.random() * 999999); + + // make sure that ffmpeg is installed and in path + Process rtmpSendingProcess = execute( + ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://" + + SERVER_ADDR + "/LiveApp/" + streamName); + + try { + Process finalProcess = rtmpSendingProcess; + Awaitility.await().pollDelay(5, TimeUnit.SECONDS).atMost(10, TimeUnit.SECONDS).until(()-> { + return finalProcess.isAlive(); + }); + } + catch (Exception e) { + //try one more time because it may give high resource usage + rtmpSendingProcess = execute( + ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://" + + SERVER_ADDR + "/LiveApp/" + streamName); + } + + + + Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + return MuxingTest.testFile("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); + }); + + + String content = getM3U8Content("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8"); + + + long now = System.currentTimeMillis(); + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd"); + String formattedTime = formatter.format(new Date(now)); + + //(now/10000) we can not guarantee we will have a ts created just now so use regex like live_test873835-20241218-173452XX.ts + String regex = streamName+"-"+formattedTime+"-"+(now/100000) + "\\d{2}\\.ts"; + System.out.println("regex for ts name:"+regex); + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(content); + assertTrue (matcher.find()); + + rtmpSendingProcess.destroyForcibly(); + + Awaitility.await().atMost(15, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> { + return null == RestServiceV2Test.getBroadcast(streamName); + }); + + appSettings.setHlsMuxingEnabled(hlsEnabled); + appSettings.setHlsSegmentFileNameFormat(hlsSegmentFileNameFormat); + ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings); + } catch (Exception e) { + e.printStackTrace(); + fail(e.getMessage()); + } + } + + private String getM3U8Content(String urlString) throws Exception { + URL url = new URL(urlString); + + // Open a connection and create a BufferedReader + BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream())); + + // Read the URL content into a StringBuilder + StringBuilder content = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + content.append(line).append("\n"); + } + + // Close the reader + reader.close(); + + // Print the content + System.out.println("URL Content:"); + System.out.println(content.toString()); + + return content.toString(); + } diff --git a/src/test/java/io/antmedia/test/AppSettingsUnitTest.java b/src/test/java/io/antmedia/test/AppSettingsUnitTest.java index 7eaf1a255..1f7dd9a0c 100644 --- a/src/test/java/io/antmedia/test/AppSettingsUnitTest.java +++ b/src/test/java/io/antmedia/test/AppSettingsUnitTest.java @@ -646,6 +646,10 @@ public void testUnsetAppSettings(AppSettings appSettings) { assertEquals("test/folder", appSettings.getSubFolder()); assertFalse(appSettings.isWriteSubscriberEventsToDatastore()); + + assertEquals("%09d", appSettings.getHlsSegmentFileSuffixFormat()); + appSettings.setHlsSegmentFileNameFormat("%s"); + assertEquals("%s", appSettings.getHlsSegmentFileSuffixFormat()); appSettings.setAppStatus(AppSettings.APPLICATION_STATUS_INSTALLED); @@ -664,8 +668,7 @@ public void testUnsetAppSettings(AppSettings appSettings) { //by also checking its default value. assertEquals("New field is added to settings. PAY ATTENTION: Please CHECK ITS DEFAULT VALUE and fix the number of fields.", - 197, numberOfFields); - + 198, numberOfFields); } diff --git a/src/test/java/io/antmedia/test/MuxerUnitTest.java b/src/test/java/io/antmedia/test/MuxerUnitTest.java index 1c127e884..002553f8e 100644 --- a/src/test/java/io/antmedia/test/MuxerUnitTest.java +++ b/src/test/java/io/antmedia/test/MuxerUnitTest.java @@ -626,10 +626,16 @@ public void testHEVCHLSMuxingInFMP4() { //check the init file and m4s files there assertTrue(hlsMuxer.getFile().exists()); - assertTrue(new File(hlsMuxer.getFile().getParentFile()+ "/" + streamId + "_init.mp4").exists()); - assertTrue(new File(hlsMuxer.getFile().getParentFile()+ "/" + streamId + "000000003.fmp4").exists()); - - + String[] filesInStreams = hlsMuxer.getFile().getParentFile().list(); + boolean initFileFound = false; + String regex = streamId + "_" + System.currentTimeMillis()/10000 + "\\d{4}_init.mp4"; + System.out.println("regex:"+regex); + + for (int i = 0; i < filesInStreams.length; i++) { + System.out.println("files:"+filesInStreams[i]); + initFileFound |= filesInStreams[i].matches(regex); + } + assertTrue(initFileFound); assertTrue(MuxingTest.testFile(hlsMuxer.getFile().getAbsolutePath(), 107000)); assertEquals(0, hlsMuxer.getAudioNotWrittenCount()); @@ -4000,6 +4006,12 @@ public void testHLSNaming() { hlsMuxer.init(appScope, "test", 300, "", 400000); assertEquals("./webapps/junit/streams/test_300p400kbps%09d.ts", hlsMuxer.getSegmentFilename()); + getAppSettings().setHlsSegmentFileNameFormat("-%Y%m%d-%s"); + hlsMuxer = new HLSMuxer(vertx, Mockito.mock(StorageClient.class), "", 7, null, false); + hlsMuxer.init(appScope, "test", 0, "", 0); + assertEquals("./webapps/junit/streams/test-%Y%m%d-%s.ts", hlsMuxer.getSegmentFilename()); + + } public void testHLSMuxing(String name) { diff --git a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java index ba6a9a264..37ba9e3b7 100644 --- a/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java +++ b/src/test/java/io/antmedia/test/StreamFetcherUnitTest.java @@ -971,8 +971,17 @@ public void testHLSSourceFmp4() { //test HLS Source String streamId = testFetchStreamSources("src/test/resources/test.m3u8", false, false, true, "fmp4"); - File f = new File("webapps/junit/streams/"+streamId +"_init.mp4"); - assertTrue(f.exists()); + String[] filesInStreams = new File("webapps/junit/streams").list(); + boolean initFileFound = false; + String regex = streamId + "_" + System.currentTimeMillis()/100000 + "\\d{5}_init.mp4"; + System.out.println("regex:"+regex); + + for (int i = 0; i < filesInStreams.length; i++) { + System.out.println("files:"+filesInStreams[i]); + initFileFound |= filesInStreams[i].matches(regex); + } + assertTrue(initFileFound); + logger.info("leaving testHLSSource"); } diff --git a/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java new file mode 100644 index 000000000..fc67f228c --- /dev/null +++ b/src/test/java/io/antmedia/test/db/ConsoleDataStoreFactoryUnitTest.java @@ -0,0 +1,98 @@ +package io.antmedia.test.db; + + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.context.ApplicationContext; + +import io.antmedia.console.datastore.AbstractConsoleDataStore; +import io.antmedia.console.datastore.ConsoleDataStoreFactory; +import io.antmedia.console.datastore.MapDBStore; +import io.antmedia.console.datastore.MongoStore; +import io.antmedia.console.datastore.RedisStore; +import io.antmedia.datastore.db.DataStoreFactory; +import io.antmedia.muxer.IAntMediaStreamHandler; +import io.vertx.core.Vertx; + +public class ConsoleDataStoreFactoryUnitTest { + + private ConsoleDataStoreFactory consoleDataStoreFactory; + + @Mock + private ApplicationContext applicationContext; + + @Mock + private Vertx vertx; + + @Before + public void setUp() { + MockitoAnnotations.openMocks(this); + consoleDataStoreFactory = new ConsoleDataStoreFactory(); + when(applicationContext.getBean(IAntMediaStreamHandler.VERTX_BEAN_NAME)).thenReturn(vertx); + } + + @Test + public void testSetAndGetDbName() { + String dbName = "testDbName"; + consoleDataStoreFactory.setDbName(dbName); + assertEquals("DB name should be set correctly", dbName, consoleDataStoreFactory.getDbName()); + } + + @Test + public void testSetAndGetDbType() { + String dbType = DataStoreFactory.DB_TYPE_MONGODB; + consoleDataStoreFactory.setDbType(dbType); + assertEquals("DB type should be set correctly", dbType, consoleDataStoreFactory.getDbType()); + } + + @Test + public void testGetDataStoreMongoDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_MONGODB); + consoleDataStoreFactory.setDbHost("127.0.0.1"); + consoleDataStoreFactory.setDbUser(null); + consoleDataStoreFactory.setDbPassword("password"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type MongoStore", dataStore instanceof MongoStore); + } + + @Test + public void testGetDataStoreMapDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_MAPDB); + + consoleDataStoreFactory.setApplicationContext(applicationContext); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type MapDBStore", dataStore instanceof MapDBStore); + dataStore.clear(); + dataStore.close(); + } + + @Test + public void testGetDataStoreRedisDB() { + consoleDataStoreFactory.setDbType(DataStoreFactory.DB_TYPE_REDISDB); + consoleDataStoreFactory.setDbHost("redis://127.0.0.1:6379"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNotNull("DataStore should not be null", dataStore); + assertTrue("DataStore should be of type RedisStore", dataStore instanceof RedisStore); + } + + @Test + public void testGetDataStoreUndefined() { + consoleDataStoreFactory.setDbType("undefined_db_type"); + + AbstractConsoleDataStore dataStore = consoleDataStoreFactory.getDataStore(); + assertNull("DataStore should be null for undefined DB type", dataStore); + } +}