Merge branch 'master' into filteringOnHW

owasp-suppressions.xml

@@ -123,4 +123,14 @@
 	   <packageUrl regex="true">^pkg:maven/org\.apache\.tomcat/tomcat-coyote@10\.1\.19$</packageUrl>
 	   <cve>CVE-2024-34750</cve>
 	</suppress>
+	<suppress>
+	   <notes><![CDATA[
+	   file name: tomcat-catalina-10.1.34.jar
+	   This vulnerability seems to be fixed in 10.1.34 Otherwise, Ant Media Server works on Linux which is a case sensitve OS. So this doesn't affect the software.   
+	   ]]></notes>
+	   <packageUrl regex="true">^pkg:maven/org\.apache\.tomcat/tomcat-catalina@10\.1\.34$</packageUrl>
+	   <cve>CVE-2024-56337</cve>
+	</suppress>
+
+
 </suppressions>

src/main/java/io/antmedia/AppSettings.java

@@ -1145,6 +1145,19 @@ public class AppSettings implements Serializable{
 	@Value( "${hlsSegmentType:mpegts}" )
 	private String hlsSegmentType = "mpegts";
 
+
+	/**
+	 * HLS segment file suffix format. 
+	 * By default: %09d which means 9 digit incremental
+	 * To add time: It can be like %Y%m%d-%s
+	 * If you want to use both incrementing numbers and date together
+	 * - Please use double % for the incrementing number suffix like: %s-%%09d
+	 * - +second_level_segment_index to HLS flags
+	 *
+	 */
+	@Value( "${hlsSegmentFileSuffixFormat:%09d}" )
+	private String hlsSegmentFileSuffixFormat = "%09d";
+
 	/**
 	 * The path for manually saved used VoDs
 	 * Determines the directory to store VOD files.
@@ -3984,6 +3997,14 @@ public void setHlsSegmentType(String hlsSegmentType) {
 		this.hlsSegmentType = hlsSegmentType;
 	}
 
+	public String getHlsSegmentFileSuffixFormat() {
+		return hlsSegmentFileSuffixFormat;
+	}
+
+	public void setHlsSegmentFileNameFormat(String hlsSegmentFileSuffixFormat) {
+		this.hlsSegmentFileSuffixFormat = hlsSegmentFileSuffixFormat;
+	}
+
 	public String getRecordingSubfolder() {
 		return recordingSubfolder;
 	}

src/main/java/io/antmedia/console/datastore/ConsoleDataStoreFactory.java

@@ -1,11 +1,14 @@
 package io.antmedia.console.datastore;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.ApplicationContext;
 import org.springframework.context.ApplicationContextAware;
 
 import io.antmedia.AppSettings;
+import io.antmedia.datastore.db.DataStoreFactory;
 import io.antmedia.muxer.IAntMediaStreamHandler;
 import io.vertx.core.Vertx;
 
@@ -43,6 +46,8 @@ public class ConsoleDataStoreFactory implements ApplicationContextAware {
 
 	private Vertx vertx;
 
+	private static Logger logger = LoggerFactory.getLogger(ConsoleDataStoreFactory.class);
+
 	public String getAppName() {
 		return appName;
 	}
@@ -97,19 +102,21 @@ public void setApplicationContext(ApplicationContext applicationContext) throws
 
 	public AbstractConsoleDataStore getDataStore() {
 		if (dataStore == null) {
-			if(dbType.contentEquals("mongodb"))
+			if(DataStoreFactory.DB_TYPE_MONGODB.contentEquals(dbType))
 			{
-
 				dataStore = new MongoStore(dbHost, dbUser, dbPassword);
 			}
-			else if(dbType.contentEquals("mapdb"))
+			else if(DataStoreFactory.DB_TYPE_MAPDB.contentEquals(dbType))
 			{
 				dataStore = new MapDBStore(vertx);
 			}
-			else if(dbType.contentEquals("redisdb"))
+			else if(DataStoreFactory.DB_TYPE_REDISDB.contentEquals(dbType))
 			{
 				dataStore = new RedisStore(dbHost);
 			}
+			else {
+				logger.error("Undefined Console Datastore:{}  db name:{}", dbType, dbName);
+			}
 		}
 		return dataStore;
 	}

src/main/java/io/antmedia/datastore/db/DataStoreFactory.java

@@ -97,19 +97,19 @@ public void setDbPassword(String dbPassword) {
 
 	public void init()  
 	{
-		if(dbType.contentEquals(DB_TYPE_MONGODB))
+		if(DB_TYPE_MONGODB.contentEquals(dbType))
 		{
 			dataStore = new MongoStore(dbHost, dbUser, dbPassword, dbName);
 		}
-		else if(dbType .contentEquals(DB_TYPE_MAPDB))
+		else if(DB_TYPE_MAPDB .contentEquals(dbType))
 		{
 			dataStore = new MapDBStore(dbName+".db", vertx);
 		}
-		else if(dbType .contentEquals(DB_TYPE_REDISDB))
+		else if(DB_TYPE_REDISDB .contentEquals(dbType))
 		{
 			dataStore = new RedisStore(dbHost, dbName);
 		}
-		else if(dbType .contentEquals(DB_TYPE_MEMORYDB))
+		else if(DB_TYPE_MEMORYDB .contentEquals(dbType))
 		{
 			dataStore = new InMemoryDataStore(dbName);
 		}

src/main/java/io/antmedia/muxer/HLSMuxer.java

@@ -29,13 +29,10 @@ public class HLSMuxer extends Muxer  {
 
 	public static final String SEI_USER_DATA = "sei_user_data";
 
+	private static final String LETTER_DOT = ".";
 	private static final String TS_EXTENSION = "ts";
 	private static final String FMP4_EXTENSION = "fmp4";
 
-	private static final String SEGMENT_SUFFIX_TS = "%0"+SEGMENT_INDEX_LENGTH+"d." + TS_EXTENSION;
-	//DASH also has m4s and ChunkTransferServlet is responsbile for streaming m4s files, so it's better to use fmp4 here
-	private static final String SEGMENT_SUFFIX_FMP4 = "%0"+SEGMENT_INDEX_LENGTH+"d."+ FMP4_EXTENSION;
-
 	private static final String HLS_SEGMENT_TYPE_MPEGTS = "mpegts";
 	private static final String HLS_SEGMENT_TYPE_FMP4 = "fmp4";
 
@@ -80,6 +77,8 @@ public class HLSMuxer extends Muxer  {
 
 	private AVPacket tmpPacketForSEI;
 
+	private String segmentFileNameSuffix;
+
 	public HLSMuxer(Vertx vertx, StorageClient storageClient, String s3StreamsFolderPath, int uploadExtensionsToS3, String httpEndpoint, boolean addDateTimeToResourceName) {
 		super(vertx);
 		this.storageClient = storageClient;
@@ -147,36 +146,41 @@ public void init(IScope scope, String name, int resolutionHeight, String subFold
 
 			logger.info("hls time:{}, hls list size:{} hls playlist type:{} for stream:{}", hlsTime, hlsListSize, this.hlsPlayListType, streamId);
 
-			if (StringUtils.isNotBlank(httpEndpoint)) 			
-			{
+			if (StringUtils.isNotBlank(httpEndpoint)) {
 				segmentFilename = httpEndpoint;
 				segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : "";
 				segmentFilename += (this.subFolder != null ? subFolder : "");
 				segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : "";
-				segmentFilename += initialResourceNameWithoutExtension;	
-			}
-			else 
-			{
+				segmentFilename += initialResourceNameWithoutExtension;
+			} else {
 				segmentFilename = file.getParentFile().toString();
 				segmentFilename += !segmentFilename.endsWith(File.separator) ? File.separator : "";
 				segmentFilename += initialResourceNameWithoutExtension;
 			}
 
+			segmentFileNameSuffix = getAppSettings().getHlsSegmentFileSuffixFormat();
+
+			if(segmentFileNameSuffix.contains("%s") || segmentFileNameSuffix.contains("%Y") || segmentFileNameSuffix.contains("%m")) {
+				options.put("strftime", "1");
+			}
+
+			segmentFilename += getAppSettings().getHlsSegmentFileSuffixFormat();
+
 			//remove double slashes with single slash because it may cause problems
 			segmentFilename = replaceDoubleSlashesWithSingleSlash(segmentFilename);
-
+
+			segmentFilename += LETTER_DOT;
 			options.put("hls_segment_type", hlsSegmentType);
 			if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) {
-				
-				segmentInitFilename = initialResourceNameWithoutExtension + "_init.mp4";
+
+				segmentInitFilename = initialResourceNameWithoutExtension + "_" + System.currentTimeMillis() + "_init.mp4";
 				options.put("hls_fmp4_init_filename", segmentInitFilename);
-				segmentFilename += SEGMENT_SUFFIX_FMP4;
-			}
-			else { //if it's mpegts
-				segmentFilename += SEGMENT_SUFFIX_TS;
+				segmentFilename += FMP4_EXTENSION;
+			} else { //if it's mpegts
+				segmentFilename += TS_EXTENSION;
 			}
 
-					
+
 			options.put("hls_segment_filename", segmentFilename);
 
 			if (hlsPlayListType != null && (hlsPlayListType.equals("event") || hlsPlayListType.equals("vod"))) 
@@ -371,15 +375,10 @@ public synchronized void writeTrailer() {
 				//convert segmentFileName to regular expression
 
 				int indexOfSuffix = 0;
-				if (HLS_SEGMENT_TYPE_FMP4.equals(hlsSegmentType)) {
-					indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_FMP4);
-				}
-				else {
-					indexOfSuffix = segmentFilename.indexOf(SEGMENT_SUFFIX_TS);
-				}
+				indexOfSuffix = segmentFilename.indexOf(segmentFileNameSuffix);
 
 				String segmentFileWithoutSuffix = segmentFilename.substring(segmentFilename.lastIndexOf("/")+1, indexOfSuffix);
-				String regularExpression = segmentFileWithoutSuffix + "[0-9]*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$";
+				String regularExpression = segmentFileWithoutSuffix + ".*\\.(?:" + TS_EXTENSION +"|" + FMP4_EXTENSION +")$";
 				File[] files = getHLSFilesInDirectory(regularExpression);
 
 				if (files != null)

src/test/java/io/antmedia/integration/MuxingTest.java

@@ -13,15 +13,22 @@
 import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
 import static org.junit.Assert.*;
 
+import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.lang.ProcessHandle.Info;
 import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
 import java.net.URL;
+import java.text.SimpleDateFormat;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import io.antmedia.AntMediaApplicationAdapter;
 import io.antmedia.AppSettings;
@@ -808,6 +815,108 @@ public static byte[] getByteArray(String address) {
 		}
 		return null;
 	}
+
+	@Test
+	public void testHLSSegmentFileName() {
+
+		try {
+			ConsoleAppRestServiceTest.resetCookieStore();
+			Result result = ConsoleAppRestServiceTest.callisFirstLogin();
+			if (result.isSuccess()) {
+				Result createInitialUser = ConsoleAppRestServiceTest.createDefaultInitialUser();
+				assertTrue(createInitialUser.isSuccess());
+			}
+
+			result = ConsoleAppRestServiceTest.authenticateDefaultUser();
+			assertTrue(result.isSuccess());
+			AppSettings appSettings = ConsoleAppRestServiceTest.callGetAppSettings("LiveApp");
+			boolean hlsEnabled = appSettings.isHlsMuxingEnabled();
+			appSettings.setHlsMuxingEnabled(true);
+			String hlsSegmentFileNameFormat = appSettings.getHlsSegmentFileSuffixFormat();
+			appSettings.setHlsSegmentFileNameFormat("-%Y%m%d-%s");
+			result = ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings);
+			assertTrue(result.isSuccess());
+
+			// send rtmp stream with ffmpeg to red5
+			String streamName = "live_test"  + (int)(Math.random() * 999999);
+
+			// make sure that ffmpeg is installed and in path
+			Process rtmpSendingProcess = execute(
+					ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://"
+							+ SERVER_ADDR + "/LiveApp/" + streamName);
+
+			try {
+				Process finalProcess = rtmpSendingProcess;
+				Awaitility.await().pollDelay(5, TimeUnit.SECONDS).atMost(10, TimeUnit.SECONDS).until(()-> {
+					return finalProcess.isAlive();
+				});
+			}
+			catch (Exception e) {
+				//try one more time because it may give high resource usage
+				 rtmpSendingProcess = execute(
+							ffmpegPath + " -re -i src/test/resources/test.flv -acodec copy -vcodec copy -f flv rtmp://"
+									+ SERVER_ADDR + "/LiveApp/" + streamName);
+            }
+
+
+
+			Awaitility.await().atMost(30, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
+				return MuxingTest.testFile("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8");
+			});
+
+
+			String content = getM3U8Content("http://" + SERVER_ADDR + ":5080/LiveApp/streams/" + streamName+ ".m3u8");
+
+
+			long now = System.currentTimeMillis();
+	        SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMdd");
+	        String formattedTime = formatter.format(new Date(now));
+
+	        //(now/10000) we can not guarantee we will have a ts created just now so use regex like live_test873835-20241218-173452XX.ts
+	        String regex = streamName+"-"+formattedTime+"-"+(now/100000) + "\\d{2}\\.ts";
+	        System.out.println("regex for ts name:"+regex);
+
+			Pattern pattern = Pattern.compile(regex);
+	        Matcher matcher = pattern.matcher(content);
+	        assertTrue (matcher.find());
+
+			rtmpSendingProcess.destroyForcibly();
+
+			Awaitility.await().atMost(15, TimeUnit.SECONDS).pollInterval(1, TimeUnit.SECONDS).until(() -> {
+				return null == RestServiceV2Test.getBroadcast(streamName);
+			});
+
+			appSettings.setHlsMuxingEnabled(hlsEnabled);
+			appSettings.setHlsSegmentFileNameFormat(hlsSegmentFileNameFormat);
+			ConsoleAppRestServiceTest.callSetAppSettings("LiveApp", appSettings);
+		} catch (Exception e) {
+			e.printStackTrace();
+			fail(e.getMessage());
+		}
+	}
+
+	private String getM3U8Content(String urlString) throws Exception {
+		URL url = new URL(urlString);
+
+        // Open a connection and create a BufferedReader
+        BufferedReader reader = new BufferedReader(new InputStreamReader(url.openStream()));
+
+        // Read the URL content into a StringBuilder
+        StringBuilder content = new StringBuilder();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            content.append(line).append("\n");
+        }
+
+        // Close the reader
+        reader.close();
+
+        // Print the content
+        System.out.println("URL Content:");
+        System.out.println(content.toString());
+
+        return content.toString();
+	}
 
 
 

src/test/java/io/antmedia/test/AppSettingsUnitTest.java

@@ -646,6 +646,10 @@ public void testUnsetAppSettings(AppSettings appSettings) {
 		assertEquals("test/folder", appSettings.getSubFolder());
 
 		assertFalse(appSettings.isWriteSubscriberEventsToDatastore());
+
+		assertEquals("%09d", appSettings.getHlsSegmentFileSuffixFormat());
+		appSettings.setHlsSegmentFileNameFormat("%s");
+		assertEquals("%s", appSettings.getHlsSegmentFileSuffixFormat());
 
 
 		appSettings.setAppStatus(AppSettings.APPLICATION_STATUS_INSTALLED);
@@ -664,8 +668,7 @@ public void testUnsetAppSettings(AppSettings appSettings) {
 		//by also checking its default value. 
 
 		assertEquals("New field is added to settings. PAY ATTENTION: Please CHECK ITS DEFAULT VALUE and fix the number of fields.", 
-				197, numberOfFields);
-
+				198, numberOfFields);
 	}