Gzip fix

binders/difido-testng/src/main/java/il/co/topq/difido/config/RemoteDifidoConfig.java

@@ -21,7 +21,11 @@ public enum RemoteDifidoOptions implements ConfigOptions {
 		USE_SHARED_EXECUTION("use.shared.execution", "false"),
 		EXISTING_EXECUTION_ID("existing.execution.id", "-1"),
 		FORCE_NEW_EXECUTION("force.new.execution", "false"),
-		APPEND_TO_EXISTING_EXECUTION("append.to.existing.execution", "false");
+		APPEND_TO_EXISTING_EXECUTION("append.to.existing.execution", "false"),
+		COMPRESS_FILES_ABOVE("compress.files.above","5000"),
+		DONT_COMPRESS_EXTENSIONS("dont.compress.extensions","7z;zip;rar;gzip;gz;jpg;jpeg;png;gif;avi;xvid;mp4;mp3;tif;tiff;pdf;wmf;svg;exe;jar");
+
+
 		// @formatter:on
 
 		private String property;

binders/difido-testng/src/main/java/il/co/topq/difido/reporters/DifidoClient.java

@@ -13,6 +13,7 @@
 import org.apache.commons.httpclient.methods.PutMethod;
 import org.apache.commons.httpclient.methods.RequestEntity;
 import org.apache.commons.httpclient.methods.StringRequestEntity;
+import org.apache.commons.httpclient.methods.multipart.ByteArrayPartSource;
 import org.apache.commons.httpclient.methods.multipart.FilePart;
 import org.apache.commons.httpclient.methods.multipart.MultipartRequestEntity;
 import org.apache.commons.httpclient.methods.multipart.Part;
@@ -83,9 +84,18 @@ public void addTestDetails(int executionId, TestDetails testDetails) throws Exce
 		handleResponseCode(method, responseCode);
 	}
 
+
+	public void addFile(final int executionId, final String uid, final byte[] bytes, String fileName) throws Exception {
+		Part[] parts = new Part[] { new FilePart("file", new ByteArrayPartSource(fileName, bytes))};
+		addFile(executionId,uid,parts);
+	}
 	public void addFile(final int executionId, final String uid, final File file) throws Exception {
-		PostMethod method = new PostMethod(baseUri + "executions/" + executionId + "/details/" + uid + "/file/");
 		Part[] parts = new Part[] { new FilePart("file", file) };
+		addFile(executionId,uid,parts);
+	}
+
+	private void addFile(final int executionId, final String uid, final Part[] parts) throws Exception {
+		PostMethod method = new PostMethod(baseUri + "executions/" + executionId + "/details/" + uid + "/file/");
 		method.setRequestEntity(new MultipartRequestEntity(parts, method.getParams()));
 		final int responseCode = client.executeMethod(method);
 		handleResponseCode(method, responseCode);

binders/difido-testng/src/main/java/il/co/topq/difido/reporters/RemoteDifidoReporter.java

@@ -2,13 +2,20 @@
 
 import java.io.File;
 import java.util.ArrayList;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.logging.Logger;
 
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
 import org.testng.ISuite;
 
+import il.co.topq.difido.ZipUtils;
+import il.co.topq.difido.config.DifidoConfig;
 import il.co.topq.difido.config.RemoteDifidoConfig;
+import il.co.topq.difido.config.DifidoConfig.DifidoOptions;
 import il.co.topq.difido.config.RemoteDifidoConfig.RemoteDifidoOptions;
 import il.co.topq.difido.model.execution.Execution;
 import il.co.topq.difido.model.execution.ScenarioNode;
@@ -34,6 +41,8 @@ public class RemoteDifidoReporter extends AbstractDifidoReporter {
 	private RemoteDifidoConfig difidoConfig;
 
 	private ExecutionDetails details;
+
+	private Set<String> extentionsToSkip = null;
 
 	/**
 	 * When files are added in the setup phase, there is no test context and no
@@ -200,11 +209,54 @@ public void addFile(File file) {
 
 	private void sendFileToServer(File file) {
 		try {
-			client.addFile(executionId, getTestDetails().getUid(), file);
+
+			int thresholdInBytes = difidoConfig.getPropertyAsInt(RemoteDifidoOptions.COMPRESS_FILES_ABOVE);
+			if (thresholdInBytes > 0 && file.length() > thresholdInBytes && isCompressable(file)){
+
+				//zip the file to an in-memory byte array and upload to server 
+				//adding .gz to the original fileName;
+				byte[] zippped = ZipUtils.gzipToBytesArray(file);
+				if (null != zippped){
+					client.addFile(executionId, getTestDetails().getUid(), zippped, file.getName().concat(".gz"));
+				}
+				else {
+					log.warning("Failed to zip file on the fly, uploading original");
+					client.addFile(executionId, getTestDetails().getUid(), file);
+				}
+			}
+			else {
+				client.addFile(executionId, getTestDetails().getUid(), file);
+			}
 		} catch (Exception e) {
 			log.warning("Failed uploading file " + file.getName() + " to remote server due to " + e.getMessage());
 		}
 	}
+
+	/**
+	 * checks whether the file extension is 'blacklisted' for compression
+	 */
+	private boolean isCompressable(File f){
+		String ext = FilenameUtils.getExtension(f.getAbsolutePath()).toLowerCase();
+		return !getSkippedExtensions().contains(ext);
+	}
+
+	private Set<String> getSkippedExtensions(){
+		initSkippedExtensions();
+		return this.extentionsToSkip;
+	}
+
+	private void initSkippedExtensions(){
+		if (this.extentionsToSkip != null)
+			return;
+
+		this.extentionsToSkip = new LinkedHashSet<>();
+		List<String> extentionsList = difidoConfig.getPropertyAsList(RemoteDifidoOptions.DONT_COMPRESS_EXTENSIONS);
+		for (String extension : extentionsList){
+			int startFrom = extension.lastIndexOf(".") + 1; //strip off any irrelevant file parts a user may have entered (e.g. tar.gz, or *.exe)
+			extentionsToSkip.add(extension.toLowerCase().substring(startFrom));
+		}
+	}
+
 
 	/**
 	 * Elements that are created in setup phases, before test context is created

binders/difido-testng/src/test/java/il/co/topq/difido/DifidoReporterTests.java

@@ -11,6 +11,7 @@
 
 import javax.imageio.ImageIO;
 
+import org.apache.commons.io.FileUtils;
 import org.testng.annotations.Test;
 
 import il.co.topq.difido.model.Enums.Status;
@@ -77,6 +78,33 @@ public void testAddFile() {
 		File file = new File("pom.xml");
 		report.addFile(file, "This is the file");
 	}
+
+	@Test 
+	public void testAddBigFile(){
+		try {
+			File file = File.createTempFile("big", ".log");
+			StringBuilder sb = new StringBuilder();
+			for (int i=0;i<1000000;i++){
+				sb.append("line: #" + i).append("\n");
+			}
+			FileUtils.write(file, sb.toString());
+
+
+			report.addFile(file, "Really big file");
+
+		} catch (Exception e) {
+			// TODO: handle exception
+		}
+
+	}
+
+	@Test
+	public void testAddImageFile() throws Exception{
+		//Reporter.log("This file should be uploaded uncompressed");
+		File image = new File(DifidoReporterTests.class.getResource("/login.png").toURI());
+		report.log("File: " + image.getAbsolutePath() + " exists()=" + image.exists());
+		report.addFile(image, "This file should be uncompressed on the server");
+	}
 
 	@Test(description = "Adding screenshot to the report")
 	public void testAddScreenshot() throws IOException, AWTException {

difido-reports-common/src/main/java/il/co/topq/difido/ZipUtils.java

@@ -1,15 +1,23 @@
 package il.co.topq.difido;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.io.IOException;
 import java.io.InputStream;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Enumeration;
 import java.util.List;
 import java.util.jar.JarEntry;
 import java.util.jar.JarFile;
+import java.util.zip.GZIPOutputStream;
 
-class ZipUtils {
+import org.apache.commons.io.FileUtils;
+
+public class ZipUtils {
 
 	/**
 	 * uncompress all the files in the specified zipFile to the specified
@@ -51,4 +59,101 @@ static List<File> decopmerss(String zipFile, String outputFolder, String filter)
 
 	}
 
+	/**
+	 * Gzips the originalFile and returns a zipped one.
+	 * @param originalFile
+	 * @return
+	 */
+	public static File gzip(File originalFile) {
+		if (originalFile == null || !originalFile.exists())
+			return null;
+
+
+		File zippedFile = getZippedFile(originalFile);
+		if (zippedFile == null)
+			return null;
+
+		try (FileInputStream input = new FileInputStream(originalFile); 
+			 FileOutputStream output = new FileOutputStream(zippedFile);
+			 GZIPOutputStream gzipOS = new GZIPOutputStream(output)) {
+
+			byte[] buffer = new byte[1024];
+			int len;
+			while((len= input.read(buffer)) != -1){
+				gzipOS.write(buffer, 0, len);
+			}
+
+			gzipOS.close();
+			output.close();
+			input.close();
+
+			return zippedFile;
+		}
+
+		catch(Exception e){
+			e.printStackTrace();
+		}
+
+		return originalFile;
+
+
+	}
+
+	/**
+	 * Gzipps the given file and returns it as byte[] 
+	 * @param file
+	 * @return
+	 */
+	public static byte[] gzipToBytesArray(File file){
+
+		try(FileInputStream input = new FileInputStream(file);
+				ByteArrayOutputStream output = new ByteArrayOutputStream();
+				GZIPOutputStream gzipOS = new GZIPOutputStream(output);){
+
+			byte[] buffer = new byte[1024];
+			int len;
+			while((len= input.read(buffer)) != -1){
+				gzipOS.write(buffer, 0, len);
+			}
+
+
+			gzipOS.flush();
+			output.flush();
+			gzipOS.close();
+			output.close();
+
+			return output.toByteArray();
+
+		} catch (FileNotFoundException e) {
+			e.printStackTrace();
+		} catch (IOException e) {
+			e.printStackTrace();
+		}
+
+
+		return null;
+
+	}
+
+
+	/**
+	 * Returns a file with .gz appended.
+	 * 
+	 * @param originalFile
+	 * @return
+	 */
+	private static File getZippedFile(File originalFile){
+		try{	
+			return new File(originalFile.getAbsolutePath().concat(".gz"));
+
+
+		} catch (Exception e) {
+			e.printStackTrace();
+			return null;
+		}
+
+
+
+
+	}
 }

difido-reports-common/src/main/resources/il.co.topq.difido.view/tree.html

@@ -85,7 +85,21 @@ <h3>Test Information</h3></div>
     $(function() {
         $("#leftpane").resizable({
             minWidth: 250,
-            maxWidth: 600
+            maxWidth: 600, 
+             //iframes interfere with resize because they  consume  
+            //our mouse events and ruin our beautiful resize. 
+            //so we must hide the iframes before the resize and 
+            //unhide when done. 
+            start: function () {
+                $("iframe").each(function (index, element) {
+                    $(element).hide();
+                });
+
+            },
+            stop: function () {
+                $('iframe').show();
+            }
+
         });
     });
 

server/difido-server/config/difido_config.properties

@@ -6,6 +6,7 @@ max.execution.idle.time.in.seconds=600
 
 # The maximum time in days that the server will keep HTML reports. 
 # After the specified period the HTML reports will be deleted but if Elastic is enabled they will be kept in the Elasticsearch
+# 0 means never. 
 days.to.keep.html.reports=0
 
 # The location of the HTML folder that includes the server pages and the HTML reports
@@ -18,6 +19,20 @@ custom.execution.properties=
 enable.html.reports=true
 
 
+# Will enable us to only store compressed (.gz) version of resources specified by {compress.extensions}
+# automatic gzip decoding is supported by all modern browsers and this feature should be enabled  
+# (as it saves about 95% of disk space) unless you have good reasons not to. 
+enable.compressed.resources=true
+
+# This setting is meaningful only if enable.compressed.resources=true
+# After how many days do you want the reports to be compressed: -1 means: as soon as possible (i.e the execution is no longer active) 
+compress.html.after.x.days=-1
+
+# make sure the extensions are lowercase, separated by semicolon
+compress.extensions=log;txt;html;js;css
+
+
+
 # List of plugin classes
 plugin.classes=il.co.topq.report.plugins.mail.DefaultMailPlugin
 

server/difido-server/src/main/java/il/co/topq/report/Configuration.java

@@ -30,6 +30,9 @@ public enum ConfigProps {
 		ELASTIC_TRANSPORT_TCP_PORT("elastic.transport.tcp.port","9300"),
 		ENABLE_HTML_REPORTS("enable.html.reports", "true"), 
 		DAYS_TO_KEEP_HTML_REPORTS("days.to.keep.html.reports","0"),
+		ENABLE_COMPRESSED_RESOURCES("enable.compressed.resources","true"),
+		COMPRESS_AFTER_X_DAYS("compress.html.after.x.days","-1"),
+		EXTENSIONS_TO_ARCHIVE("compress.extensions","log;txt;html;js;css"),
 		EXTERNAL_LINKS("external.links",""),
 		ENABLE_MAIL("enable.mail", "false"), MAIL_USER_NAME("mail.user.name",""), 
 		MAIL_PASSWORD("mail.password", ""),
@@ -51,6 +54,7 @@ public enum ConfigProps {
 		LAST_REPORTS_INTERVAL_IN_SEC("last.reports.interval.in.sec", "10"),
 		LAST_REPORTS_NUM_OF_EXECUTIONS("last.reports.num.of.executions", "4"),
 		LAST_REPORTS_FILTER("last.reports.filter","");
+
 		// @formatter:off
 
 		private final String propName;
@@ -140,7 +144,7 @@ public int readInt(ConfigProps prop) {
 	}
 
 	public List<String> readList(ConfigProps prop) {
-		final String value = configProperties.getProperty(prop.getPropName());
+		final String value = readString(prop);
 		if (StringUtils.isEmpty(value)) {
 			return new ArrayList<String>();
 		}