Release 2021-08-13-1100

BEA#143 PIPELINE: data in archive BEA#147 SDB/BEV/BEI: JS file anti-cache timestamps BEA#366 NGCHM: html/ngchm? BEA#390 Data: some legacy data lack titles BEA#404 Docker: tomcat startup BEA#405 MBA: duplicate features BEA#407 MQA: next at last dataset BEA#408 MBatch: Add UMAP BEA#409 MQA: Add UMAP BEA#410 MQA: GUI Buttons BEA#411 MOB: More MW Data BEA#412 NGCHM: widget loading BEA#413 NGCHM/SuperClust: one only? BEA#414 MQA: Small Space Support BEA#415 NGCHM: annotation bar height BEA#417 ALL: WebInspect Remediation BEA#418 MtOB: allfactors support BEA#419 MQA: missing somatic mutation data BEA#420 Docker: PanDoc and TexLive revisit BEA#428 ALL: CentOS to AlmaLinux BEA#430 MQA: make query scroll BEA#433 BEV: resize results better BEA#434 BEV: no results message locks GUI BEA#435 MBatch: static PNG legend title BEA#436 LR: log report CI/CD BEA#437 MBatch: UTF-8 features and batch names BEA#439 BEV: fix histogram plots BEA#440 BEV: handle no results indexes
MD-Anderson-Bioinformatics · Aug 20, 2021 · e908cf9 · e908cf9
1 parent 82dee29
commit e908cf9
Show file tree

Hide file tree

Showing 24 changed files with 5,080 additions and 130 deletions.
diff --git a/apps/SamplesValidation/src/main/java/edu/mda/bcb/samval/matrix/BufferedRandomAccessFile.java b/apps/SamplesValidation/src/main/java/edu/mda/bcb/samval/matrix/BufferedRandomAccessFile.java
@@ -207,7 +207,8 @@ public final String getNextLine() throws IOException
 			}
 			return input.toString();
 		}
-		line = new String(this.buffer, 0, this.bufferPosition, lineEnd - this.bufferPosition);
+		//line = new String(this.buffer, 0, this.bufferPosition, lineEnd - this.bufferPosition);
+		line = new String(this.buffer, this.bufferPosition, lineEnd - this.bufferPosition);
 		this.bufferPosition = lineEnd;
 		return line;
 	}

diff --git a/apps/SamplesValidation/src/main/java/edu/mda/bcb/samval/matrix/Matrix.java b/apps/SamplesValidation/src/main/java/edu/mda/bcb/samval/matrix/Matrix.java
@@ -23,6 +23,7 @@
 import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -228,23 +229,26 @@ public void sort(ArrayList<Header> headers)
 		Collections.sort(headers, Header.HEADER_SORTED_ORDER);
 	}
 
-	public void write(String outPath) throws FileNotFoundException, IOException
+	public int write(String outPath) throws FileNotFoundException, IOException
 	{
-		this.write(outPath, this.delim, false, null, null);
+		int col = this.write(outPath, this.delim, false, null, null);
+		return col;
 	}
 
-	public void write(String outPath, boolean theCleanHeaders, String theOriginalColumn, String theNewColumn) throws FileNotFoundException, IOException
+	public int write(String outPath, boolean theCleanHeaders, String theOriginalColumn, String theNewColumn) throws FileNotFoundException, IOException
 	{
-		this.write(outPath, this.delim, theCleanHeaders, theOriginalColumn, theNewColumn);
+		int col = this.write(outPath, this.delim, theCleanHeaders, theOriginalColumn, theNewColumn);
+		return col;
 	}
 
+	public int write(String outPath, String delimiter, boolean theCleanHeaders, String theOriginalColumn, String theNewColumn) throws FileNotFoundException, IOException
+	{
 	/**
 	 * Write the matrix. Will write in sorted order if columns/rows were sorted.
 	 * Iterates on the ArrayList row/columns instances, but checks HashSet
 	 * rowSet/columnSet instances to see if an entry was filtered out.
 	 */
-	public void write(String outPath, String delimiter, boolean theCleanHeaders, String theOriginalColumn, String theNewColumn) throws FileNotFoundException, IOException
-	{
+		int colCount = 0;
 		BufferedRandomAccessFile braf = new BufferedRandomAccessFile(this.path, "r");
 		BufferedWriter brw = new BufferedWriter(new FileWriter(new File(outPath)));
 		String idCol = this.idHeader;
@@ -268,10 +272,12 @@ public void write(String outPath, String delimiter, boolean theCleanHeaders, Str
 				String colLabel = col.label;
 				if (theCleanHeaders)
 				{
+					colCount += 1;
 					brw.write(delimiter + checkname_contents(colLabel));
 				}
 				else
 				{
+					colCount += 1;
 					brw.write(delimiter + colLabel);
 				}
 			}
@@ -327,6 +333,7 @@ public void write(String outPath, String delimiter, boolean theCleanHeaders, Str
 			braf.close();
 			brw.close();
 		}
+		return colCount;
 	}
 
 	/**
@@ -392,6 +399,88 @@ public boolean removeRow(String row)
 	{
 		return this.rowSet.remove(row);
 	}
+
+	public void removeNonBatches(String theKeepColumn) throws IOException
+	{
+		BufferedRandomAccessFile braf = new BufferedRandomAccessFile(this.path, "r");
+		try
+		{
+			for (Header col : this.columns)
+			{
+				if (!col.label.equals(theKeepColumn))
+				{
+					HashMap<String, Integer> batchToCount = new HashMap<>();
+					for (Header row : this.rows)
+					{
+						if (row.index != null)
+						{
+							braf.seek((long) row.index);
+							String line = braf.getNextLine().replaceAll("\r", "").replaceAll("\n", "");
+							String[] toks = line.split(delim);
+							if (col.index != null)
+							{
+								String batch = toks[(int) col.index];
+								Integer count = batchToCount.get(batch);
+								if (null==count)
+								{
+									count = 0;
+								}
+								count += 1;
+								batchToCount.put(batch, count);
+							}
+						}
+					}
+					// remove non batch types
+					// first get total number of batches
+					int batchCount = batchToCount.size();
+					// check if any batches are "-" or ""
+					if (null!=batchToCount.get("-"))
+					{
+						// remove that from count
+						batchCount -= 1;
+					}
+					if (null!=batchToCount.get(""))
+					{
+						// remove that from count
+						batchCount -= 1;
+					}
+					if (batchCount<2)
+					{
+						// if only one batch, then remove column
+						this.removeColumn(col.label);
+					}
+					else if ((batchCount <= this.rows.size()) &&
+							 (batchCount > this.rows.size()*.9) )
+					{
+						// if number of batches equals number of samples or close to it, then remove column
+						this.removeColumn(col.label);
+					}
+					// count number of samples with non "-" and non "" batches
+					int samplesWithBatch = 0;
+					for (String batch : batchToCount.keySet())
+					{
+						if ((!"-".equals(batch))&&(!"".equals(batch)))
+						{
+							samplesWithBatch += batchToCount.get(batch);
+						}
+					}
+					// if less than 60% of samples have batches, remove column
+					if (samplesWithBatch < this.rows.size()*.6)
+					{
+						this.removeColumn(col.label);
+					}
+				}
+			}
+		}
+		catch (Exception e)
+		{
+			throw e;
+		}
+		finally
+		{
+			braf.close();
+		}
+	}
 
 	/**
 	 * Add a column. All column values for the added column will be set to the

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataDC.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataDC.java
@@ -72,10 +72,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet AnalysisDataDC returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting analysis data from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting analysis data from MetabolomicsWorkbench", exp);
+			log("AnalysisDataDC", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataMSC.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataMSC.java
@@ -72,10 +72,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet AnalysisDataMSC returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting analysis data from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting analysis data from MetabolomicsWorkbench", exp);
+			log("AnalysisDataMSC", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataRaw.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisDataRaw.java
@@ -72,10 +72,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet AnalysisDataRaw returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting analysis data from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting analysis data from MetabolomicsWorkbench", exp);
+			log("AnalysisDataRaw", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisTableData.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/AnalysisTableData.java
@@ -63,10 +63,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet AnalysisTableData returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting AnalysisTableData from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting AnalysisTableData from MetabolomicsWorkbench", exp);
+			log("AnalysisTableData", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/Factors.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/Factors.java
@@ -68,10 +68,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet Factors returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting list of factors from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting list of factors from MetabolomicsWorkbench", exp);
+			log("Factors", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/MetaboliteMap.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/MetaboliteMap.java
@@ -83,8 +83,9 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 		}
 		catch (Exception exp)
 		{
-			log("Error getting MetaboliteMap from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting MetaboliteMap from MetabolomicsWorkbench", exp);
+			log("MetaboliteMap", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/Studies.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/Studies.java
@@ -55,10 +55,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet Studies returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting list of summaries from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting list of summaries from MetabolomicsWorkbench", exp);
+			log("Studies", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/ZipConversion.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/ZipConversion.java
@@ -61,6 +61,7 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 				RefMetUtil refmetUtil = (RefMetUtil) (this.getServletContext().getAttribute("REFMET"));
 				OtherIdsUtil otherIdsUtil = (OtherIdsUtil) (this.getServletContext().getAttribute("OTHERIDS"));
 				DownloadConvertSingle dcs = new DownloadConvertSingle(analysis, analysisUtil, refmetUtil, otherIdsUtil, metaUtil);
+				// returns mAnalysis.hash only
 				String zip = dcs.dAndC();
 				response.setContentType("application/text;charset=UTF-8");
 				try (OutputStream out = response.getOutputStream())
@@ -74,10 +75,11 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet ZipConversion returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting analysis data from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting analysis data from MetabolomicsWorkbench", exp);
+			log("ZipConversion", exp);
+			response.setStatus(400);
+			response.sendError(400);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/ZipDownload.java b/apps/StdMW/src/main/java/edu/mda/bcb/stdmw/servlets/ZipDownload.java
@@ -16,6 +16,8 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
 import javax.servlet.ServletException;
 import javax.servlet.annotation.WebServlet;
 import javax.servlet.http.HttpServlet;
@@ -48,11 +50,16 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 		try
 		{
 			log("Servlet ZipDownload " + MWUrls.M_VERSION);
-			String loc = request.getParameter("loc");
-			if (null!=loc)
+			String analysis_hash = request.getParameter("analysis_hash");
+			String study_hash = request.getParameter("study_hash");
+			if ((null!=analysis_hash)&&(null!=study_hash))
 			{
 				File dataDir = new File(MWUrls.M_MW_ZIPTMP);
-				File zipFile = new File(dataDir, loc);
+				checkPathExistsSafely(dataDir, study_hash);
+				File subDir = new File(dataDir, study_hash);
+				checkPathExistsSafely(subDir, analysis_hash);
+				File lastDir = new File(subDir, analysis_hash);
+				File zipFile = new File(lastDir, (analysis_hash + ".zip"));
 				if (zipFile.exists())
 				{
 					response.setContentType("application/zip;charset=UTF-8");
@@ -75,10 +82,21 @@ protected void processRequest(HttpServletRequest request, HttpServletResponse re
 			}
 			log("Servlet ZipDownload returning");
 		}
-		catch(Exception exp)
+		catch (Exception exp)
 		{
-			log("Error getting analysis data from MetabolomicsWorkbench", exp);
-			throw new ServletException("Error getting analysis data from MetabolomicsWorkbench", exp);
+			log("ZipDownload", exp);
+			response.setStatus(400);
+			response.sendError(400);
+		}
+	}
+
+	protected void checkPathExistsSafely(File theDir, String theCheckDir) throws Exception
+	{
+		String [] dirs = theDir.list();
+		ArrayList<String> dirList = new ArrayList<>(Arrays.asList(dirs));
+		if (!dirList.contains(theCheckDir))
+		{
+			throw new Exception("Dir not found:" + theCheckDir);
 		}
 	}
 

diff --git a/apps/StdMW/src/main/webapp/index.html b/apps/StdMW/src/main/webapp/index.html
@@ -5,14 +5,14 @@
 		<meta charset="UTF-8">
 		<meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-		<link href="lib/DataTables/DataTables-1.10.20/css/jquery.dataTables.min.css" rel="stylesheet" type="text/css">
-		<link href="lib/jquery-ui-1.12.1/jquery-ui.min.css" rel="stylesheet" type="text/css">
-		<link href="StdMW.css" rel="stylesheet" type="text/css">
+		<link href="lib/DataTables/DataTables-1.10.20/css/jquery.dataTables.min.css?v=BEA_VERSION_TIMESTAMP" rel="stylesheet" type="text/css">
+		<link href="lib/jquery-ui-1.12.1/jquery-ui.min.css?v=BEA_VERSION_TIMESTAMP" rel="stylesheet" type="text/css">
+		<link href="StdMW.css?v=BEA_VERSION_TIMESTAMP" rel="stylesheet" type="text/css">
 
-		<script src="lib/knockout-3.5.1.js" type="text/javascript" charset="utf-8"></script>
-		<script src="lib/jquery-3.5.0.min.js" type="text/javascript" charset="utf-8"></script>
-		<script src="lib/jquery-ui-1.12.1/jquery-ui.min.js" type="text/javascript" charset="utf-8"></script>
-		<script src="lib/DataTables/DataTables-1.10.20/js/jquery.dataTables.min.js" type="text/javascript" charset="utf-8"></script>
+		<script src="lib/knockout-3.5.1.js?v=BEA_VERSION_TIMESTAMP" type="text/javascript" charset="utf-8"></script>
+		<script src="lib/jquery-3.5.0.min.js?v=BEA_VERSION_TIMESTAMP" type="text/javascript" charset="utf-8"></script>
+		<script src="lib/jquery-ui-1.12.1/jquery-ui.min.js?v=BEA_VERSION_TIMESTAMP" type="text/javascript" charset="utf-8"></script>
+		<script src="lib/DataTables/DataTables-1.10.20/js/jquery.dataTables.min.js?v=BEA_VERSION_TIMESTAMP" type="text/javascript" charset="utf-8"></script>
 
 		<script type="text/javascript">
 			/* global ko */
@@ -65,7 +65,7 @@
 							success: function (theString)
 							{
 								console.log("zipc :" + theString);
-								window.location.href = 'zipdl?loc=' + theString;
+								window.location.href = 'zipdl?analysis_hash=' + theAnalysisHash + '&study_hash=' + theStudyHash;
 							},
 							error: function (jqXHR, textStatus, errorThrown)
 							{
@@ -291,7 +291,7 @@
 	</head>
 	<body style="display: none;" data-bind="visible: $root.makeGuiVisible()">
 		<div id='alrt'>
-			<p onclick="document.getElementById('alrt').style.display = 'none';"><small>Legal notice: Unauthorized access to the network is prohibited. This system is for the use of authorized users only. 
+			<p onclick="document.getElementById('alrt').style.display = 'none';"><small>Privacy Notice - Legal notice: Unauthorized access to the network is prohibited. This system is for the use of authorized users only. 
 					Individuals using this computer system without authority, or in excess of their authority, are subject to having all of 
 					their activities on this system monitored and recorded by system personnel. In the course of monitoring individuals 
 					improperly using this system, or in the course of system maintenance, the activities of authorized users may also be 
@@ -324,7 +324,7 @@
 					<a href="https://bioinformatics.mdanderson.org/main/TCGABatchEffects:Overview" target="_blank"><small>Batch Effects Overview</small><br></a>
 					<strong>Documentation</strong><br>
 					<a href="https://bioinformatics.mdanderson.org/public-software/tcga-batch-effects/" target="_blank"><small>MBatch R Package</small></a><br>
-					<a href="" onclick="document.getElementById('alrt').style.display = 'block'; return false;"><small>Legal Notice</small></a><br>
+					<a href="" onclick="document.getElementById('alrt').style.display = 'block'; return false;"><small>Legal - Privacy Notice</small></a><br>
 					<a href="https://github.com/MD-Anderson-Bioinformatics/" target="_blank"><small>GitHub</small></a><br>
 					<strong>About</strong><br>
 					<a href="https://bioinformatics.mdanderson.org" target="_blank"><small>About BCB at MD Anderson</small></a><br>

diff --git a/apps/StdMWUtils/src/main/java/edu/mda/bcb/stdmwutils/MWStack.java b/apps/StdMWUtils/src/main/java/edu/mda/bcb/stdmwutils/MWStack.java
@@ -298,6 +298,7 @@ public static File downloadDataset(File theBaseDir, String theAnalysisId, String
 			OtherIdsUtil otherIdsUtil = OtherIdsUtil.readNewestOtherIdsFile();
 			DownloadConvertSingle dcs = new DownloadConvertSingle(analysis, analysisUtil, refmetUtil, otherIdsUtil, metaUtil);
 			String zip = dcs.dAndC();
+			zip = dcs.mAnalysis.study_hash + "/" + dcs.mAnalysis.hash + "/" + dcs.mAnalysis.hash + ".zip";
 			return new File(MWUrls.M_MW_ZIPTMP, zip);
 		}
 		return null;