Skip to content

Commit

Permalink
Merge branch 'USN_REASON_TIMESTAMP'
Browse files Browse the repository at this point in the history
  • Loading branch information
@lfcnassif
lfcnassif committed May 27, 2024
2 parents 548215b + b33d5dc commit b56fd6e
Show file tree
Hide file tree
Showing 7 changed files with 246 additions and 124 deletions.
14 changes: 14 additions & 0 deletions iped-app/resources/config/profiles/forensic/conf/ParserConfig.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>

<!-- this config is merged with default config XML -->
<properties>

<parsers>
<parser class="iped.parsers.usnjrnl.UsnJrnlParser">
<params>
<param name="extractEntries" type="bool">true</param>
</params>
</parser>
</parsers>

</properties>
14 changes: 14 additions & 0 deletions iped-app/resources/config/profiles/pedo/conf/ParserConfig.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>

<!-- this config is merged with default config XML -->
<properties>

<parsers>
<parser class="iped.parsers.usnjrnl.UsnJrnlParser">
<params>
<param name="extractEntries" type="bool">true</param>
</params>
</parser>
</parsers>

</properties>
98 changes: 96 additions & 2 deletions iped-engine/src/main/java/iped/engine/config/ParsersConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,34 @@

import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream.Filter;
import java.nio.file.Files;
import java.nio.file.Path;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import iped.configuration.Configurable;

public class ParsersConfig implements Configurable<String> {
Expand All @@ -18,6 +41,8 @@ public class ParsersConfig implements Configurable<String> {

private static final String PARSER_CONFIG = "ParserConfig.xml"; //$NON-NLS-1$

public static final String PARSER_DISABLED_ATTR = "iped:disabled";

private String parserConfigXml;
private transient Path tmp;

Expand All @@ -33,7 +58,53 @@ public boolean accept(Path entry) throws IOException {

@Override
public void processConfig(Path resource) throws IOException {
parserConfigXml = new String(Files.readAllBytes(resource), StandardCharsets.UTF_8);
if (parserConfigXml == null) {
parserConfigXml = new String(Files.readAllBytes(resource), StandardCharsets.UTF_8);
parserConfigXml = parserConfigXml.trim().replaceFirst("^([\\W]+)<", "<");
} else {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newDefaultInstance();
dbf.setNamespaceAware(false);
DocumentBuilder db;
try {
db = dbf.newDocumentBuilder();
Document doc = db.parse(new InputSource(new StringReader(parserConfigXml)));
Document changedDoc = db.parse(resource.toFile());

Element root = changedDoc.getDocumentElement();
NodeList rootNl = root.getElementsByTagName("parsers").item(0).getChildNodes();
for (int i = 0; i < rootNl.getLength(); i++) {
Node child = rootNl.item(i);
if (child instanceof Element) {
Element element = (Element) child;
if (element.getTagName().equals("parser")) {
String className = element.getAttribute("class");
XPath xPath = XPathFactory.newInstance().newXPath();
String expression = "/properties/parsers/parser[@class='" + className + "']";
NodeList nlParser = (NodeList) xPath.compile(expression).evaluate(doc, XPathConstants.NODESET);

expression = "/properties/parsers";
NodeList nlParsers = (NodeList) xPath.compile(expression).evaluate(doc, XPathConstants.NODESET);
Node newnode = doc.importNode(element, true);
for (int j = 0; j < nlParsers.getLength(); j++) {
for (int k = 0; k < nlParser.getLength(); k++) {
nlParsers.item(j).removeChild(nlParser.item(k));
}
nlParsers.item(j).appendChild(newnode);
}
}
}
}

TransformerFactory tf = TransformerFactory.newInstance();
Transformer transformer = tf.newTransformer();
transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
StringWriter writer = new StringWriter();
transformer.transform(new DOMSource(doc), new StreamResult(writer));
parserConfigXml = writer.getBuffer().toString();
} catch (ParserConfigurationException | SAXException | XPathExpressionException | TransformerException e) {
throw new RuntimeException(e);
}
}
}

@Override
Expand All @@ -46,11 +117,34 @@ public void setConfiguration(String config) {
parserConfigXml = config;
}

public String removeDisabledParsers(String parserConfigXml) {
String[] slices = parserConfigXml.split(PARSER_DISABLED_ATTR + "=\"true\"");
StringBuffer result = new StringBuffer();
for (int i = 0; i < slices.length; i++) {
String part = slices[i];
if (i > 0) {
int disabledParserEndIndex = part.indexOf(">");
if (disabledParserEndIndex == 0 || part.charAt(disabledParserEndIndex - 1) != '/') {
disabledParserEndIndex = part.indexOf("</parser>");
}
part = part.substring(disabledParserEndIndex + 1);
}
if (i < slices.length - 1) {
int disabledParserIndex = part.lastIndexOf("<parser");
result.append(part.substring(0, disabledParserIndex));
} else {
result.append(part);
}
}
return result.toString();
}

public synchronized File getTmpConfigFile() {
if (tmp == null) {
try {
tmp = Files.createTempFile("parser-config", ".xml");
Files.write(tmp, parserConfigXml.getBytes(StandardCharsets.UTF_8));

Files.write(tmp, removeDisabledParsers(parserConfigXml).getBytes(StandardCharsets.UTF_8));
tmp.toFile().deleteOnExit();

} catch (IOException e) {
Expand Down
7 changes: 5 additions & 2 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/usnjrnl/ReportGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public void endHTMLDocument(PrintWriter out) {
out.println("</BODY></HTML>"); //$NON-NLS-1$
}

public InputStream createHTMLReport(List<UsnJrnlEntry> entries) {
public InputStream createHTMLReport(List<UsnJrnlEntry> entries, Exception entriesReadError) {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
PrintWriter out = new PrintWriter(new OutputStreamWriter(bout, StandardCharsets.UTF_8)); // $NON-NLS-1$

Expand Down Expand Up @@ -90,14 +90,17 @@ public InputStream createHTMLReport(List<UsnJrnlEntry> entries) {

out.print("</table>");

out.print("<b>Error during additional entries read:</b>" + entriesReadError.getMessage());

endHTMLDocument(out);

out.close();
return new ByteArrayInputStream(bout.toByteArray());

}

public InputStream createCSVReport(List<UsnJrnlEntry> entries, TemporaryResources tmp) throws IOException {
public InputStream createCSVReport(List<UsnJrnlEntry> entries, TemporaryResources tmp, Exception entriesReadError)
throws IOException {
Path path = tmp.createTempFile();
try (OutputStream os = Files.newOutputStream(path);
Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
Expand Down
72 changes: 48 additions & 24 deletions iped-parsers/iped-parsers-impl/src/main/java/iped/parsers/usnjrnl/UsnJrnlParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
Expand All @@ -25,6 +26,7 @@
import iped.data.IItemReader;
import iped.io.SeekableInputStream;
import iped.parsers.standard.StandardParser;
import iped.parsers.util.MetadataUtil;
import iped.properties.BasicProps;
import iped.properties.ExtraProperties;
import iped.search.IItemSearcher;
Expand Down Expand Up @@ -55,6 +57,8 @@ public enum ReportType {
public static final MediaType USNJRNL_REPORT_CSV = MediaType.parse("application/x-usnjournal-report-csv");
public static final MediaType USNJRNL_REGISTRY = MediaType.parse("application/x-usnjournal-registry");

static final String USN_REASON_PREFIX = "usnJrnl:";

private static Set<MediaType> SUPPORTED_TYPES = MediaType.set(USNJRNL_$J);

@Override
Expand Down Expand Up @@ -136,7 +140,8 @@ public UsnJrnlEntry readEntry(SeekableInputStream in) throws IOException {
return null;
}

private void createReport(ArrayList<UsnJrnlEntry> entries, int n, ParseContext context, ContentHandler handler)
private void createReport(ArrayList<UsnJrnlEntry> entries, int n, ParseContext context, ContentHandler handler,
Exception entriesReadError)
throws SAXException, IOException {
ReportGenerator rg = new ReportGenerator();
EmbeddedDocumentExtractor extractor = context.get(EmbeddedDocumentExtractor.class,
Expand All @@ -149,11 +154,11 @@ private void createReport(ArrayList<UsnJrnlEntry> entries, int n, ParseContext c
try (TemporaryResources tmp = new TemporaryResources()) {
if (reportType == ReportType.CSV) {
cMetadata.set(StandardParser.INDEXER_CONTENT_TYPE, USNJRNL_REPORT_CSV.toString());
is = rg.createCSVReport(entries, tmp);
is = rg.createCSVReport(entries, tmp, entriesReadError);

} else if (reportType == ReportType.HTML) {
cMetadata.set(StandardParser.INDEXER_CONTENT_TYPE, USNJRNL_REPORT_HTML.toString());
is = rg.createHTMLReport(entries);
is = rg.createHTMLReport(entries, entriesReadError);
name += " " + n;
}

Expand All @@ -179,12 +184,15 @@ private void createReport(ArrayList<UsnJrnlEntry> entries, int n, ParseContext c

String[] props = ReportGenerator.cols;

metadataItem.set(TikaCoreProperties.CREATED, rg.timeFormat.format(entry.getFileTime()));
metadataItem.set(ReportGenerator.cols[0], String.format("0x%016X", entry.getOffset()));
metadataItem.set(props[1], entry.getFileName());
metadataItem.set(props[2], entry.getFullPath());
metadataItem.set(props[3], Long.toString(entry.getUSN()));
String formatedDate = rg.timeFormat.format(entry.getFileTime());
for (String value : entry.getReasons()) {
value = value.toLowerCase();
MetadataUtil.setMetadataType(USN_REASON_PREFIX + value, Date.class);
metadataItem.set(USN_REASON_PREFIX + value, formatedDate);
metadataItem.add(props[5], value);
}
metadataItem.set(props[6], "0x" + Util.byteArrayToHex(entry.getMftRef()));
Expand Down Expand Up @@ -231,28 +239,33 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
int n = 1;
IItemSearcher searcher = context.get(IItemSearcher.class);
IItemReader item = context.get(IItemReader.class);
Exception entriesReadError = null;
try (SeekableInputStream sis = item.getSeekableInputStream()) {
jumpZeros(sis, 0, sis.size());
while (findNextEntry(sis)) {
UsnJrnlEntry u = readEntry(sis);
// do not insert empty registries in the list
if (u == null) {
continue;
}

entries.add(u);

if (entries.size() % MAX_ENTRIES == 0) {
int baseIndex = ((entries.size() / MAX_ENTRIES) - 1) * MAX_ENTRIES;
rebuildFullPaths(entries.subList(baseIndex, baseIndex + MAX_ENTRIES), searcher, item);
}

// limits the html table size
if (entries.size() == MAX_ENTRIES && reportType == ReportType.HTML) {
createReport(entries, n, context, handler);
entries.clear();
n++;
try {
while (findNextEntry(sis)) {
UsnJrnlEntry u = readEntry(sis);
// do not insert empty registries in the list
if (u == null) {
continue;
}

entries.add(u);

if (entries.size() % MAX_ENTRIES == 0) {
int baseIndex = ((entries.size() / MAX_ENTRIES) - 1) * MAX_ENTRIES;
rebuildFullPaths(entries.subList(baseIndex, baseIndex + MAX_ENTRIES), searcher, item);
}

// limits the html table size
if (entries.size() == MAX_ENTRIES && reportType == ReportType.HTML) {
createReport(entries, n, context, handler, entriesReadError);
entries.clear();
n++;
}
}
} catch (Exception e) {
entriesReadError = e;
}
}

Expand All @@ -261,7 +274,18 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,
int baseIndex = (entries.size() / MAX_ENTRIES) * MAX_ENTRIES;
rebuildFullPaths(entries.subList(baseIndex, entries.size()), searcher, item);
}
createReport(entries, n, context, handler);
createReport(entries, n, context, handler, entriesReadError);
}
if (entriesReadError instanceof TikaException) {
throw (TikaException) entriesReadError;
} else if (entriesReadError instanceof IOException) {
throw (IOException) entriesReadError;
} else if (entriesReadError instanceof SAXException) {
throw (SAXException) entriesReadError;
} else if (entriesReadError instanceof RuntimeException) {
throw (RuntimeException) entriesReadError;
} else if (entriesReadError != null) {
throw new RuntimeException(entriesReadError);
}

}
Expand Down
3 changes: 3 additions & 0 deletions iped-parsers/iped-parsers-impl/src/test/java/iped/parsers/usnjrnl/AbstractPkgTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ protected static class EmbeddedUsnParser extends AbstractParser {
protected List<String> contenttype = new ArrayList<String>();
protected List<String> title = new ArrayList<String>();
protected List<String> created = new ArrayList<String>();
protected List<Metadata> metadata = new ArrayList<Metadata>();

public Set<MediaType> getSupportedTypes(ParseContext context) {
return (new AutoDetectParser()).getSupportedTypes(context);
Expand All @@ -55,6 +56,8 @@ public void parse(InputStream stream, ContentHandler handler, Metadata metadata,

if (metadata.get(TikaCoreProperties.CREATED) != null)
created.add(metadata.get(TikaCoreProperties.CREATED));

this.metadata.add(metadata);
}

}
Expand Down
Loading

0 comments on commit b56fd6e

Please sign in to comment.