Merge pull request #24 from MaibornWolff/scmlogparser

#35, #36, #129: Added an importer rom existing sources  which extract…

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/)
 ### Added
 - Adding Labels and UI
 - Support for links to source page of SonarQube in sonarimporter
+- Added SCMLogParser
 
 ### Changed
 

analysis/README.md

@@ -10,10 +10,11 @@ CodeCharta analysis tools generally follow the pipes and filters architecture pr
 
 Components that import data from an external source, e.g. SonarQube, and generate visualisation data.
 
-| Source        | Projekt                                                 |
+| Source        | Project                                                 |
 | ---           | ---                                                     |
+| SCM log       | [SCMLogParser](import/SCMLogParser/README.md)            |
 | SonarQube     | [SonarImporter](import/SonarImporter/README.md)         |
-| SourceMonitor | [SonarImporter](import/SourceMonitorImporter/README.md) |
+| SourceMonitor | [SourceMonitorImporter](import/SourceMonitorImporter/README.md) |
 
 ### Filter
 
@@ -54,5 +55,3 @@ Via gradle:
 - Integration tests:
 
 > ./gradlew integrationTest
-
-gestartet werden.

analysis/gradle/wrapper/gradle-wrapper.properties

@@ -1,4 +1,4 @@
-#Tue Apr 04 15:18:55 CEST 2017
+#Sun Jun 18 12:20:36 CEST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME

analysis/import/SCMLogParser/README.md

@@ -0,0 +1,26 @@
+# CVSLogParser
+
+Generates visualisation data from repository (Git or SVN) logs. It supports the following metrics per file:
+
+| Metric               | Description |
+| ---                  | --- |
+| `number_of_commits`  | total number of commits |
+| `weeks_with_commits` | weeks with commits |
+| `number_of_authors`  | number of authors with commits |
+
+Additionally it saves the names of authors.
+
+## Usage
+
+### Creating the repository log for metric generation  
+
+* Git:   `git log --name-status`
+* SVN:   `svn log --verbose`
+
+The generated logs must be in UTF-8 encoding.
+
+### Executing the SCMLogParser
+
+> `ccsh scmlogparser <log file> [--git|--svn] [<output file>]`
+
+The result is written as JSON to standard out or into the specified output file.

analysis/import/SCMLogParser/build.gradle

@@ -0,0 +1,22 @@
+apply plugin: 'application'
+apply plugin: 'java'
+
+dependencies {
+    compile project(':model')
+    compile 'org.apache.commons:commons-lang3:3.4'
+    testCompile 'junit:junit:4.12'
+    testCompile 'org.assertj:assertj-core:3.5.2'
+    testCompile 'org.mockito:mockito-all:1.9.5'
+}
+
+mainClassName = "de.maibornwolff.codecharta.importer.scmlogparser.SCMLogParser"
+applicationName = 'codecharta-scmlogparser'
+
+jar {
+    baseName = "${applicationName}"
+    manifest {
+        attributes 'Main-Class': mainClassName
+    }
+
+    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+}

analysis/import/SCMLogParser/src/main/java/de/maibornwolff/codecharta/importer/scmlogparser/ProjectConverter.java

@@ -0,0 +1,51 @@
+package de.maibornwolff.codecharta.importer.scmlogparser;
+
+import de.maibornwolff.codecharta.model.Node;
+import de.maibornwolff.codecharta.model.NodeType;
+import de.maibornwolff.codecharta.model.Project;
+import de.maibornwolff.codecharta.model.input.VersionControlledFile;
+import de.maibornwolff.codecharta.nodeinserter.FileSystemPath;
+import de.maibornwolff.codecharta.nodeinserter.NodeInserter;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public final class ProjectConverter {
+
+    private ProjectConverter() {
+        // utility class
+    }
+
+    public static Project convert(String projectName, List<VersionControlledFile> versionControlledFiles) {
+        Project project = new Project(projectName);
+        versionControlledFiles.forEach(vcFile -> ProjectConverter.addVersionControlledFile(project, vcFile));
+        return project;
+    }
+
+    private static void addVersionControlledFile(Project project, VersionControlledFile versionControlledFile) {
+        Map<String, Object> attributes = extractAttributes(versionControlledFile);
+        Node newNode = new Node(extractFilenamePart(versionControlledFile), NodeType.File, attributes, "", Collections.emptyList());
+        NodeInserter.insertByPath(project, new FileSystemPath(extractPathPart(versionControlledFile)), newNode);
+    }
+
+    private static Map<String, Object> extractAttributes(VersionControlledFile versionControlledFile) {
+        HashMap<String, Object> attributes = new HashMap<>();
+        attributes.put("number_of_commits", versionControlledFile.getNumberOfOccurrencesInCommits());
+        attributes.put("weeks_with_commits", versionControlledFile.getNumberOfWeeksWithCommits());
+        attributes.put("authors", versionControlledFile.getAuthors());
+        attributes.put("number_of_authors", versionControlledFile.getNumberOfAuthors());
+        return attributes;
+    }
+
+    private static String extractFilenamePart(VersionControlledFile versionControlledFile) {
+        String path = versionControlledFile.getFilename();
+        return path.substring(path.lastIndexOf('/') + 1);
+    }
+
+    private static String extractPathPart(VersionControlledFile versionControlledFile) {
+        String path = versionControlledFile.getFilename();
+        return path.substring(0, path.lastIndexOf('/') + 1);
+    }
+}

analysis/import/SCMLogParser/src/main/java/de/maibornwolff/codecharta/importer/scmlogparser/SCMLogParser.java

@@ -0,0 +1,67 @@
+package de.maibornwolff.codecharta.importer.scmlogparser;
+
+import de.maibornwolff.codecharta.importer.scmlogparser.parser.GitLogParserStrategy;
+import de.maibornwolff.codecharta.importer.scmlogparser.parser.LogParser;
+import de.maibornwolff.codecharta.importer.scmlogparser.parser.LogParserStrategy;
+import de.maibornwolff.codecharta.importer.scmlogparser.parser.SVNLogParserStrategy;
+import de.maibornwolff.codecharta.model.Project;
+import de.maibornwolff.codecharta.serialization.ProjectSerializer;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.stream.Stream;
+
+public class SCMLogParser {
+
+    public static void main(String[] args) throws IOException {
+        if (args.length >= 1) {
+            if (args[0].equals("-h") || args[0].equals("--help")) {
+                showHelpAndTerminate();
+            }
+        }
+        if (args.length >= 2) {
+            String pathToLog = args[0];
+            String gitOrSvn = args[1];
+
+            Project project = parseDataFromLog(pathToLog, gitOrSvn);
+            if (args.length >= 3) {
+                ProjectSerializer.serializeProjectAndWriteToFile(project, args[2]);
+            } else {
+                ProjectSerializer.serializeProject(project, new OutputStreamWriter(System.out));
+            }
+        } else {
+            showErrorAndTerminate();
+        }
+    }
+
+    private static Project parseDataFromLog(String pathToLog, String gitOrSvn) throws IOException {
+        LogParserStrategy parserStrategy = null;
+        switch (gitOrSvn) {
+            case "--git":
+                parserStrategy = new GitLogParserStrategy();
+                break;
+            case "--svn":
+                parserStrategy = new SVNLogParserStrategy();
+                break;
+            default:
+                showErrorAndTerminate();
+        }
+        Stream<String> lines = Files.lines(Paths.get(pathToLog));
+        return new LogParser(parserStrategy).parse(lines);
+    }
+
+    private static void showErrorAndTerminate() {
+        System.out.println("Invalid arguments!\n");
+        showHelpAndTerminate();
+    }
+
+    private static void showHelpAndTerminate() {
+        System.out.println("Please use the following syntax\n\"SCMLogParser-x.x.jar <pathToLogFile> --git/--svn\" [<pathToOutputfile>]\n" +
+                "The log file must have been created by using \"svn log --verbose\" or \"git log --name-status\"\n" +
+                "If no output file was specified, the output will be piped to standard out");
+        System.exit(0);
+    }
+
+}

analysis/import/SCMLogParser/src/main/java/de/maibornwolff/codecharta/importer/scmlogparser/parser/CommitCollector.java

@@ -0,0 +1,68 @@
+package de.maibornwolff.codecharta.importer.scmlogparser.parser;
+
+import de.maibornwolff.codecharta.model.input.Commit;
+import de.maibornwolff.codecharta.model.input.VersionControlledFile;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collector;
+
+class CommitCollector {
+
+    static Collector<Commit, ?, List<VersionControlledFile>> create() {
+        CommitCollector collector = new CommitCollector();
+        return Collector.of(ArrayList::new, collector::collectCommit, collector::combineForParallelExecution);
+    }
+
+    private void collectCommit(List<VersionControlledFile> versionControlledFiles, Commit commit) {
+        removeEmptyFiles(commit);
+        if (isEmpty(commit)) {
+            return;
+        }
+        addYetUnknownFilesToVersionControlledFiles(versionControlledFiles, commit.getFilenames());
+        addCommitMetadataToMatchingVersionControlledFiles(commit, versionControlledFiles);
+    }
+
+    private void removeEmptyFiles(Commit commit) {
+        commit.getFilenames().removeIf(String::isEmpty);
+    }
+
+    private boolean isEmpty(Commit commit) {
+        return commit.getFilenames().isEmpty();
+    }
+
+    private void addYetUnknownFilesToVersionControlledFiles(List<VersionControlledFile> versionControlledFiles, List<String> filenamesOfCommit) {
+        filenamesOfCommit.stream()
+            .filter(filename -> !versionControlledFilesContainsFile(versionControlledFiles, filename))
+            .forEach(unknownFilename-> addYetUnknownFile(versionControlledFiles, unknownFilename));
+    }
+
+    private boolean versionControlledFilesContainsFile(List<VersionControlledFile> versionControlledFiles, String filename) {
+        return findVersionControlledFileByFilename(versionControlledFiles, filename).isPresent();
+    }
+
+    private Optional<VersionControlledFile> findVersionControlledFileByFilename(List<VersionControlledFile> versionControlledFiles, String filename) {
+        return versionControlledFiles.stream()
+            .filter(commit -> commit.getFilename().equals(filename))
+            .findFirst();
+    }
+
+    private boolean addYetUnknownFile(List<VersionControlledFile> versionControlledFiles, String filenameOfYetUnversionedFile) {
+        VersionControlledFile missingVersionControlledFile = new VersionControlledFile(filenameOfYetUnversionedFile);
+        return versionControlledFiles.add(missingVersionControlledFile);
+    }
+
+    private void addCommitMetadataToMatchingVersionControlledFiles(Commit commit, List<VersionControlledFile> versionControlledFiles) {
+        commit.getFilenames().stream()
+                .map(file -> findVersionControlledFileByFilename(versionControlledFiles, file))
+                .filter(Optional::isPresent)
+                .forEach(vcFile -> vcFile.get().registerCommit(commit));
+
+    }
+
+    private List<VersionControlledFile> combineForParallelExecution(List<VersionControlledFile> firstCommits, List<VersionControlledFile> secondCommits) {
+        throw new UnsupportedOperationException("parallel collection of commits not supported");
+    }
+
+}

analysis/import/SCMLogParser/src/main/java/de/maibornwolff/codecharta/importer/scmlogparser/parser/GitLogParserStrategy.java

@@ -0,0 +1,92 @@
+package de.maibornwolff.codecharta.importer.scmlogparser.parser;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+import java.util.function.Predicate;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class GitLogParserStrategy implements LogParserStrategy {
+
+    /*
+     * see "diff-raw status letters" at https://github.com/git/git/blob/35f6318d44379452d8d33e880d8df0267b4a0cd0/diff.h#L326
+     */
+    private static final List<Character> STATUS_LETTERS = Arrays.asList('A', 'C', 'D', 'M', 'R', 'T', 'X', 'U');
+
+    private static final String AUTHOR_ROW_INDICATOR = "Author: ";
+
+    private static final String DATE_ROW_INDICATOR = "Date: ";
+
+    public static final Predicate<String> GIT_COMMIT_SEPARATOR_TEST = logLine -> logLine.startsWith("commit");
+
+    private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy ZZZ", Locale.US);
+
+    private boolean isFileLine(String commitLine) {
+        if (commitLine.length() < 2) {
+            return false;
+        }
+        char firstChar = commitLine.charAt(0);
+        char secondChar = commitLine.charAt(1);
+        return isStatusLetter(firstChar) && Character.isWhitespace(secondChar);
+    }
+
+    private static boolean isStatusLetter(char character) {
+        return STATUS_LETTERS.contains(character);
+    }
+
+    String parseFilename(String fileLine) {
+        if (fileLine.isEmpty()) {
+            return fileLine;
+        }
+        String filename = fileLine.substring(1);
+        return filename.trim();
+    }
+
+    public Collector<String, ?, Stream<List<String>>> createLogLineCollector() {
+        return LogLineCollector.create(GIT_COMMIT_SEPARATOR_TEST);
+    }
+
+    @Override
+    public Optional<String> parseAuthor(List<String> commitLines) {
+        return commitLines.stream()
+                .filter(commitLine -> commitLine.startsWith(AUTHOR_ROW_INDICATOR))
+                .map(this::parseAuthor)
+                .findFirst();
+
+    }
+
+    String parseAuthor(String authorLine) {
+        String authorWithEmail = authorLine.substring(AUTHOR_ROW_INDICATOR.length());
+        int beginOfEmail = authorWithEmail.indexOf('<');
+        if (beginOfEmail < 0) {
+            return authorWithEmail;
+        }
+        return authorWithEmail.substring(0, beginOfEmail).trim();
+    }
+
+    @Override
+    public List<String> parseFilenames(List<String> commitLines) {
+        return commitLines.stream()
+                .filter(this::isFileLine)
+                .map(this::parseFilename)
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public Optional<LocalDateTime> parseDate(List<String> commitLines) {
+        return commitLines.stream()
+                .filter(commitLine -> commitLine.startsWith(DATE_ROW_INDICATOR))
+                .map(this::parseCommitDate)
+                .findFirst();
+    }
+
+    private LocalDateTime parseCommitDate(String metadataDateLine) {
+        String commitDateAsString = metadataDateLine.replace(DATE_ROW_INDICATOR, "").trim();
+        return LocalDateTime.parse(commitDateAsString, DATE_TIME_FORMATTER );
+    }
+}