Merge pull request #99 from SOBotics/release/0.8.0

Release/0.8.0

.travis.yml

@@ -3,7 +3,7 @@ jdk:
  - oraclejdk8
 env:
   global:
-    - secure: "Ojj6thhMCgqLWNbppzw4GqgWXtKRgVQC9cZD2Qw2UrQeJiPS6MfA0cKcLrizPDzDSKAGEiLUc/W2I9cMT5PZwawdchDwjGS2b2SQWAEVrAEMcKhMf51jqgU3gRLt45C9AcK9N9td15CFFqxrkFmNUmsaiZjFGElxKQb2tjMAZMyAN0GLGqJkvTcolHWbaCfkNbo33q5tYp9kshFBrLHdxtUFvy0b0rMaGTP8uiZU0OcAy8JqyRGQ8ZCku7Bup6TRoD5q9HIepoPF+hquZOnBoxeL6RPiIujiKUfXRh2VCt6z88UKbHaUIudZRmndjQI2O2li5tJZ0jbnLOHt7lihem3NdCiQGRKVPj0/UPo75EB+Q/IoxJb+pv1SG0K/SonXvjgV7Kls03SL05LD4aEb3es0D63GK29jx9AUYd4f2FYUPS0QTUdNHDs6F9UkcjLzAsH2CwnQAjqTqauO2xDo6ERUOBgpKsGfVHlkR4l9R67OALaRQ0zndiRmJwa0piOlz8M51gC/bGFDFT8bXvBaCjP20Ia6LoPwOAuFlBbq9YvOSbXhk3G5PiX8NGDN7/z5fM8TJXqOTdljcpUN2NE/deeYW09NZoCBSzAuY/wTFVfImsgDLh+TD93CJOUVAudvSUDe9plUNO3pncnfyXMtoMJfDbO6MXb9iyRnTjCwt4c="
-    - secure: "YcX0LO4cfhl/8uchu0O+kwr1BTZP1103WNNKaybuiW0HIDD5NpR/4bFAeqORyfIMrx6OZjwirnJp4O1DrZizOo1TM2wRwJ1BEYOPY86WYsXoYz5VXyyC0laYDGbfb03c5lhKcqZ4pTo9Z2Xu0Pty29/ZpeqscUbyR3nkCsagkYdqa+QrCqqh/XmFX2MwxOZO+IfDVRhe3JfEzveSNFYtElpPDIfCE+Lv8PE1K6wy2vu41fhKCWuP+rvZJZWvkZauectlfQZcZ2Gh8+a0t73Jz8OngjaBow9mqzz/u4dqbcKCsxoxVEkfkkaQk+3ui0+CkDUxspwAErWyptHSrnmov30t2wjgH0diR/p2M3fnqWSxpOsdmOvtmwsKEGmquNhzEVfL3P2iosXRwsjSxQ37fVMOMcQCw9z2yds7ErdzkVbE8sV+zbzRkqnmRLNxgnnyHZI3bvnAnVGIhG2wK/7gakI/kLvWzjYciqUNmwkMsSODVaX2bzt2++9tUL2yp0AZZzU7cQzyl/+QuMQ6jw/I/lh6gzW7dcV3yhtl0wF1HFM8L2olqL9u1KTlIoFe9oZxI5yBYjHJ5QZ0kLivOWQcmHztnGZeDtRQ20rvWxDHzRa7P0Jq6+ttlshqlfLc5xZyiactjlULwdFdSxe6kVHTK9w6YXjabU3cu7ipG0CVIfQ="
+    - secure: "IXm8Q0NKfM0T+fpszZ/4NI3y7ul22SxTe5q8QLomLTakYcTCMuhvBQy8RKbRl3ElWk8f0n7aG7OBKhlhuKbqQJHAfIwP/ro/m4gw06Nf2vsakVeNJHUe7G0u32RxArd3VzVvHBRt1j7jXNK9TgK9KJkdYn16/1wKr+rvgxh1Jxp1NzrnqR3HLiBrNj4Z08GLm3gRSt0CTGEr2iFbujmuHvjSOfzt3JZnfKZqojxTjTZ/a9OjrHrrHcpvFGssXu0O3+MnHP3P/AhWzZFac11rZy2tX+YgPNPxJGnSJX5XzGi34W/ONur+Lc38pJDBH9vu0j/fAe3PkuHY1drOZ0cwhlvJSMT4CS32USBbg3eJbfqyngtGeCHw97OKYJ33jkDqKoRtiyWIoxoxxJx8O1dLeJ8lzH+2i7UDYYfuhGePXZBxw/VS2u14s0B7NQ5BMM+PyI3KLNBy/Bl+2rnxyBUaLeJ9qtIE09eT78WKweZ3daLbFLj8sy6XeaFUTUSvPZSKs86zW6RnHFPRVyogxNJQxKbEV9KHWkPWIpIlxY/bohyf1n02bTcjAxhjt4zn/F6Q12Qp/jclNG1+4xWi/g7DQX003F4du49IwX6nKMEWwVn+NUaBbB/+tTenbn+ZQ7qIoruvhC71kVLGpCYNQklB34BGjIHvPyLL4B8uGDSjP58="
+    - secure: "tKxo7j+IDrYmjE6JfCQCuzeXOXzxb/GfhspzR+GK3js70CQL1T7Ybp1ZBGJqg+Em5rVMOO1liUyD0xLOpYVU1Au2N/KM+ozrGbOv30C0KHIBT090nYloO6AttY0tpnTT+WMOM62Jq30FMY27eo2UTZ6tWKhd47/OKCDWvW1CtmMq7+GhBCcGv1ERqF35cgbhMzDc9AuzDsTkavYU+zH3Y/YowyysR7vEJU4tHcNciUbAM1bz/viQwPHSn4ZucaHqRlMI0ljDOruBfpUKFHKI2WG49mxrfjCXXMr6Gl2bWOPTrWhmM1abmi/I75OfDI8vREchbYSjLNKZZ8QQk3bRk6vDD0eVWCmRa28qA7agK50jTgFLjYElT+ktIHnGfprruJyZK6Y2cE3q5xgVp6vpGdVPdHPkeVMajzoVfxYn3LmOmCaXiAZRnnpgi3hD7qvuHpZIFPDz2qYjjOs6KhaGKRzpK938V9vrqWplJRysyLDMP3L08kKgfNIqfJvmXdwZfjGbbm5IdW69ubwQWlSeBN6aFd05fRXZkS8GM3P/EVunHJHA8BK6ydWHbeME1zrXaSnGrc54kdrYaD1Ub3P6uS+V7YGQfcHBIZ2DjczSBtDp9L/QH+8rwbv9NOEs/Ha8vUP6NSK3pu5Wtv4XCGOOBClk/uup2jdHNIICZzOzWfE="
 after_success:
-  "curl --ftp-create-dirs -T target/guttenberg-$TRAVIS_TAG.jar -u $FTP_USER:$FTP_PASSWORD ftp://deltasf.ddns.net/"
+  "curl --ftp-create-dirs -T target/guttenberg-$TRAVIS_TAG.jar -u $FTP_USER:$FTP_PASSWORD ftp://sobotics.org/bot"

README.md

@@ -7,21 +7,20 @@ Guttenberg is a bot that searches for plagiarism or duplicated answers on Stack
 
 ## Implementation
 
-Every 60 seconds, Guttenberg fetches the most recent answers (the "targets") on Stack Overflow. For each of these answers, possibly related posts (for example answers to related questions) are collected. Each related post will be split into the full markdown, code-blocks, plaintext paragraphs and blockquotes. These parts are compared with the "target" and the [Jaro-Winkler distance](https://en.wikipedia.org/wiki/Jaro–Winkler_distance) will be calculated. If one of the comparison reaches a certain score (which is defined in `general.properties`), a message like this will be posted in chat:
+Every 60 seconds, Guttenberg fetches the most recent answers (the "targets") on Stack Overflow. For each of these answers, possibly related posts (for example answers to related questions) are collected. All those posts will be checked for different characteristics (such as the [Jaro-Winkler distance](https://en.wikipedia.org/wiki/Jaro–Winkler_distance) of the posts). If at least one of the characteristics mets the requirements, a message like this will be posted in chat:
 
-![](https://i.imgur.com/HhwCWJr.png)
+![sample chat message](http://i.stack.imgur.com/hB7Hz.png)
 
 
 ## Accuracy
 
-At the moment, we are testing a very early version of the bot. That's why we get lot's of false positives. But we did already find copied answers without proper attribution and users posting the same answer on multiple questions.
-
+We have no statistics about the accuracy yet.
 
 ## Plans for the future
 
-### Feedback
+### Feedback statistics
 
-We have no statistics about the amount of tps and fps yet. In a future release, we want to allow users replying to a report to give feedback.
+Although users can send feedback, we are not logging it yet, so we can't provide statistics. In the future, this should be done by a dashboard like Sentinel.
 
 
   [1]: http://chat.stackoverflow.com/rooms/111347/sobotics

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>de.felixsfd.stackoverflow</groupId>
     <artifactId>guttenberg</artifactId>
-    <version>0.7.0</version>
+    <version>0.8.0</version>
 
     <dependencies>
         <dependency>
@@ -17,7 +17,7 @@
         <dependency>
             <groupId>fr.tunaki.stackoverflow</groupId>
             <artifactId>chatexchange</artifactId>
-            <version>1.0.2-SNAPSHOT</version>
+            <version>1.1.0</version>
         </dependency>
         <dependency>
             <groupId>com.optimaize.languagedetector</groupId>
@@ -48,7 +48,7 @@
         <dependency>
         	<groupId>org.sobotics</groupId>
         	<artifactId>redunda-lib</artifactId>
-        	<version>0.3.0</version>
+        	<version>1.0.0</version>
         </dependency>
     </dependencies>
     <repositories>

src/main/java/org/sobotics/guttenberg/clients/Guttenberg.java

@@ -11,14 +11,13 @@
 import org.slf4j.LoggerFactory;
 import org.sobotics.guttenberg.finders.NewAnswersFinder;
 import org.sobotics.redunda.PingService;
-import org.sobotics.guttenberg.entities.OptedInUser;
 import org.sobotics.guttenberg.entities.Post;
+import org.sobotics.guttenberg.entities.PostMatch;
 import org.sobotics.guttenberg.finders.PlagFinder;
 import org.sobotics.guttenberg.finders.RelatedAnswersFinder;
 import org.sobotics.guttenberg.printers.SoBoticsPostPrinter;
 import org.sobotics.guttenberg.utils.FilePathUtils;
 import org.sobotics.guttenberg.utils.StatusUtils;
-import org.sobotics.guttenberg.utils.UserUtils;
 
 import fr.tunaki.stackoverflow.chat.Room;
 
@@ -58,7 +57,6 @@ public void execute() throws Throwable {
 		Instant startTime = Instant.now();
 		Properties props = new Properties();
 		LOGGER.info("Executing at - "+startTime);
-		//NewAnswersFinder answersFinder = new NewAnswersFinder();
 
 		try {
 			props.load(new FileInputStream(FilePathUtils.generalPropertiesFile));
@@ -69,6 +67,7 @@ public void execute() throws Throwable {
 
 		//Fetch recent answers / The targets
 		List<Post> recentAnswers = NewAnswersFinder.findRecentAnswers();
+		StatusUtils.numberOfCheckedTargets.addAndGet(recentAnswers.size());
 		//Fetch their question_ids
 		List<Integer> ids = new ArrayList<Integer>();
 		for (Post answer : recentAnswers) {
@@ -113,52 +112,27 @@ public void execute() throws Throwable {
 		LOGGER.info("There are "+plagFinders.size()+" PlagFinders");
 		LOGGER.info("Find the duplicates...");
 		//Let PlagFinders find the best match
+		List<PostMatch> allMatches = new ArrayList<PostMatch>();
 		for (PlagFinder finder : plagFinders) {
-			@SuppressWarnings("unused")
-			Post originalAnswer = finder.getMostSimilarAnswer();
-			double score = finder.getJaroScore();
-			double minimumScore = 0.78;
+			List<PostMatch> matchesInFinder = finder.matchesForReasons();
 
-			try {
-				double s = new Double(props.getProperty("minimumScore", "0.78"));
-				if (s > 0) {
-					minimumScore = s;
-				}
-			} catch (Throwable e) {
-				LOGGER.warn("Could not convert score from properties-file to double. Using hardcoded", e);
-			}
-
-			if (score > minimumScore) {
-				for (Room room : this.chatRooms) {
-					List<OptedInUser> pingUsersList = UserUtils.pingUserIfApplicable(score, room.getRoomId());
-					if (room.getRoomId() == 111347) {
+			if (matchesInFinder != null) {
+				LOGGER.info("Found "+matchesInFinder.size()+" PostMatches in this PlagFinder");
+				allMatches.addAll(matchesInFinder);
+
+				for (PostMatch match : matchesInFinder) {
+					if (match.isValidMatch()) {
+						StatusUtils.numberOfReportedPosts.incrementAndGet();
 						SoBoticsPostPrinter printer = new SoBoticsPostPrinter();
-						String report = printer.print(finder);
-						String pings = " (";
+						String message = printer.print(match);
 
-						if (!finder.matchedPostIsRepost()) {
-							//only ping if it's not a repost
-							for (OptedInUser user : pingUsersList) {
-	                            if (!user.isWhenInRoom() || (user.isWhenInRoom() && UserUtils.checkIfUserInRoom(room, user.getUser().getUserId()))) {
-	                                pings+=(" @"+user.getUser().getUsername().replace(" ",""));
-	                            }
-	                        }
+						for (Room room : this.chatRooms) {
+							room.send(message);
 						}
-
-						if (pings.length() > 2) {
-							report += pings + " )";
-						}
-
-						room.send(report);
-						StatusUtils.numberOfReportedPosts.incrementAndGet();
 					}
 				}
-			} else {
-				//LOGGER.info("Score "+finder.getJaroScore()+" too low");
 			}
 
-			StatusUtils.numberOfCheckedTargets.incrementAndGet();
-
 		}
 
 		StatusUtils.lastSucceededExecutionStarted = startTime;

src/main/java/org/sobotics/guttenberg/commandlists/SoBoticsCommandsList.java

@@ -5,7 +5,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Optional;
 import java.util.Properties;
 
 import org.slf4j.Logger;
@@ -64,10 +63,10 @@ public void mention(Room room, PingMessageEvent event, boolean isReply, RunnerSe
     }
 
     public void globalCommand(Room room, MessagePostedEvent event,  RunnerService instance) {
-    	//only ROs should execute global commands!
+    	//only ROs (and Generic Bot) should execute global commands!
     	try {
     		User user = event.getUser().get();
-    		if (!user.isModerator() && !user.isRoomOwner())
+    		if (!user.isModerator() && !user.isRoomOwner() && user.getId() != 7481043)
     			return;
     	} catch (Throwable e) {
     		LOGGER.warn("Could not verify privileges of that user. Don't execute the command.", e);

src/main/java/org/sobotics/guttenberg/commands/Alive.java

@@ -1,11 +1,6 @@
 package org.sobotics.guttenberg.commands;
 
 import org.sobotics.guttenberg.utils.CommandUtils;
-import org.sobotics.guttenberg.utils.FilePathUtils;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.util.Properties;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,18 +28,9 @@ public boolean validate() {
     }
 
     @Override
-    public void execute(Room room, RunnerService instance) {
-    	Properties prop = new Properties();
-
-        try{
-            prop.load(new FileInputStream(FilePathUtils.loginPropertiesFile));
-        }
-        catch (IOException e){
-            LOGGER.error("Error: ", e);
-            room.replyTo(message.getId(), "Maybe. But something strange is going on!");
-            return;
-        }
-        room.send("The instance "+prop.getProperty("location", "undefined")+ " is running.\nStandby: "+PingService.standby.toString());
+    public void execute(Room room, RunnerService instance) {  
+    	LOGGER.info("Someone wants to know, if I'm alive");
+        room.send("The instance "+PingService.location+ " is running.\nStandby: "+PingService.standby.toString());
     }
 
     @Override

src/main/java/org/sobotics/guttenberg/commands/Check.java

@@ -2,10 +2,14 @@
 
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
 import java.util.Properties;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.sobotics.guttenberg.entities.PostMatch;
 import org.sobotics.guttenberg.finders.PlagFinder;
 import org.sobotics.guttenberg.services.RunnerService;
 import org.sobotics.guttenberg.utils.ApiUtils;
@@ -62,16 +66,29 @@ public void execute(Room room, RunnerService instance) {
             JsonObject target = answer.get("items").getAsJsonArray().get(0).getAsJsonObject();
             PlagFinder finder = new PlagFinder(target);
             finder.collectData();
-            finder.getMostSimilarAnswer();
-            double score = Math.round(finder.getJaroScore()*100.0)/100.0;
-            String link = "http://stackoverflow.com/a/"+finder.getJaroAnswer().getAnswerID();
-
-            if (score > 0) {
-                String reply = "The closest match with a score of **"+score+"** is [this post]("+link+").";
-                room.replyTo(message.getId(), reply);
+            List<PostMatch> matches = finder.matchesForReasons(true);
+
+            if (matches.size() > 0) {
+            	//sort the matches
+            	Collections.sort(matches, 
+            		    Comparator.comparingDouble(PostMatch::getTotalScore).reversed());
+
+
+            	//prepare printing them
+            	int i = 0;
+            	String reply = "These posts are similar to the target: ";
+            	for (PostMatch match : matches) {
+            		if (i > 5)
+            			break;
+            		double roundedTotalScore = Math.round(match.getTotalScore()*100.0)/100.0;
+            		reply += "["+match.getOriginal().getAnswerID()+"](http://stackoverflow.com/a/"+match.getOriginal().getAnswerID()+") ("+roundedTotalScore+"); ";
+            		i++;
+            	}
+
+            	room.replyTo(message.getId(), reply);
             } else {
-                room.replyTo(message.getId(), "There are no similar posts.");
-            }  
+            	room.replyTo(message.getId(), "No similar posts found.");
+            }
         } catch (IOException e) {
             LOGGER.error("ERROR", e);
         }

src/main/java/org/sobotics/guttenberg/commands/Kill.java

@@ -25,7 +25,7 @@ public void execute(Room room, RunnerService instance) {
 		User user = message.getUser();
 
     	if (!user.isModerator() && !user.isRoomOwner()) {
-    		room.replyTo(message.getId(), "Sorry, but only room-owners and moderators can use this command");
+    		room.replyTo(message.getId(), "Sorry, but only room-owners and moderators can use this command (@FelixSFD)");
     		return;
     	}
 

src/main/java/org/sobotics/guttenberg/commands/OptIn.java

@@ -74,7 +74,7 @@ public void execute(Room room, RunnerService instance) {
 
 	@Override
 	public String description() {
-		return "Get notified about possible plagiarism with a certain score. Usage: `opt-in <score> <always?>`";
+		return "Get notified about possible plagiarism with a certain score. Usage: opt-in <score> <always?>";
 	}
 
 	@Override

src/main/java/org/sobotics/guttenberg/commands/OptOut.java

@@ -47,7 +47,7 @@ public void execute(Room room, RunnerService instance) {
 
 	@Override
 	public String description() {
-		return "Removed you from the list";
+		return "Removes you from the list";
 	}
 
 	@Override

src/main/java/org/sobotics/guttenberg/commands/Quota.java

@@ -3,6 +3,7 @@
 import org.sobotics.guttenberg.utils.CommandUtils;
 import org.sobotics.guttenberg.utils.FilePathUtils;
 import org.sobotics.guttenberg.utils.StatusUtils;
+import org.sobotics.redunda.PingService;
 
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -41,7 +42,7 @@ public void execute(Room room, RunnerService instance) {
             return;
         }
 
-        room.send("The remaining quota on "+prop.getProperty("location", "undefined")+" is: "+StatusUtils.remainingQuota);
+        room.send("The remaining quota on "+PingService.location+" is: "+StatusUtils.remainingQuota);
     }
 
     @Override

src/main/java/org/sobotics/guttenberg/commands/Reboot.java

@@ -32,7 +32,7 @@ public void execute(Room room, RunnerService instance) {
     	User user = message.getUser();
 
     	if (!user.isModerator() && !user.isRoomOwner()) {
-    		room.replyTo(message.getId(), "Sorry, but only room-owners and moderators can use this command");
+    		room.replyTo(message.getId(), "Sorry, but only room-owners and moderators can use this command (@FelixSFD)");
     		return;
     	}
 

src/main/java/org/sobotics/guttenberg/commands/Status.java

@@ -10,6 +10,7 @@
 import org.sobotics.guttenberg.utils.CommandUtils;
 import org.sobotics.guttenberg.utils.FilePathUtils;
 import org.sobotics.guttenberg.utils.StatusUtils;
+import org.sobotics.redunda.PingService;
 import org.sobotics.guttenberg.services.RunnerService;
 
 import fr.tunaki.stackoverflow.chat.Message;
@@ -46,19 +47,19 @@ public void execute(Room room, RunnerService instance) {
 
 
         StringBuilder status = new StringBuilder();
-        status.append("Running since: ").append(StatusUtils.startupDate);
+        status.append("Location: ").append(PingService.location);
+        status.append("\nRunning since: ").append(StatusUtils.startupDate);
 
         if (room.getRoomId() == 111347) {
             status.append("\nLast execution finished: ").append(StatusUtils.lastExecutionFinished);
-            status.append("\nLocation: ").append(prop.getProperty("location", "undefined"));
         }
 
         String version = prop2.getProperty("version", "undefined");
         status.append("\nVersion: ").append(version);
         status.append("\nChecked ").append(StatusUtils.numberOfCheckedTargets).append(" targets and reported ").append(StatusUtils.numberOfReportedPosts);
         status.append("\nRemaining quota: ").append(StatusUtils.remainingQuota);
+        status.append("\n---");
 
-        //room.replyTo(message.getId(), status.toString());
         room.send(status.toString());
     }