Multiwarc

ivy.xml

@@ -52,7 +52,7 @@
 
 		<dependency org="org.softee" name="pojo-mbean" rev="latest.release"/>
 		<!-- <dependency org="org.jgroups" name="jgroups" rev="latest.release"/> -->
-
+		<dependency org="com.meltmedia.jgroups" name="jgroups-aws" rev="1.4.0"/>
 		<dependency org="com.google.guava" name="guava" rev="latest.release"/>
 
 		<dependency org="net.java.dev.javacc" name="javacc" rev="5.0" conf="compile"/>

src/it/unimi/di/law/bubing/Agent.java

@@ -121,8 +121,9 @@ public Agent(final String hostname, final int jmxPort, final RuntimeConfiguratio
 		ByteArrayList list = new ByteArrayList();
 		while(rc.seed.hasNext()) {
 			final URI nextSeed = rc.seed.next();
-			if (nextSeed != null) frontier.enqueue(BURL.toByteArrayList(nextSeed, list));
+			if (nextSeed != null) frontier.enqueueIfLocal(BURL.toByteArrayList(nextSeed, list));
 		}
+		LOGGER.info("Finished reading seeds");
 
 		// We wait for the notification of a stop event, usually caused by a call to stop().
 		synchronized(this) {

src/it/unimi/di/law/bubing/frontier/FetchingThread.java

@@ -4,11 +4,13 @@
 import java.io.IOException;
 import java.net.URI;
 import java.security.NoSuchAlgorithmException;
-import java.util.List;
+import java.security.cert.CertificateException;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.Lock;
 
 import javax.net.ssl.SSLContext;
+import java.security.cert.X509Certificate;
 
 import org.apache.http.client.CookieStore;
 import org.apache.http.client.HttpClient;
@@ -18,6 +20,7 @@
 import org.apache.http.conn.DnsResolver;
 import org.apache.http.conn.socket.ConnectionSocketFactory;
 import org.apache.http.conn.socket.PlainConnectionSocketFactory;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
 import org.apache.http.conn.ssl.TrustSelfSignedStrategy;
 import org.apache.http.cookie.Cookie;
@@ -27,7 +30,9 @@
 import org.apache.http.impl.client.HttpClients;
 import org.apache.http.impl.conn.BasicHttpClientConnectionManager;
 import org.apache.http.message.BasicHeader;
+import org.apache.http.ssl.SSLContextBuilder;
 import org.apache.http.ssl.SSLContexts;
+import org.apache.http.ssl.TrustStrategy;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -112,22 +117,40 @@ public final class FetchingThread extends Thread implements Closeable {
 			throw new RuntimeException(cantHappen.getMessage(), cantHappen);
 		}
 	}
+	/** An SSL context that accepts all certificates */
+        private static final SSLContext TRUST_ALL_CERTIFICATES_SSL_CONTEXT;
+	static {
+		try {
+			TRUST_ALL_CERTIFICATES_SSL_CONTEXT = SSLContexts.custom().loadTrustMaterial(null, new TrustStrategy() {
+                                        public boolean isTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+                                                return true;
+                                        }}).build();
+		}
+		catch (Exception cantHappen) {
+			throw new RuntimeException(cantHappen.getMessage(), cantHappen);
+                }
+        }
+
 
 	/** A support class that makes it possible to plug in a custom DNS resolver. */
 	protected static final class BasicHttpClientConnectionManagerWithAlternateDNS
 			extends BasicHttpClientConnectionManager {
+
 		static Registry<ConnectionSocketFactory> getDefaultRegistry() {
+			// setup a Trust Strategy that allows all certificates.
+			//
+			SSLContext sslContext = TRUST_ALL_CERTIFICATES_SSL_CONTEXT;
 			return RegistryBuilder.<ConnectionSocketFactory> create()
 					.register("http", PlainConnectionSocketFactory.getSocketFactory())
 					.register("https",
-							new SSLConnectionSocketFactory(SSLContexts.createSystemDefault(),
+							new SSLConnectionSocketFactory(sslContext,
 									new String[] {
 											"TLSv1.2",
 											"TLSv1.1",
 											"TLSv1",
 											"SSLv3",
 											"SSLv2Hello",
-									}, null, SSLConnectionSocketFactory.getDefaultHostnameVerifier()))
+									}, null, new NoopHostnameVerifier()))
 					.build();
 		}
 
@@ -136,7 +159,6 @@ public BasicHttpClientConnectionManagerWithAlternateDNS(final DnsResolver dnsRes
 		}
 	}
 
-
 	private static int length(final String s) {
 		return s == null ? 0 : s.length();
 	}
@@ -184,14 +206,19 @@ public FetchingThread(final Frontier frontier, final int index) throws NoSuchAlg
 		connManager.closeIdleConnections(0, TimeUnit.MILLISECONDS);
 		connManager.setConnectionConfig(ConnectionConfig.custom().setBufferSize(8 * 1024).build()); // TODO: make this configurable
 
+		List<BasicHeader> headers = new ArrayList<BasicHeader>();
+		headers.add(new BasicHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.95,text/*;q=0.9,*/*;q=0.8"));
+		headers.add(new BasicHeader("Accept-Language","fr-FR,fr;q=0.8,en-US;q=0.6,en;q=0.4"));
+		headers.add(new BasicHeader("From", frontier.rc.userAgentFrom));
+
 		cookieStore = new BasicCookieStore();
 		httpClient = HttpClients.custom()
-				.setSSLContext(TRUST_SELF_SIGNED_SSL_CONTEXT)
+				.setSSLContext(TRUST_ALL_CERTIFICATES_SSL_CONTEXT)
 				.setConnectionManager(connManager)
 				.setConnectionReuseStrategy(frontier.rc.keepAliveTime == 0 ? NoConnectionReuseStrategy.INSTANCE : DefaultConnectionReuseStrategy.INSTANCE)
 				.setUserAgent(frontier.rc.userAgent)
 				.setDefaultCookieStore(cookieStore)
-				.setDefaultHeaders(ObjectLists.singleton(new BasicHeader("From", frontier.rc.userAgentFrom)))
+				.setDefaultHeaders(headers)
 				.build();
    		fetchData = new FetchData(frontier.rc);
 	}

src/it/unimi/di/law/bubing/frontier/Frontier.java

@@ -641,6 +641,42 @@ public void enqueue(final ByteArrayList url) throws IOException, InterruptedExce
 		return;
 	}
 
+	/** Enqueues a URL to the the BUbiNG crawl if it is local
+	 *
+	 * <p>Before {@linkplain AbstractSieve#enqueue(Object, Object) enqueueing the URL to the sieve}
+	 * we perform a number of checks:
+	 *
+	 * <ul>
+	 *
+	 * <li>if there are too many URLs for the URL scheme+authority, we discard the URL;
+	 *
+	 * <li>if the URL appears in the URL cache, we discard it; otherwise, we add it to the cache;
+	 *
+	 * <li>if another agent is responsible for the URL, we disregard it
+	 *
+	 * </ul>
+	 *
+	 * <p>The difference between this method and {@link #enqueueLocal(ByteArrayList)} is that the
+	 * latter does not check whether the argument is
+	 * {@linkplain JobManager#local(it.unimi.dsi.jai4j.Job) local}.
+	 *
+	 * @param url a {@linkplain BURL BUbiNG URL} to be enqueued to the BUbiNG crawl.
+	 * @throws InterruptedException from {@link AbstractSieve#enqueue(Object, Object)}. */
+	public void enqueueIfLocal(final ByteArrayList url) throws IOException, InterruptedException {
+		final byte[] urlBuffer = url.elements();
+		final int inStore = schemeAuthority2Count.get(urlBuffer, 0, BURL.startOfpathAndQuery(urlBuffer));
+		if (inStore >= rc.maxUrlsPerSchemeAuthority) return;
+
+		if (!urlCache.add(url)) return;
+
+		final BubingJob job = new BubingJob(url);
+
+		if (agent.local(job)) {
+			if (sieve.enqueue(url, null)) nextFlush = System.currentTimeMillis() + MIN_FLUSH_INTERVAL;
+		}
+		return;
+	}
+
 	/** Returns whether the workbench is full.
 	 *
 	 * @return whether the workbench is full. */