Headless mode

.babelrc

@@ -0,0 +1,3 @@
+{
+  "plugins": ["meaningful-logs"]
+}

.eslintrc

@@ -0,0 +1,16 @@
+{  "env": {
+  "node": true
+},
+  "globals": {
+    "d3": true,
+    "$": true,
+    "chrome": true,
+    "jQuery": true,
+    "describe": true,
+    "it": true,
+    "beforeEach": true,
+    "afterEach": true,
+    "after": true,
+    "before": true
+  },
+  "extends": ["standard"]}

.gitignore

@@ -1,4 +1,5 @@
 .idea
 projectFilesBackup
 extension.zip
-
+node_modules
+npm-debug.log

README.md

@@ -1,11 +1,13 @@
 # Web Scraper
-Web Scraper is a chrome browser extension built for data extraction from web 
+Web Scraper is a chrome browser extension and a library built for data extraction from web 
 pages. Using this extension you can create a plan (sitemap) how a web site 
 should be traversed and what should be extracted. Using these sitemaps the 
 Web Scraper will navigate the site accordingly and extract all data. Scraped 
 data later can be exported as CSV.
 
-Install the extension from [Chrome store] [chrome-store]
+To use it as an extension install it from [Chrome store] [chrome-store]
+
+To use it as a library do `npm i web-scraper-headless`
 
 ### Features
 
@@ -26,6 +28,31 @@ Install the extension from [Chrome store] [chrome-store]
 
  Submit bugs and suggest features on [bug tracker] [github-issues]
 
+#### Headless mode
+To use it as a library you need a sitemap, for example exported from the app.
+
+    const webscraper = require('webscraper-headless')
+    const sitemap = {
+      id: 'test',
+      startUrl: 'http://test.lv/',
+      selectors: [
+        {
+          'id': 'a',
+          'selector': '#scraper-test-one-page a',
+          'multiple': false,
+          type: 'SelectorText',
+          'parentSelectors': [
+            '_root'
+          ]
+        }
+      ]
+    }
+    const options = {} // optional delay and pageLoadDelay
+    webscraper(sitemap, options)
+        .then(function (scraped) {
+            // This is your scraped info
+        })
+
 #### Bugs
 When submitting a bug please attach an exported sitemap if possible.
 

extension/assets/base64.js

@@ -1,36 +1,37 @@
+var jquery = require('jquery-deferred')
 /**
  * @url http://jsperf.com/blob-base64-conversion
  * @type {{blobToBase64: blobToBase64, base64ToBlob: base64ToBlob}}
  */
 var Base64 = {
 
-	blobToBase64: function(blob) {
+  blobToBase64: function (blob) {
+    var deferredResponse = jquery.Deferred()
+    var reader = new FileReader()
+    reader.onload = function () {
+      var dataUrl = reader.result
+      var base64 = dataUrl.split(',')[1]
+      deferredResponse.resolve(base64)
+    }
+    reader.readAsDataURL(blob)
 
-		var deferredResponse = $.Deferred();
-		var reader = new FileReader();
-		reader.onload = function() {
-			var dataUrl = reader.result;
-			var base64 = dataUrl.split(',')[1];
-			deferredResponse.resolve(base64);
-		};
-		reader.readAsDataURL(blob);
+    return deferredResponse.promise()
+  },
 
-		return deferredResponse.promise();
-	},
+  base64ToBlob: function (base64, mimeType) {
+    var deferredResponse = jquery.Deferred()
+    var binary = atob(base64)
+    var len = binary.length
+    var buffer = new ArrayBuffer(len)
+    var view = new Uint8Array(buffer)
+    for (var i = 0; i < len; i++) {
+      view[i] = binary.charCodeAt(i)
+    }
+    var blob = new Blob([view], {type: mimeType})
+    deferredResponse.resolve(blob)
 
-	base64ToBlob: function(base64, mimeType) {
+    return deferredResponse.promise()
+  }
+}
 
-		var deferredResponse = $.Deferred();
-		var binary = atob(base64);
-		var len = binary.length;
-		var buffer = new ArrayBuffer(len);
-		var view = new Uint8Array(buffer);
-		for (var i = 0; i < len; i++) {
-			view[i] = binary.charCodeAt(i);
-		}
-		var blob = new Blob([view], {type: mimeType});
-		deferredResponse.resolve(blob);
-
-		return deferredResponse.promise();
-	}
-};
+module.exports = Base64

extension/assets/jquery.whencallsequentially.js

@@ -1,48 +1,48 @@
+var jquery = require('jquery-deferred')
 /**
  * @author Martins Balodis
  *
  * An alternative version of $.when which can be used to execute asynchronous
  * calls sequentially one after another.
  *
- * @returns $.Deferred().promise()
+ * @returns jqueryDeferred().promise()
  */
-$.whenCallSequentially = function (functionCalls) {
-
-	var deferredResonse = $.Deferred();
-	var resultData = new Array();
+module.exports = function whenCallSequentially (functionCalls) {
+  var deferredResonse = jquery.Deferred()
+  var resultData = []
 
 	// nothing to do
-	if (functionCalls.length === 0) {
-		return deferredResonse.resolve(resultData).promise();
-	}
+  if (functionCalls.length === 0) {
+    return deferredResonse.resolve(resultData).promise()
+  }
 
-	var currentDeferred = functionCalls.shift()();
+  var currentDeferred = functionCalls.shift()()
 	// execute synchronous calls synchronously
-	while (currentDeferred.state() === 'resolved') {
-		currentDeferred.done(function (data) {
-			resultData.push(data);
-		});
-		if (functionCalls.length === 0) {
-			return deferredResonse.resolve(resultData).promise();
-		}
-		currentDeferred = functionCalls.shift()();
-	}
+  while (currentDeferred.state() === 'resolved') {
+    currentDeferred.done(function (data) {
+      resultData.push(data)
+    })
+    if (functionCalls.length === 0) {
+      return deferredResonse.resolve(resultData).promise()
+    }
+    currentDeferred = functionCalls.shift()()
+  }
 
 	// handle async calls
-	var interval = setInterval(function () {
+  var interval = setInterval(function () {
 		// handle mixed sync calls
-		while (currentDeferred.state() === 'resolved') {
-			currentDeferred.done(function (data) {
-				resultData.push(data);
-			});
-			if (functionCalls.length === 0) {
-				clearInterval(interval);
-				deferredResonse.resolve(resultData);
-				break;
-			}
-			currentDeferred = functionCalls.shift()();
-		}
-	}, 10);
+    while (currentDeferred.state() === 'resolved') {
+      currentDeferred.done(function (data) {
+        resultData.push(data)
+      })
+      if (functionCalls.length === 0) {
+        clearInterval(interval)
+        deferredResonse.resolve(resultData)
+        break
+      }
+      currentDeferred = functionCalls.shift()()
+    }
+  }, 10)
 
-	return deferredResonse.promise();
-};
+  return deferredResonse.promise()
+}