Merge pull request #12 from armand1m/fix/export-utilities

fix: export utilities and add managed jsdom example

examples/typescript/src/hacker-news/scraper.ts

@@ -14,7 +14,10 @@ const main = async () => {
     baseUrl: "https://news.ycombinator.com/",
     target: ".athing",
     selectors: {
-      rank: ({ text }) => text('.rank'),
+      rank: (utils) => {
+        const value = utils.text('.rank').replace(/^\D+/g, '');
+        return Number(value);
+      },
       name: ({ text }) => text('.titlelink'),
       url: ({ href }) => href('.titlelink'),
       score: ({ element }) => {

examples/typescript/src/managed-jsdom/scraper.ts

@@ -0,0 +1,55 @@
+import pino from 'pino'
+import { scrape, fetchPage, createWindow } from '@armand1m/papercut';
+
+const main = async () => {
+  const logger = pino({
+    name: 'Hacker News',
+    enabled: false
+  });
+
+  const rawHTML = await fetchPage('https://news.ycombinator.com/')
+  const window = createWindow(rawHTML);
+
+  const results = await scrape({
+    strict: true,
+    logger,
+    document: window.document,
+    target: ".athing",
+    selectors: {
+      rank: (utils) => {
+        const value = utils.text('.rank').replace(/^\D+/g, '');
+        return Number(value);
+      },
+      name: ({ text }) => text('.titlelink'),
+      url: ({ href }) => href('.titlelink'),
+      score: ({ element }) => {
+        return element.nextElementSibling?.querySelector('.score')
+          ?.textContent;
+      },
+      createdBy: ({ element }) => {
+        return element.nextElementSibling?.querySelector('.hnuser')
+          ?.textContent;
+      },
+      createdAt: ({ element }) => {
+        return element.nextElementSibling
+          ?.querySelector('.age')
+          ?.getAttribute('title');
+      },
+    },
+    options: {
+      log: false,
+      cache: true,
+      concurrency: {
+        page: 2,
+        node: 2,
+        selector: 2
+      }
+    }
+  });
+
+  window.close();
+
+  console.log(JSON.stringify(results, null, 2));
+};
+
+main();

src/index.ts

@@ -21,3 +21,6 @@ export type { SelectorUtilities } from './selectors/createSelectorUtilities';
 
 export { geosearch } from './http/geosearch';
 export type { GeosearchResult } from './http/geosearch';
+
+export { fetchPage } from './http/fetchPage';
+export { createWindow } from './utilities/createWindow';

template.md

@@ -104,6 +104,21 @@ Then run it using `node` or `ts-node`:
 npx ts-node ./paginated-scraper.ts
 ```
 
+#### Managed JSDOM
+
+In case you want to use your own JSDOM and Pino instance and tweak/configure as much as you prefer, you can use the `scrape` function instead.
+
+In the example below, we use the exposed `createWindow` and `fetchPage` utilities for convenience. You can use JSDOM constructor directly and any other strategy to fetch your page HTML as desired.
+
+```ts file=./examples/typescript/src/managed-jsdom/scraper.ts
+```
+
+Then run it using `node` or `ts-node`:
+
+```sh
+npx ts-node ./managed-jsdom.ts
+```
+
 ## API Reference
 
 [Click here to open the API reference.](https://armand1m.github.io/papercut)