feat: add stream query parameter (#152)

* refactor: sort fields

* refactor: add optimizeForSpeed

* feat: add stream mode

* fix: ignore build files

lightweight/index.d.ts

@@ -10,10 +10,10 @@ type ScreenshotOverlay = {
 }
 
 type PdfMargin = {
+  bottom?: string | number;
+  left?: string | number;
+  right?: string | number;
   top?: string | number;
-   bottom?: string | number;
-   left?: string | number;
-   right?: string | number;
 }
 
 type PdfOptions = {
@@ -28,18 +28,19 @@ type PdfOptions = {
 
 type ScreenshotOptions = {
   codeScheme?: string;
-  omitBackground?: boolean;
-  type?: "jpeg" | "png";
   element?: string;
   fullPage?: boolean;
-  overlay?: ScreenshotOverlay
+  omitBackground?: boolean;
+  optimizeForSpeed?: boolean;
+  overlay?: ScreenshotOverlay,
+  type?: "jpeg" | "png";
 }
 
 type MqlClientOptions = {
-  endpoint?: string;
   apiKey?: string;
-  retry?: number;
   cache?: Map<string, any>;
+  endpoint?: string;
+  retry?: number;
 }
 
 type MqlQuery = {
@@ -83,8 +84,9 @@ type MicrolinkApiOptions = {
   screenshot?: boolean | ScreenshotOptions;
   scripts?: string | string[];
   scroll?: string;
-  styles?: string | string[];
   staleTtl?: string | number;
+  stream?: boolean;
+  styles?: string | string[];
   timeout?: number;
   ttl?: string | number;
   video?: boolean;
@@ -100,41 +102,33 @@ type IframeInfo = {
 }
 
 type MediaInfo = {
-  url: string;
-  type?: string;
-  palette?: string[];
+  alternative_color?: string;
   background_color?: string;
   color?: string;
-  alternative_color?: string;
-  width?: number;
-  height?: number;
-  duration?: number;
   duration_pretty?: string;
+  duration?: number;
+  height?: number;
+  palette?: string[];
+  type?: string;
+  url: string;
+  width?: number;
 }
 
 type MqlResponseData = {
-  // A human-readable representation of the author's name.
+  audio?: MediaInfo | null;
   author?: string | null;
-  // An ISO 8601 representation of the date the article was published.
   date?: string | null;
-  // The publisher's chosen description of the article.
   description?: string | null;
-  // An ISO 639-1 representation of the url content language.
+  function?: MqlFunctionResult;
+  iframe?: IframeInfo | null;
+  image?: MediaInfo | null;
   lang?: string | null;
-  // An image URL that best represents the publisher brand.
   logo?: MediaInfo | null;
-  // A human-readable representation of the publisher's name.
   publisher?: string | null;
-  // The publisher's chosen title of the article.
+  screenshot?: MediaInfo | null;
   title?: string | null;
-  // The URL of the article.
   url?: string;
-  image?: MediaInfo | null;
-  screenshot?: MediaInfo | null;
   video?: MediaInfo | null;
-  audio?: MediaInfo | null;
-  iframe?: IframeInfo | null;
-  function?: MqlFunctionResult;
 }
 
 type MqlFunctionResult = {
@@ -165,25 +159,26 @@ type HTTPResponseRaw = HTTPResponse & { body: ArrayBuffer };
 export type MqlResponse = MqlPayload & { response: HTTPResponseWithBody };
 
 export type MqlError = {
-  headers: { [key: string]: string };
+  code: string;
   data?: MqlResponseData;
-  statusCode?: number;
-  name: string;
-  message: string;
   description: string;
-  status: MqlStatus;
-  code: string;
+  headers: { [key: string]: string };
+  message: string;
   more: string;
+  name: string;
+  status: MqlStatus;
+  statusCode?: number;
   url: string;
 }
 
 export type MqlOptions = MqlClientOptions & MicrolinkApiOptions;
 
 interface mql {
+  (url: string, opts?: MqlOptions & { stream: true }, gotOpts?: object): Promise<Response>;
   (url: string, opts?: MqlOptions, gotOpts?: object): Promise<MqlResponse>;
+  arrayBuffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise<HTTPResponseRaw>;
   extend: (gotOpts?: object) => mql;
   stream: (input: RequestInfo, init?: RequestInit) => ReadableStream;
-  arrayBuffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise<HTTPResponseRaw>;
 }
 
 declare const mql: mql;

package.json

@@ -139,6 +139,7 @@
   "standard": {
     "ignore": [
       "lightweight/index.js",
+      "lightweight/index.umd.js",
       "src/node.mjs"
     ]
   },

src/factory.js

@@ -28,7 +28,13 @@ const parseBody = (input, error, url) => {
   }
 }
 
-const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => {
+const factory = streamResponseType => ({
+  VERSION,
+  MicrolinkError,
+  urlHttp,
+  got,
+  flatten
+}) => {
   const assertUrl = (url = '') => {
     if (!urlHttp(url)) {
       const message = `The \`url\` as \`${url}\` is not valid. Ensure it has protocol (http or https) and hostname.`
@@ -55,8 +61,8 @@ const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => {
   const fetchFromApi = async (apiUrl, opts = {}) => {
     try {
       const response = await got(apiUrl, opts)
-      return opts.responseType === 'buffer'
-        ? { body: response.body, response }
+      return opts.responseType === streamResponseType
+        ? response
         : { ...response.body, response }
     } catch (err) {
       const { response = {} } = err
@@ -100,6 +106,10 @@ const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => {
     const headers = isPro
       ? { ...gotHeaders, 'x-api-key': apiKey }
       : { ...gotHeaders }
+
+    if (opts.stream) {
+      responseType = streamResponseType
+    }
     return [apiUrl, { ...gotOpts, responseType, cache, retry, headers }]
   }
 

src/lightweight.js

@@ -2,9 +2,10 @@
 
 const urlHttp = require('url-http/lightweight')
 const { flattie: flatten } = require('flattie')
-const factory = require('./factory')
 const { default: ky } = require('ky')
 
+const factory = require('./factory')('arrayBuffer')
+
 class MicrolinkError extends Error {
   constructor (props) {
     super()
@@ -24,10 +25,10 @@ const got = async (url, { responseType, ...opts }) => {
     const body = await response[responseType]()
     const { headers, status: statusCode } = response
     return { url: response.url, body, headers, statusCode }
-  } catch (err) {
-    if (err.response) {
-      const { response } = err
-      err.response = {
+  } catch (error) {
+    if (error.response) {
+      const { response } = error
+      error.response = {
         ...response,
         headers: Array.from(response.headers.entries()).reduce(
           (acc, [key, value]) => {
@@ -40,7 +41,7 @@ const got = async (url, { responseType, ...opts }) => {
         body: await response.text()
       }
     }
-    throw err
+    throw error
   }
 }
 

src/node.d.ts

@@ -2,12 +2,9 @@ import { MqlPayload, MqlOptions } from '../lightweight';
 
 export { MqlError, MqlPayload } from '../lightweight'
 
-type HTTPResponse = {
-  url: string;
-  isFromCache?: boolean;
-  statusCode: number;
-  headers: { [key: string]: string };
-};
+import { Response, Options as GotOpts } from 'got/dist/source/core'
+
+type HTTPResponse = Response<Buffer>
 
 type HTTPResponseWithBody = HTTPResponse & { body: MqlPayload };
 
@@ -16,10 +13,11 @@ type HTTPResponseRaw = HTTPResponse & { body: Buffer };
 type MqlResponse = MqlPayload & { response: HTTPResponseWithBody };
 
 interface Mql {
-  (url: string, opts?: MqlOptions, gotOpts?: object): Promise<MqlResponse>;
-  extend: (gotOpts?: object) => Mql;
-  stream: (url: string, gotOpts?: object) => NodeJS.ReadableStream;
-  buffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise<HTTPResponseRaw>;
+  (url: string, opts?: MqlOptions & { stream: true }, gotOpts?: GotOpts): Promise<HTTPResponse>;
+  (url: string, opts?: MqlOptions, gotOpts?: GotOpts): Promise<MqlResponse>;
+  extend: (gotOpts?: GotOpts) => Mql;
+  stream: (url: string, gotOpts?: GotOpts) => NodeJS.ReadableStream;
+  buffer: (url: string, opts?: MqlOptions, gotOpts?: GotOpts) => Promise<HTTPResponseRaw>;
 }
 
 declare const mql: Mql;

src/node.js

@@ -1,4 +1,4 @@
-const mql = require('./factory')({
+const mql = require('./factory')('buffer')({
   MicrolinkError: require('whoops')('MicrolinkError'),
   urlHttp: require('url-http/lightweight'),
   got: require('got').extend({ headers: { 'user-agent': undefined } }),

test/lightweight.test-d.ts

@@ -3,12 +3,28 @@ import type { MqlError } from '../lightweight'
 
 /** mql */
 
-mql('https://example.com', {
-  endpoint: 'https://pro.microlink.io',
-  apiKey: '123',
-  retry: 2,
-  cache: new Map()
-})
+;async () => {
+  const result = await mql('https://example.com', {
+    endpoint: 'https://pro.microlink.io',
+    apiKey: '123',
+    retry: 2,
+    cache: new Map()
+  })
+
+  console.log(result.status)
+  console.log(result.data)
+  console.log(result.statusCode)
+  console.log(result.headers)
+  console.log(result.response)
+}
+
+;(async () => {
+  const response = await mql('https://example.com', {
+    stream: true,
+    screenshot: true
+  })
+  console.log(response.body)
+})()
 
 /** data */
 
@@ -35,8 +51,7 @@ mql('https://github.com/microlinkhq', {
           type: 'number'
         },
         stars: {
-          selector:
-            '.js-responsive-underlinenav a[data-tab-item="stars"] span',
+          selector: '.js-responsive-underlinenav a[data-tab-item="stars"] span',
           type: 'number'
         }
       }
@@ -73,6 +88,7 @@ mql('https://example.com', {
   screenshot: {
     codeScheme: 'atom-dark',
     type: 'png',
+    optimizeForSpeed: true,
     overlay: {
       background: '#000',
       browser: 'light'
@@ -99,7 +115,6 @@ console.log(result.statusCode)
 console.log(result.headers)
 
 /** error */
-
 ;({
   status: 'error',
   data: { url: 'fetch failed' },
@@ -111,8 +126,7 @@ console.log(result.headers)
   name: 'MicrolinkError',
   message: 'EFATALCLIENT, fetch failed',
   description: 'fetch failed'
-}) as MqlError
-
+} as MqlError)
 ;({
   status: 'fail',
   code: 'EAUTH',
@@ -128,7 +142,8 @@ console.log(result.headers)
     'content-type': 'application/json',
     date: 'Thu, 05 Oct 2023 11:02:35 GMT',
     nel: '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}',
-    'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=6tEv%2Fk7XkC0so782muCCxAfbFeaMusFvyv839c8Xv74aKQFy1jD%2Fd8hRrldtfntrhuzCi5HG8W%2FlBxk1a9qKqxHObl79FhxBnK6pAOF6gGXc9Vi0wHnXb1hayCkTxolfpR7yoH89el9W34r1T8E%3D"}],"group":"cf-nel","max_age":604800}',
+    'report-to':
+      '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=6tEv%2Fk7XkC0so782muCCxAfbFeaMusFvyv839c8Xv74aKQFy1jD%2Fd8hRrldtfntrhuzCi5HG8W%2FlBxk1a9qKqxHObl79FhxBnK6pAOF6gGXc9Vi0wHnXb1hayCkTxolfpR7yoH89el9W34r1T8E%3D"}],"group":"cf-nel","max_age":604800}',
     server: 'cloudflare',
     'transfer-encoding': 'chunked',
     vary: 'Accept-Encoding',
@@ -141,17 +156,19 @@ console.log(result.headers)
     'x-cache': 'Error from cloudfront'
   },
   name: 'MicrolinkError',
-  message: 'EAUTH, Invalid API key. Make sure you are attaching your API key as `x-api-key` header.',
-  description: 'Invalid API key. Make sure you are attaching your API key as `x-api-key` header.'
-}) as MqlError
+  message:
+    'EAUTH, Invalid API key. Make sure you are attaching your API key as `x-api-key` header.',
+  description:
+    'Invalid API key. Make sure you are attaching your API key as `x-api-key` header.'
+} as MqlError)
 
 /* extend */
 
 mql.extend({ responseType: 'text' })
 
 /* stream */
 
-mql.stream('https://example.com', { headers: { 'user-agent': 'foo' }})
+mql.stream('https://example.com', { headers: { 'user-agent': 'foo' } })
 
 /* arrraBuffer */