diff --git a/lightweight/index.d.ts b/lightweight/index.d.ts index 7c01b0b..445269b 100644 --- a/lightweight/index.d.ts +++ b/lightweight/index.d.ts @@ -10,10 +10,10 @@ type ScreenshotOverlay = { } type PdfMargin = { + bottom?: string | number; + left?: string | number; + right?: string | number; top?: string | number; - bottom?: string | number; - left?: string | number; - right?: string | number; } type PdfOptions = { @@ -28,18 +28,19 @@ type PdfOptions = { type ScreenshotOptions = { codeScheme?: string; - omitBackground?: boolean; - type?: "jpeg" | "png"; element?: string; fullPage?: boolean; - overlay?: ScreenshotOverlay + omitBackground?: boolean; + optimizeForSpeed?: boolean; + overlay?: ScreenshotOverlay, + type?: "jpeg" | "png"; } type MqlClientOptions = { - endpoint?: string; apiKey?: string; - retry?: number; cache?: Map; + endpoint?: string; + retry?: number; } type MqlQuery = { @@ -83,8 +84,9 @@ type MicrolinkApiOptions = { screenshot?: boolean | ScreenshotOptions; scripts?: string | string[]; scroll?: string; - styles?: string | string[]; staleTtl?: string | number; + stream?: boolean; + styles?: string | string[]; timeout?: number; ttl?: string | number; video?: boolean; @@ -100,41 +102,33 @@ type IframeInfo = { } type MediaInfo = { - url: string; - type?: string; - palette?: string[]; + alternative_color?: string; background_color?: string; color?: string; - alternative_color?: string; - width?: number; - height?: number; - duration?: number; duration_pretty?: string; + duration?: number; + height?: number; + palette?: string[]; + type?: string; + url: string; + width?: number; } type MqlResponseData = { - // A human-readable representation of the author's name. + audio?: MediaInfo | null; author?: string | null; - // An ISO 8601 representation of the date the article was published. date?: string | null; - // The publisher's chosen description of the article. description?: string | null; - // An ISO 639-1 representation of the url content language. + function?: MqlFunctionResult; + iframe?: IframeInfo | null; + image?: MediaInfo | null; lang?: string | null; - // An image URL that best represents the publisher brand. logo?: MediaInfo | null; - // A human-readable representation of the publisher's name. publisher?: string | null; - // The publisher's chosen title of the article. + screenshot?: MediaInfo | null; title?: string | null; - // The URL of the article. url?: string; - image?: MediaInfo | null; - screenshot?: MediaInfo | null; video?: MediaInfo | null; - audio?: MediaInfo | null; - iframe?: IframeInfo | null; - function?: MqlFunctionResult; } type MqlFunctionResult = { @@ -165,25 +159,26 @@ type HTTPResponseRaw = HTTPResponse & { body: ArrayBuffer }; export type MqlResponse = MqlPayload & { response: HTTPResponseWithBody }; export type MqlError = { - headers: { [key: string]: string }; + code: string; data?: MqlResponseData; - statusCode?: number; - name: string; - message: string; description: string; - status: MqlStatus; - code: string; + headers: { [key: string]: string }; + message: string; more: string; + name: string; + status: MqlStatus; + statusCode?: number; url: string; } export type MqlOptions = MqlClientOptions & MicrolinkApiOptions; interface mql { + (url: string, opts?: MqlOptions & { stream: true }, gotOpts?: object): Promise; (url: string, opts?: MqlOptions, gotOpts?: object): Promise; + arrayBuffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise; extend: (gotOpts?: object) => mql; stream: (input: RequestInfo, init?: RequestInit) => ReadableStream; - arrayBuffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise; } declare const mql: mql; diff --git a/package.json b/package.json index fb1639e..3e27350 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,7 @@ "standard": { "ignore": [ "lightweight/index.js", + "lightweight/index.umd.js", "src/node.mjs" ] }, diff --git a/src/factory.js b/src/factory.js index 48accfa..10f282d 100644 --- a/src/factory.js +++ b/src/factory.js @@ -28,7 +28,13 @@ const parseBody = (input, error, url) => { } } -const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => { +const factory = streamResponseType => ({ + VERSION, + MicrolinkError, + urlHttp, + got, + flatten +}) => { const assertUrl = (url = '') => { if (!urlHttp(url)) { const message = `The \`url\` as \`${url}\` is not valid. Ensure it has protocol (http or https) and hostname.` @@ -55,8 +61,8 @@ const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => { const fetchFromApi = async (apiUrl, opts = {}) => { try { const response = await got(apiUrl, opts) - return opts.responseType === 'buffer' - ? { body: response.body, response } + return opts.responseType === streamResponseType + ? response : { ...response.body, response } } catch (err) { const { response = {} } = err @@ -100,6 +106,10 @@ const factory = ({ VERSION, MicrolinkError, urlHttp, got, flatten }) => { const headers = isPro ? { ...gotHeaders, 'x-api-key': apiKey } : { ...gotHeaders } + + if (opts.stream) { + responseType = streamResponseType + } return [apiUrl, { ...gotOpts, responseType, cache, retry, headers }] } diff --git a/src/lightweight.js b/src/lightweight.js index dd1a05d..ed352ae 100644 --- a/src/lightweight.js +++ b/src/lightweight.js @@ -2,9 +2,10 @@ const urlHttp = require('url-http/lightweight') const { flattie: flatten } = require('flattie') -const factory = require('./factory') const { default: ky } = require('ky') +const factory = require('./factory')('arrayBuffer') + class MicrolinkError extends Error { constructor (props) { super() @@ -24,10 +25,10 @@ const got = async (url, { responseType, ...opts }) => { const body = await response[responseType]() const { headers, status: statusCode } = response return { url: response.url, body, headers, statusCode } - } catch (err) { - if (err.response) { - const { response } = err - err.response = { + } catch (error) { + if (error.response) { + const { response } = error + error.response = { ...response, headers: Array.from(response.headers.entries()).reduce( (acc, [key, value]) => { @@ -40,7 +41,7 @@ const got = async (url, { responseType, ...opts }) => { body: await response.text() } } - throw err + throw error } } diff --git a/src/node.d.ts b/src/node.d.ts index b96a9d2..492e9dc 100644 --- a/src/node.d.ts +++ b/src/node.d.ts @@ -2,12 +2,9 @@ import { MqlPayload, MqlOptions } from '../lightweight'; export { MqlError, MqlPayload } from '../lightweight' -type HTTPResponse = { - url: string; - isFromCache?: boolean; - statusCode: number; - headers: { [key: string]: string }; -}; +import { Response, Options as GotOpts } from 'got/dist/source/core' + +type HTTPResponse = Response type HTTPResponseWithBody = HTTPResponse & { body: MqlPayload }; @@ -16,10 +13,11 @@ type HTTPResponseRaw = HTTPResponse & { body: Buffer }; type MqlResponse = MqlPayload & { response: HTTPResponseWithBody }; interface Mql { - (url: string, opts?: MqlOptions, gotOpts?: object): Promise; - extend: (gotOpts?: object) => Mql; - stream: (url: string, gotOpts?: object) => NodeJS.ReadableStream; - buffer: (url: string, opts?: MqlOptions, gotOpts?: object) => Promise; + (url: string, opts?: MqlOptions & { stream: true }, gotOpts?: GotOpts): Promise; + (url: string, opts?: MqlOptions, gotOpts?: GotOpts): Promise; + extend: (gotOpts?: GotOpts) => Mql; + stream: (url: string, gotOpts?: GotOpts) => NodeJS.ReadableStream; + buffer: (url: string, opts?: MqlOptions, gotOpts?: GotOpts) => Promise; } declare const mql: Mql; diff --git a/src/node.js b/src/node.js index 2d978cd..0d83208 100644 --- a/src/node.js +++ b/src/node.js @@ -1,4 +1,4 @@ -const mql = require('./factory')({ +const mql = require('./factory')('buffer')({ MicrolinkError: require('whoops')('MicrolinkError'), urlHttp: require('url-http/lightweight'), got: require('got').extend({ headers: { 'user-agent': undefined } }), diff --git a/test/lightweight.test-d.ts b/test/lightweight.test-d.ts index 0d4b095..4d8e4e3 100644 --- a/test/lightweight.test-d.ts +++ b/test/lightweight.test-d.ts @@ -3,12 +3,28 @@ import type { MqlError } from '../lightweight' /** mql */ -mql('https://example.com', { - endpoint: 'https://pro.microlink.io', - apiKey: '123', - retry: 2, - cache: new Map() -}) +;async () => { + const result = await mql('https://example.com', { + endpoint: 'https://pro.microlink.io', + apiKey: '123', + retry: 2, + cache: new Map() + }) + + console.log(result.status) + console.log(result.data) + console.log(result.statusCode) + console.log(result.headers) + console.log(result.response) +} + +;(async () => { + const response = await mql('https://example.com', { + stream: true, + screenshot: true + }) + console.log(response.body) +})() /** data */ @@ -35,8 +51,7 @@ mql('https://github.com/microlinkhq', { type: 'number' }, stars: { - selector: - '.js-responsive-underlinenav a[data-tab-item="stars"] span', + selector: '.js-responsive-underlinenav a[data-tab-item="stars"] span', type: 'number' } } @@ -73,6 +88,7 @@ mql('https://example.com', { screenshot: { codeScheme: 'atom-dark', type: 'png', + optimizeForSpeed: true, overlay: { background: '#000', browser: 'light' @@ -99,7 +115,6 @@ console.log(result.statusCode) console.log(result.headers) /** error */ - ;({ status: 'error', data: { url: 'fetch failed' }, @@ -111,8 +126,7 @@ console.log(result.headers) name: 'MicrolinkError', message: 'EFATALCLIENT, fetch failed', description: 'fetch failed' -}) as MqlError - +} as MqlError) ;({ status: 'fail', code: 'EAUTH', @@ -128,7 +142,8 @@ console.log(result.headers) 'content-type': 'application/json', date: 'Thu, 05 Oct 2023 11:02:35 GMT', nel: '{"success_fraction":0,"report_to":"cf-nel","max_age":604800}', - 'report-to': '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=6tEv%2Fk7XkC0so782muCCxAfbFeaMusFvyv839c8Xv74aKQFy1jD%2Fd8hRrldtfntrhuzCi5HG8W%2FlBxk1a9qKqxHObl79FhxBnK6pAOF6gGXc9Vi0wHnXb1hayCkTxolfpR7yoH89el9W34r1T8E%3D"}],"group":"cf-nel","max_age":604800}', + 'report-to': + '{"endpoints":[{"url":"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=6tEv%2Fk7XkC0so782muCCxAfbFeaMusFvyv839c8Xv74aKQFy1jD%2Fd8hRrldtfntrhuzCi5HG8W%2FlBxk1a9qKqxHObl79FhxBnK6pAOF6gGXc9Vi0wHnXb1hayCkTxolfpR7yoH89el9W34r1T8E%3D"}],"group":"cf-nel","max_age":604800}', server: 'cloudflare', 'transfer-encoding': 'chunked', vary: 'Accept-Encoding', @@ -141,9 +156,11 @@ console.log(result.headers) 'x-cache': 'Error from cloudfront' }, name: 'MicrolinkError', - message: 'EAUTH, Invalid API key. Make sure you are attaching your API key as `x-api-key` header.', - description: 'Invalid API key. Make sure you are attaching your API key as `x-api-key` header.' -}) as MqlError + message: + 'EAUTH, Invalid API key. Make sure you are attaching your API key as `x-api-key` header.', + description: + 'Invalid API key. Make sure you are attaching your API key as `x-api-key` header.' +} as MqlError) /* extend */ @@ -151,7 +168,7 @@ mql.extend({ responseType: 'text' }) /* stream */ -mql.stream('https://example.com', { headers: { 'user-agent': 'foo' }}) +mql.stream('https://example.com', { headers: { 'user-agent': 'foo' } }) /* arrraBuffer */ diff --git a/test/node.test-d.ts b/test/node.test-d.ts index 362b5d5..b9996dd 100644 --- a/test/node.test-d.ts +++ b/test/node.test-d.ts @@ -1,10 +1,39 @@ import mql from '../src/node' -/** response */ +/** mql */ -const result = await mql('https://example.com', { meta: true }) -console.log(result.response.isFromCache) +;(async () => { + const result = await mql('https://example.com', { meta: true }) + console.log(result.status) + console.log(result.data) + console.log(result.statusCode) + console.log(result.headers) + console.log(result.response) + console.log(result.response.isFromCache) +})() + +;(async () => { + const response = await mql('https://example.com', { + stream: true, + screenshot: true + }) + console.log(response.body) +})() + +/** got options */ + +await mql( + 'https://example.com', + { meta: true }, + { + timeout: 1000 + } +) /** stream */ -mql.stream('https://example.com', { meta: true }) +mql.stream('https://cdn.microlink.io/logo/logo.png', { + headers: { + accept: 'image/webp' + } +}) diff --git a/test/opts.mjs b/test/opts.mjs index de9bc49..84a2033 100644 --- a/test/opts.mjs +++ b/test/opts.mjs @@ -17,6 +17,26 @@ clients.forEach(({ constructor: mql, target }) => { t.snapshot(response.statusCode) }) + test(`${target} » stream`, async t => { + const response = await mql( + 'https://kikobeats.com?ref=mql', { + screenshot: { optimizedForSpeed: true }, + stream: true + } + ) + + t.is(typeof response.headers, 'object') + t.is(typeof response.statusCode, 'number') + t.is(typeof response.url, 'string') + + if (target === 'node') { + t.true(Buffer.isBuffer(response.body)) + t.is(typeof response.isFromCache, 'boolean') + } else { + t.true((response.body instanceof ArrayBuffer)) + } + }) + if (target === 'node') { test('node » cache support', async t => { const cache = new Map()