v4.0.5

- Add auto discovery support (#181)
- Sync to the last provider list
- Update examples dependencies

README.md

@@ -29,7 +29,6 @@ yarn add @extractus/oembed-extractor
 ```
 
 ```ts
-// es6 module
 import { extract } from '@extractus/oembed-extractor'
 
 const result = await extract('https://www.youtube.com/watch?v=x2bqscVkGxk')
@@ -39,10 +38,6 @@ console.log(result)
 ### Deno
 
 ```ts
-// deno < 1.28
-import { extract } from 'https://esm.sh/@extractus/oembed-extractor'
-
-// deno > 1.28
 import { extract } from 'npm:@extractus/oembed-extractor'
 ```
 

examples/browser-oembed-parser/package.json

@@ -6,7 +6,7 @@
     "start": "node server"
   },
   "dependencies": {
-    "express": "^4.18.2",
-    "got": "^13.0.0"
+    "express": "latest",
+    "got": "latest"
   }
 }

examples/bun-oembed-parser/package.json

@@ -5,10 +5,10 @@
     "start": "bun run index.ts"
   },
   "devDependencies": {
-    "bun-types": "^0.6.13"
+    "bun-types": "latest"
   },
   "dependencies": {
-    "hono": "^3.2.7",
+    "hono": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }

examples/deno-oembed-parser/index.ts

@@ -1,6 +1,6 @@
 import { serve } from 'https://deno.land/std/http/server.ts'
 
-import { Hono } from 'https://deno.land/x/hono@v3.2.7/mod.ts'
+import { Hono } from 'https://deno.land/x/hono/mod.ts'
 
 import { extract } from 'npm:@extractus/oembed-extractor'
 

examples/node-oembed-parser/package.json

@@ -7,7 +7,7 @@
     "start": "node index.js"
   },
   "dependencies": {
-    "express": "^4.18.2",
+    "express": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }

examples/tsnode-oembed-parser/package.json

@@ -7,10 +7,10 @@
     "start": "node dist/index.js"
   },
   "devDependencies": {
-    "typescript": "^5.1.6"
+    "typescript": "latest"
   },
   "dependencies": {
-    "express": "^4.18.2",
+    "express": "latest",
     "@extractus/oembed-extractor": "latest"
   }
 }

package.json

@@ -1,5 +1,5 @@
 {
-  "version": "4.0.4",
+  "version": "4.0.5",
   "name": "@extractus/oembed-extractor",
   "description": "Get oEmbed data from given URL.",
   "homepage": "https://github.com/extractus/oembed-extractor",
@@ -11,10 +11,12 @@
   "main": "./src/main.js",
   "type": "module",
   "imports": {
-    "cross-fetch": "./src/deno/cross-fetch.js"
+    "cross-fetch": "./src/deno/cross-fetch.js",
+    "linkedom": "https://deno.land/x/[email protected]/deno-dom-wasm.ts"
   },
   "browser": {
-    "cross-fetch": "./src/deno/cross-fetch.js"
+    "cross-fetch": "./src/deno/cross-fetch.js",
+    "linkedom": "./src/browser/linkedom.js"
   },
   "types": "./index.d.ts",
   "engines": {
@@ -30,11 +32,12 @@
     "reset": "node reset"
   },
   "dependencies": {
-    "cross-fetch": "^4.0.0"
+    "cross-fetch": "^4.0.0",
+    "linkedom": "^0.16.11"
   },
   "devDependencies": {
-    "eslint": "^9.1.1",
-    "globals": "^15.0.0",
+    "eslint": "^9.2.0",
+    "globals": "^15.1.0",
     "https-proxy-agent": "^7.0.4",
     "jest": "^29.7.0",
     "nock": "^13.5.4"

src/main.js

@@ -1,6 +1,7 @@
 // main.js
 
 import { isValid as isValidURL } from './utils/linker.js'
+import extractWithDiscovery from './utils/autoDiscovery.js'
 import fetchEmbed from './utils/fetchEmbed.js'
 
 import { getEndpoint } from './utils/provider.js'
@@ -10,12 +11,10 @@ export const extract = async (url, params = {}, options = {}) => {
     throw new Error('Invalid input URL')
   }
   const endpoint = getEndpoint(url)
-  if (!endpoint) {
-    throw new Error(`No provider found with given url "${url}"`)
-  }
 
-  const data = await fetchEmbed(url, params, endpoint, options)
-  return data
+  return endpoint
+    ? fetchEmbed(url, params, endpoint, options)
+    : extractWithDiscovery(url, params, options)
 }
 
 export {

src/utils/fetchEmbed.js

@@ -1,6 +1,6 @@
 // utils -> fetchEmbed
 
-import retrieve from './retrieve.js'
+import { getJson } from './retrieve.js'
 import { getDomain } from './linker.js'
 
 const isFacebookGraphDependent = (url) => {
@@ -34,6 +34,7 @@ export default async (url, params = {}, endpoint = '', options = {}) => { // esl
 
   const queryParams = new URLSearchParams(query).toString()
   const link = endpoint + '?' + queryParams
-  const body = retrieve(link, options)
+  const body = await getJson(link, options)
+  body.method = 'provider-api'
   return body
 }

src/utils/providers.latest.js

@@ -1,4 +1,4 @@
-// provider data, synchronized at 2024-04-26T08:46:02.055Z
+// provider data, synchronized at 2024-05-07T10:41:10.221Z
 
 /* eslint-disable */ 
 
@@ -198,6 +198,12 @@ export const providers = [
     ],
     "e": "www.behance.net/services/oembed"
   },
+  {
+    "s": [
+      "cloud\\.biqapp\\.com/*"
+    ],
+    "e": "biqapp.com/api/v1/video/oembed"
+  },
   {
     "s": [
       "blackfire\\.io/profiles/*/graph",
@@ -277,6 +283,12 @@ export const providers = [
     ],
     "e": "img.catbo.at/oembed.json"
   },
+  {
+    "s": [
+      "embeds\\.celero\\.io/*"
+    ],
+    "e": "api.celero.io/api/oembed"
+  },
   {
     "s": [
       "view\\.ceros\\.com/*"
@@ -816,6 +828,12 @@ export const providers = [
     ],
     "e": "www.hulu.com/api/oembed.json"
   },
+  {
+    "s": [
+      "oembed\\.ideamapper\\.com/*"
+    ],
+    "e": "oembed.ideamapper.com"
+  },
   {
     "s": [
       "*\\.idomoo\\.com/*"

src/utils/providers.orginal.json

@@ -415,6 +415,19 @@
       }
     ]
   },
+  {
+    "provider_name": "biqnetwork",
+    "provider_url": "https://biqapp.com/",
+    "endpoints": [
+      {
+        "schemes": [
+          "https://cloud.biqapp.com/*"
+        ],
+        "url": "https://biqapp.com/api/v1/video/oembed",
+        "discovery": true
+      }
+    ]
+  },
   {
     "provider_name": "Blackfire.io",
     "provider_url": "https://blackfire.io",
@@ -591,6 +604,22 @@
       }
     ]
   },
+  {
+    "provider_name": "Celero",
+    "provider_url": "https://www.celero.io",
+    "endpoints": [
+      {
+        "schemes": [
+          "https://embeds.celero.io/*"
+        ],
+        "url": "https://api.celero.io/api/oembed",
+        "discovery": true,
+        "formats": [
+          "json"
+        ]
+      }
+    ]
+  },
   {
     "provider_name": "Ceros",
     "provider_url": "http://www.ceros.com/",
@@ -1698,6 +1727,19 @@
       }
     ]
   },
+  {
+    "provider_name": "Ideamapper",
+    "provider_url": "http://oembed.ideamapper.com",
+    "endpoints": [
+      {
+        "schemes": [
+          "http://oembed.ideamapper.com/*"
+        ],
+        "url": "http://oembed.ideamapper.com",
+        "discovery": true
+      }
+    ]
+  },
   {
     "provider_name": "Idomoo",
     "provider_url": "https://idomoo.com/",

src/utils/providers.prev.js

@@ -1,4 +1,4 @@
-// provider data, synchronized at 2024-03-29T04:36:42.975Z
+// provider data, synchronized at 2024-04-26T08:46:02.055Z
 
 /* eslint-disable */ 
 
@@ -212,6 +212,12 @@ export const providers = [
     ],
     "e": "blogcast.host/oembed"
   },
+  {
+    "s": [
+      "bsky\\.app/profile/*/post/*"
+    ],
+    "e": "embed.bsky.app/oembed"
+  },
   {
     "s": [
       "www\\.bookingmood\\.com/embed/*/*"
@@ -1198,6 +1204,12 @@ export const providers = [
     ],
     "e": "ndla.no/oembed"
   },
+  {
+    "s": [
+      "*\\.neetorecord\\.com/watch/*"
+    ],
+    "e": "api.neetorecord.com/api/v1/oembed"
+  },
   {
     "s": [
       "*\\.nfb\\.ca/film/*"
@@ -1559,6 +1571,13 @@ export const providers = [
     ],
     "e": "embed.sendtonews.com/services/oembed"
   },
+  {
+    "s": [
+      "shopshare\\.tv/shopboard/*",
+      "shopshare\\.tv/shopcast/*"
+    ],
+    "e": "shopshare.tv/api/shopcast/oembed"
+  },
   {
     "s": [
       "www\\.shortnote\\.jp/view/notes/*"
@@ -1830,6 +1849,12 @@ export const providers = [
     ],
     "e": "app-test.totango.com/oembed"
   },
+  {
+    "s": [
+      "trackspace\\.upitup\\.com/*"
+    ],
+    "e": "trackspace.upitup.com/oembed"
+  },
   {
     "s": [
       "trinitymedia\\.ai/player/*",

src/utils/retrieve.js

@@ -15,7 +15,28 @@ const profetch = async (url, options = {}) => {
   return res
 }
 
-export default async (url, options = {}) => {
+export const getHtml = async (url, options = {}) => {
+  const {
+    headers = {
+      'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',
+    },
+    proxy = null,
+    agent = null,
+    signal = null,
+  } = options
+
+  const res = proxy ? await profetch(url, { proxy, signal }) : await fetch(url, { headers, agent, signal })
+
+  const status = res.status
+  if (status >= 400) {
+    throw new Error(`Request failed with error code ${status}`)
+  }
+
+  const text = await res.text()
+  return text
+}
+
+export const getJson = async (url, options = {}) => {
   const {
     headers = {
       'user-agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0',