Skip to content

Commit

Permalink
feat(app/web): bypass metrics.api.github.overuse with OAuth (#1171)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lowlighter
lowlighter authored Aug 6, 2022
1 parent 0937317 commit 587ceec
Show file tree
Hide file tree
Showing 13 changed files with 657 additions and 111 deletions.
17 changes: 10 additions & 7 deletions source/app/metrics/metadata.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ metadata.plugin = async function({__plugins, __templates, name, logger}) {
if ((meta.category === "github") && (!meta.disclaimer))
meta.disclaimer = "This plugin is not affiliated, associated, authorized, endorsed by, or in any way officially connected with [GitHub](https://github.com).\nAll product and company names are trademarks™ or registered® trademarks of their respective holders."

//Deprecation
meta.deprecated = !!meta?.deprecation

//Inputs parser
{
meta.inputs = function({data: {user = null} = {}, q, account}, defaults = {}) {
Expand Down Expand Up @@ -345,30 +348,30 @@ metadata.plugin = async function({__plugins, __templates, name, logger}) {
//Web metadata
{
meta.web = Object.fromEntries(
Object.entries(inputs).map(([key, {type, description: text, example, default: defaulted, min = 0, max = 9999, values}]) => [
Object.entries(inputs).map(([key, {type, description: text, example, default: defaulted, min = 0, max = 9999, values, extras}]) => [
//Format key
metadata.to.query(key),
//Value descriptor
(() => {
switch (type) {
case "boolean":
return {text, type: "boolean", defaulted: /^(?:[Tt]rue|[Oo]n|[Yy]es|1)$/.test(defaulted) ? true : /^(?:[Ff]alse|[Oo]ff|[Nn]o|0)$/.test(defaulted) ? false : defaulted}
return {text, type: "boolean", defaulted: /^(?:[Tt]rue|[Oo]n|[Yy]es|1)$/.test(defaulted) ? true : /^(?:[Ff]alse|[Oo]ff|[Nn]o|0)$/.test(defaulted) ? false : defaulted, extras}
case "number":
return {text, type: "number", min, max, defaulted}
return {text, type: "number", min, max, defaulted, extras}
case "array":
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
case "string": {
if (Array.isArray(values))
return {text, type: "select", values, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
}
case "json":
return {text, type: "text", placeholder: example ?? defaulted, defaulted}
return {text, type: "text", placeholder: example ?? defaulted, defaulted, extras}
default:
return null
}
})(),
]).filter(([key, value]) => (value) && (key !== name)),
]).filter(([key, value]) => (value)&&(!((name === "base")&&(key === "repositories")))),
)
}

Expand Down
163 changes: 143 additions & 20 deletions source/app/web/instance.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@ import express from "express"
import ratelimit from "express-rate-limit"
import cache from "memory-cache"
import util from "util"
import url from "url"
import axios from "axios"
import mocks from "../../../tests/mocks/index.mjs"
import metrics from "../metrics/index.mjs"
import presets from "../metrics/presets.mjs"
import setup from "../metrics/setup.mjs"
import crypto from "crypto"

/**App */
export default async function({sandbox = false} = {}) {
Expand Down Expand Up @@ -55,7 +58,20 @@ export default async function({sandbox = false} = {}) {
//Apply mocking if needed
if (mock)
Object.assign(api, await mocks(api))
const {graphql, rest} = api
//Custom user octokits sessions
const authenticated = new Map()
const uapi = session => {
if (!/^[a-f0-9]+$/i.test(`${session}`))
return null
if (authenticated.has(session)) {
const {login, token} = authenticated.get(session)
console.debug(`metrics/app/session/${login} > authenticated with session ${session.substring(0, 6)}, using custom octokit`)
return {login, graphql: octokit.graphql.defaults({headers: {authorization: `token ${token}`}}), rest: new OctokitRest.Octokit({auth: token})}
}
else if (session)
console.debug(`metrics/app/session > unknown session ${session.substring(0, 6)}, using default octokit`)
return null
}

//Setup server
const app = express()
Expand Down Expand Up @@ -87,22 +103,19 @@ export default async function({sandbox = false} = {}) {
const limiter = ratelimit({max: debug ? Number.MAX_SAFE_INTEGER : 60, windowMs: 60 * 1000, headers: false})
const metadata = Object.fromEntries(
Object.entries(conf.metadata.plugins)
.map(([key, value]) => [key, Object.fromEntries(Object.entries(value).filter(([key]) => ["name", "icon", "category", "web", "supports", "scopes"].includes(key)))])
.map(([key, value]) => [key, Object.fromEntries(Object.entries(value).filter(([key]) => ["name", "icon", "category", "web", "supports", "scopes", "deprecated"].includes(key)))])
.map(([key, value]) => [key, key === "core" ? {...value, web: Object.fromEntries(Object.entries(value.web).filter(([key]) => /^config[.]/.test(key)).map(([key, value]) => [key.replace(/^config[.]/, ""), value]))} : value]),
)
const enabled = Object.entries(metadata).filter(([_name, {category}]) => category !== "core").map(([name]) => ({name, category: metadata[name]?.category ?? "community", enabled: plugins[name]?.enabled ?? false}))
const enabled = Object.entries(metadata).filter(([_name, {category}]) => category !== "core").map(([name]) => ({name, category: metadata[name]?.category ?? "community", deprecated: metadata[name]?.deprecated ?? false, enabled: plugins[name]?.enabled ?? false}))
const templates = Object.entries(Templates).map(([name]) => ({name, enabled: (conf.settings.templates.enabled.length ? conf.settings.templates.enabled.includes(name) : true) ?? false}))
const actions = {flush: new Map()}
const requests = {rest: {limit: 0, used: 0, remaining: 0, reset: NaN}, graphql: {limit: 0, used: 0, remaining: 0, reset: NaN}}
const requests = {rest: {limit: 0, used: 0, remaining: 0, reset: NaN}, graphql: {limit: 0, used: 0, remaining: 0, reset: NaN}, search: {limit: 0, used: 0, remaining: 0, reset: NaN}}
let _requests_refresh = false
if (!conf.settings.notoken) {
const refresh = async () => {
try {
const {limit} = await graphql("{ limit:rateLimit {limit remaining reset:resetAt used} }")
Object.assign(requests, {
rest: (await rest.rateLimit.get()).data.rate,
graphql: {...limit, reset: new Date(limit.reset).getTime()},
})
const {resources} = (await api.rest.rateLimit.get()).data
Object.assign(requests, {rest: resources.core, graphql: resources.graphql, search: resources.search})
}
catch {
console.debug("metrics/app > failed to update remaining requests")
Expand Down Expand Up @@ -130,8 +143,16 @@ export default async function({sandbox = false} = {}) {
app.get("/.templates/:template", limiter, (req, res) => req.params.template in conf.templates ? res.status(200).json(conf.templates[req.params.template]) : res.sendStatus(404))
for (const template in conf.templates)
app.use(`/.templates/${template}/partials`, express.static(`${conf.paths.templates}/${template}/partials`))
//Modes
//Modes and extras
app.get("/.modes", limiter, (req, res) => res.status(200).json(conf.settings.modes))
app.get("/.extras", limiter, async (req, res) => {
if ((authenticated.has(req.headers["x-metrics-session"]))&&(conf.settings.extras?.logged)) {
if (conf.settings.extras?.features !== true)
return res.status(200).json([...conf.settings.extras.features, ...conf.settings.extras.logged])
}
res.status(200).json(conf.settings.extras?.features ?? conf.settings?.extras?.default ?? false)
})
app.get("/.extras.logged", limiter, async (req, res) => res.status(200).json(conf.settings.extras?.logged ?? []))
//Styles
app.get("/.css/style.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.css`))
app.get("/.css/style.vars.css", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/style.vars.css`))
Expand All @@ -152,7 +173,18 @@ export default async function({sandbox = false} = {}) {
app.get("/.js/clipboard.min.js", limiter, (req, res) => res.sendFile(`${conf.paths.node_modules}/clipboard/dist/clipboard.min.js`))
//Meta
app.get("/.version", limiter, (req, res) => res.status(200).send(conf.package.version))
app.get("/.requests", limiter, (req, res) => res.status(200).json(requests))
app.get("/.requests", limiter, async (req, res) => {
try {
const custom = uapi(req.headers["x-metrics-session"])
if (custom) {
const {data:{resources}} = await custom.rest.rateLimit.get()
if (resources)
return res.status(200).json({rest:resources.core, graphql:resources.graphql, search:resources.search, login:custom.login})
}
}
catch {} //eslint-disable-line no-empty
return res.status(200).json(requests)
})
app.get("/.hosted", limiter, (req, res) => res.status(200).json(conf.settings.hosted || null))
//Cache
app.get("/.uncache", limiter, (req, res) => {
Expand All @@ -172,6 +204,84 @@ export default async function({sandbox = false} = {}) {
}
})

//OAuth
if (conf.settings.oauth) {
console.debug("metrics/app/oauth > enabled")
const states = new Map()
app.get("/.oauth/", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/index.html`))
app.get("/.oauth/index.html", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/index.html`))
app.get("/.oauth/script.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/script.js`))
app.get("/.oauth/authenticate", (req, res) => {
//Create a state to protect against cross-site request forgery attacks
const state = crypto.randomBytes(64).toString("hex")
const scopes = new url.URLSearchParams(req.query).get("scopes")
const from = new url.URLSearchParams(req.query).get("scopes")
states.set(state, {from, scopes})
console.debug(`metrics/app/oauth > request ${state}`)
//OAuth through GitHub
return res.redirect(`https://github.com/login/oauth/authorize?${new url.URLSearchParams({
client_id:conf.settings.oauth.id,
state,
redirect_uri:`${conf.settings.oauth.url}/.oauth/authorize`,
allow_signup:false,
scope:scopes,
})}`)
})
app.get("/.oauth/authorize", async (req, res) => {
//Check state
const {code, state} = req.query
if ((!state)||(!states.has(state))) {
console.debug("metrics/app/oauth > 400 (invalid state)")
return res.status(400).send("Bad request: invalid state")
}
//OAuth
try {
//Authorize user
console.debug("metrics/app/oauth > authorization")
const {data} = await axios.post("https://github.com/login/oauth/access_token", `${new url.URLSearchParams({
client_id:conf.settings.oauth.id,
client_secret:conf.settings.oauth.secret,
code,
})}`)
const token = new url.URLSearchParams(data).get("access_token")
//Validate user
const {data:{login}} = await axios.get("https://api.github.com/user", {headers:{Authorization:`token ${token}`}})
console.debug(`metrics/app/oauth > authorization success for ${login}`)
const session = crypto.randomBytes(128).toString("hex")
authenticated.set(session, {login, token})
console.debug(`metrics/app/oauth > created session ${session.substring(0, 6)}`)
//Redirect user back
const {from} = states.get(state)
return res.redirect(`/.oauth/redirect?${new url.URLSearchParams({to:from, session})}`)
}
catch {
console.debug("metrics/app/oauth > authorization failed")
return res.status(401).send("Unauthorized: oauth failed")
}
finally {
states.delete(state)
}
})
app.get("/.oauth/revoke/:session", limiter, async (req, res) => {
const session = req.params.session?.replace(/[\n\r]/g, "")
if (authenticated.has(session)) {
const {token} = authenticated.get(session)
try {
console.log(await axios.delete(`https://api.github.com/applications/${conf.settings.oauth.id}/grant`, {auth:{username:conf.settings.oauth.id, password:conf.settings.oauth.secret}, headers:{Accept:"application/vnd.github+json"}, data:{access_token:token}}))
authenticated.delete(session)
console.debug(`metrics/app/oauth > deleted session ${session.substring(0, 6)}`)
return res.redirect("/.oauth")
}
catch {} //eslint-disable-line no-empty
}
return res.status(400).send("Bad request: invalid session")
})
app.get("/.oauth/redirect", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/oauth/redirect.html`))
app.get("/.oauth/enabled", limiter, (req, res) => res.json(true))
}
else
app.get("/.oauth/enabled", limiter, (req, res) => res.json(false))

//Pending requests
const pending = new Map()

Expand Down Expand Up @@ -236,7 +346,7 @@ export default async function({sandbox = false} = {}) {
}
;(async () => {
try {
const json = await metrics.insights({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates})
const json = await metrics.insights({login}, {...api, ...uapi(req.headers["x-metrics-session"]), conf, callbacks}, {Plugins, Templates})
//Cache
cache.put(`insights.${login}`, json)
if ((!debug) && (cached)) {
Expand Down Expand Up @@ -289,12 +399,14 @@ export default async function({sandbox = false} = {}) {
app.get("/.js/embed/app.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/embed/app.js`))
app.get("/.js/embed/app.placeholder.js", limiter, (req, res) => res.sendFile(`${conf.paths.statics}/embed/app.placeholder.js`))
//App routes
app.get("/:login/:repository?", ...middlewares, async (req, res) => {
app.get("/:login/:repository?", ...middlewares, async (req, res, next) => {
//Request params
const login = req.params.login?.replace(/[\n\r]/g, "")
const repository = req.params.repository?.replace(/[\n\r]/g, "")
let solve = null
//Check username
if ((login.startsWith("."))||(login.includes("/")))
return next()
if (!/^[-\w]+$/i.test(login)) {
console.debug(`metrics/app/${login} > 400 (invalid username)`)
return res.status(400).send("Bad request: username seems invalid")
Expand Down Expand Up @@ -335,19 +447,28 @@ export default async function({sandbox = false} = {}) {

//Compute rendering
try {
//Render
//Prepare settings
const q = req.query
console.debug(`metrics/app/${login} > ${util.inspect(q, {depth: Infinity, maxStringLength: 256})}`)
if ((q["config.presets"]) && ((conf.settings.extras?.features?.includes("metrics.setup.community.presets")) || (conf.settings.extras?.features === true) || (conf.settings.extras?.default))) {
const octokit = {...api, ...uapi(req.headers["x-metrics-session"])}
let uconf = conf
if ((octokit.login)&&(conf.settings.extras?.logged)&&(uconf.settings.extras?.features !== true)) {
console.debug(`metrics/app/${login} > session is authenticated, adding additional permissions ${conf.settings.extras.logged}`)
uconf = {...conf, settings:{...conf.settings, extras:{...conf.settings.extras}}}
uconf.settings.extras.features = uconf.settings.extras.features ?? []
uconf.settings.extras.features.push(...conf.settings.extras.logged)
}
//Preset
if ((q["config.presets"]) && ((uconf.settings.extras?.features?.includes("metrics.setup.community.presets")) || (uconf.settings.extras?.features === true) || (uconf.settings.extras?.default))) {
console.debug(`metrics/app/${login} > presets have been specified, loading them`)
Object.assign(q, await presets(q["config.presets"]))
}
const convert = conf.settings.outputs.includes(q["config.output"]) ? q["config.output"] : conf.settings.outputs[0]
//Render
const convert = uconf.settings.outputs.includes(q["config.output"]) ? q["config.output"] : uconf.settings.outputs[0]
const {rendered, mime} = await metrics({login, q}, {
graphql,
rest,
...octokit,
plugins,
conf,
conf:uconf,
die: q["plugins.errors.fatal"] ?? false,
verify: q.verify ?? false,
convert: convert !== "auto" ? convert : null,
Expand Down Expand Up @@ -441,9 +562,11 @@ export default async function({sandbox = false} = {}) {
"── Content ────────────────────────────────────────────────────────",
`Plugins enabled │ ${enabled.map(({name}) => name).join(", ")}`,
`Templates enabled │ ${templates.filter(({enabled}) => enabled).map(({name}) => name).join(", ")}`,
"── OAuth ──────────────────────────────────────────────────────────",
`Client id │ ${conf.settings.oauth?.id ?? "(none)"}`,
"── Extras ─────────────────────────────────────────────────────────",
`Default │ ${conf.settings.extras?.default ?? false}`,
`Features │ ${conf.settings.extras?.features ?? "(none)"}`,
`Features │ ${Array.isArray(conf.settings.extras?.features) ? conf.settings.extras.features?.length ? conf.settings.extras?.features : "(none)" : "(default)"}`,
"───────────────────────────────────────────────────────────────────",
"Server ready !",
].join("\n")))
Expand Down
Loading

0 comments on commit 587ceec

Please sign in to comment.