Points docs to last stable release

.circleci/config.yml

@@ -17,6 +17,7 @@ jobs:
           keys:
             - v1-dependencies-{{ checksum "package.json" }}
             - v1-dependencies-
+            - v1-releases-cache
 
       - run:
           name: Install dependencies
@@ -27,6 +28,11 @@ jobs:
             - node_modules
           key: v1-dependencies-{{ checksum "package.json" }}
 
+      - save_cache:
+          paths:
+            - build-temp/releases
+          key: v1-releases-cache
+
       - run:
           name: Test
           command: npm test

README.md

@@ -7,7 +7,7 @@ This project is used to build the website for [fastify web framework](https://gi
 
 ## Requirements
 
- - Node 8.3.0+
+ - Node 12+
  - Install dependencies with `npm install`
 
 

package.json

@@ -5,11 +5,11 @@
   "scripts": {
     "build:cleanup": "rimraf build-temp",
     "build:create-temp-folder": "mkdirp build-temp",
-    "build:get-releases": "node src/scripts/downloadReleases.js fastify/fastify build-temp/releases v0.11.0",
+    "build:get-releases": "node src/scripts/downloadReleases.js fastify/fastify build-temp/releases v1.13.0",
     "build:process-releases": "node src/scripts/processReleases.js build-temp/releases src/website",
     "build:website": "node src/scripts/buildWebsite.js",
     "build:website:watch": "npm run build:website && chokidar 'src/website' -c 'npm run build:website'",
-    "build": "npm run build:cleanup && npm run build:create-temp-folder && npm run build:get-releases && npm run build:process-releases && npm run build:website",
+    "build": "npm run build:create-temp-folder && npm run build:get-releases && npm run build:process-releases && npm run build:website",
     "deploy": "gh-pages -d build --dotfiles",
     "test:lint": "eslint src --ignore-pattern 'src/website/content/'",
     "test:images": "node __tests__/organisations-images",
@@ -36,9 +36,11 @@
   },
   "homepage": "https://github.com/fastify/website#readme",
   "dependencies": {
-    "async": "^3.1.0",
+    "async": "^3.2.0",
+    "axios": "^0.19.2",
     "clean-css": "^4.2.1",
     "clone": "^2.1.2",
+    "compare-versions": "^3.6.0",
     "debug": "^4.1.1",
     "js-yaml": "^3.13.1",
     "lodash": "^4.17.15",
@@ -55,10 +57,9 @@
     "metalsmith-writemetadata": "^0.4.5",
     "mkdirp": "^0.5.1",
     "node-sass": "^4.13.1",
-    "nunjucks": "^3.2.0",
+    "nunjucks": "^3.2.1",
     "nunjucks-markdown-filter": "^0.1.0",
-    "request": "^2.88.0",
-    "request-promise-native": "^1.0.8",
+    "parse-link-header": "^1.0.1",
     "rimraf": "^3.0.0",
     "svgo": "^1.3.2",
     "unzip-stream": "^0.3.0"

src/scripts/downloadReleases.js

@@ -1,87 +1,153 @@
 #!/usr/bin/env node
-
-const request = require('request-promise-native')
-const mapLimit = require('async/mapLimit')
+const crypto = require('crypto')
 const { resolve, join } = require('path')
+const fs = require('fs')
+const { promisify } = require('util')
+const finished = promisify(require('stream').finished)
+const axios = require('axios')
 const unzip = require('unzip-stream')
-const { cpus } = require('os')
-const sortVersionTags = require('./utils/sortVersionTags')
+const parseLinkHeader = require('parse-link-header')
+const compareVersions = require('compare-versions')
+const { fileExists } = require('./utils')
 
 const repository = process.argv[2] || 'fastify/fastify'
 const dest = resolve(process.argv[3] || 'releases')
-const minRelease = process.argv[4] || 'v0.11.0'
+const minRelease = process.argv[4] || 'v1.13.0'
+
+function md5 (str) {
+  return crypto.createHash('md5').update(str).digest('hex')
+}
 
-console.log(`downloading releases into ${dest}`)
+async function getAllReleases (repository, requestConfig) {
+  let releases = []
+  let nextUrl = `https://api.github.com/repos/${repository}/releases`
+  while (nextUrl) {
+    const { data, headers } = await axios.get(nextUrl, requestConfig)
+    const links = parseLinkHeader(headers.link)
+    nextUrl = links.next ? links.next.url : null
+    releases = releases.concat(data)
+  }
 
-const requestConfig = {
-  json: true,
-  headers: { 'User-Agent': 'fastify-website-builder-v1' }
+  return releases
 }
 
-if (process.env.GH_NAME && process.env.GH_TOKEN) {
-  console.log(`  » GitHub API requests authenticated as "${process.env.GH_NAME}"`)
-  requestConfig.user = process.env.GH_NAME
-  requestConfig.pass = process.env.GH_TOKEN
+function downloadReleases (releases, requestConfig) {
+  const releasesDownload = Object.values(releases).map(async (release) => {
+    const destPath = join(dest, release.dest)
+    if (!release.ignoreCache) {
+      const hasCache = await fileExists(destPath)
+      if (hasCache) {
+        console.log(`Skipped ${release.name} (already in cache)`)
+        return release
+      }
+    }
+    const { data } = await axios.get(release.url, { ...requestConfig, responseType: 'stream' })
+    const stream = data.pipe(unzip.Extract({ path: destPath }))
+    await finished(stream)
+    console.log(`Downloaded ${release.name}`)
+    return release
+  })
+  return Promise.all(releasesDownload)
 }
 
-request(Object.assign(requestConfig, { url: `https://api.github.com/repos/${repository}/releases` }))
-  .then((releases) => {
-    const selectedReleases = releases
-      // creates version map and label per every release
-      .map((release) => {
-        const [major, minor, patch] = release.name.substr(1).split('.').map(Number)
-        return {
-          name: release.name,
-          url: release.zipball_url,
-          label: `v${major}.${minor}.x`,
-          version: { major, minor, patch }
-        }
-      })
-      // removes release prior to a given release
-      .filter(({ version }) => {
-        const [major, minor, patch] = minRelease.substr(1).split('.').map(Number)
-        return (
-          version.major > major ||
-          (version.major === major && version.minor > minor) ||
-          (version.major === major && version.minor === minor && version.patch >= patch)
-        )
-      })
-      // Keeps only the latest patch per release and creates a map
-      .reduce((acc, curr) => {
-        if (!acc[curr.label] || acc[curr.label].version.patch < curr.version.patch) {
-          acc[curr.label] = curr
-        }
-
-        return acc
-      }, {})
-
-    // Create an alias of the latest release as `latest`
-    const latestReleaseKey = Object.keys(selectedReleases).sort(sortVersionTags).reverse()[0]
-    selectedReleases.latest = selectedReleases[latestReleaseKey]
-
-    // Adds current master
-    selectedReleases.master = {
-      label: 'master',
-      name: 'master',
-      url: `https://github.com/${repository}/archive/master.zip`
+async function main () {
+  console.log(`Downloading releases into ${dest}`)
+
+  const requestConfig = {
+    responseType: 'json',
+    headers: { 'User-Agent': 'fastify-website-builder-v2' }
+  }
+
+  if (process.env.GH_NAME && process.env.GH_TOKEN) {
+    console.log(`GitHub API requests authenticated as "${process.env.GH_NAME}"`)
+    requestConfig.auth = {
+      username: process.env.GH_NAME,
+      password: process.env.GH_TOKEN
     }
+  }
 
-    // downloads the releases
-    mapLimit(Object.keys(selectedReleases), cpus().length * 2, (name, done) => {
-      const release = selectedReleases[name]
-      request(Object.assign(requestConfig, { url: release.url }))
-        .pipe(unzip.Extract({ path: join(dest, name) }))
-        .on('finish', () => {
-          console.log(` - ${name}`)
-          done()
-        })
-    }, (err, data) => {
-      if (err) throw err
-
-      console.log('All release downloaded')
+  const releases = await getAllReleases(repository, requestConfig)
+  console.log(`Found ${releases.length} releases`)
+  const selectedReleases = releases
+    // removes draft releases and pre-releases
+    .filter((release) => !release.draft && !release.prerelease)
+    // removes releases with invalid names (e.g. 0.2.0 did not have a release name)
+    .filter((release) => compareVersions.validate(release.name))
+    // sorts releases by name descendant
+    .sort((a, b) => compareVersions(a.name, b.name) * -1)
+    // creates version map and label per every release
+    .map((release) => {
+      const [major, minor, patch] = release.name.split('-', 1)[0].substr(1).split('.').map(Number)
+      const annotation = release.name.split('-').slice(1).join('-') // catches annotations like `-alpha.1` or `-pre-release.2`
+      return {
+        name: release.name,
+        url: release.zipball_url,
+        dest: md5(release.zipball_url),
+        label: `v${major}.${minor}.x`,
+        docsPath: `v${major}.${minor}.x`,
+        fullVersion: release.name,
+        version: { major, minor, patch, annotation }
+      }
     })
+    // removes release prior to a given release
+    .filter(({ name, version }) => {
+      const skip = compareVersions.compare(name, minRelease, '<')
+      if (skip) {
+        console.log(`Ignoring "${name}" as it's older than minimum version "${minRelease}"`)
+      }
+      return !skip
+    })
+    // Keeps only the latest patch per release and creates a map
+    .reduce((acc, curr) => {
+      if (!acc[curr.label] || acc[curr.label].version.patch < curr.version.patch) {
+        acc[curr.label] = curr
+      }
+
+      return acc
+    }, {})
+
+  // selected the "latest" release at the last release without annotations
+  const latestRelease = Object.values(selectedReleases)
+    .sort((a, b) => compareVersions(a.name, b.name) * -1)
+    .find((r) => r.version.annotation === '')
+
+  // Create an alias of the latest release as `latest`
+  selectedReleases.latest = { ...latestRelease, name: 'latest', docsPath: 'latest' }
+
+  // Add current master
+  const masterUrl = `https://github.com/${repository}/archive/master.zip`
+  selectedReleases.master = {
+    label: 'master',
+    name: 'master',
+    docsPath: 'master',
+    fullVersion: 'master',
+    url: masterUrl,
+    dest: md5(masterUrl),
+    ignoreCache: true
+  }
+
+  // downloads the releases
+  const manifest = await downloadReleases(selectedReleases, requestConfig)
+  // sort the releases by version and makes sure latest and then master are the first 2 entries
+  manifest.sort(({ name: a }, { name: b }) => {
+    switch (true) {
+      case a === 'latest' && b === 'master': return -1 // latest always goes first
+      case b === 'latest' && a === 'master': return 1
+      case a === 'latest' || a === 'master': return -1 // master is the second
+      case b === 'latest' || b === 'master': return 1
+      default:
+        return compareVersions(a, b) * -1 // otherwise compare by versions descendant
+    }
   })
-  .catch((err) => {
-    console.error(err, err.stack)
-    process.exit(1)
-  })
+  // saves a manifest with all the current releases in the dest folder
+  const manifestFile = join(dest, 'releases.json')
+  await fs.promises.writeFile(manifestFile, JSON.stringify(manifest, null, 2))
+  console.log(`Manifest file created: ${manifestFile}`)
+
+  console.log(`Completed: downloaded ${Object.keys(selectedReleases).length} releases`)
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})