Skip to content

Commit

Permalink
Adjust cleanup job to also look at IDL names folders (#88)
Browse files Browse the repository at this point in the history
The "idlnames" and "idlnameparsed" folders have recently been added, see:
w3c/reffy#489 (comment)

They contain files per IDL Name, which need to be dropped when the IDL names no
longer appear in any of the crawled specs.

Also adjust jobs execution schedules to have the cleanup job run after the
weekly tr crawl, see:
#86 (review)
  • Loading branch information
tidoust authored Feb 10, 2021
1 parent d40ac3b commit bb3a03e
Show file tree
Hide file tree
Showing 3 changed files with 41 additions and 2 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/cleanup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ name: Clean up abandoned files

on:
schedule:
- cron: '30 0 * * 1'
- cron: '0 2 * * 1'
workflow_dispatch:
jobs:
update:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/update-tr.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
on:
schedule:
- cron: '11 11 * * 1'
- cron: '0 1 * * 1'
workflow_dispatch:
name: Update TR report
jobs:
Expand Down
39 changes: 39 additions & 0 deletions tools/clean-abandoned-files.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ const ed = require("../ed/index.json");
const tr = require("../tr/index.json");

const subdirs = ["dfns", "css", "headings", "idl", "idlparsed", "links", "refs"];
const idlnamesSubdirs = ["idlnames", "idlnamesparsed"];


const removeExtension = f => {
Expand All @@ -21,7 +22,45 @@ function checkDir(path, index) {
}
}

function checkIdlNames(path, index) {
// Build the list of IDL names from idlparsed extracts
const idlFiles = new Set(index.results.map(spec => spec.idlparsed).filter(s => !!s));
let idlNames = new Set();
for (const idlFile of idlFiles) {
const idlparsed = require("../" + path + "/" + idlFile);
if (!idlparsed || !idlparsed.idlparsed) {
continue;
}
if (idlparsed.idlparsed.idlNames) {
for (const name of Object.keys(idlparsed.idlparsed.idlNames)) {
idlNames.add(name);
}
}
if (idlparsed.idlparsed.idlExtendedNames) {
for (const name of Object.keys(idlparsed.idlparsed.idlExtendedNames)) {
idlNames.add(name);
}
}
}

// Parse subfolders that contain files named after IDL names
for (const subdir of idlnamesSubdirs) {
if (!fs.existsSync(path + "/" + subdir)) {
continue;
}
const filenames = fs.readdirSync(path + "/" + subdir);
for (const filename of filenames) {
const name = filename.match(/(.+)\.[^\.]+$/)[1];
if (!idlNames.has(name)) {
fs.unlinkSync(path + "/" + subdir + "/" + filename);
}
}
}
}

for (let dir of subdirs) {
checkDir("ed/" + dir, ed);
checkDir("tr/" + dir, tr);
}
checkIdlNames("ed", ed);
checkIdlNames("tr", tr);

0 comments on commit bb3a03e

Please sign in to comment.