Feat/sync deleted repos

docs/sync.md

@@ -12,7 +12,7 @@
       - [File renamed/moves/deleted](#file-renamedmovesdeleted)
       - [node\_modules, tests \& fixtures](#node_modules-tests--fixtures)
   - [Detecting \& importing new files not already monitored in Snyk](#detecting--importing-new-files-not-already-monitored-in-snyk)
-  - [Repository is archived](#repository-is-archived)
+  - [Repository is archived / deleted](#repository-is-archived--deleted)
 - [Kick off sync](#kick-off-sync)
   - [1. Set the env vars](#1-set-the-env-vars)
   - [2. Download \& run](#2-download--run)
@@ -77,9 +77,9 @@ Any projects that were imported but match the default exclusions list (deemed to
 While analyzing each target known to Snyk any new Snyk supported files found in the repo that do not have a corresponding project in Snyk will be imported in batches. Any files matching the default or user provided `exclusionGlobs` will be ignored.
 If a file has a corresponding de-activated project in Snyk, it will not be brought in again. Activate manually or via API if it should be active.
 
-## Repository is archived
+## Repository is archived / deleted
 
-If the repository is now marked as archived, all relevant Snyk projects will be de-activated.
+If the repository is now marked as archived or has been deleted (API returns 404), all relevant Snyk projects will be de-activated.
 # Kick off sync
 
 `sync` command will analyze existing projects & targets (repos) in Snyk organization and determine if any changes are needed.

src/lib/source-handlers/github/index.ts

@@ -5,3 +5,4 @@ export * from './get-repo-metadata';
 export * from './types';
 export * from './is-configured';
 export * from './git-clone-url';
+export * from './validate-token';

src/lib/source-handlers/github/validate-token.ts

@@ -0,0 +1,28 @@
+import { Octokit } from '@octokit/rest';
+import { retry } from '@octokit/plugin-retry';
+import * as debugLib from 'debug';
+import type { Target } from '../../types';
+import { getGithubToken } from './get-github-token';
+import { getGithubBaseUrl } from './github-base-url';
+
+const githubClient = Octokit.plugin(retry);
+const debug = debugLib('snyk:get-github-defaultBranch-script');
+
+export async function validateToken(
+  target: Target,
+  host?: string,
+): Promise<{ valid: true }> {
+  const githubToken = getGithubToken();
+  const baseUrl = getGithubBaseUrl(host);
+  const octokit: Octokit = new githubClient({
+    baseUrl,
+    auth: githubToken,
+  });
+
+  debug(`Fetching organization info to validate token access: ${target.owner}`);
+
+  await octokit.orgs.get({
+    org: target.owner!,
+  });
+  return { valid: true };
+}

src/scripts/sync/sync-projects-per-target.ts

@@ -1,7 +1,10 @@
 import type { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 
-import { getGithubRepoMetaData } from '../../lib/source-handlers/github';
+import {
+  getGithubRepoMetaData,
+  validateToken,
+} from '../../lib/source-handlers/github';
 import { updateBranch } from '../../lib/project/update-branch';
 import type {
   SnykProject,
@@ -28,6 +31,16 @@ export function getMetaDataGenerator(
   return getDefaultBranchGenerators[origin];
 }
 
+export function getTokenValidator(
+  origin: SupportedIntegrationTypesUpdateProject,
+): (target: Target, host?: string | undefined) => Promise<{ valid: true }> {
+  const generator = {
+    [SupportedIntegrationTypesUpdateProject.GITHUB]: validateToken,
+    [SupportedIntegrationTypesUpdateProject.GHE]: validateToken,
+  };
+  return generator[origin];
+}
+
 export function getTargetConverter(
   origin: SupportedIntegrationTypesUpdateProject,
 ): (target: SnykTarget) => Target {
@@ -60,27 +73,56 @@ export async function syncProjectsForTarget(
 
   debug(`Syncing projects for target ${target.attributes.displayName}`);
   let targetMeta: RepoMetaData;
+  let isDeleted = false;
   const origin = target.attributes
     .origin as SupportedIntegrationTypesUpdateProject;
   const targetData = getTargetConverter(origin)(target);
+  const isTokenValid = await getTokenValidator(origin);
 
   try {
     targetMeta = await getMetaDataGenerator(origin)(targetData, host);
   } catch (e) {
-    //TODO: if repo is deleted, deactivate all projects
-    debug(e);
-    const error = `Getting default branch via ${origin} API failed with error: ${e.message}`;
-    projects.map((project) => {
-      failed.add({
-        errorMessage: error,
-        projectPublicId: project.id,
-        type: ProjectUpdateType.BRANCH,
-        from: project.branch!,
-        to: targetMeta.branch,
-        dryRun: config.dryRun,
-        target,
+    debug(`Failed to get metadata ${JSON.stringify(targetData)}: ` + e);
+    if (
+      e.status === 404 &&
+      (await isTokenValid(targetData, host)).valid === true
+    ) {
+      isDeleted = true;
+    } else {
+      const error = `Getting metadata from ${origin} API failed with error: ${e.message}`;
+      projects.map((project) => {
+        failed.add({
+          errorMessage: error,
+          projectPublicId: project.id,
+          type: ProjectUpdateType.BRANCH,
+          from: project.branch!,
+          to: 'unknown',
+          dryRun: config.dryRun,
+          target,
+        });
       });
-    });
+      return {
+        updated: Array.from(updated).map((t) => ({ ...t, target })),
+        failed: Array.from(failed).map((t) => ({ ...t, target })),
+      };
+    }
+  }
+
+  if (isDeleted || targetMeta!.archived) {
+    const res = await bulkDeactivateProjects(
+      requestManager,
+      orgId,
+      projects,
+      config.dryRun,
+    );
+
+    res.updated.map((t) => ({ ...t, target })).forEach((i) => updated.add(i));
+    res.failed.map((t) => ({ ...t, target })).forEach((i) => failed.add(i));
+
+    return {
+      updated: Array.from(updated).map((t) => ({ ...t, target })),
+      failed: Array.from(failed).map((t) => ({ ...t, target })),
+    };
   }
 
   const deactivate = [];

test/scripts/sync/clone-and-analyze.spec.ts

@@ -67,8 +67,8 @@ describe('cloneAndAnalyze', () => {
       ];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
         sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
       };
@@ -130,8 +130,8 @@ describe('cloneAndAnalyze', () => {
       ];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
         sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
       };
@@ -205,8 +205,8 @@ describe('cloneAndAnalyze', () => {
       ];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
         sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
       };
@@ -298,8 +298,8 @@ describe('cloneAndAnalyze', () => {
       ];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: 'https://github.com/snyk-fixtures/monorepo-simple.git',
         sshUrl: '[email protected]:snyk-fixtures/monorepo-simple.git',
       };
@@ -324,8 +324,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'main',
         archived: false,
+        branch: 'main',
         cloneUrl: 'https://github.com/snyk-fixtures/no-supported-manifests.git',
         sshUrl: '[email protected]:snyk-fixtures/no-supported-manifests.git',
       };
@@ -350,8 +350,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: 'https://github.com/snyk-fixtures/empty-repo.git',
         sshUrl: '[email protected]:snyk-fixtures/empty-repo.git',
       };
@@ -376,8 +376,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl:
           'https://github.com/snyk-fixtures/python-requirements-custom-name-inside-folder.git',
         sshUrl:
@@ -414,8 +414,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/mono-repo.git`,
         sshUrl: `git@${GHE_URL.host}/snyk-fixtures/mono-repo.git`,
       };
@@ -451,8 +451,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`,
         sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`,
       };
@@ -481,8 +481,8 @@ describe('cloneAndAnalyze', () => {
       const projects: SnykProject[] = [];
 
       const repoMeta: RepoMetaData = {
-        branch: 'master',
         archived: false,
+        branch: 'master',
         cloneUrl: `https://${GHE_URL.host}/snyk-fixtures/docker-goof.git`,
         sshUrl: `git@${GHE_URL.host}/snyk-fixtures/docker-goof.git`,
       };