Revert "1983 remove unused scans (#1996)" (#2091)

This reverts commit 9534d75.

backend/Dockerfile.worker

@@ -32,6 +32,7 @@ RUN unzip findomain-linux.zip && chmod +x findomain && cp findomain /usr/bin/fin
 RUN go mod init crossfeed-worker
 
 RUN go install github.com/facebookincubator/nvdtools/...@latest
+RUN go install -v github.com/owasp-amass/amass/v3/...@master
 
 FROM ruby:alpine as rubyBuild
 
@@ -73,6 +74,7 @@ RUN wget https://publicsuffix.org/list/public_suffix_list.dat --no-use-server-ti
 COPY --from=build /app/dist/worker.bundle.js worker.bundle.js
 
 COPY --from=deps /usr/bin/findomain /usr/bin/
+COPY --from=deps /go/bin/amass /usr/bin/
 COPY --from=deps /go/bin/csv2cpe /go/bin/nvdsync /go/bin/cpe2cve /usr/bin/
 
 COPY --from=deps /etc/ssl/certs /etc/ssl/certs

backend/src/api/scans.ts

@@ -67,6 +67,13 @@ export const SCAN_SCHEMA: ScanSchema = {
     global: false,
     description: 'Passive discovery of subdomains from public certificates'
   },
+  amass: {
+    type: 'fargate',
+    isPassive: false,
+    global: false,
+    description:
+      'Open source tool that integrates passive APIs and active subdomain enumeration in order to discover target subdomains'
+  },
   findomain: {
     type: 'fargate',
     isPassive: true,
@@ -165,6 +172,21 @@ export const SCAN_SCHEMA: ScanSchema = {
     memory: '4096',
     description: 'Scrapes all webpages on a given domain, respecting robots.txt'
   },
+  hibp: {
+    type: 'fargate',
+    isPassive: true,
+    global: false,
+    cpu: '2048',
+    memory: '16384',
+    description:
+      'Finds emails that have appeared in breaches related to a given domain'
+  },
+  lookingGlass: {
+    type: 'fargate',
+    isPassive: true,
+    global: false,
+    description: 'Finds vulnerabilities and malware from the LookingGlass API'
+  },
   dnstwist: {
     type: 'fargate',
     isPassive: true,
@@ -174,6 +196,13 @@ export const SCAN_SCHEMA: ScanSchema = {
     description:
       'Domain name permutation engine for detecting similar registered domains.'
   },
+  rootDomainSync: {
+    type: 'fargate',
+    isPassive: true,
+    global: false,
+    description:
+      'Creates domains from root domains by doing a single DNS lookup for each root domain.'
+  },
   savedSearch: {
     type: 'fargate',
     isPassive: true,

backend/src/models/domain.ts

@@ -46,7 +46,7 @@ export class Domain extends BaseEntity {
   })
   fromRootDomain: string;
 
-  /** Scan that discovered this domain (findomain, etc) */
+  /** Scan that discovered this domain (findomain, amass) */
   @Column({
     nullable: true
   })

backend/src/tasks/amass.ts

@@ -0,0 +1,57 @@
+import { Domain } from '../models';
+import { spawnSync } from 'child_process';
+import { readFileSync } from 'fs';
+import { plainToClass } from 'class-transformer';
+import { CommandOptions } from './ecs-client';
+import getRootDomains from './helpers/getRootDomains';
+import saveDomainsToDb from './helpers/saveDomainsToDb';
+import * as path from 'path';
+
+const OUT_PATH = path.join(__dirname, 'out-' + Math.random() + '.txt');
+
+export const handler = async (commandOptions: CommandOptions) => {
+  const { organizationId, organizationName, scanId } = commandOptions;
+
+  console.log('Running amass on organization', organizationName);
+
+  const rootDomains = await getRootDomains(organizationId!);
+
+  for (const rootDomain of rootDomains) {
+    try {
+      const args = [
+        'enum',
+        '-ip',
+        '-active',
+        '-d',
+        rootDomain,
+        '-json',
+        OUT_PATH
+      ];
+      console.log('Running amass with args', args);
+      spawnSync('amass', args, { stdio: 'pipe' });
+      const output = String(readFileSync(OUT_PATH));
+      const lines = output.split('\n');
+      const domains: Domain[] = [];
+      for (const line of lines) {
+        if (line == '') continue;
+        const parsed = JSON.parse(line);
+        domains.push(
+          plainToClass(Domain, {
+            ip: parsed.addresses[0].ip,
+            name: parsed.name,
+            asn: parsed.addresses[0].asn,
+            organization: { id: organizationId },
+            fromRootDomain: rootDomain,
+            subdomainSource: 'amass',
+            discoveredBy: { id: scanId }
+          })
+        );
+      }
+      await saveDomainsToDb(domains);
+      console.log(`amass created/updated ${domains.length} new domains`);
+    } catch (e) {
+      console.error(e);
+      continue;
+    }
+  }
+};

backend/src/tasks/ecs-client.ts

@@ -100,6 +100,7 @@ class ECSClient {
             `CENSYS_API_SECRET=${process.env.CENSYS_API_SECRET}`,
             `WORKER_USER_AGENT=${process.env.WORKER_USER_AGENT}`,
             `SHODAN_API_KEY=${process.env.SHODAN_API_KEY}`,
+            `HIBP_API_KEY=${process.env.HIBP_API_KEY}`,
             `SIXGILL_CLIENT_ID=${process.env.SIXGILL_CLIENT_ID}`,
             `SIXGILL_CLIENT_SECRET=${process.env.SIXGILL_CLIENT_SECRET}`,
             `PE_SHODAN_API_KEYS=${process.env.PE_SHODAN_API_KEYS}`,

backend/src/tasks/hibp.ts

@@ -0,0 +1,151 @@
+import { Domain, Service, Vulnerability, connectToDatabase } from '../models';
+import { CommandOptions } from './ecs-client';
+import { plainToClass } from 'class-transformer';
+import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb';
+import axios from 'axios';
+
+/**
+ * The hibp scan looks up emails from a particular .gov domain
+ * that have shown up in breaches using the Have I
+ * Been Pwned Enterprise API.
+ * Be aware this scan can only query breaches from .gov domains
+ */
+
+interface breachResults {
+  Name: string;
+  Title: string;
+  Domain: string;
+  BreachDate: string;
+  AddedDate: string;
+  ModifiedDate: string;
+  PwnCount: number;
+  Description: string;
+  LogoPath: string;
+  DataClasses: string[];
+  IsVerified: boolean;
+  IsFabricated: boolean;
+  IsSensitive: boolean;
+  IsRetired: boolean;
+  IsSpamList: boolean;
+  passwordIncluded: boolean;
+}
+
+async function getIps(organizationId?: String): Promise<Domain[]> {
+  await connectToDatabase();
+
+  let domains = Domain.createQueryBuilder('domain')
+    .leftJoinAndSelect('domain.organization', 'organization')
+    .andWhere('ip IS NOT NULL')
+    .andWhere('domain.name LIKE :gov', { gov: '%.gov' })
+    .andWhere('domain.ipOnly=:bool', { bool: false });
+
+  if (organizationId) {
+    domains = domains.andWhere('domain.organization=:org', {
+      org: organizationId
+    });
+  }
+
+  return domains.getMany();
+}
+
+async function lookupEmails(
+  breachesDict: { [key: string]: breachResults },
+  domain: Domain
+) {
+  try {
+    const { data } = await axios.get(
+      'https://haveibeenpwned.com/api/v2/enterprisesubscriber/domainsearch/' +
+        domain.name,
+      {
+        headers: {
+          Authorization: 'Bearer ' + process.env.HIBP_API_KEY!
+        }
+      }
+    );
+
+    const addressResults = {};
+    const breachResults = {};
+    const finalResults = {};
+
+    const shouldCountBreach = (breach) =>
+      breach.IsVerified === true && breach.BreachDate > '2016-01-01';
+
+    for (const email in data) {
+      const filtered = (data[email] || []).filter((e) =>
+        shouldCountBreach(breachesDict[e])
+      );
+      if (filtered.length > 0) {
+        addressResults[email + '@' + domain.name] = filtered;
+        for (const breach of filtered) {
+          if (!(breach in breachResults)) {
+            breachResults[breach] = breachesDict[breach];
+            breachResults[breach].passwordIncluded =
+              breachResults[breach].DataClasses.indexOf('Passwords') > -1;
+          }
+        }
+      }
+    }
+
+    finalResults['Emails'] = addressResults;
+    finalResults['Breaches'] = breachResults;
+    return finalResults;
+  } catch (error) {
+    console.error(
+      `An error occured when trying to access the HIPB API using the domain: ${domain.name}: ${error} `
+    );
+    return null;
+  }
+}
+
+export const handler = async (commandOptions: CommandOptions) => {
+  const { organizationId, organizationName } = commandOptions;
+
+  console.log('Running hibp on organization', organizationName);
+  const domainsWithIPs = await getIps(organizationId);
+  const { data } = await axios.get(
+    'https://haveibeenpwned.com/api/v2/breaches',
+    {
+      headers: {
+        Authorization: 'Bearer ' + process.env.HIBP_API_KEY!
+      }
+    }
+  );
+  const breachesDict: { [key: string]: breachResults } = {};
+  for (const breach of data) {
+    breachesDict[breach.Name] = breach;
+  }
+
+  const services: Service[] = [];
+  const vulns: Vulnerability[] = [];
+  for (const domain of domainsWithIPs) {
+    const results = await lookupEmails(breachesDict, domain);
+
+    if (results) {
+      console.log(
+        `Got ${Object.keys(results['Emails']).length} emails for domain ${
+          domain.name
+        }`
+      );
+
+      if (Object.keys(results['Emails']).length !== 0) {
+        vulns.push(
+          plainToClass(Vulnerability, {
+            domain: domain,
+            lastSeen: new Date(Date.now()),
+            title: 'Exposed Emails',
+            state: 'open',
+            source: 'hibp',
+            severity: 'Low',
+            needsPopulation: false,
+            structuredData: {
+              emails: results['Emails'],
+              breaches: results['Breaches']
+            },
+            description: `Emails associated with ${domain.name} have been exposed in a breach.`
+          })
+        );
+        await saveVulnerabilitiesToDb(vulns, false);
+      }
+    }
+  }
+};