Update DNSTwist to use the pe-reports module and run on all P&E orgs

backend/Dockerfile.worker

@@ -60,7 +60,7 @@ RUN apt remove dav1d && apt autoclean && apt autoremove
 
 # Install pe-source module
 # Sync the latest from cf-staging branch
-RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout c9cbbd73b22ef38cabe1da6ba50aeb2dc0be4f99 && sed -i 's/"pandas == 1.1.5"/"pandas == 1.5.1"/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup_reports.py  && pip install .
+RUN git clone -b cf-source-staging https://github.com/cisagov/pe-reports.git && cd pe-reports && git checkout eedd406b8ce92519c8bbd8672b86cf000ba38af4 && sed -i 's/"pandas == 1.1.5"/"pandas == 1.5.1"/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup.py && sed -i 's/psycopg2-binary == 2.9.3/psycopg2-binary == 2.9.5/g' setup_reports.py  && pip install .
 # Python dependencies
 
 COPY worker/requirements.txt worker/requirements.txt

backend/src/api/scans.ts

@@ -190,7 +190,7 @@ export const SCAN_SCHEMA: ScanSchema = {
   dnstwist: {
     type: 'fargate',
     isPassive: true,
-    global: false,
+    global: true,
     cpu: '2048',
     memory: '16384',
     description:

backend/src/tasks/dnstwist.ts

@@ -1,111 +1,17 @@
-import { connectToDatabase, Domain, Vulnerability } from '../models';
-import getRootDomains from './helpers/getRootDomains';
 import { CommandOptions } from './ecs-client';
-import { plainToClass } from 'class-transformer';
-import saveVulnerabilitiesToDb from './helpers/saveVulnerabilitiesToDb';
 import { spawnSync } from 'child_process';
-import saveDomainsToDb from './helpers/saveDomainsToDb';
-import * as dns from 'dns';
-
-async function runDNSTwist(domain: string) {
-  const child = spawnSync(
-    'dnstwist',
-    ['-r', '--tld', './worker/common_tlds.dict', '-f', 'json', domain],
-    {
-      stdio: 'pipe',
-      encoding: 'utf-8'
-    }
-  );
-  const savedOutput = child.stdout;
-  const finalResults = JSON.parse(savedOutput);
-  console.log(
-    `Got ${
-      Object.keys(finalResults).length
-    } similar domains for root domain ${domain}`
-  );
-  return finalResults;
-}
+import { getPeEnv } from './helpers/getPeEnv';
 
 export const handler = async (commandOptions: CommandOptions) => {
-  const { organizationId, organizationName } = commandOptions;
-  await connectToDatabase();
-  const dateNow = new Date(Date.now());
-  console.log('Running dnstwist on organization', organizationName);
-  const root_domains = await getRootDomains(organizationId!);
-  const vulns: Vulnerability[] = [];
-  console.log('Root domains:', root_domains);
-  for (const root_domain of root_domains) {
-    try {
-      const results = await runDNSTwist(root_domain);
-
-      // Fetch existing domain object
-      let domain = await Domain.findOne({
-        organization: { id: organizationId },
-        name: root_domain
-      });
-
-      if (!domain) {
-        let ipAddress;
-        const new_domain: Domain[] = [];
-        try {
-          ipAddress = (await dns.promises.lookup(root_domain)).address;
-        } catch (e) {
-          ipAddress = null;
-        }
-        new_domain.push(
-          plainToClass(Domain, {
-            name: root_domain,
-            ip: ipAddress,
-            organization: { id: organizationId }
-          })
-        );
-        await saveDomainsToDb(new_domain);
-        domain = await Domain.findOne({
-          organization: { id: organizationId },
-          name: root_domain
-        });
-      }
-
-      // Fetch existing dnstwist vulnerability
-      const existingVuln = await Vulnerability.findOne({
-        domain: { id: domain?.id },
-        source: 'dnstwist'
-      });
+  console.log('Running dnstwist on P&E organizations');
 
-      // Map date-first-observed to any domain-name that already exists
-      const existingVulnsMap = {};
-      if (existingVuln) {
-        for (const domain of existingVuln.structuredData['domains']) {
-          existingVulnsMap[domain['domain']] = domain['date_first_observed'];
-        }
-      }
-      // If an existing domain-name is in the new results, set date-first-observed to the mapped value
-      // Else, set date-first-observed to today's date (dateNow)
-      for (const domain of results) {
-        domain['date_first_observed'] =
-          existingVulnsMap[domain['domain']] || dateNow;
-      }
-      if (Object.keys(results).length !== 0) {
-        vulns.push(
-          plainToClass(Vulnerability, {
-            domain: domain,
-            lastSeen: new Date(Date.now()),
-            title: 'DNS Twist Domains',
-            state: 'open',
-            source: 'dnstwist',
-            severity: 'Low',
-            needsPopulation: false,
-            structuredData: { domains: results },
-            description: `Registered domains similar to ${root_domain}.`
-          })
-        );
-        await saveVulnerabilitiesToDb(vulns, false);
-      } else {
-        continue;
-      }
-    } catch (e) {
-      console.error(e);
-      continue;
-    }
-  }
+  const child = spawnSync('python', ['-m', 'pe_source', 'dnstwist'], {
+    stdio: 'inherit',
+    encoding: 'utf-8',
+    env: {
+      ...getPeEnv()
+    },
+    cwd: '/app/pe-reports'
+  });
+  console.log('Done');
 };

backend/src/tasks/helpers/getPeEnv.ts

@@ -0,0 +1,7 @@
+// Gets PE environment variables to send to PE scripts.
+export const getPeEnv = () => ({
+  DB_HOST: process.env.DB_HOST!,
+  PE_DB_NAME: process.env.PE_DB_NAME!,
+  PE_DB_USERNAME: process.env.PE_DB_USERNAME!,
+  PE_DB_PASSWORD: process.env.PE_DB_PASSWORD!
+});