feat: additional metadata from manifest file

lib/analyzer/applications/node-modules-utils.ts

@@ -34,7 +34,8 @@ async function createTempProjectDir(
   };
 }
 
-const manifestName: string = "package.json";
+export const manifestName: string = "package.json";
+export const manifestLockName: string = "package-lock.json";
 
 async function fileExists(path: string): Promise<boolean> {
   return await stat(path)

lib/analyzer/applications/runtime-common.ts

@@ -1,23 +1,103 @@
 import * as path from "path";
+import { parsePkgJson } from "snyk-nodejs-lockfile-parser";
 import { ApplicationFilesFact } from "../../facts";
+import {
+  manifestLockName as nodeManifestLockName,
+  manifestName as nodeManifestName,
+} from "./node-modules-utils";
 import {
   AppDepsScanResultWithoutTarget,
+  AppFileType,
   ApplicationFileInfo,
   FilePathToContent,
+  ManifestMetadata,
 } from "./types";
 
-export function getAppFilesRootDir(
-  filePaths: string[],
-): [string, ApplicationFileInfo[]] {
+interface AppFileMetadataExtractor {
+  manifestFileMatcher: (filePath: string) => boolean;
+  metadataExtractor: (fileContent: string) => ManifestMetadata | undefined;
+}
+
+export const filesMetadataExtractorPerLanguage: Record<
+  string,
+  AppFileMetadataExtractor
+> = {
+  node: {
+    manifestFileMatcher: (filePath: string) => {
+      return [nodeManifestName, nodeManifestLockName].some(
+        (mf) => path.basename(filePath) === mf,
+      );
+    },
+    metadataExtractor: (fileContent: string) => {
+      try {
+        const pkgJson = parsePkgJson(fileContent);
+        const moduleName = pkgJson.name;
+        const repoUrl = (pkgJson as any).repository?.url;
+
+        if (!repoUrl && !moduleName) {
+          return undefined;
+        }
+        const metadata: ManifestMetadata = {
+          moduleName: pkgJson.name,
+        };
+        if (repoUrl) {
+          metadata.repoUrl = repoUrl;
+        }
+        return metadata;
+      } catch (err) {
+        return undefined;
+      }
+    },
+  },
+};
+
+export function getAppFileInfos(
+  filePathToContent: FilePathToContent,
+  rootDir: string,
+  manifestMetadataExtractor?: AppFileMetadataExtractor,
+): ApplicationFileInfo[] {
   const appFiles: ApplicationFileInfo[] = [];
-  const splitPaths: string[][] = [];
 
+  const filePaths = Object.keys(filePathToContent);
+  if (!filePaths.length) {
+    return appFiles;
+  }
+
+  for (const filePath of filePaths) {
+    const appFile: ApplicationFileInfo = { path: filePath };
+    if (manifestMetadataExtractor) {
+      const { manifestFileMatcher, metadataExtractor } =
+        manifestMetadataExtractor;
+      if (manifestFileMatcher(filePath)) {
+        appFile.type = AppFileType.Manifest;
+        const manifestContent = filePathToContent[filePath];
+        appFile.metadata = metadataExtractor(manifestContent);
+      }
+    }
+
+    appFiles.push(appFile);
+  }
+
+  // Remove the common path prefix from each appFile
+  appFiles.forEach((file) => {
+    const prefix = rootDir.endsWith(path.sep)
+      ? rootDir
+      : `${rootDir}${path.sep}`;
+    if (file.path.startsWith(prefix)) {
+      file.path = file.path.substring(prefix.length); // Remove rootDir from path
+    }
+  });
+
+  return appFiles;
+}
+
+export function getRootDir(filePaths: string[]): string {
   if (!filePaths.length) {
-    return [path.sep, appFiles];
+    return path.sep;
   }
 
+  const splitPaths: string[][] = [];
   for (const filePath of filePaths) {
-    appFiles.push({ path: filePath });
     splitPaths.push(filePath.split("/").filter(Boolean));
   }
 
@@ -37,16 +117,7 @@ export function getAppFilesRootDir(
 
   // Join the common parts to form the common directory
   const rootDir = "/" + commonParts.join("/");
-
-  // Remove the common path prefix from each appFile
-  appFiles.forEach((file) => {
-    const prefix = rootDir === path.sep ? rootDir : `${rootDir}${path.sep}`;
-    if (file.path.startsWith(prefix)) {
-      file.path = file.path.substring(prefix.length); // Remove rootDir from path
-    }
-  });
-
-  return [rootDir || path.sep, appFiles];
+  return rootDir || path.sep;
 }
 
 export function getApplicationFiles(
@@ -56,9 +127,14 @@ export function getApplicationFiles(
 ): AppDepsScanResultWithoutTarget[] {
   const scanResults: AppDepsScanResultWithoutTarget[] = [];
 
-  const [appFilesRootDir, appFiles] = getAppFilesRootDir(
-    Object.keys(filePathToContent),
+  const manifestMetadataExtractor = filesMetadataExtractorPerLanguage[language];
+  const appFilesRootDir = getRootDir(Object.keys(filePathToContent));
+  const appFiles = getAppFileInfos(
+    filePathToContent,
+    appFilesRootDir,
+    manifestMetadataExtractor,
   );
+
   if (appFiles.length) {
     scanResults.push({
       facts: [

lib/analyzer/applications/types.ts

@@ -31,12 +31,24 @@ export interface FilePathToElfContent {
 export interface AggregatedJars {
   [path: string]: JarBuffer[];
 }
+
+export interface ManifestMetadata {
+  repoUrl?: string;
+  moduleName?: string;
+}
+
+export enum AppFileType {
+  Manifest = "manifest",
+  Code = "code",
+}
+
 export interface ApplicationFileInfo {
   path: string;
+  type?: AppFileType;
+  metadata?: ManifestMetadata;
 }
 export interface ApplicationFiles {
   fileHierarchy: ApplicationFileInfo[];
-  moduleName?: string;
   language: string;
 }
 

test/lib/analyzer/runtime-commons.spec.ts

@@ -0,0 +1,170 @@
+import {
+  filesMetadataExtractorPerLanguage,
+  getAppFileInfos,
+  getRootDir,
+} from "../../../lib/analyzer/applications/runtime-common";
+import { AppFileType } from "../../../lib/analyzer/applications/types";
+
+describe("application files root dir extraction", () => {
+  it("should correctly get root dir for js ts files", async () => {
+    let nodeProjectFiles = [
+      "/aaa/bbb/ccc/y.js",
+      "/aaa/bbb/ccc/z.js",
+      "/aaa/x.js",
+    ];
+
+    expect(getRootDir(nodeProjectFiles)).toBe("/aaa");
+
+    nodeProjectFiles = [
+      "/srv/dist/index.js",
+      "/srv/dist/src/app.js",
+      "/srv/dist/src/utils/helpers.js",
+      "/srv/dist/src/components/header.ts",
+      "/srv/dist/src/components/footer.js",
+      "/srv/dist/src/services/api.js",
+      "/srv/dist/src/models/user.js",
+      "/srv/dist/src/config/config.ts",
+      "/srv/dist/package.json",
+      "/srv/dist/package-lock.json",
+    ];
+
+    expect(getRootDir(nodeProjectFiles)).toBe("/srv/dist");
+  });
+
+  it("should return / as the root dir in case nothing's found", async () => {
+    const nodeProjectFiles = ["/srv/dist/index.js", "/opt/app.js"];
+    expect(getRootDir(nodeProjectFiles)).toBe("/");
+  });
+
+  it("should only consider full path segments for common prefix", async () => {
+    const nodeProjectFiles = ["/srv/dist/index.js", "/srv2/app.js"];
+    expect(getRootDir(nodeProjectFiles)).toBe("/");
+  });
+
+  it("should correctly get root dir for python application files", async () => {
+    const pythonProjectFiles = [
+      "/app/index.py",
+      "/app/src/app.py",
+      "/app/src/utils/helpers.py",
+      "/app/src/components/header.py",
+      "/app/src/components/footer.py",
+      "/app/src/services/api.py",
+      "/app/src/models/user.py",
+      "/app/src/config/config.py",
+      "/app/requirements.txt",
+    ];
+    expect(getRootDir(pythonProjectFiles)).toBe("/app");
+  });
+});
+
+describe("application files info extraction", () => {
+  it("should correctly get app infos for js ts files", async () => {
+    const nodeProjectFiles = {
+      "/aaa/bbb/ccc/y.js": "",
+      "/aaa/bbb/ccc/z.js": "",
+      "/aaa/x.js": "",
+    };
+
+    const appFiles = getAppFileInfos(
+      nodeProjectFiles,
+      "/aaa",
+      filesMetadataExtractorPerLanguage.node,
+    );
+    expect(appFiles.length).toBe(3);
+    expect(appFiles).toEqual([
+      { path: "bbb/ccc/y.js" },
+      { path: "bbb/ccc/z.js" },
+      { path: "x.js" },
+    ]);
+  });
+
+  it("should correctly identify node manifest files", async () => {
+    const nodeProjectFiles = {
+      "/srv/dist/index.js": "",
+      "/srv/dist/src/app.js": "",
+      "/srv/dist/src/utils/helpers.js": "",
+      "/srv/dist/src/components/header.ts": "",
+      "/srv/dist/src/components/footer.js": "",
+      "/srv/dist/src/services/api.js": "",
+      "/srv/dist/src/models/user.js": "",
+      "/srv/dist/src/config/config.ts": "",
+      "/srv/dist/package.json": "{}",
+      "/srv/dist/package-lock.json": "{", // check that we do not fail bad formatted files.
+    };
+
+    const appFiles = getAppFileInfos(
+      nodeProjectFiles,
+      "/srv/dist",
+      filesMetadataExtractorPerLanguage.node,
+    );
+    expect(appFiles.length).toBe(10);
+    expect(appFiles).toEqual([
+      { path: "index.js" },
+      { path: "src/app.js" },
+      { path: "src/utils/helpers.js" },
+      { path: "src/components/header.ts" },
+      { path: "src/components/footer.js" },
+      { path: "src/services/api.js" },
+      { path: "src/models/user.js" },
+      { path: "src/config/config.ts" },
+      {
+        path: "package.json",
+        type: AppFileType.Manifest,
+        metadata: { moduleName: "package.json" },
+      },
+      {
+        path: "package-lock.json",
+        type: AppFileType.Manifest,
+      },
+    ]);
+  });
+
+  it("should not change app files path when root dir is /", async () => {
+    const nodeProjectFiles = {
+      "/srv/dist/index.js": "",
+      "/opt/app.js": "",
+    };
+    const appFiles = getAppFileInfos(
+      nodeProjectFiles,
+      "/",
+      filesMetadataExtractorPerLanguage.node,
+    );
+    expect(appFiles.length).toBe(2);
+    expect(appFiles).toEqual([
+      { path: "srv/dist/index.js" },
+      { path: "opt/app.js" },
+    ]);
+  });
+
+  it("should correctly get app infos for python files", async () => {
+    const pythonProjectFiles = {
+      "/app/index.py": "",
+      "/app/src/app.py": "",
+      "/app/src/utils/helpers.py": "",
+      "/app/src/components/header.py": "",
+      "/app/src/components/footer.py": "",
+      "/app/src/services/api.py": "",
+      "/app/src/models/user.py": "",
+      "/app/src/config/config.py": "",
+      "/app/requirements.txt": "",
+    };
+    const appFiles = getAppFileInfos(
+      pythonProjectFiles,
+      "/app",
+      filesMetadataExtractorPerLanguage.python,
+    );
+
+    expect(appFiles.length).toBe(9);
+    expect(appFiles).toEqual([
+      { path: "index.py" },
+      { path: "src/app.py" },
+      { path: "src/utils/helpers.py" },
+      { path: "src/components/header.py" },
+      { path: "src/components/footer.py" },
+      { path: "src/services/api.py" },
+      { path: "src/models/user.py" },
+      { path: "src/config/config.py" },
+      { path: "requirements.txt" },
+    ]);
+  });
+});