feat: assume txt files with "requirements" in their name are `requi…

…rements.txt` files (#1271) This relaxes our logic for deciding if we can parse a file as a `requirements.txt`: we now assume any file that has "requirements" in its name and ends with `.txt` is probably a `requirements.txt`, which matches logic found elsewhere in the wild (e.g. dependabot). Resolves #1266 Resolves #370 Resolves #67
google · Sep 25, 2024 · f7d5076 · f7d5076
1 parent 6d17d98
commit f7d5076
Show file tree

Hide file tree

Showing 10 changed files with 135 additions and 1 deletion.
diff --git a/cmd/osv-scanner/__snapshots__/main_test.snap b/cmd/osv-scanner/__snapshots__/main_test.snap
@@ -714,6 +714,42 @@ No issues found
 
 ---
 
+[TestRun/requirements.txt_can_have_all_kinds_of_names - 1]
+Scanning dir ./fixtures/locks-requirements
+Scanned <rootdir>/fixtures/locks-requirements/my-requirements.txt file and found 1 package
+Scanned <rootdir>/fixtures/locks-requirements/requirements-dev.txt file and found 1 package
+Scanned <rootdir>/fixtures/locks-requirements/requirements.prod.txt file and found 1 package
+Scanned <rootdir>/fixtures/locks-requirements/requirements.txt file and found 3 packages
+Scanned <rootdir>/fixtures/locks-requirements/the_requirements_for_test.txt file and found 1 package
++-------------------------------------+------+-----------+---------+---------+---------------------------------------------------+
+| OSV URL                             | CVSS | ECOSYSTEM | PACKAGE | VERSION | SOURCE                                            |
++-------------------------------------+------+-----------+---------+---------+---------------------------------------------------+
+| https://osv.dev/PYSEC-2022-190      | 9.8  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-2gwj-7jmv-h26r |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2022-1        | 8.7  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-53qw-q765-4fww |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2022-20       | 8.7  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-6cw3-g6wv-c2xv |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2022-2        | 8.7  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-8c5j-9r9f-c6w8 |      |           |         |         |                                                   |
+| https://osv.dev/GHSA-8x94-hmjh-97hq | 8.8  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/PYSEC-2022-19       | 6.1  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-95rw-fx8r-36v6 |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2022-3        | 6.9  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-jrh2-hc4r-7jwx |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2021-439      | 7.3  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-v6rh-hp5x-86rv |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2022-191      | 9.8  | PyPI      | django  | 2.2.24  | fixtures/locks-requirements/requirements.prod.txt |
+| https://osv.dev/GHSA-w24h-v9qh-8gxj |      |           |         |         |                                                   |
+| https://osv.dev/PYSEC-2020-73       |      | PyPI      | pandas  | 0.23.4  | fixtures/locks-requirements/requirements.txt      |
++-------------------------------------+------+-----------+---------+---------+---------------------------------------------------+
+
+---
+
+[TestRun/requirements.txt_can_have_all_kinds_of_names - 2]
+
+---
+
 [TestRun/verbosity_level_=_error - 1]
 No issues found
 

diff --git a/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt b/cmd/osv-scanner/fixtures/locks-requirements/my-requirements.txt
@@ -0,0 +1 @@
+flask
diff --git a/cmd/osv-scanner/fixtures/locks-requirements/osv-scanner.toml b/cmd/osv-scanner/fixtures/locks-requirements/osv-scanner.toml
@@ -0,0 +1,2 @@
+[[PackageOverrides]]
+ignore = true
diff --git a/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt b/cmd/osv-scanner/fixtures/locks-requirements/requirements-dev.txt
@@ -0,0 +1 @@
+black
diff --git a/cmd/osv-scanner/fixtures/locks-requirements/requirements.prod.txt b/cmd/osv-scanner/fixtures/locks-requirements/requirements.prod.txt
@@ -0,0 +1,7 @@
+django==2.2.24
+    # via
+    #   -r requirements.in
+    #   django-debug-toolbar
+    #   django-filter
+    #   django-storages
+    #   easy-thumbnails
diff --git a/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt b/cmd/osv-scanner/fixtures/locks-requirements/requirements.txt
@@ -0,0 +1,3 @@
+flask
+flask-cors
+pandas==0.23.4
diff --git a/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt b/cmd/osv-scanner/fixtures/locks-requirements/the_requirements_for_test.txt
@@ -0,0 +1 @@
+pytest
diff --git a/cmd/osv-scanner/main_test.go b/cmd/osv-scanner/main_test.go
@@ -341,6 +341,12 @@ func TestRun(t *testing.T) {
 			args: []string{"", "--config=./fixtures/osv-scanner-unknown-config.toml", "./fixtures/locks-many"},
 			exit: 127,
 		},
+		// a bunch of requirements.txt files with different names
+		{
+			name: "requirements.txt can have all kinds of names",
+			args: []string{"", "--config=./fixtures/osv-scanner-empty-config.toml", "./fixtures/locks-requirements"},
+			exit: 1,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

diff --git a/pkg/lockfile/parse-requirements-txt.go b/pkg/lockfile/parse-requirements-txt.go
@@ -103,7 +103,9 @@ func isLineContinuation(line string) bool {
 type RequirementsTxtExtractor struct{}
 
 func (e RequirementsTxtExtractor) ShouldExtract(path string) bool {
-	return filepath.Base(path) == "requirements.txt"
+	base := filepath.Base(path)
+
+	return strings.Contains(base, "requirements") && strings.HasSuffix(base, ".txt")
 }
 
 func (e RequirementsTxtExtractor) Extract(f DepFile) ([]PackageDetails, error) {

diff --git a/pkg/lockfile/parse-requirements-txt_test.go b/pkg/lockfile/parse-requirements-txt_test.go
@@ -43,8 +43,83 @@ func TestRequirementsTxtExtractor_ShouldExtract(t *testing.T) {
 		{
 			name: "",
 			path: "path.to.my.requirements.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "requirements-dev.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "requirements.dev.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "dev-requirements.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "requirements-ci.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "requirements.ci.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "ci-requirements.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "dev.txt",
+			want: false,
+		},
+		{
+			name: "",
+			path: "ci.txt",
+			want: false,
+		},
+		{
+			name: "",
+			path: "requirements",
+			want: false,
+		},
+		{
+			name: "",
+			path: "requirements.json",
+			want: false,
+		},
+		{
+			name: "",
+			path: "requirements/txt",
+			want: false,
+		},
+		{
+			name: "",
+			path: "requirements/txt.txt",
+			want: false,
+		},
+		{
+			name: "",
+			path: "path/requirements/txt.txt",
 			want: false,
 		},
+		{
+			name: "",
+			path: "path/requirements/requirements.txt",
+			want: true,
+		},
+		{
+			name: "",
+			path: "path/requirements/requirements-dev.txt",
+			want: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {