diff --git a/checks/evaluation/license.go b/checks/evaluation/license.go index 185c38da018e..713b76780060 100644 --- a/checks/evaluation/license.go +++ b/checks/evaluation/license.go @@ -18,7 +18,9 @@ import ( "github.com/ossf/scorecard/v4/checker" sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/hasApprovedLicenseFile" "github.com/ossf/scorecard/v4/probes/hasLicenseFile" + "github.com/ossf/scorecard/v4/probes/hasLicenseFileAtTopDir" ) // License applies the score policy for the Fuzzing check. @@ -29,6 +31,8 @@ func License(name string, // We have 7 unique probes, each should have a finding. expectedProbes := []string{ hasLicenseFile.Probe, + hasApprovedLicenseFile.Probe, + hasLicenseFileAtTopDir.Probe, } if !finding.UniqueProbesEqual(findings, expectedProbes) { @@ -36,14 +40,33 @@ func License(name string, return checker.CreateRuntimeErrorResult(name, e) } - // Compute the score. This is currently configured for a single probe - // that returns positive or negative (whether the project has a license - // file at all). + // Compute the score. + score := 0 + m := make(map[string]bool) for i := range findings { f := &findings[i] if f.Outcome == finding.OutcomePositive { - return checker.CreateMaxScoreResult(name, "project has license file") + switch f.Probe { + case hasApprovedLicenseFile.Probe: + score += scoreProbeOnce(f.Probe, m, 1) + case hasLicenseFileAtTopDir.Probe: + score += scoreProbeOnce(f.Probe, m, 3) + case hasLicenseFile.Probe: + score += scoreProbeOnce(f.Probe, m, 6) + m[f.Probe] = true + default: + e := sce.WithMessage(sce.ErrScorecardInternal, "unknown probe results") + return checker.CreateRuntimeErrorResult(name, e) + } } } - return checker.CreateMinScoreResult(name, "project does not have a license file") + _, defined := m[hasLicenseFile.Probe] + if !defined { + if score > 0 { + e := sce.WithMessage(sce.ErrScorecardInternal, "score calculation problem") + return checker.CreateRuntimeErrorResult(name, e) + } + return checker.CreateMinScoreResult(name, "license file not detected") + } + return checker.CreateResultWithScore(name, "license file detected", score) } diff --git a/checks/evaluation/license_test.go b/checks/evaluation/license_test.go index d6c7cb760933..9c292b990fa6 100644 --- a/checks/evaluation/license_test.go +++ b/checks/evaluation/license_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/ossf/scorecard/v4/checker" + sce "github.com/ossf/scorecard/v4/errors" "github.com/ossf/scorecard/v4/finding" scut "github.com/ossf/scorecard/v4/utests" ) @@ -35,23 +36,111 @@ func TestLicense(t *testing.T) { Probe: "hasLicenseFile", Outcome: finding.OutcomePositive, }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasLicenseFileAtTopDir", + Outcome: finding.OutcomePositive, + }, }, result: scut.TestReturn{ Score: checker.MaxResultScore, NumberOfInfo: 0, }, }, { - name: "Negative outcome = Min score", findings: []finding.Finding{ { Probe: "hasLicenseFile", Outcome: finding.OutcomeNegative, }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "hasLicenseFileAtTopDir", + Outcome: finding.OutcomeNegative, + }, }, result: scut.TestReturn{ Score: checker.MinResultScore, NumberOfInfo: 0, }, + }, { + findings: []finding.Finding{ + { + Probe: "hasLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "hasLicenseFileAtTopDir", + Outcome: finding.OutcomeNegative, + }, + }, + result: scut.TestReturn{ + Score: 6, + NumberOfInfo: 0, + }, + }, { + findings: []finding.Finding{ + { + Probe: "hasLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomeNegative, + }, + }, + result: scut.TestReturn{ + Score: -1, + NumberOfInfo: 0, + Error: sce.ErrScorecardInternal, + }, + }, { + findings: []finding.Finding{ + { + Probe: "hasLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomeNegative, + }, + { + Probe: "hasLicenseFileAtTopDir", + Outcome: finding.OutcomePositive, + }, + }, + result: scut.TestReturn{ + Score: 9, + NumberOfInfo: 0, + }, + }, { + findings: []finding.Finding{ + { + Probe: "hasLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasApprovedLicenseFile", + Outcome: finding.OutcomePositive, + }, + { + Probe: "hasLicenseFileAtTopDir", + Outcome: finding.OutcomeNegative, + }, + }, + result: scut.TestReturn{ + Score: 7, + NumberOfInfo: 0, + }, }, } for _, tt := range tests { diff --git a/checks/license_test.go b/checks/license_test.go index e55a79a17b56..5a15dbf2a15b 100644 --- a/checks/license_test.go +++ b/checks/license_test.go @@ -42,9 +42,9 @@ func TestLicenseFileSubdirectory(t *testing.T) { inputFolder: "testdata/licensedir/withlicense", expected: scut.TestReturn{ Error: nil, - Score: checker.MaxResultScore, - NumberOfInfo: 1, - NumberOfWarn: 0, + Score: 9, // Does not have approved format + NumberOfInfo: 2, + NumberOfWarn: 1, }, err: nil, }, @@ -54,7 +54,7 @@ func TestLicenseFileSubdirectory(t *testing.T) { expected: scut.TestReturn{ Error: nil, Score: checker.MinResultScore, - NumberOfWarn: 1, + NumberOfWarn: 3, }, err: nil, }, diff --git a/probes/entries.go b/probes/entries.go index c29b64cf3bc8..16b4b75eab81 100644 --- a/probes/entries.go +++ b/probes/entries.go @@ -24,7 +24,9 @@ import ( "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript" "github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript" + "github.com/ossf/scorecard/v4/probes/hasApprovedLicenseFile" "github.com/ossf/scorecard/v4/probes/hasLicenseFile" + "github.com/ossf/scorecard/v4/probes/hasLicenseFileAtTopDir" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsText" "github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure" @@ -68,6 +70,8 @@ var ( } License = []ProbeImpl{ hasLicenseFile.Run, + hasApprovedLicenseFile.Run, + hasLicenseFileAtTopDir.Run, } ) diff --git a/probes/hasApprovedLicenseFile/def.yml b/probes/hasApprovedLicenseFile/def.yml new file mode 100644 index 000000000000..ef9165ab80ef --- /dev/null +++ b/probes/hasApprovedLicenseFile/def.yml @@ -0,0 +1,26 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: hasApprovedLicenseFile +short: Check that the project has a license file of approved format +motivation: > + A license can give users information about how the source code may or may not be used. The lack of a license will impede any kind of security review or audit and creates a legal risk for potential users. +implementation: > + The implementation checks whether a license file is present and is of an approved format +outcome: + - If a license file is found, the outcome is positive. +remediation: + effort: Low + text: + - Update the license file format in the Github repository. \ No newline at end of file diff --git a/probes/hasApprovedLicenseFile/impl.go b/probes/hasApprovedLicenseFile/impl.go new file mode 100644 index 000000000000..3110a175c83a --- /dev/null +++ b/probes/hasApprovedLicenseFile/impl.go @@ -0,0 +1,70 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package hasApprovedLicenseFile + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "hasApprovedLicenseFile" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + if raw.LicenseResults.LicenseFiles == nil || len(raw.LicenseResults.LicenseFiles) == 0 { + f, err := finding.NewWith(fs, Probe, + "project doe not have a license file", nil, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil + } + + for _, licenseFile := range raw.LicenseResults.LicenseFiles { + if licenseFile.LicenseInformation.Approved { + if licenseFile.LicenseInformation.Approved { + msg := "License file is of approved type" + + f, err := finding.NewWith(fs, Probe, + msg, nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil + } + } + } + + f, err := finding.NewWith(fs, Probe, + "project license files has no approved information", nil, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil +} diff --git a/probes/hasApprovedLicenseFile/impl_test.go b/probes/hasApprovedLicenseFile/impl_test.go new file mode 100644 index 000000000000..80990a77f2b9 --- /dev/null +++ b/probes/hasApprovedLicenseFile/impl_test.go @@ -0,0 +1,158 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package hasApprovedLicenseFile + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "License file found and is approved: outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Approved: true, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "License file found and is not approved: outcome should be negative", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Approved: false, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "License file not found and outcome should be negative", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "License file found and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Attribution: "wrong attribution", + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: nil, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "0 license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +} diff --git a/probes/hasLicenseFile/impl_test.go b/probes/hasLicenseFile/impl_test.go index 61f06f18b5db..24bf689a39ce 100644 --- a/probes/hasLicenseFile/impl_test.go +++ b/probes/hasLicenseFile/impl_test.go @@ -62,6 +62,28 @@ func Test_Run(t *testing.T) { finding.OutcomeNegative, }, }, + { + name: "nil license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: nil, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "0 license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, } for _, tt := range tests { tt := tt // Re-initializing variable so it is not changed while executing the closure below diff --git a/probes/hasLicenseFileAtTopDir/def.yml b/probes/hasLicenseFileAtTopDir/def.yml new file mode 100644 index 000000000000..e5a0f0a349fa --- /dev/null +++ b/probes/hasLicenseFileAtTopDir/def.yml @@ -0,0 +1,26 @@ +# Copyright 2023 OpenSSF Scorecard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +id: hasLicenseFileAtTopDir +short: Check that the project has a license file +motivation: > + A license can give users information about how the source code may or may not be used. The lack of a license will impede any kind of security review or audit and creates a legal risk for potential users. +implementation: > + This check will detect files in the top-level directory with any combination of the following names and extensions:LICENSE, LICENCE, COPYING, COPYRIGHT and having common extensions such as .html, .txt, or .md. It will also detect these files in a directory named LICENSES. (Files in a LICENSES directory are typically named as their SPDX license identifier followed by an appropriate file extension, as described in the REUSE Specification.) +outcome: + - If the projects license file is found at the top level, then the outcome is positive. +remediation: + effort: Low + text: + - Place the license file at the top level of the project source tree. \ No newline at end of file diff --git a/probes/hasLicenseFileAtTopDir/impl.go b/probes/hasLicenseFileAtTopDir/impl.go new file mode 100644 index 000000000000..4740a858014c --- /dev/null +++ b/probes/hasLicenseFileAtTopDir/impl.go @@ -0,0 +1,85 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package hasLicenseFileAtTopDir + +import ( + "embed" + "fmt" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" + "github.com/ossf/scorecard/v4/probes/internal/utils/uerror" +) + +//go:embed *.yml +var fs embed.FS + +const Probe = "hasLicenseFileAtTopDir" + +func Run(raw *checker.RawResults) ([]finding.Finding, string, error) { + if raw == nil { + return nil, "", fmt.Errorf("%w: raw", uerror.ErrNil) + } + + if raw.LicenseResults.LicenseFiles == nil || len(raw.LicenseResults.LicenseFiles) == 0 { + f, err := finding.NewWith(fs, Probe, + "project doe not have a license file", nil, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil + } + + for _, licenseFile := range raw.LicenseResults.LicenseFiles { + switch licenseFile.LicenseInformation.Attribution { + case checker.LicenseAttributionTypeAPI, checker.LicenseAttributionTypeHeuristics: + // both repoAPI and scorecard (not using the API) follow checks.md + // for a file to be found it must have been in the correct location + // award location points. + msg := "License file found in expected location" + + f, err := finding.NewWith(fs, Probe, + msg, nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil + + case checker.LicenseAttributionTypeOther: + // TODO ascertain location found + msg := "License file found in unexpected location" + + f, err := finding.NewWith(fs, Probe, + msg, nil, + finding.OutcomePositive) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil + default: + } + } + + f, err := finding.NewWith(fs, Probe, + "project license file has no attribution", nil, + finding.OutcomeNegative) + if err != nil { + return nil, Probe, fmt.Errorf("create finding: %w", err) + } + return []finding.Finding{*f}, Probe, nil +} diff --git a/probes/hasLicenseFileAtTopDir/impl_test.go b/probes/hasLicenseFileAtTopDir/impl_test.go new file mode 100644 index 000000000000..3c8bad6e3996 --- /dev/null +++ b/probes/hasLicenseFileAtTopDir/impl_test.go @@ -0,0 +1,161 @@ +// Copyright 2023 OpenSSF Scorecard Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// nolint:stylecheck +package hasLicenseFileAtTopDir + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "github.com/ossf/scorecard/v4/checker" + "github.com/ossf/scorecard/v4/finding" +) + +func Test_Run(t *testing.T) { + t.Parallel() + // nolint:govet + tests := []struct { + name string + raw *checker.RawResults + outcomes []finding.Outcome + err error + }{ + { + name: "License file found and correct attribution 1", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Attribution: checker.LicenseAttributionTypeAPI, + Approved: true, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "License file not found and outcome should be negative", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "License file found and correct attribution 2", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Attribution: checker.LicenseAttributionTypeHeuristics, + Approved: true, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomePositive, + }, + }, + { + name: "License file found and wrong attribution", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{ + { + File: checker.File{ + Path: "LICENSE.md", + }, + LicenseInformation: checker.License{ + Attribution: "wrong_attribution", + Approved: true, + }, + }, + }, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "nil license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: nil, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + { + name: "0 license files and outcome should be positive", + raw: &checker.RawResults{ + LicenseResults: checker.LicenseData{ + LicenseFiles: []checker.LicenseFile{}, + }, + }, + outcomes: []finding.Outcome{ + finding.OutcomeNegative, + }, + }, + } + for _, tt := range tests { + tt := tt // Re-initializing variable so it is not changed while executing the closure below + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + findings, s, err := Run(tt.raw) + if !cmp.Equal(tt.err, err, cmpopts.EquateErrors()) { + t.Errorf("mismatch (-want +got):\n%s", cmp.Diff(tt.err, err, cmpopts.EquateErrors())) + } + if err != nil { + return + } + if diff := cmp.Diff(Probe, s); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + if diff := cmp.Diff(len(tt.outcomes), len(findings)); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + for i := range tt.outcomes { + outcome := &tt.outcomes[i] + f := &findings[i] + if diff := cmp.Diff(*outcome, f.Outcome); diff != "" { + t.Errorf("mismatch (-want +got):\n%s", diff) + } + } + }) + } +}