Skip to content

Commit

Permalink
Merge pull request #420 from MartinBasti/reconciliation-invalid-results
Browse files Browse the repository at this point in the history 
fix(STONEINTG-639): report invalid results
  • Loading branch information
@MartinBasti
MartinBasti authored Nov 20, 2023
2 parents 3b841aa + 8f601ad commit 6c9bd8e
Show file tree
Hide file tree
Showing 6 changed files with 302 additions and 56 deletions.
4 changes: 4 additions & 0 deletions controllers/integrationpipeline/integrationpipeline_adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package integrationpipeline
import (
"context"
"fmt"
"strings"

applicationapiv1alpha1 "github.com/redhat-appstudio/application-api/api/v1alpha1"
"github.com/redhat-appstudio/integration-service/gitops"
Expand Down Expand Up @@ -170,6 +171,9 @@ func GetIntegrationPipelineRunStatus(adapterClient client.Client, ctx context.Co
}

if !outcome.HasPipelineRunPassedTesting() {
if !outcome.HasPipelineRunValidTestOutputs() {
return intgteststat.IntegrationTestStatusTestFail, strings.Join(outcome.GetValidationErrorsList(), "; "), nil
}
return intgteststat.IntegrationTestStatusTestFail, "Integration test failed", nil
}

Expand Down
120 changes: 120 additions & 0 deletions controllers/integrationpipeline/integrationpipeline_adapter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -970,4 +970,124 @@ var _ = Describe("Pipeline Adapter", Ordered, func() {
Expect(intgPipelineRunWithDeletionTimestamp.DeletionTimestamp).ToNot(BeNil())
})
})

When("GetIntegrationPipelineRunStatus is called with an Integration PLR with invalid TEST_OUTPUT result", func() {
var (
taskRunInvalidResult *tektonv1.TaskRun
intgPipelineInvalidResult *tektonv1.PipelineRun
)

BeforeEach(func() {

taskRunInvalidResult = &tektonv1.TaskRun{
ObjectMeta: metav1.ObjectMeta{
Name: "test-taskrun-invalid",
Namespace: "default",
},
Spec: tektonv1.TaskRunSpec{
TaskRef: &tektonv1.TaskRef{
Name: "test-taskrun-invalid",
ResolverRef: tektonv1.ResolverRef{
Resolver: "bundle",
Params: tektonv1.Params{
{
Name: "bundle",
Value: tektonv1.ParamValue{Type: "string", StringVal: "quay.io/redhat-appstudio/example-tekton-bundle:test"},
},
{
Name: "name",
Value: tektonv1.ParamValue{Type: "string", StringVal: "test-task"},
},
},
},
},
},
}
Expect(k8sClient.Create(ctx, taskRunInvalidResult)).Should(Succeed())

now := time.Now()
taskRunInvalidResult.Status = tektonv1.TaskRunStatus{
TaskRunStatusFields: tektonv1.TaskRunStatusFields{
StartTime: &metav1.Time{Time: now},
CompletionTime: &metav1.Time{Time: now.Add(5 * time.Minute)},
Results: []tektonv1.TaskRunResult{
{
Name: "TEST_OUTPUT",
Value: *tektonv1.NewStructuredValues(`{
"result": "INVALID",
}`),
},
},
},
}
Expect(k8sClient.Status().Update(ctx, taskRunInvalidResult)).Should(Succeed())

intgPipelineInvalidResult = &tektonv1.PipelineRun{
ObjectMeta: metav1.ObjectMeta{
Name: "pipelinerun-component-sample-invalid-result",
Namespace: "default",
Annotations: map[string]string{
"pac.test.appstudio.openshift.io/on-target-branch": "[main]",
},
},
Spec: tektonv1.PipelineRunSpec{
PipelineRef: &tektonv1.PipelineRef{
Name: "component-pipeline-invalid",
ResolverRef: tektonv1.ResolverRef{
Resolver: "bundle",
Params: tektonv1.Params{
{
Name: "bundle",
Value: tektonv1.ParamValue{Type: "string", StringVal: "quay.io/redhat-appstudio/example-tekton-bundle:component-pipeline-fail"},
},
{
Name: "name",
Value: tektonv1.ParamValue{Type: "string", StringVal: "test-task"},
},
},
},
},
},
}
Expect(k8sClient.Create(ctx, intgPipelineInvalidResult)).Should(Succeed())

intgPipelineInvalidResult.Status = tektonv1.PipelineRunStatus{
PipelineRunStatusFields: tektonv1.PipelineRunStatusFields{
CompletionTime: &metav1.Time{Time: time.Now()},
ChildReferences: []tektonv1.ChildStatusReference{
{
Name: taskRunInvalidResult.Name,
PipelineTaskName: "task1",
},
},
},
Status: v1.Status{
Conditions: v1.Conditions{
apis.Condition{
Reason: "Completed",
Status: "True",
Type: apis.ConditionSucceeded,
},
},
},
}
Expect(k8sClient.Status().Update(ctx, intgPipelineInvalidResult)).Should(Succeed())

})

AfterEach(func() {
err := k8sClient.Delete(ctx, intgPipelineInvalidResult)
Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue())
err = k8sClient.Delete(ctx, taskRunInvalidResult)
Expect(err == nil || k8serrors.IsNotFound(err)).To(BeTrue())
})

It("ensures test status in snapshot is updated to failed", func() {
status, detail, err := GetIntegrationPipelineRunStatus(adapter.client, adapter.context, intgPipelineInvalidResult)

Expect(err).To(BeNil())
Expect(status).To(Equal(intgteststat.IntegrationTestStatusTestFail))
Expect(detail).To(ContainSubstring("Invalid result:"))
})
})
})
99 changes: 69 additions & 30 deletions helpers/integration.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,13 @@ type AppStudioTestResult struct {
Warnings int `json:"warnings"`
}

// IntegrationTestTaskResult provides results from integration test task
// including metadata about validity of results
type IntegrationTestTaskResult struct {
TestOutput *AppStudioTestResult
ValidationError error
}

var testResultSchema = `{
"$schema": "http://json-schema.org/draft/2020-12/schema#",
"type": "object",
Expand Down Expand Up @@ -109,7 +116,7 @@ var testResultSchema = `{
type TaskRun struct {
pipelineTaskName string
trStatus *tektonv1.TaskRunStatus
testResult *AppStudioTestResult
testResult *IntegrationTestTaskResult
}

// NewTaskRunFromTektonTaskRun creates and returns am integration TaskRun from the TaskRunStatus.
Expand Down Expand Up @@ -144,8 +151,8 @@ func (t *TaskRun) GetDuration() time.Duration {
return end.Sub(start)
}

// GetTestResult returns a AppStudioTestResult if the TaskRun produced the result. It will return nil otherwise.
func (t *TaskRun) GetTestResult() (*AppStudioTestResult, error) {
// GetTestResult returns a IntegrationTestTaskResult if the TaskRun produced the result. It will return nil otherwise.
func (t *TaskRun) GetTestResult() (*IntegrationTestTaskResult, error) {
// Check for an already parsed result.
if t.testResult != nil {
return t.testResult, nil
Expand All @@ -158,20 +165,21 @@ func (t *TaskRun) GetTestResult() (*AppStudioTestResult, error) {

for _, taskRunResult := range t.trStatus.TaskRunStatusFields.Results {
if taskRunResult.Name == LegacyTestOutputName || taskRunResult.Name == TestOutputName {
var result AppStudioTestResult
var testOutput AppStudioTestResult
var testResult IntegrationTestTaskResult = IntegrationTestTaskResult{}
var v interface{}
err := json.Unmarshal([]byte(taskRunResult.Value.StringVal), &result)
if err != nil {
return nil, fmt.Errorf("error while mapping json data from taskRun %s: to AppStudioTestResult %w", taskRunResult.Name, err)
}
if err := json.Unmarshal([]byte(taskRunResult.Value.StringVal), &v); err != nil {
return nil, fmt.Errorf("error while mapping json data from taskRun %s: %w", taskRunResult.Name, err)
}
if err = sch.Validate(v); err != nil {
return nil, fmt.Errorf("error validating schema of results from taskRun %s: %w", taskRunResult.Name, err)

if err := json.Unmarshal([]byte(taskRunResult.Value.StringVal), &testOutput); err != nil {
testResult.ValidationError = fmt.Errorf("error while mapping json data from task %s result %s to AppStudioTestResult: %w", t.GetPipelineTaskName(), taskRunResult.Name, err)
} else if err := json.Unmarshal([]byte(taskRunResult.Value.StringVal), &v); err != nil {
testResult.ValidationError = fmt.Errorf("error while mapping json data from task %s result %s: %w", t.GetPipelineTaskName(), taskRunResult.Name, err)
} else if err = sch.Validate(v); err != nil {
testResult.ValidationError = fmt.Errorf("error validating schema of results from task %s result %s: %w", t.GetPipelineTaskName(), taskRunResult.Name, err)
} else {
testResult.TestOutput = &testOutput
}
t.testResult = &result
return &result, nil
t.testResult = &testResult
return &testResult, nil
}
}
return nil, nil
Expand Down Expand Up @@ -200,24 +208,48 @@ type IntegrationPipelineRunOutcome struct {
pipelineRunSucceeded bool
pipelineRun *tektonv1.PipelineRun
// map: task name to results
results map[string]*AppStudioTestResult
results map[string]*IntegrationTestTaskResult
}

// HasPipelineRunSucceeded returns true when pipeline in outcome succeeded
func (ipro *IntegrationPipelineRunOutcome) HasPipelineRunSucceeded() bool {
return ipro.pipelineRunSucceeded
}

// HasPipelineRunValidTestOutputs returns false when we failed to parse results of TEST_OUTPUT in tasks
func (ipro *IntegrationPipelineRunOutcome) HasPipelineRunValidTestOutputs() bool {
for _, result := range ipro.results {
if result.ValidationError != nil {
return false
}
}
return true
}

// GetValidationErrorsList returns validation error messages for each invalid task result in a list.
func (ipro *IntegrationPipelineRunOutcome) GetValidationErrorsList() []string {
var errors []string
for _, result := range ipro.results {
if result.ValidationError != nil {
errors = append(errors, fmt.Sprintf("Invalid result: %s", result.ValidationError))
}
}
return errors
}

// HasPipelineRunPassedTesting returns general outcome
// If any of the tasks with the TEST_OUTPUT result don't have the `result` field set to SUCCESS or SKIPPED, it returns false.
func (ipro *IntegrationPipelineRunOutcome) HasPipelineRunPassedTesting() bool {
if !ipro.HasPipelineRunSucceeded() {
return false
}
if !ipro.HasPipelineRunValidTestOutputs() {
return false
}
for _, result := range ipro.results {
if result.Result != AppStudioTestOutputSuccess &&
result.Result != AppStudioTestOutputSkipped &&
result.Result != AppStudioTestOutputWarning {
if result.TestOutput.Result != AppStudioTestOutputSuccess &&
result.TestOutput.Result != AppStudioTestOutputSkipped &&
result.TestOutput.Result != AppStudioTestOutputWarning {
return false
}
}
Expand All @@ -227,10 +259,17 @@ func (ipro *IntegrationPipelineRunOutcome) HasPipelineRunPassedTesting() bool {
// LogResults writes tasks names with results into given logger, each task on separate line
func (ipro *IntegrationPipelineRunOutcome) LogResults(logger logr.Logger) {
for k, v := range ipro.results {
logger.Info(fmt.Sprintf("Found task results for pipeline run %s", ipro.pipelineRun.Name),
"pipelineRun.Name", ipro.pipelineRun.Name,
"pipelineRun.Namespace", ipro.pipelineRun.Namespace,
"task.Name", k, "task.Result", v)
if v.TestOutput != nil {
logger.Info(fmt.Sprintf("Found task results for pipeline run %s", ipro.pipelineRun.Name),
"pipelineRun.Name", ipro.pipelineRun.Name,
"pipelineRun.Namespace", ipro.pipelineRun.Namespace,
"task.Name", k, "task.TestOutput", v.TestOutput)
} else if v.ValidationError != nil {
logger.Info(fmt.Sprintf("Invalid task results for pipeline run %s", ipro.pipelineRun.Name),
"pipelineRun.Name", ipro.pipelineRun.Name,
"pipelineRun.Namespace", ipro.pipelineRun.Namespace,
"task.Name", k, "task.ValidationError", v.ValidationError.Error())
}
}
}

Expand All @@ -244,13 +283,13 @@ func GetIntegrationPipelineRunOutcome(adapterClient client.Client, ctx context.C
return &IntegrationPipelineRunOutcome{
pipelineRunSucceeded: false,
pipelineRun: pipelineRun,
results: map[string]*AppStudioTestResult{},
results: map[string]*IntegrationTestTaskResult{},
}, nil
}
// Check if the pipelineRun.Status contains the childReferences to TaskRuns
if !reflect.ValueOf(pipelineRun.Status.ChildReferences).IsZero() {
// If the pipelineRun.Status contains the childReferences, parse them in the new way by querying for TaskRuns
results, err := GetAppStudioTestResultsFromPipelineRunWithChildReferences(adapterClient, ctx, pipelineRun)
results, err := GetIntegrationTestTaskResultsFromPipelineRunWithChildReferences(adapterClient, ctx, pipelineRun)
if err != nil {
return nil, fmt.Errorf("error while getting test results from pipelineRun %s: %w", pipelineRun.Name, err)
}
Expand All @@ -265,20 +304,20 @@ func GetIntegrationPipelineRunOutcome(adapterClient client.Client, ctx context.C
return &IntegrationPipelineRunOutcome{
pipelineRunSucceeded: true,
pipelineRun: pipelineRun,
results: map[string]*AppStudioTestResult{},
results: map[string]*IntegrationTestTaskResult{},
}, nil
}

// GetAppStudioTestResultsFromPipelineRunWithChildReferences finds all TaskRuns from childReferences of the PipelineRun
// that also contain a TEST_OUTPUT result and returns the parsed data
// GetIntegrationTestTaskResultsFromPipelineRunWithChildReferences finds all TaskRuns from childReferences of the PipelineRun
// that also contain a TEST_OUTPUT result and returns the parsed data or validation error
// returns map taskName: result
func GetAppStudioTestResultsFromPipelineRunWithChildReferences(adapterClient client.Client, ctx context.Context, pipelineRun *tektonv1.PipelineRun) (map[string]*AppStudioTestResult, error) {
func GetIntegrationTestTaskResultsFromPipelineRunWithChildReferences(adapterClient client.Client, ctx context.Context, pipelineRun *tektonv1.PipelineRun) (map[string]*IntegrationTestTaskResult, error) {
taskRuns, err := GetAllChildTaskRunsForPipelineRun(adapterClient, ctx, pipelineRun)
if err != nil {
return nil, err
}

results := map[string]*AppStudioTestResult{}
results := map[string]*IntegrationTestTaskResult{}
for _, tr := range taskRuns {
r, err := tr.GetTestResult()
if err != nil {
Expand Down
Loading

0 comments on commit 6c9bd8e

Please sign in to comment.