From a0cc6326f2ff3b7f1c5c7d14b552dfa50e4d6b23 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 10:42:05 +0700 Subject: [PATCH 001/268] Run go fmt for All code --- cmd/config/manager.go | 2 +- cmd/config/view/view.go | 2 +- cmd/description/descriptor.go | 2 +- cmd/execution/executioner.go | 4 ++-- cmd/execution/executioner_test.go | 6 +++--- cmd/list/lister.go | 2 +- cmd/list/lister_test.go | 6 +++--- cmd/root.go | 4 ++-- cmd/root_test.go | 4 ++-- cmd/schedule/describe/describe.go | 2 +- cmd/schedule/describe/describe_test.go | 12 ++++++------ cmd/schedule/list/list.go | 4 ++-- cmd/schedule/list/list_test.go | 12 ++++++------ cmd/schedule/remove/remove.go | 2 +- cmd/schedule/remove/remove_test.go | 12 ++++++------ cmd/schedule/schedule.go | 17 ++++++++--------- cmd/schedule/schedule_test.go | 12 ++++++------ cmd/version/version.go | 2 +- cmd/version/version_test.go | 4 ++-- config/config.go | 2 +- daemon/client.go | 16 ++++++++-------- daemon/client_mock.go | 4 ++-- daemon/client_test.go | 8 ++++---- exec/cli/cli.go | 2 +- proctord/audit/auditor_mock.go | 2 +- proctord/config/config.go | 2 +- proctord/config/config_test.go | 2 +- proctord/http/client_test.go | 2 +- proctord/instrumentation/newrelic.go | 2 +- proctord/jobs/execution/executioner_mock.go | 2 +- proctord/jobs/execution/executioner_test.go | 6 +++--- proctord/jobs/execution/handler.go | 4 ++-- proctord/jobs/execution/handler_test.go | 6 +++--- proctord/jobs/execution/job.go | 1 - proctord/jobs/logs/handler_test.go | 4 ++-- proctord/jobs/metadata/handler_test.go | 2 +- proctord/jobs/metadata/store_test.go | 2 +- proctord/jobs/schedule/handler.go | 4 ++-- proctord/jobs/schedule/handler_test.go | 10 +++++----- proctord/jobs/schedule/worker.go | 6 +++--- proctord/jobs/schedule/worker_test.go | 6 +++--- proctord/jobs/secrets/handler_test.go | 2 +- proctord/jobs/secrets/store_test.go | 2 +- proctord/kubernetes/client.go | 10 +++++----- proctord/kubernetes/client_mock.go | 2 +- proctord/kubernetes/client_test.go | 6 +++--- proctord/middleware/validate_client_version.go | 6 +++--- .../middleware/validate_client_version_test.go | 10 +++++----- proctord/server/router.go | 7 +++---- proctord/storage/postgres/client.go | 2 +- proctord/storage/postgres/client_test.go | 2 +- proctord/storage/postgres/migrations.go | 2 +- proctord/storage/store.go | 2 +- proctord/storage/store_mock.go | 2 +- proctord/storage/store_test.go | 14 +++++++------- proctord/utility/utils.go | 4 ++-- utility/sort/sort_test.go | 2 +- 57 files changed, 139 insertions(+), 142 deletions(-) diff --git a/cmd/config/manager.go b/cmd/config/manager.go index 3cd1cd3e..e518fa1b 100644 --- a/cmd/config/manager.go +++ b/cmd/config/manager.go @@ -9,9 +9,9 @@ import ( "strings" "github.com/fatih/color" + "github.com/spf13/cobra" proctor_config "proctor/config" "proctor/io" - "github.com/spf13/cobra" ) func CreateDirIfNotExist(dir string) { diff --git a/cmd/config/view/view.go b/cmd/config/view/view.go index 293ddd66..8adf6e50 100644 --- a/cmd/config/view/view.go +++ b/cmd/config/view/view.go @@ -7,9 +7,9 @@ import ( "path/filepath" "github.com/fatih/color" + "github.com/spf13/cobra" proctor_config "proctor/config" "proctor/io" - "github.com/spf13/cobra" ) func CreateDirIfNotExist(dir string) { diff --git a/cmd/description/descriptor.go b/cmd/description/descriptor.go index 5d00fc0a..44d28591 100644 --- a/cmd/description/descriptor.go +++ b/cmd/description/descriptor.go @@ -5,10 +5,10 @@ import ( "strings" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" proc_metadata "proctor/proctord/jobs/metadata" - "github.com/spf13/cobra" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/execution/executioner.go b/cmd/execution/executioner.go index aa45b60c..113fa74a 100644 --- a/cmd/execution/executioner.go +++ b/cmd/execution/executioner.go @@ -5,10 +5,10 @@ import ( "strings" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" proctord_utility "proctor/proctord/utility" - "github.com/spf13/cobra" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { @@ -50,7 +50,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in osExitFunc(1) return } - + printer.Println("Proc submitted for execution. \nStreaming logs:", color.FgGreen) err = proctorDClient.StreamProcLogs(executedProcName) if err != nil { diff --git a/cmd/execution/executioner_test.go b/cmd/execution/executioner_test.go index 298ae01f..d65a377d 100644 --- a/cmd/execution/executioner_test.go +++ b/cmd/execution/executioner_test.go @@ -7,12 +7,12 @@ import ( "testing" "github.com/fatih/color" - "proctor/daemon" - "proctor/io" - "proctor/proctord/utility" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" + "proctor/proctord/utility" ) type ExecutionCmdTestSuite struct { diff --git a/cmd/list/lister.go b/cmd/list/lister.go index 51e35267..bdef68bf 100644 --- a/cmd/list/lister.go +++ b/cmd/list/lister.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" - "github.com/spf13/cobra" "proctor/utility/sort" ) diff --git a/cmd/list/lister_test.go b/cmd/list/lister_test.go index f162ccee..e0e2a7a6 100644 --- a/cmd/list/lister_test.go +++ b/cmd/list/lister_test.go @@ -6,12 +6,12 @@ import ( "testing" "github.com/fatih/color" - "proctor/daemon" - "proctor/io" - proc_metadata "proctor/proctord/jobs/metadata" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" + proc_metadata "proctor/proctord/jobs/metadata" ) type ListCmdTestSuite struct { diff --git a/cmd/root.go b/cmd/root.go index 44c3d864..29d83799 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -2,8 +2,8 @@ package cmd import ( "fmt" - "proctor/cmd/schedule/remove" "os" + "proctor/cmd/schedule/remove" "proctor/cmd/config" "proctor/cmd/config/view" @@ -11,8 +11,8 @@ import ( "proctor/cmd/execution" "proctor/cmd/list" "proctor/cmd/schedule" - schedule_list "proctor/cmd/schedule/list" schedule_describe "proctor/cmd/schedule/describe" + schedule_list "proctor/cmd/schedule/list" "proctor/cmd/version" "proctor/daemon" "proctor/io" diff --git a/cmd/root_test.go b/cmd/root_test.go index de58895a..f00f4bb1 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -3,11 +3,11 @@ package cmd import ( "testing" - "proctor/daemon" - "proctor/io" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "proctor/cmd/version/github" + "proctor/daemon" + "proctor/io" ) func TestRootCmdUsage(t *testing.T) { diff --git a/cmd/schedule/describe/describe.go b/cmd/schedule/describe/describe.go index 4c6be1f4..d337ad4b 100644 --- a/cmd/schedule/describe/describe.go +++ b/cmd/schedule/describe/describe.go @@ -3,9 +3,9 @@ package describe import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" - "github.com/spf13/cobra" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/schedule/describe/describe_test.go b/cmd/schedule/describe/describe_test.go index e9cc7431..473c2dc7 100644 --- a/cmd/schedule/describe/describe_test.go +++ b/cmd/schedule/describe/describe_test.go @@ -1,19 +1,19 @@ package describe import ( - "proctor/daemon" - "proctor/io" "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" "testing" - "github.com/stretchr/testify/assert" ) type ScheduleCreateCmdTestSuite struct { suite.Suite - mockPrinter *io.MockPrinter - mockProctorDClient *daemon.MockClient - testScheduleDescribeCmd *cobra.Command + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testScheduleDescribeCmd *cobra.Command } func (s *ScheduleCreateCmdTestSuite) SetupTest() { diff --git a/cmd/schedule/list/list.go b/cmd/schedule/list/list.go index 2aa49ed7..df42f334 100644 --- a/cmd/schedule/list/list.go +++ b/cmd/schedule/list/list.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" - "github.com/spf13/cobra" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { @@ -25,7 +25,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { printer.Println(fmt.Sprintf("%-40s %-30s %-20s %s", "ID", "PROC NAME", "GROUP NAME", "TAGS"), color.FgGreen) for _, scheduledProc := range scheduledProcs { - printer.Println(fmt.Sprintf("%-40s %-30s %-20s %s", scheduledProc.ID, scheduledProc.Name, scheduledProc.Group,scheduledProc.Tags), color.Reset) + printer.Println(fmt.Sprintf("%-40s %-30s %-20s %s", scheduledProc.ID, scheduledProc.Name, scheduledProc.Group, scheduledProc.Tags), color.Reset) } }, } diff --git a/cmd/schedule/list/list_test.go b/cmd/schedule/list/list_test.go index d63433e7..4b030e89 100644 --- a/cmd/schedule/list/list_test.go +++ b/cmd/schedule/list/list_test.go @@ -1,19 +1,19 @@ package list import ( - "proctor/daemon" - "proctor/io" "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" "testing" - "github.com/stretchr/testify/assert" ) type ScheduleCreateCmdTestSuite struct { suite.Suite - mockPrinter *io.MockPrinter - mockProctorDClient *daemon.MockClient - testScheduleListCmd *cobra.Command + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testScheduleListCmd *cobra.Command } func (s *ScheduleCreateCmdTestSuite) SetupTest() { diff --git a/cmd/schedule/remove/remove.go b/cmd/schedule/remove/remove.go index e46c56a7..334bbeb8 100644 --- a/cmd/schedule/remove/remove.go +++ b/cmd/schedule/remove/remove.go @@ -3,9 +3,9 @@ package remove import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" - "github.com/spf13/cobra" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/schedule/remove/remove_test.go b/cmd/schedule/remove/remove_test.go index 0d4d6bed..025aea5f 100644 --- a/cmd/schedule/remove/remove_test.go +++ b/cmd/schedule/remove/remove_test.go @@ -1,19 +1,19 @@ package remove import ( - "proctor/daemon" - "proctor/io" "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" "testing" - "github.com/stretchr/testify/assert" ) type ScheduleCreateCmdTestSuite struct { suite.Suite - mockPrinter *io.MockPrinter - mockProctorDClient *daemon.MockClient - testScheduleRemoveCmd *cobra.Command + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testScheduleRemoveCmd *cobra.Command } func (s *ScheduleCreateCmdTestSuite) SetupTest() { diff --git a/cmd/schedule/schedule.go b/cmd/schedule/schedule.go index 232fc4cc..c0b54f3e 100644 --- a/cmd/schedule/schedule.go +++ b/cmd/schedule/schedule.go @@ -3,41 +3,41 @@ package schedule import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/daemon" "proctor/io" - "github.com/spf13/cobra" "strings" ) -func NewCmd(printer io.Printer,proctorDClient daemon.Client) *cobra.Command { +func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { return &cobra.Command{ Use: "schedule", Short: "Create scheduled jobs", Long: "This command helps to create scheduled jobs", - Example: fmt.Sprintf("proctor schedule run-sample -g my-group -t '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar"), - Args: cobra.MinimumNArgs(1), + Example: fmt.Sprintf("proctor schedule run-sample -g my-group -t '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar"), + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { procName := args[0] printer.Println(fmt.Sprintf("%-40s %-100s", "Creating Scheduled Job", procName), color.Reset) time, err := cmd.Flags().GetString("time") if err != nil { - printer.Println(err.Error(),color.FgRed) + printer.Println(err.Error(), color.FgRed) } notificationEmails, err := cmd.Flags().GetString("notify") if err != nil { - printer.Println(err.Error(),color.FgRed) + printer.Println(err.Error(), color.FgRed) } tags, err := cmd.Flags().GetString("tags") if err != nil { - printer.Println(err.Error(),color.FgRed) + printer.Println(err.Error(), color.FgRed) } group, err := cmd.Flags().GetString("group") if err != nil { - printer.Println(err.Error(),color.FgRed) + printer.Println(err.Error(), color.FgRed) } jobArgs := make(map[string]string) @@ -70,4 +70,3 @@ func NewCmd(printer io.Printer,proctorDClient daemon.Client) *cobra.Command { }, } } - diff --git a/cmd/schedule/schedule_test.go b/cmd/schedule/schedule_test.go index 48331d0a..54611051 100644 --- a/cmd/schedule/schedule_test.go +++ b/cmd/schedule/schedule_test.go @@ -1,19 +1,19 @@ package schedule import ( - "proctor/daemon" - "proctor/io" "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "proctor/daemon" + "proctor/io" "testing" - "github.com/stretchr/testify/assert" ) type ScheduleCreateCmdTestSuite struct { suite.Suite - mockPrinter *io.MockPrinter - mockProctorDClient *daemon.MockClient - testScheduleCreateCmd *cobra.Command + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testScheduleCreateCmd *cobra.Command } func (s *ScheduleCreateCmdTestSuite) SetupTest() { diff --git a/cmd/version/version.go b/cmd/version/version.go index e483dd7b..656a92ad 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -4,9 +4,9 @@ import ( "fmt" "github.com/fatih/color" + "github.com/spf13/cobra" "proctor/cmd/version/github" "proctor/io" - "github.com/spf13/cobra" ) const ClientVersion = "v0.6.0" diff --git a/cmd/version/version_test.go b/cmd/version/version_test.go index 70e2cf03..c37a708e 100644 --- a/cmd/version/version_test.go +++ b/cmd/version/version_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/fatih/color" - gh "proctor/cmd/version/github" - "proctor/io" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + gh "proctor/cmd/version/github" + "proctor/io" ) func TestVersionCmdUsage(t *testing.T) { diff --git a/config/config.go b/config/config.go index 78fd43cf..bc0e63f1 100644 --- a/config/config.go +++ b/config/config.go @@ -5,8 +5,8 @@ import ( "os" "time" - "proctor/proctord/utility" "github.com/pkg/errors" + "proctor/proctord/utility" "github.com/spf13/viper" ) diff --git a/daemon/client.go b/daemon/client.go index 9805fc1e..fb4231b1 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -18,12 +18,12 @@ import ( "github.com/briandowns/spinner" "github.com/fatih/color" + "github.com/gorilla/websocket" "proctor/config" "proctor/io" proc_metadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/schedule" "proctor/proctord/utility" - "github.com/gorilla/websocket" ) type Client interface { @@ -31,7 +31,7 @@ type Client interface { ExecuteProc(string, map[string]string) (string, error) StreamProcLogs(string) error GetDefinitiveProcExecutionStatus(string) (string, error) - ScheduleJob(string, string, string, string,string, map[string]string) (string, error) + ScheduleJob(string, string, string, string, string, map[string]string) (string, error) ListScheduledProcs() ([]schedule.ScheduledJob, error) DescribeScheduledProc(string) (schedule.ScheduledJob, error) RemoveScheduledProc(string) error @@ -104,7 +104,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { - return "", buildHTTPError(c, resp) + return "", buildHTTPError(c, resp) } var scheduledJob ScheduleJobPayload @@ -401,19 +401,19 @@ func buildHTTPError(c *client, resp *http.Response) error { return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorMissingConfig) } return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorInvalidConfig) - } - + } + if resp.StatusCode == http.StatusBadRequest { return getHttpResponseError(resp.Body) - } + } if resp.StatusCode == http.StatusNoContent { return fmt.Errorf(utility.NoScheduledJobsError) - } + } if resp.StatusCode == http.StatusNotFound { return fmt.Errorf(utility.JobNotFoundError) - } + } if resp.StatusCode == http.StatusForbidden { return fmt.Errorf(utility.JobForbiddenErrorHeader) diff --git a/daemon/client_mock.go b/daemon/client_mock.go index bb6f386d..d9ba5a33 100644 --- a/daemon/client_mock.go +++ b/daemon/client_mock.go @@ -1,9 +1,9 @@ package daemon import ( + "github.com/stretchr/testify/mock" proc_metadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/schedule" - "github.com/stretchr/testify/mock" ) type MockClient struct { @@ -35,7 +35,7 @@ func (m *MockClient) GetDefinitiveProcExecutionStatus(name string) (string, erro return args.Get(0).(string), args.Error(1) } -func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string,group string, jobArgs map[string]string) (string, error) { +func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, group string, jobArgs map[string]string) (string, error) { args := m.Called(name, tags, time, notificationEmails, group, jobArgs) return args.Get(0).(string), args.Error(1) } diff --git a/daemon/client_test.go b/daemon/client_test.go index c826f08a..8c3255c9 100644 --- a/daemon/client_test.go +++ b/daemon/client_test.go @@ -10,16 +10,16 @@ import ( "proctor/cmd/version" - "proctor/config" - "proctor/io" "github.com/gorilla/websocket" "github.com/thingful/httpmock" + "proctor/config" + "proctor/io" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" proc_metadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/metadata/env" "proctor/proctord/utility" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" ) type TestConnectionError struct { diff --git a/exec/cli/cli.go b/exec/cli/cli.go index a8db7cd5..5d53ec8b 100644 --- a/exec/cli/cli.go +++ b/exec/cli/cli.go @@ -2,10 +2,10 @@ package main import ( "proctor/cmd" + "proctor/cmd/version/github" "proctor/config" "proctor/daemon" "proctor/io" - "proctor/cmd/version/github" ) func main() { diff --git a/proctord/audit/auditor_mock.go b/proctord/audit/auditor_mock.go index 79afaf39..296d98f1 100644 --- a/proctord/audit/auditor_mock.go +++ b/proctord/audit/auditor_mock.go @@ -1,8 +1,8 @@ package audit import ( - "proctor/proctord/storage/postgres" "github.com/stretchr/testify/mock" + "proctor/proctord/storage/postgres" ) type MockAuditor struct { diff --git a/proctord/config/config.go b/proctord/config/config.go index 100cbdfd..96384d45 100644 --- a/proctord/config/config.go +++ b/proctord/config/config.go @@ -147,4 +147,4 @@ func SentryDSN() string { func DocsPath() string { return viper.GetString("DOCS_PATH") -} \ No newline at end of file +} diff --git a/proctord/config/config_test.go b/proctord/config/config_test.go index ee4148e2..122ccb80 100644 --- a/proctord/config/config_test.go +++ b/proctord/config/config_test.go @@ -256,4 +256,4 @@ func TestDocsPath(t *testing.T) { viper.AutomaticEnv() assert.Equal(t, "path1", DocsPath()) -} \ No newline at end of file +} diff --git a/proctord/http/client_test.go b/proctord/http/client_test.go index 8a89c456..01ada998 100644 --- a/proctord/http/client_test.go +++ b/proctord/http/client_test.go @@ -7,8 +7,8 @@ import ( "net/http" "testing" - "proctor/proctord/config" "github.com/stretchr/testify/assert" + "proctor/proctord/config" ) func TestNewClient(t *testing.T) { diff --git a/proctord/instrumentation/newrelic.go b/proctord/instrumentation/newrelic.go index e36322c8..5e8510d3 100644 --- a/proctord/instrumentation/newrelic.go +++ b/proctord/instrumentation/newrelic.go @@ -3,8 +3,8 @@ package instrumentation import ( "net/http" - "proctor/proctord/config" "github.com/newrelic/go-agent" + "proctor/proctord/config" ) var NewRelicApp newrelic.Application diff --git a/proctord/jobs/execution/executioner_mock.go b/proctord/jobs/execution/executioner_mock.go index fc741c26..d5ff7b61 100644 --- a/proctord/jobs/execution/executioner_mock.go +++ b/proctord/jobs/execution/executioner_mock.go @@ -1,8 +1,8 @@ package execution import ( - "proctor/proctord/storage/postgres" "github.com/stretchr/testify/mock" + "proctor/proctord/storage/postgres" ) type MockExecutioner struct { diff --git a/proctord/jobs/execution/executioner_test.go b/proctord/jobs/execution/executioner_test.go index 982a1676..b696bd24 100644 --- a/proctord/jobs/execution/executioner_test.go +++ b/proctord/jobs/execution/executioner_test.go @@ -4,14 +4,14 @@ import ( "errors" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" "proctor/proctord/utility" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" ) type ExecutionerTestSuite struct { diff --git a/proctord/jobs/execution/handler.go b/proctord/jobs/execution/handler.go index fec503d7..04faefab 100644 --- a/proctord/jobs/execution/handler.go +++ b/proctord/jobs/execution/handler.go @@ -5,13 +5,13 @@ import ( "encoding/json" "fmt" "github.com/getsentry/raven-go" + "github.com/gorilla/mux" + "net/http" "proctor/proctord/audit" "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/storage/postgres" "proctor/proctord/utility" - "github.com/gorilla/mux" - "net/http" "time" ) diff --git a/proctord/jobs/execution/handler_test.go b/proctord/jobs/execution/handler_test.go index 063a6740..2e23f763 100644 --- a/proctord/jobs/execution/handler_test.go +++ b/proctord/jobs/execution/handler_test.go @@ -5,9 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "proctor/proctord/audit" - "proctor/proctord/storage" - "proctor/proctord/utility" "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -15,6 +12,9 @@ import ( "github.com/urfave/negroni" "net/http" "net/http/httptest" + "proctor/proctord/audit" + "proctor/proctord/storage" + "proctor/proctord/utility" "testing" ) diff --git a/proctord/jobs/execution/job.go b/proctord/jobs/execution/job.go index b5244931..d31ef4d3 100644 --- a/proctord/jobs/execution/job.go +++ b/proctord/jobs/execution/job.go @@ -5,4 +5,3 @@ type Job struct { Args map[string]string `json:"args"` CallbackURL string `json:"callback_url"` } - diff --git a/proctord/jobs/logs/handler_test.go b/proctord/jobs/logs/handler_test.go index 6b3ded6b..549470d6 100644 --- a/proctord/jobs/logs/handler_test.go +++ b/proctord/jobs/logs/handler_test.go @@ -7,12 +7,12 @@ import ( "strings" "testing" - "proctor/proctord/kubernetes" - "proctor/proctord/utility" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "proctor/proctord/kubernetes" + "proctor/proctord/utility" ) type LoggerTestSuite struct { diff --git a/proctord/jobs/metadata/handler_test.go b/proctord/jobs/metadata/handler_test.go index bd7c8411..de07fce8 100644 --- a/proctord/jobs/metadata/handler_test.go +++ b/proctord/jobs/metadata/handler_test.go @@ -11,10 +11,10 @@ import ( "proctor/proctord/jobs/metadata/env" - "proctor/proctord/utility" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "proctor/proctord/utility" ) type MetadataHandlerTestSuite struct { diff --git a/proctord/jobs/metadata/store_test.go b/proctord/jobs/metadata/store_test.go index af9fd973..b108fc77 100644 --- a/proctord/jobs/metadata/store_test.go +++ b/proctord/jobs/metadata/store_test.go @@ -5,10 +5,10 @@ import ( "errors" "testing" - "proctor/proctord/redis" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "proctor/proctord/redis" ) type MetadataStoreTestSuite struct { diff --git a/proctord/jobs/schedule/handler.go b/proctord/jobs/schedule/handler.go index c40e75fc..3f72be7b 100644 --- a/proctord/jobs/schedule/handler.go +++ b/proctord/jobs/schedule/handler.go @@ -10,11 +10,11 @@ import ( "github.com/badoux/checkmail" + "github.com/robfig/cron" "proctor/proctord/jobs/metadata" "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/utility" - "github.com/robfig/cron" ) type scheduler struct { @@ -90,7 +90,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { _, err = scheduler.metadataStore.GetJobMetadata(scheduledJob.Name) if err != nil { if err.Error() == "redigo: nil returned" { - logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", scheduledJob.Tags), scheduledJob.Name, ) + logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", scheduledJob.Tags), scheduledJob.Name) w.WriteHeader(http.StatusNotFound) w.Write([]byte(utility.NonExistentProcClientError)) diff --git a/proctord/jobs/schedule/handler_test.go b/proctord/jobs/schedule/handler_test.go index 30eaa947..73597f08 100644 --- a/proctord/jobs/schedule/handler_test.go +++ b/proctord/jobs/schedule/handler_test.go @@ -12,12 +12,12 @@ import ( "net/http/httptest" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" "proctor/proctord/jobs/metadata" "proctor/proctord/storage" "proctor/proctord/storage/postgres" "proctor/proctord/utility" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" ) type SchedulerTestSuite struct { @@ -66,7 +66,7 @@ func (suite *SchedulerTestSuite) TestSuccessfulJobScheduling() { suite.mockMetadataStore.On("GetJobMetadata", scheduledJob.Name).Return(&metadata.Metadata{}, nil) insertedScheduledJobID := "123" - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, userEmail,scheduledJob.Group, scheduledJob.Args).Return(insertedScheduledJobID, nil) + suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, userEmail, scheduledJob.Group, scheduledJob.Args).Return(insertedScheduledJobID, nil) suite.testScheduler.Schedule()(responseRecorder, req) @@ -250,7 +250,7 @@ func (suite *SchedulerTestSuite) TestUniqnessConstrainOnJobNameAndArg() { req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) suite.mockMetadataStore.On("GetJobMetadata", scheduledJob.Name).Return(&metadata.Metadata{}, nil) - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "",scheduledJob.Group, scheduledJob.Args).Return("", errors.New("pq: duplicate key value violates unique constraint \"unique_jobs_schedule_name_args\"")) + suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "", scheduledJob.Group, scheduledJob.Args).Return("", errors.New("pq: duplicate key value violates unique constraint \"unique_jobs_schedule_name_args\"")) suite.testScheduler.Schedule()(responseRecorder, req) @@ -276,7 +276,7 @@ func (suite *SchedulerTestSuite) TestErrorPersistingScheduledJob() { req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) suite.mockMetadataStore.On("GetJobMetadata", scheduledJob.Name).Return(&metadata.Metadata{}, nil) - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "",scheduledJob.Group, scheduledJob.Args).Return("", errors.New("any-error")) + suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "", scheduledJob.Group, scheduledJob.Args).Return("", errors.New("any-error")) suite.testScheduler.Schedule()(responseRecorder, req) diff --git a/proctord/jobs/schedule/worker.go b/proctord/jobs/schedule/worker.go index da41ffe3..d8cd4c6f 100644 --- a/proctord/jobs/schedule/worker.go +++ b/proctord/jobs/schedule/worker.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/robfig/cron" "proctor/proctord/audit" "proctor/proctord/jobs/execution" "proctor/proctord/logger" @@ -14,7 +15,6 @@ import ( "proctor/proctord/storage" "proctor/proctord/storage/postgres" "proctor/proctord/utility" - "github.com/robfig/cron" ) type worker struct { @@ -86,7 +86,7 @@ func (worker *worker) enableScheduledJobIfItDoesNotExist(scheduledJob postgres.J if err != nil { logger.Error(fmt.Sprintf("Error notifying job: %s `", scheduledJob.Tags), scheduledJob.Name, "` ID: `", jobExecutionID, "` execution status: `", jobExecutionStatus, "` to users: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name,"job_id":jobExecutionID, "job_execution_status": jobExecutionStatus}) + raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name, "job_id": jobExecutionID, "job_execution_status": jobExecutionStatus}) return } }) @@ -109,7 +109,7 @@ func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Sig scheduledJobs, err := worker.store.GetScheduledJobs() if err != nil { logger.Error("Error getting scheduled jobs from store: ", err.Error()) - raven.CaptureError(err,nil) + raven.CaptureError(err, nil) continue } diff --git a/proctord/jobs/schedule/worker_test.go b/proctord/jobs/schedule/worker_test.go index b4b12869..70cc92b9 100644 --- a/proctord/jobs/schedule/worker_test.go +++ b/proctord/jobs/schedule/worker_test.go @@ -9,15 +9,15 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" "proctor/proctord/audit" "proctor/proctord/jobs/execution" "proctor/proctord/mail" "proctor/proctord/storage" "proctor/proctord/storage/postgres" "proctor/proctord/utility" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" ) type WorkerTestSuite struct { diff --git a/proctord/jobs/secrets/handler_test.go b/proctord/jobs/secrets/handler_test.go index 91766132..dd776444 100644 --- a/proctord/jobs/secrets/handler_test.go +++ b/proctord/jobs/secrets/handler_test.go @@ -9,10 +9,10 @@ import ( "errors" - "proctor/proctord/utility" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "proctor/proctord/utility" ) type SecretsHandlerTestSuite struct { diff --git a/proctord/jobs/secrets/store_test.go b/proctord/jobs/secrets/store_test.go index 5929ac11..d94de955 100644 --- a/proctord/jobs/secrets/store_test.go +++ b/proctord/jobs/secrets/store_test.go @@ -4,11 +4,11 @@ import ( "encoding/json" "testing" - "proctor/proctord/redis" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "proctor/proctord/redis" ) type SecretsStoreTestSuite struct { diff --git a/proctord/kubernetes/client.go b/proctord/kubernetes/client.go index d8473351..b824929c 100644 --- a/proctord/kubernetes/client.go +++ b/proctord/kubernetes/client.go @@ -6,15 +6,15 @@ import ( "net/http" "time" - "proctor/proctord/config" - "proctor/proctord/logger" - "proctor/proctord/utility" uuid "github.com/satori/go.uuid" batch_v1 "k8s.io/api/batch/v1" "k8s.io/api/core/v1" meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" + "proctor/proctord/config" + "proctor/proctord/logger" + "proctor/proctord/utility" //Package needed for kubernetes cluster in google cloud _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/clientcmd" @@ -105,8 +105,8 @@ func (client *client) ExecuteJob(imageName string, envMap map[string]string) (st } objectMeta := meta_v1.ObjectMeta{ - Name: uniqueJobName, - Labels: label, + Name: uniqueJobName, + Labels: label, Annotations: config.JobPodAnnotations(), } diff --git a/proctord/kubernetes/client_mock.go b/proctord/kubernetes/client_mock.go index 3851fe94..25edf517 100644 --- a/proctord/kubernetes/client_mock.go +++ b/proctord/kubernetes/client_mock.go @@ -3,8 +3,8 @@ package kubernetes import ( "io" - "proctor/proctord/utility" "github.com/stretchr/testify/mock" + "proctor/proctord/utility" ) type MockClient struct { diff --git a/proctord/kubernetes/client_test.go b/proctord/kubernetes/client_test.go index 8389a2cb..1cc7e372 100644 --- a/proctord/kubernetes/client_test.go +++ b/proctord/kubernetes/client_test.go @@ -7,8 +7,6 @@ import ( "testing" "time" - "proctor/proctord/config" - "proctor/proctord/utility" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -16,6 +14,8 @@ import ( meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" batch_v1 "k8s.io/client-go/kubernetes/typed/batch/v1" + "proctor/proctord/config" + "proctor/proctord/utility" batchV1 "k8s.io/api/batch/v1" fakeclientset "k8s.io/client-go/kubernetes/fake" @@ -98,7 +98,7 @@ func (suite *ClientTestSuite) TestJobExecution() { expectedLabel := jobLabel(executedJobname) assert.Equal(t, expectedLabel, executedJob.ObjectMeta.Labels) assert.Equal(t, expectedLabel, executedJob.Spec.Template.ObjectMeta.Labels) - assert.Equal(t, map[string]string{"key.one":"true"}, executedJob.Spec.Template.Annotations) + assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) assert.Equal(t, config.KubeJobActiveDeadlineSeconds(), executedJob.Spec.ActiveDeadlineSeconds) assert.Equal(t, config.KubeJobRetries(), executedJob.Spec.BackoffLimit) diff --git a/proctord/middleware/validate_client_version.go b/proctord/middleware/validate_client_version.go index 742941c3..01819ef6 100644 --- a/proctord/middleware/validate_client_version.go +++ b/proctord/middleware/validate_client_version.go @@ -2,11 +2,11 @@ package middleware import ( "fmt" - "proctor/proctord/config" - "proctor/proctord/utility" "github.com/hashicorp/go-version" - "proctor/proctord/logger" "net/http" + "proctor/proctord/config" + "proctor/proctord/logger" + "proctor/proctord/utility" ) func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { diff --git a/proctord/middleware/validate_client_version_test.go b/proctord/middleware/validate_client_version_test.go index bbbaa8ee..c7bb0411 100644 --- a/proctord/middleware/validate_client_version_test.go +++ b/proctord/middleware/validate_client_version_test.go @@ -1,12 +1,12 @@ package middleware import ( - "proctor/proctord/utility" "github.com/stretchr/testify/assert" "io/ioutil" "net/http" "net/http/httptest" "os" + "proctor/proctord/utility" "testing" ) @@ -18,7 +18,7 @@ func getTestHandler() http.HandlerFunc { func TestValidClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION","0.2.0") + os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -36,7 +36,7 @@ func TestValidClientVersionHttpHeader(t *testing.T) { func TestEmptyClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION","0.2.0") + os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -53,7 +53,7 @@ func TestEmptyClientVersionHttpHeader(t *testing.T) { func TestInvalidClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION","0.3.0") + os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.3.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -69,5 +69,5 @@ func TestInvalidClientVersionHttpHeader(t *testing.T) { assert.Equal(t, http.StatusBadRequest, resp.StatusCode) body, _ := ioutil.ReadAll(resp.Body) bodyString := string(body) - assert.Equal(t,bodyString,"Your Proctor client is using an outdated version: 0.1.0 . To continue using proctor, please upgrade to latest version.") + assert.Equal(t, bodyString, "Your Proctor client is using an outdated version: 0.1.0 . To continue using proctor, please upgrade to latest version.") } diff --git a/proctord/server/router.go b/proctord/server/router.go index 3fe96839..994d7643 100644 --- a/proctord/server/router.go +++ b/proctord/server/router.go @@ -2,6 +2,8 @@ package server import ( "fmt" + "net/http" + "path" "proctor/proctord/audit" "proctor/proctord/config" "proctor/proctord/docs" @@ -16,11 +18,9 @@ import ( "proctor/proctord/redis" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "net/http" - "path" - "proctor/proctord/instrumentation" "github.com/gorilla/mux" + "proctor/proctord/instrumentation" ) var postgresClient postgres.Client @@ -55,7 +55,6 @@ func NewRouter() (*mux.Router, error) { fmt.Fprintf(w, "pong") }) - router.HandleFunc("/docs", docs.APIDocHandler) router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.DocsPath())))) router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { diff --git a/proctord/storage/postgres/client.go b/proctord/storage/postgres/client.go index 465bd99d..5fd0d4eb 100644 --- a/proctord/storage/postgres/client.go +++ b/proctord/storage/postgres/client.go @@ -4,9 +4,9 @@ import ( "fmt" "time" + "github.com/jmoiron/sqlx" "proctor/proctord/config" "proctor/proctord/logger" - "github.com/jmoiron/sqlx" //postgres driver _ "github.com/lib/pq" ) diff --git a/proctord/storage/postgres/client_test.go b/proctord/storage/postgres/client_test.go index d6ffb29c..1195e0f1 100644 --- a/proctord/storage/postgres/client_test.go +++ b/proctord/storage/postgres/client_test.go @@ -4,9 +4,9 @@ import ( "fmt" "testing" - "proctor/proctord/config" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" + "proctor/proctord/config" ) func TestNamedExec(t *testing.T) { diff --git a/proctord/storage/postgres/migrations.go b/proctord/storage/postgres/migrations.go index d5f5c50e..624d0c0d 100644 --- a/proctord/storage/postgres/migrations.go +++ b/proctord/storage/postgres/migrations.go @@ -3,9 +3,9 @@ package postgres import ( "fmt" + "github.com/mattes/migrate" "proctor/proctord/config" "proctor/proctord/logger" - "github.com/mattes/migrate" //postgres driver _ "github.com/mattes/migrate/database/postgres" //driver for reading migrations from file diff --git a/proctord/storage/store.go b/proctord/storage/store.go index 8cc16e27..e7362f57 100644 --- a/proctord/storage/store.go +++ b/proctord/storage/store.go @@ -5,8 +5,8 @@ import ( "encoding/json" "time" - "proctor/proctord/storage/postgres" "github.com/satori/go.uuid" + "proctor/proctord/storage/postgres" ) type Store interface { diff --git a/proctord/storage/store_mock.go b/proctord/storage/store_mock.go index b7e1d3b9..ac689f21 100644 --- a/proctord/storage/store_mock.go +++ b/proctord/storage/store_mock.go @@ -1,8 +1,8 @@ package storage import ( - "proctor/proctord/storage/postgres" "github.com/stretchr/testify/mock" + "proctor/proctord/storage/postgres" ) type MockStore struct { diff --git a/proctord/storage/store_test.go b/proctord/storage/store_test.go index 2d37ebf0..f714bc60 100644 --- a/proctord/storage/store_test.go +++ b/proctord/storage/store_test.go @@ -4,11 +4,11 @@ import ( "bytes" "encoding/gob" "errors" - "proctor/proctord/storage/postgres" - "proctor/proctord/utility" "github.com/satori/go.uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "proctor/proctord/storage/postgres" + "proctor/proctord/utility" "testing" ) @@ -159,7 +159,7 @@ func TestJobsScheduleInsertionSuccessfull(t *testing.T) { postgresClient := postgres.NewClient() testStore := New(postgresClient) - scheduledJobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com","group1", map[string]string{}) + scheduledJobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) assert.NoError(t, err) _, err = uuid.FromString(scheduledJobID) assert.NoError(t, err) @@ -181,12 +181,12 @@ func TestJobsScheduleInsertionFailed(t *testing.T) { mockPostgresClient.On("NamedExec", "INSERT INTO jobs_schedule (id, name, tags, time, notification_emails, user_email, group_name, args, enabled) "+ - "VALUES (:id, :name, :tags, :time, :notification_emails, :user_email, :group_name, :args, :enabled)", + "VALUES (:id, :name, :tags, :time, :notification_emails, :user_email, :group_name, :args, :enabled)", mock.AnythingOfType("*postgres.JobsSchedule")).Run(func(args mock.Arguments) { }).Return(int64(0), errors.New("any-error")). Once() - _, err := testStore.InsertScheduledJob(jobName, tag, time, notificationEmail, userEmail,groupName, map[string]string{}) + _, err := testStore.InsertScheduledJob(jobName, tag, time, notificationEmail, userEmail, groupName, map[string]string{}) assert.Error(t, err) @@ -197,7 +197,7 @@ func TestGetScheduledJobByID(t *testing.T) { postgresClient := postgres.NewClient() testStore := New(postgresClient) - jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com","group1", map[string]string{}) + jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) assert.NoError(t, err) resultJob, err := testStore.GetScheduledJob(jobID) @@ -223,7 +223,7 @@ func TestRemoveScheduledJobByID(t *testing.T) { postgresClient := postgres.NewClient() testStore := New(postgresClient) - jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com","group1", map[string]string{}) + jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) assert.NoError(t, err) removedJobsCount, err := testStore.RemoveScheduledJob(jobID) diff --git a/proctord/utility/utils.go b/proctord/utility/utils.go index 9a8613ab..b55f31fb 100644 --- a/proctord/utility/utils.go +++ b/proctord/utility/utils.go @@ -40,10 +40,10 @@ const JobNotFoundError = "Job not found" const JobSucceeded = "SUCCEEDED" const JobFailed = "FAILED" const JobWaiting = "WAITING" -const JobNotFound="NOT_FOUND" +const JobNotFound = "NOT_FOUND" const JobExecutionStatusFetchError = "JOB_EXECUTION_STATUS_FETCH_ERROR" const NoDefinitiveJobExecutionStatusFound = "NO_DEFINITIVE_JOB_EXECUTION_STATUS_FOUND" -const GroupNameMissingError = "Group Name is missing" +const GroupNameMissingError = "Group Name is missing" const JobNameContextKey = "job_name" const UserEmailContextKey = "user_email" const JobArgsContextKey = "job_args" diff --git a/utility/sort/sort_test.go b/utility/sort/sort_test.go index 958b1dfd..fcb1e951 100644 --- a/utility/sort/sort_test.go +++ b/utility/sort/sort_test.go @@ -1,8 +1,8 @@ package sort import ( - "proctor/proctord/jobs/metadata" "github.com/stretchr/testify/assert" + "proctor/proctord/jobs/metadata" "testing" ) From 3f0682c843f8d60d2c582e34e69c104190244a88 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 11:41:35 +0700 Subject: [PATCH 002/268] [Jasoet|Bimo] refactor: rename cmd to cli --- {cmd => cli}/config/manager.go | 32 ++++++++------- {cmd => cli}/config/view/view.go | 15 ++----- {cmd => cli}/description/descriptor.go | 6 +-- {cmd => cli}/description/descriptor_test.go | 12 +++--- {cmd => cli}/execution/executioner.go | 6 +-- {cmd => cli}/execution/executioner_test.go | 2 +- {cmd => cli}/list/lister.go | 2 +- {cmd => cli}/list/lister_test.go | 12 +++--- {cmd => cli}/root.go | 41 +++++++++---------- {cmd => cli}/root_test.go | 6 +-- {cmd => cli}/schedule/describe/describe.go | 2 +- .../schedule/describe/describe_test.go | 2 +- {cmd => cli}/schedule/list/list.go | 2 +- {cmd => cli}/schedule/list/list_test.go | 2 +- {cmd => cli}/schedule/remove/remove.go | 2 +- {cmd => cli}/schedule/remove/remove_test.go | 2 +- {cmd => cli}/schedule/schedule.go | 2 +- {cmd => cli}/schedule/schedule_test.go | 2 +- {cmd => cli}/version/github/client.go | 8 +++- {cmd => cli}/version/github/client_mock.go | 0 {cmd => cli}/version/version.go | 4 +- {cmd => cli}/version/version_test.go | 4 +- daemon/client.go | 4 +- daemon/client_test.go | 13 +++--- exec/cli/cli.go | 8 ++-- 25 files changed, 94 insertions(+), 97 deletions(-) rename {cmd => cli}/config/manager.go (71%) rename {cmd => cli}/config/view/view.go (76%) rename {cmd => cli}/description/descriptor.go (94%) rename {cmd => cli}/description/descriptor_test.go (91%) rename {cmd => cli}/execution/executioner.go (94%) rename {cmd => cli}/execution/executioner_test.go (99%) rename {cmd => cli}/list/lister.go (97%) rename {cmd => cli}/list/lister_test.go (87%) rename {cmd => cli}/root.go (71%) rename {cmd => cli}/root_test.go (94%) rename {cmd => cli}/schedule/describe/describe.go (98%) rename {cmd => cli}/schedule/describe/describe_test.go (97%) rename {cmd => cli}/schedule/list/list.go (97%) rename {cmd => cli}/schedule/list/list_test.go (97%) rename {cmd => cli}/schedule/remove/remove.go (97%) rename {cmd => cli}/schedule/remove/remove_test.go (97%) rename {cmd => cli}/schedule/schedule.go (98%) rename {cmd => cli}/schedule/schedule_test.go (98%) rename {cmd => cli}/version/github/client.go (83%) rename {cmd => cli}/version/github/client_mock.go (100%) rename {cmd => cli}/version/version.go (94%) rename {cmd => cli}/version/version_test.go (96%) diff --git a/cmd/config/manager.go b/cli/config/manager.go similarity index 71% rename from cmd/config/manager.go rename to cli/config/manager.go index e518fa1b..08acaa20 100644 --- a/cmd/config/manager.go +++ b/cli/config/manager.go @@ -6,12 +6,12 @@ import ( "io/ioutil" "os" "path/filepath" + "proctor/shared/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" - proctor_config "proctor/config" - "proctor/io" + proctorConfig "proctor/config" ) func CreateDirIfNotExist(dir string) { @@ -28,11 +28,11 @@ func NewCmd(printer io.Printer) *cobra.Command { Use: "config", Short: "Configure proctor client", Long: "This command helps configure client with proctord host, email id and access token", - Example: fmt.Sprintf("proctor config %s=example.proctor.com %s=example@proctor.com %s=XXXXX", proctor_config.ProctorHost, proctor_config.EmailId, proctor_config.AccessToken), + Example: fmt.Sprintf("proctor config %s=example.proctor.com %s=example@proctor.com %s=XXXXX", proctorConfig.ProctorHost, proctorConfig.EmailId, proctorConfig.AccessToken), Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - configFile := filepath.Join(proctor_config.ConfigFileDir(), "proctor.yaml") + configFile := filepath.Join(proctorConfig.ConfigFileDir(), "proctor.yaml") if _, err := os.Stat(configFile); err == nil { printer.Println("[Warning] This will overwrite current config:", color.FgYellow) existingProctorConfig, err := ioutil.ReadFile(configFile) @@ -58,7 +58,7 @@ func NewCmd(printer io.Printer) *cobra.Command { } } - CreateDirIfNotExist(proctor_config.ConfigFileDir()) + CreateDirIfNotExist(proctorConfig.ConfigFileDir()) var configFileContent string for _, v := range args { arg := strings.Split(v, "=") @@ -69,16 +69,16 @@ func NewCmd(printer io.Printer) *cobra.Command { } switch arg[0] { - case proctor_config.ProctorHost: - configFileContent += fmt.Sprintf("%s: %s\n", proctor_config.ProctorHost, arg[1]) - case proctor_config.EmailId: - configFileContent += fmt.Sprintf("%s: %s\n", proctor_config.EmailId, arg[1]) - case proctor_config.AccessToken: - configFileContent += fmt.Sprintf("%s: %s\n", proctor_config.AccessToken, arg[1]) - case proctor_config.ConnectionTimeoutSecs: - configFileContent += fmt.Sprintf("%s: %s\n", proctor_config.ConnectionTimeoutSecs, arg[1]) - case proctor_config.ProcExecutionStatusPollCount: - configFileContent += fmt.Sprintf("%s: %s\n", proctor_config.ProcExecutionStatusPollCount, arg[1]) + case proctorConfig.ProctorHost: + configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ProctorHost, arg[1]) + case proctorConfig.EmailId: + configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.EmailId, arg[1]) + case proctorConfig.AccessToken: + configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.AccessToken, arg[1]) + case proctorConfig.ConnectionTimeoutSecs: + configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ConnectionTimeoutSecs, arg[1]) + case proctorConfig.ProcExecutionStatusPollCount: + configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ProcExecutionStatusPollCount, arg[1]) default: printer.Println(fmt.Sprintf("Proctor doesn't support config key: %s", arg[0]), color.FgYellow) } @@ -88,10 +88,12 @@ func NewCmd(printer io.Printer) *cobra.Command { f, err := os.Create(configFile) if err != nil { printer.Println(fmt.Sprintf("Error creating config file %s: %s", configFile, err.Error()), color.FgRed) + return } _, err = f.Write(configFileContentBytes) if err != nil { printer.Println(fmt.Sprintf("Error writing content %v \n to config file %s: %s", configFileContentBytes, configFile, err.Error()), color.FgRed) + return } defer f.Close() printer.Println("Proctor client configured successfully", color.FgGreen) diff --git a/cmd/config/view/view.go b/cli/config/view/view.go similarity index 76% rename from cmd/config/view/view.go rename to cli/config/view/view.go index 8adf6e50..3ca49df5 100644 --- a/cmd/config/view/view.go +++ b/cli/config/view/view.go @@ -8,19 +8,10 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - proctor_config "proctor/config" - "proctor/io" + proctorConfig "proctor/config" + "proctor/shared/io" ) -func CreateDirIfNotExist(dir string) { - if _, err := os.Stat(dir); os.IsNotExist(err) { - err = os.MkdirAll(dir, 0755) - if err != nil { - panic(err) - } - } -} - func NewCmd(printer io.Printer) *cobra.Command { return &cobra.Command{ Use: "show", @@ -29,7 +20,7 @@ func NewCmd(printer io.Printer) *cobra.Command { Example: fmt.Sprintf("proctor config show"), Run: func(cmd *cobra.Command, args []string) { - configFile := filepath.Join(proctor_config.ConfigFileDir(), "proctor.yaml") + configFile := filepath.Join(proctorConfig.ConfigFileDir(), "proctor.yaml") if _, err := os.Stat(configFile); os.IsNotExist(err) { printer.Println(fmt.Sprintf("Client Config is absent: %s", configFile), color.FgRed) printer.Println(fmt.Sprintf("Setup config using `proctor config PROCTOR_HOST=some.host ...`"), color.FgRed) diff --git a/cmd/description/descriptor.go b/cli/description/descriptor.go similarity index 94% rename from cmd/description/descriptor.go rename to cli/description/descriptor.go index 44d28591..24874089 100644 --- a/cmd/description/descriptor.go +++ b/cli/description/descriptor.go @@ -2,13 +2,13 @@ package description import ( "fmt" + "proctor/shared/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" - proc_metadata "proctor/proctord/jobs/metadata" + procMetadata "proctor/proctord/jobs/metadata" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { @@ -31,7 +31,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { } userProvidedProcName := args[0] - desiredProc := proc_metadata.Metadata{} + desiredProc := procMetadata.Metadata{} for _, proc := range procList { if userProvidedProcName == proc.Name { desiredProc = proc diff --git a/cmd/description/descriptor_test.go b/cli/description/descriptor_test.go similarity index 91% rename from cmd/description/descriptor_test.go rename to cli/description/descriptor_test.go index 3f327364..19702ea0 100644 --- a/cmd/description/descriptor_test.go +++ b/cli/description/descriptor_test.go @@ -3,12 +3,12 @@ package description import ( "errors" "fmt" + "proctor/shared/io" "strings" "testing" "proctor/daemon" - "proctor/io" - proc_metadata "proctor/proctord/jobs/metadata" + procMetadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/metadata/env" "github.com/fatih/color" @@ -51,7 +51,7 @@ func (s *DescribeCmdTestSuite) TestDescribeCmdRun() { Description: "secret one description", } - anyProc := proc_metadata.Metadata{ + anyProc := procMetadata.Metadata{ Name: "do-something", Description: "does something", Contributors: "user@example.com", @@ -62,7 +62,7 @@ func (s *DescribeCmdTestSuite) TestDescribeCmdRun() { Secrets: []env.VarMetadata{secret}, }, } - procList := []proc_metadata.Metadata{anyProc} + procList := []procMetadata.Metadata{anyProc} s.mockProctorDClient.On("ListProcs").Return(procList, nil).Once() @@ -89,7 +89,7 @@ func (s *DescribeCmdTestSuite) TestDescribeCmdForIncorrectUsage() { } func (s *DescribeCmdTestSuite) TestDescribeCmdRunProctorDClientFailure() { - s.mockProctorDClient.On("ListProcs").Return([]proc_metadata.Metadata{}, errors.New("test error")).Once() + s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, errors.New("test error")).Once() s.mockPrinter.On("Println", "test error", color.FgRed).Once() s.testDescribeCmd.Run(&cobra.Command{}, []string{"do-something"}) @@ -99,7 +99,7 @@ func (s *DescribeCmdTestSuite) TestDescribeCmdRunProctorDClientFailure() { } func (s *DescribeCmdTestSuite) TestDescribeCmdRunProcNotSupported() { - s.mockProctorDClient.On("ListProcs").Return([]proc_metadata.Metadata{}, nil).Once() + s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, nil).Once() testProcName := "do-something" s.mockPrinter.On("Println", fmt.Sprintf("Proctor doesn't support Proc `%s`\nRun `proctor list` to view supported Procs", testProcName), color.FgRed).Once() diff --git a/cmd/execution/executioner.go b/cli/execution/executioner.go similarity index 94% rename from cmd/execution/executioner.go rename to cli/execution/executioner.go index 113fa74a..f8fd6ab8 100644 --- a/cmd/execution/executioner.go +++ b/cli/execution/executioner.go @@ -2,13 +2,13 @@ package execution import ( "fmt" + "proctor/shared/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" - proctord_utility "proctor/proctord/utility" + proctordUtility "proctor/proctord/utility" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { @@ -68,7 +68,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in return } - if procExecutionStatus != proctord_utility.JobSucceeded { + if procExecutionStatus != proctordUtility.JobSucceeded { printer.Println("Proc execution failed", color.FgRed) osExitFunc(1) return diff --git a/cmd/execution/executioner_test.go b/cli/execution/executioner_test.go similarity index 99% rename from cmd/execution/executioner_test.go rename to cli/execution/executioner_test.go index d65a377d..bb17e137 100644 --- a/cmd/execution/executioner_test.go +++ b/cli/execution/executioner_test.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "github.com/stretchr/testify/mock" + "proctor/shared/io" "testing" "github.com/fatih/color" @@ -11,7 +12,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" "proctor/proctord/utility" ) diff --git a/cmd/list/lister.go b/cli/list/lister.go similarity index 97% rename from cmd/list/lister.go rename to cli/list/lister.go index bdef68bf..8a87e23e 100644 --- a/cmd/list/lister.go +++ b/cli/list/lister.go @@ -2,11 +2,11 @@ package list import ( "fmt" + "proctor/shared/io" "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" "proctor/utility/sort" ) diff --git a/cmd/list/lister_test.go b/cli/list/lister_test.go similarity index 87% rename from cmd/list/lister_test.go rename to cli/list/lister_test.go index e0e2a7a6..6962a051 100644 --- a/cmd/list/lister_test.go +++ b/cli/list/lister_test.go @@ -3,6 +3,7 @@ package list import ( "errors" "fmt" + "proctor/shared/io" "testing" "github.com/fatih/color" @@ -10,8 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" - proc_metadata "proctor/proctord/jobs/metadata" + procMetadata "proctor/proctord/jobs/metadata" ) type ListCmdTestSuite struct { @@ -38,15 +38,15 @@ func (s *ListCmdTestSuite) TestListCmdHelp() { } func (s *ListCmdTestSuite) TestListCmdRun() { - procOne := proc_metadata.Metadata{ + procOne := procMetadata.Metadata{ Name: "one", Description: "proc one description", } - procTwo := proc_metadata.Metadata{ + procTwo := procMetadata.Metadata{ Name: "two", Description: "proc two description", } - procList := []proc_metadata.Metadata{procOne, procTwo} + procList := []procMetadata.Metadata{procOne, procTwo} s.mockProctorDClient.On("ListProcs").Return(procList, nil).Once() @@ -61,7 +61,7 @@ func (s *ListCmdTestSuite) TestListCmdRun() { } func (s *ListCmdTestSuite) TestListCmdRunProctorDClientFailure() { - s.mockProctorDClient.On("ListProcs").Return([]proc_metadata.Metadata{}, errors.New("Error!!!\nUnknown Error.")).Once() + s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, errors.New("Error!!!\nUnknown Error.")).Once() s.mockPrinter.On("Println", "Error!!!\nUnknown Error.", color.FgRed).Once() s.testListCmd.Run(&cobra.Command{}, []string{}) diff --git a/cmd/root.go b/cli/root.go similarity index 71% rename from cmd/root.go rename to cli/root.go index 29d83799..16069874 100644 --- a/cmd/root.go +++ b/cli/root.go @@ -1,24 +1,23 @@ -package cmd +package cli import ( "fmt" "os" - "proctor/cmd/schedule/remove" - - "proctor/cmd/config" - "proctor/cmd/config/view" - "proctor/cmd/description" - "proctor/cmd/execution" - "proctor/cmd/list" - "proctor/cmd/schedule" - schedule_describe "proctor/cmd/schedule/describe" - schedule_list "proctor/cmd/schedule/list" - "proctor/cmd/version" - "proctor/daemon" - "proctor/io" + "proctor/cli/schedule/remove" + "proctor/shared/io" "github.com/spf13/cobra" - "proctor/cmd/version/github" + "proctor/cli/config" + "proctor/cli/config/view" + "proctor/cli/description" + "proctor/cli/execution" + "proctor/cli/list" + "proctor/cli/schedule" + scheduleDescribe "proctor/cli/schedule/describe" + scheduleList "proctor/cli/schedule/list" + "proctor/cli/version" + "proctor/cli/version/github" + "proctor/daemon" ) var ( @@ -50,9 +49,9 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith scheduleCmd := schedule.NewCmd(printer, proctorDClient) rootCmd.AddCommand(scheduleCmd) - scheduleListCmd := schedule_list.NewCmd(printer, proctorDClient) + scheduleListCmd := scheduleList.NewCmd(printer, proctorDClient) scheduleCmd.AddCommand(scheduleListCmd) - scheduleDescribeCmd := schedule_describe.NewCmd(printer, proctorDClient) + scheduleDescribeCmd := scheduleDescribe.NewCmd(printer, proctorDClient) scheduleCmd.AddCommand(scheduleDescribeCmd) scheduleRemoveCmd := remove.NewCmd(printer, proctorDClient) scheduleCmd.AddCommand(scheduleRemoveCmd) @@ -60,13 +59,13 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith var Time, NotifyEmails, Tags, Group string scheduleCmd.PersistentFlags().StringVarP(&Time, "time", "t", "", "Schedule time") - scheduleCmd.MarkFlagRequired("time") + _ = scheduleCmd.MarkFlagRequired("time") scheduleCmd.PersistentFlags().StringVarP(&Group, "group", "g", "", "Group Name") - scheduleCmd.MarkFlagRequired("group") + _ = scheduleCmd.MarkFlagRequired("group") scheduleCmd.PersistentFlags().StringVarP(&NotifyEmails, "notify", "n", "", "Notifier Email ID's") - scheduleCmd.MarkFlagRequired("notify") + _ = scheduleCmd.MarkFlagRequired("notify") scheduleCmd.PersistentFlags().StringVarP(&Tags, "tags", "T", "", "Tags") - scheduleCmd.MarkFlagRequired("tags") + _ = scheduleCmd.MarkFlagRequired("tags") if err := rootCmd.Execute(); err != nil { fmt.Println(err) diff --git a/cmd/root_test.go b/cli/root_test.go similarity index 94% rename from cmd/root_test.go rename to cli/root_test.go index f00f4bb1..882b56f7 100644 --- a/cmd/root_test.go +++ b/cli/root_test.go @@ -1,13 +1,13 @@ -package cmd +package cli import ( + "proctor/shared/io" "testing" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - "proctor/cmd/version/github" + "proctor/cli/version/github" "proctor/daemon" - "proctor/io" ) func TestRootCmdUsage(t *testing.T) { diff --git a/cmd/schedule/describe/describe.go b/cli/schedule/describe/describe.go similarity index 98% rename from cmd/schedule/describe/describe.go rename to cli/schedule/describe/describe.go index d337ad4b..a9b34c31 100644 --- a/cmd/schedule/describe/describe.go +++ b/cli/schedule/describe/describe.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" + "proctor/shared/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/schedule/describe/describe_test.go b/cli/schedule/describe/describe_test.go similarity index 97% rename from cmd/schedule/describe/describe_test.go rename to cli/schedule/describe/describe_test.go index 473c2dc7..706b23a7 100644 --- a/cmd/schedule/describe/describe_test.go +++ b/cli/schedule/describe/describe_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" + "proctor/shared/io" "testing" ) diff --git a/cmd/schedule/list/list.go b/cli/schedule/list/list.go similarity index 97% rename from cmd/schedule/list/list.go rename to cli/schedule/list/list.go index df42f334..afc88d12 100644 --- a/cmd/schedule/list/list.go +++ b/cli/schedule/list/list.go @@ -2,11 +2,11 @@ package list import ( "fmt" + "proctor/shared/io" "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/schedule/list/list_test.go b/cli/schedule/list/list_test.go similarity index 97% rename from cmd/schedule/list/list_test.go rename to cli/schedule/list/list_test.go index 4b030e89..b3a3e4ce 100644 --- a/cmd/schedule/list/list_test.go +++ b/cli/schedule/list/list_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" + "proctor/shared/io" "testing" ) diff --git a/cmd/schedule/remove/remove.go b/cli/schedule/remove/remove.go similarity index 97% rename from cmd/schedule/remove/remove.go rename to cli/schedule/remove/remove.go index 334bbeb8..07115108 100644 --- a/cmd/schedule/remove/remove.go +++ b/cli/schedule/remove/remove.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" + "proctor/shared/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cmd/schedule/remove/remove_test.go b/cli/schedule/remove/remove_test.go similarity index 97% rename from cmd/schedule/remove/remove_test.go rename to cli/schedule/remove/remove_test.go index 025aea5f..33343d24 100644 --- a/cmd/schedule/remove/remove_test.go +++ b/cli/schedule/remove/remove_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" + "proctor/shared/io" "testing" ) diff --git a/cmd/schedule/schedule.go b/cli/schedule/schedule.go similarity index 98% rename from cmd/schedule/schedule.go rename to cli/schedule/schedule.go index c0b54f3e..acb20d6e 100644 --- a/cmd/schedule/schedule.go +++ b/cli/schedule/schedule.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/io" + "proctor/shared/io" "strings" ) diff --git a/cmd/schedule/schedule_test.go b/cli/schedule/schedule_test.go similarity index 98% rename from cmd/schedule/schedule_test.go rename to cli/schedule/schedule_test.go index 54611051..477df824 100644 --- a/cmd/schedule/schedule_test.go +++ b/cli/schedule/schedule_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/io" + "proctor/shared/io" "testing" ) diff --git a/cmd/version/github/client.go b/cli/version/github/client.go similarity index 83% rename from cmd/version/github/client.go rename to cli/version/github/client.go index 20ee8d51..10b0cad1 100644 --- a/cmd/version/github/client.go +++ b/cli/version/github/client.go @@ -19,5 +19,11 @@ func NewClient() *client { func (gc *client) LatestRelease(owner, repository string) (string, error) { release, _, err := gc.client.Repositories.GetLatestRelease(context.Background(), owner, repository) - return *release.TagName, err + releaseTag := "" + + if err == nil { + releaseTag = *release.TagName + } + + return releaseTag, err } diff --git a/cmd/version/github/client_mock.go b/cli/version/github/client_mock.go similarity index 100% rename from cmd/version/github/client_mock.go rename to cli/version/github/client_mock.go diff --git a/cmd/version/version.go b/cli/version/version.go similarity index 94% rename from cmd/version/version.go rename to cli/version/version.go index 656a92ad..3487a6c2 100644 --- a/cmd/version/version.go +++ b/cli/version/version.go @@ -2,11 +2,11 @@ package version import ( "fmt" + "proctor/shared/io" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cmd/version/github" - "proctor/io" + "proctor/cli/version/github" ) const ClientVersion = "v0.6.0" diff --git a/cmd/version/version_test.go b/cli/version/version_test.go similarity index 96% rename from cmd/version/version_test.go rename to cli/version/version_test.go index c37a708e..525290ef 100644 --- a/cmd/version/version_test.go +++ b/cli/version/version_test.go @@ -2,13 +2,13 @@ package version import ( "fmt" + "proctor/shared/io" "testing" "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - gh "proctor/cmd/version/github" - "proctor/io" + gh "proctor/cli/version/github" ) func TestVersionCmdUsage(t *testing.T) { diff --git a/daemon/client.go b/daemon/client.go index fb4231b1..99335320 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -12,15 +12,15 @@ import ( "net/url" "os" "os/signal" + "proctor/shared/io" "time" - "proctor/cmd/version" + "proctor/cli/version" "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/gorilla/websocket" "proctor/config" - "proctor/io" proc_metadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/schedule" "proctor/proctord/utility" diff --git a/daemon/client_test.go b/daemon/client_test.go index 8c3255c9..64b33072 100644 --- a/daemon/client_test.go +++ b/daemon/client_test.go @@ -5,18 +5,17 @@ import ( "fmt" "net/http" "net/http/httptest" + "proctor/shared" "strings" "testing" - "proctor/cmd/version" + "proctor/cli/version" "github.com/gorilla/websocket" - "github.com/thingful/httpmock" - "proctor/config" - "proctor/io" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "github.com/thingful/httpmock" + "proctor/config" proc_metadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/metadata/env" "proctor/proctord/utility" @@ -39,12 +38,12 @@ type ClientTestSuite struct { suite.Suite testClient Client mockConfigLoader *config.MockLoader - mockPrinter *io.MockPrinter + mockPrinter *shared.MockPrinter } func (s *ClientTestSuite) SetupTest() { s.mockConfigLoader = &config.MockLoader{} - s.mockPrinter = &io.MockPrinter{} + s.mockPrinter = &shared.MockPrinter{} s.testClient = NewClient(s.mockPrinter, s.mockConfigLoader) } diff --git a/exec/cli/cli.go b/exec/cli/cli.go index 5d53ec8b..6328de98 100644 --- a/exec/cli/cli.go +++ b/exec/cli/cli.go @@ -1,11 +1,11 @@ package main import ( - "proctor/cmd" - "proctor/cmd/version/github" + "proctor/cli" + "proctor/cli/version/github" "proctor/config" "proctor/daemon" - "proctor/io" + "proctor/shared/io" ) func main() { @@ -14,5 +14,5 @@ func main() { proctorDClient := daemon.NewClient(printer, proctorConfigLoader) githubClient := github.NewClient() - cmd.Execute(printer, proctorDClient, githubClient) + cli.Execute(printer, proctorDClient, githubClient) } From db185536657a7ef0af36feb56a36bcbcae9f39ce Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 11:42:03 +0700 Subject: [PATCH 003/268] [Jasoet|Bimo] refactor: move io package to shared --- {io => shared/io}/printer.go | 0 {io => shared/io}/printer_mock.go | 0 shared/shared.go | 1 + 3 files changed, 1 insertion(+) rename {io => shared/io}/printer.go (100%) rename {io => shared/io}/printer_mock.go (100%) create mode 100644 shared/shared.go diff --git a/io/printer.go b/shared/io/printer.go similarity index 100% rename from io/printer.go rename to shared/io/printer.go diff --git a/io/printer_mock.go b/shared/io/printer_mock.go similarity index 100% rename from io/printer_mock.go rename to shared/io/printer_mock.go diff --git a/shared/shared.go b/shared/shared.go new file mode 100644 index 00000000..a29b5e40 --- /dev/null +++ b/shared/shared.go @@ -0,0 +1 @@ +package shared From 112df9bc81d46fd8c484380b92a2530a8884684a Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 12:24:12 +0700 Subject: [PATCH 004/268] [Jasoet|Bimo] refactor: move command related code to cli/command --- .gitignore | 5 ---- cli/cli.go | 1 + cli/{ => command}/config/manager.go | 0 cli/{ => command}/config/view/view.go | 0 cli/{ => command}/description/descriptor.go | 0 .../description/descriptor_test.go | 0 cli/{ => command}/execution/executioner.go | 0 .../execution/executioner_test.go | 0 cli/{ => command}/list/lister.go | 0 cli/{ => command}/list/lister_test.go | 0 cli/{ => command}/root.go | 24 +++++++++---------- cli/{ => command}/root_test.go | 4 ++-- .../schedule/describe/describe.go | 0 .../schedule/describe/describe_test.go | 0 cli/{ => command}/schedule/list/list.go | 0 cli/{ => command}/schedule/list/list_test.go | 0 cli/{ => command}/schedule/remove/remove.go | 0 .../schedule/remove/remove_test.go | 0 cli/{ => command}/schedule/schedule.go | 0 cli/{ => command}/schedule/schedule_test.go | 0 cli/{ => command}/version/github/client.go | 0 .../version/github/client_mock.go | 0 cli/{ => command}/version/version.go | 2 +- cli/{ => command}/version/version_test.go | 2 +- daemon/client.go | 2 +- daemon/client_test.go | 8 +++---- exec/cli/cli.go | 6 ++--- 27 files changed, 25 insertions(+), 29 deletions(-) create mode 100644 cli/cli.go rename cli/{ => command}/config/manager.go (100%) rename cli/{ => command}/config/view/view.go (100%) rename cli/{ => command}/description/descriptor.go (100%) rename cli/{ => command}/description/descriptor_test.go (100%) rename cli/{ => command}/execution/executioner.go (100%) rename cli/{ => command}/execution/executioner_test.go (100%) rename cli/{ => command}/list/lister.go (100%) rename cli/{ => command}/list/lister_test.go (100%) rename cli/{ => command}/root.go (82%) rename cli/{ => command}/root_test.go (95%) rename cli/{ => command}/schedule/describe/describe.go (100%) rename cli/{ => command}/schedule/describe/describe_test.go (100%) rename cli/{ => command}/schedule/list/list.go (100%) rename cli/{ => command}/schedule/list/list_test.go (100%) rename cli/{ => command}/schedule/remove/remove.go (100%) rename cli/{ => command}/schedule/remove/remove_test.go (100%) rename cli/{ => command}/schedule/schedule.go (100%) rename cli/{ => command}/schedule/schedule_test.go (100%) rename cli/{ => command}/version/github/client.go (100%) rename cli/{ => command}/version/github/client_mock.go (100%) rename cli/{ => command}/version/version.go (95%) rename cli/{ => command}/version/version_test.go (97%) diff --git a/.gitignore b/.gitignore index c9729aab..3a0f0003 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,9 @@ - - homebrew-gojek scripts/proctor.rb _output/* -proctord .env -proctord - *.swp *.swo .idea diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 00000000..7f1e458c --- /dev/null +++ b/cli/cli.go @@ -0,0 +1 @@ +package cli diff --git a/cli/config/manager.go b/cli/command/config/manager.go similarity index 100% rename from cli/config/manager.go rename to cli/command/config/manager.go diff --git a/cli/config/view/view.go b/cli/command/config/view/view.go similarity index 100% rename from cli/config/view/view.go rename to cli/command/config/view/view.go diff --git a/cli/description/descriptor.go b/cli/command/description/descriptor.go similarity index 100% rename from cli/description/descriptor.go rename to cli/command/description/descriptor.go diff --git a/cli/description/descriptor_test.go b/cli/command/description/descriptor_test.go similarity index 100% rename from cli/description/descriptor_test.go rename to cli/command/description/descriptor_test.go diff --git a/cli/execution/executioner.go b/cli/command/execution/executioner.go similarity index 100% rename from cli/execution/executioner.go rename to cli/command/execution/executioner.go diff --git a/cli/execution/executioner_test.go b/cli/command/execution/executioner_test.go similarity index 100% rename from cli/execution/executioner_test.go rename to cli/command/execution/executioner_test.go diff --git a/cli/list/lister.go b/cli/command/list/lister.go similarity index 100% rename from cli/list/lister.go rename to cli/command/list/lister.go diff --git a/cli/list/lister_test.go b/cli/command/list/lister_test.go similarity index 100% rename from cli/list/lister_test.go rename to cli/command/list/lister_test.go diff --git a/cli/root.go b/cli/command/root.go similarity index 82% rename from cli/root.go rename to cli/command/root.go index 16069874..561e5253 100644 --- a/cli/root.go +++ b/cli/command/root.go @@ -1,22 +1,22 @@ -package cli +package command import ( "fmt" "os" - "proctor/cli/schedule/remove" + "proctor/cli/command/schedule/remove" "proctor/shared/io" "github.com/spf13/cobra" - "proctor/cli/config" - "proctor/cli/config/view" - "proctor/cli/description" - "proctor/cli/execution" - "proctor/cli/list" - "proctor/cli/schedule" - scheduleDescribe "proctor/cli/schedule/describe" - scheduleList "proctor/cli/schedule/list" - "proctor/cli/version" - "proctor/cli/version/github" + "proctor/cli/command/config" + "proctor/cli/command/config/view" + "proctor/cli/command/description" + "proctor/cli/command/execution" + "proctor/cli/command/list" + "proctor/cli/command/schedule" + scheduleDescribe "proctor/cli/command/schedule/describe" + scheduleList "proctor/cli/command/schedule/list" + "proctor/cli/command/version" + "proctor/cli/command/version/github" "proctor/daemon" ) diff --git a/cli/root_test.go b/cli/command/root_test.go similarity index 95% rename from cli/root_test.go rename to cli/command/root_test.go index 882b56f7..bfdbbb74 100644 --- a/cli/root_test.go +++ b/cli/command/root_test.go @@ -1,4 +1,4 @@ -package cli +package command import ( "proctor/shared/io" @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - "proctor/cli/version/github" + "proctor/cli/command/version/github" "proctor/daemon" ) diff --git a/cli/schedule/describe/describe.go b/cli/command/schedule/describe/describe.go similarity index 100% rename from cli/schedule/describe/describe.go rename to cli/command/schedule/describe/describe.go diff --git a/cli/schedule/describe/describe_test.go b/cli/command/schedule/describe/describe_test.go similarity index 100% rename from cli/schedule/describe/describe_test.go rename to cli/command/schedule/describe/describe_test.go diff --git a/cli/schedule/list/list.go b/cli/command/schedule/list/list.go similarity index 100% rename from cli/schedule/list/list.go rename to cli/command/schedule/list/list.go diff --git a/cli/schedule/list/list_test.go b/cli/command/schedule/list/list_test.go similarity index 100% rename from cli/schedule/list/list_test.go rename to cli/command/schedule/list/list_test.go diff --git a/cli/schedule/remove/remove.go b/cli/command/schedule/remove/remove.go similarity index 100% rename from cli/schedule/remove/remove.go rename to cli/command/schedule/remove/remove.go diff --git a/cli/schedule/remove/remove_test.go b/cli/command/schedule/remove/remove_test.go similarity index 100% rename from cli/schedule/remove/remove_test.go rename to cli/command/schedule/remove/remove_test.go diff --git a/cli/schedule/schedule.go b/cli/command/schedule/schedule.go similarity index 100% rename from cli/schedule/schedule.go rename to cli/command/schedule/schedule.go diff --git a/cli/schedule/schedule_test.go b/cli/command/schedule/schedule_test.go similarity index 100% rename from cli/schedule/schedule_test.go rename to cli/command/schedule/schedule_test.go diff --git a/cli/version/github/client.go b/cli/command/version/github/client.go similarity index 100% rename from cli/version/github/client.go rename to cli/command/version/github/client.go diff --git a/cli/version/github/client_mock.go b/cli/command/version/github/client_mock.go similarity index 100% rename from cli/version/github/client_mock.go rename to cli/command/version/github/client_mock.go diff --git a/cli/version/version.go b/cli/command/version/version.go similarity index 95% rename from cli/version/version.go rename to cli/command/version/version.go index 3487a6c2..f5d78e1a 100644 --- a/cli/version/version.go +++ b/cli/command/version/version.go @@ -6,7 +6,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/version/github" + "proctor/cli/command/version/github" ) const ClientVersion = "v0.6.0" diff --git a/cli/version/version_test.go b/cli/command/version/version_test.go similarity index 97% rename from cli/version/version_test.go rename to cli/command/version/version_test.go index 525290ef..66ef7dbc 100644 --- a/cli/version/version_test.go +++ b/cli/command/version/version_test.go @@ -8,7 +8,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - gh "proctor/cli/version/github" + gh "proctor/cli/command/version/github" ) func TestVersionCmdUsage(t *testing.T) { diff --git a/daemon/client.go b/daemon/client.go index 99335320..df955339 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -15,7 +15,7 @@ import ( "proctor/shared/io" "time" - "proctor/cli/version" + "proctor/cli/command/version" "github.com/briandowns/spinner" "github.com/fatih/color" diff --git a/daemon/client_test.go b/daemon/client_test.go index 64b33072..8752fe16 100644 --- a/daemon/client_test.go +++ b/daemon/client_test.go @@ -5,11 +5,11 @@ import ( "fmt" "net/http" "net/http/httptest" - "proctor/shared" + "proctor/shared/io" "strings" "testing" - "proctor/cli/version" + "proctor/cli/command/version" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" @@ -38,12 +38,12 @@ type ClientTestSuite struct { suite.Suite testClient Client mockConfigLoader *config.MockLoader - mockPrinter *shared.MockPrinter + mockPrinter *io.MockPrinter } func (s *ClientTestSuite) SetupTest() { s.mockConfigLoader = &config.MockLoader{} - s.mockPrinter = &shared.MockPrinter{} + s.mockPrinter = &io.MockPrinter{} s.testClient = NewClient(s.mockPrinter, s.mockConfigLoader) } diff --git a/exec/cli/cli.go b/exec/cli/cli.go index 6328de98..b5848122 100644 --- a/exec/cli/cli.go +++ b/exec/cli/cli.go @@ -1,8 +1,8 @@ package main import ( - "proctor/cli" - "proctor/cli/version/github" + "proctor/cli/command" + "proctor/cli/command/version/github" "proctor/config" "proctor/daemon" "proctor/shared/io" @@ -14,5 +14,5 @@ func main() { proctorDClient := daemon.NewClient(printer, proctorConfigLoader) githubClient := github.NewClient() - cli.Execute(printer, proctorDClient, githubClient) + command.Execute(printer, proctorDClient, githubClient) } From ab16c53a1e3da7510167640b5afc81eb94f5df88 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 12:37:22 +0700 Subject: [PATCH 005/268] [Jasoet|Bimo] refactor: extract proc metadata to shared/model --- cli/command/description/descriptor.go | 2 +- cli/command/description/descriptor_test.go | 4 ++-- cli/command/list/lister_test.go | 2 +- daemon/client.go | 2 +- daemon/client_mock.go | 2 +- daemon/client_test.go | 4 ++-- proctord/jobs/execution/executioner_test.go | 3 ++- proctord/jobs/metadata/handler.go | 4 ++-- proctord/jobs/metadata/handler_test.go | 2 +- proctord/jobs/metadata/store.go | 18 +++++++++--------- proctord/jobs/metadata/store_mock.go | 11 ++++++----- .../jobs => shared/model}/metadata/env/vars.go | 0 .../jobs => shared/model}/metadata/metadata.go | 2 +- shared/model/model.go | 1 + utility/sort/sort.go | 2 +- utility/sort/sort_test.go | 2 +- 16 files changed, 32 insertions(+), 29 deletions(-) rename {proctord/jobs => shared/model}/metadata/env/vars.go (100%) rename {proctord/jobs => shared/model}/metadata/metadata.go (90%) create mode 100644 shared/model/model.go diff --git a/cli/command/description/descriptor.go b/cli/command/description/descriptor.go index 24874089..557ef9b1 100644 --- a/cli/command/description/descriptor.go +++ b/cli/command/description/descriptor.go @@ -8,7 +8,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - procMetadata "proctor/proctord/jobs/metadata" + procMetadata "proctor/shared/model/metadata" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/description/descriptor_test.go b/cli/command/description/descriptor_test.go index 19702ea0..0d98d453 100644 --- a/cli/command/description/descriptor_test.go +++ b/cli/command/description/descriptor_test.go @@ -8,8 +8,8 @@ import ( "testing" "proctor/daemon" - procMetadata "proctor/proctord/jobs/metadata" - "proctor/proctord/jobs/metadata/env" + procMetadata "proctor/shared/model/metadata" + "proctor/shared/model/metadata/env" "github.com/fatih/color" "github.com/spf13/cobra" diff --git a/cli/command/list/lister_test.go b/cli/command/list/lister_test.go index 6962a051..f37094fb 100644 --- a/cli/command/list/lister_test.go +++ b/cli/command/list/lister_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - procMetadata "proctor/proctord/jobs/metadata" + procMetadata "proctor/shared/model/metadata" ) type ListCmdTestSuite struct { diff --git a/daemon/client.go b/daemon/client.go index df955339..b31c4559 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -21,7 +21,7 @@ import ( "github.com/fatih/color" "github.com/gorilla/websocket" "proctor/config" - proc_metadata "proctor/proctord/jobs/metadata" + proc_metadata "proctor/shared/model/metadata" "proctor/proctord/jobs/schedule" "proctor/proctord/utility" ) diff --git a/daemon/client_mock.go b/daemon/client_mock.go index d9ba5a33..8ead6ce8 100644 --- a/daemon/client_mock.go +++ b/daemon/client_mock.go @@ -2,7 +2,7 @@ package daemon import ( "github.com/stretchr/testify/mock" - proc_metadata "proctor/proctord/jobs/metadata" + proc_metadata "proctor/shared/model/metadata" "proctor/proctord/jobs/schedule" ) diff --git a/daemon/client_test.go b/daemon/client_test.go index 8752fe16..a3d68529 100644 --- a/daemon/client_test.go +++ b/daemon/client_test.go @@ -16,8 +16,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/thingful/httpmock" "proctor/config" - proc_metadata "proctor/proctord/jobs/metadata" - "proctor/proctord/jobs/metadata/env" + proc_metadata "proctor/shared/model/metadata" + "proctor/shared/model/metadata/env" "proctor/proctord/utility" ) diff --git a/proctord/jobs/execution/executioner_test.go b/proctord/jobs/execution/executioner_test.go index b696bd24..45c3bb0f 100644 --- a/proctord/jobs/execution/executioner_test.go +++ b/proctord/jobs/execution/executioner_test.go @@ -7,7 +7,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "proctor/proctord/jobs/metadata" + modelMetadata "proctor/shared/model/metadata" + "proctor/shared/model/metadata" "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" diff --git a/proctord/jobs/metadata/handler.go b/proctord/jobs/metadata/handler.go index 59829a2c..30afcfec 100644 --- a/proctord/jobs/metadata/handler.go +++ b/proctord/jobs/metadata/handler.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/getsentry/raven-go" "net/http" - + procMetadata "proctor/shared/model/metadata" "proctor/proctord/logger" "proctor/proctord/utility" ) @@ -26,7 +26,7 @@ func NewHandler(store Store) Handler { func (handler *handler) HandleSubmission() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - var jobMetadata []Metadata + var jobMetadata []procMetadata.Metadata err := json.NewDecoder(req.Body).Decode(&jobMetadata) defer req.Body.Close() if err != nil { diff --git a/proctord/jobs/metadata/handler_test.go b/proctord/jobs/metadata/handler_test.go index de07fce8..7c8e6830 100644 --- a/proctord/jobs/metadata/handler_test.go +++ b/proctord/jobs/metadata/handler_test.go @@ -9,7 +9,7 @@ import ( "net/http/httptest" "testing" - "proctor/proctord/jobs/metadata/env" + "proctor/shared/model/metadata/env" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" diff --git a/proctord/jobs/metadata/store.go b/proctord/jobs/metadata/store.go index 9aef873d..4a6c741f 100644 --- a/proctord/jobs/metadata/store.go +++ b/proctord/jobs/metadata/store.go @@ -2,16 +2,16 @@ package metadata import ( "encoding/json" - + modelMetadata "proctor/shared/model/metadata" "proctor/proctord/redis" ) const JobNameKeySuffix = "-metadata" type Store interface { - CreateOrUpdateJobMetadata(metadata Metadata) error - GetAllJobsMetadata() ([]Metadata, error) - GetJobMetadata(jobName string) (*Metadata, error) + CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error + GetAllJobsMetadata() ([]modelMetadata.Metadata, error) + GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) } type store struct { @@ -28,7 +28,7 @@ func jobMetadataKey(jobName string) string { return jobName + JobNameKeySuffix } -func (store *store) CreateOrUpdateJobMetadata(metadata Metadata) error { +func (store *store) CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error { jobNameKey := jobMetadataKey(metadata.Name) binaryJobMetadata, err := json.Marshal(metadata) @@ -39,7 +39,7 @@ func (store *store) CreateOrUpdateJobMetadata(metadata Metadata) error { return store.redisClient.SET(jobNameKey, binaryJobMetadata) } -func (store *store) GetAllJobsMetadata() ([]Metadata, error) { +func (store *store) GetAllJobsMetadata() ([]modelMetadata.Metadata, error) { jobNameKeyRegex := "*" + JobNameKeySuffix keys, err := store.redisClient.KEYS(jobNameKeyRegex) @@ -56,7 +56,7 @@ func (store *store) GetAllJobsMetadata() ([]Metadata, error) { return nil, err } - jobsMetadata := make([]Metadata, len(values)) + jobsMetadata := make([]modelMetadata.Metadata, len(values)) for i := range values { err = json.Unmarshal(values[i], &jobsMetadata[i]) if err != nil { @@ -67,13 +67,13 @@ func (store *store) GetAllJobsMetadata() ([]Metadata, error) { return jobsMetadata, nil } -func (store *store) GetJobMetadata(jobName string) (*Metadata, error) { +func (store *store) GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) { binaryJobMetadata, err := store.redisClient.GET(jobMetadataKey(jobName)) if err != nil { return nil, err } - var jobMetadata Metadata + var jobMetadata modelMetadata.Metadata err = json.Unmarshal(binaryJobMetadata, &jobMetadata) if err != nil { return nil, err diff --git a/proctord/jobs/metadata/store_mock.go b/proctord/jobs/metadata/store_mock.go index 74dbcadb..c8b2e86a 100644 --- a/proctord/jobs/metadata/store_mock.go +++ b/proctord/jobs/metadata/store_mock.go @@ -2,23 +2,24 @@ package metadata import ( "github.com/stretchr/testify/mock" + modelMetadata "proctor/shared/model/metadata" ) type MockStore struct { mock.Mock } -func (m *MockStore) CreateOrUpdateJobMetadata(metadata Metadata) error { +func (m *MockStore) CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error { args := m.Called(metadata) return args.Error(0) } -func (m *MockStore) GetAllJobsMetadata() ([]Metadata, error) { +func (m *MockStore) GetAllJobsMetadata() ([]modelMetadata.Metadata, error) { args := m.Called() - return args.Get(0).([]Metadata), args.Error(1) + return args.Get(0).([]modelMetadata.Metadata), args.Error(1) } -func (m *MockStore) GetJobMetadata(jobName string) (*Metadata, error) { +func (m *MockStore) GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) { args := m.Called(jobName) - return args.Get(0).(*Metadata), args.Error(1) + return args.Get(0).(*modelMetadata.Metadata), args.Error(1) } diff --git a/proctord/jobs/metadata/env/vars.go b/shared/model/metadata/env/vars.go similarity index 100% rename from proctord/jobs/metadata/env/vars.go rename to shared/model/metadata/env/vars.go diff --git a/proctord/jobs/metadata/metadata.go b/shared/model/metadata/metadata.go similarity index 90% rename from proctord/jobs/metadata/metadata.go rename to shared/model/metadata/metadata.go index 4505e227..c4cd0af1 100644 --- a/proctord/jobs/metadata/metadata.go +++ b/shared/model/metadata/metadata.go @@ -1,6 +1,6 @@ package metadata -import "proctor/proctord/jobs/metadata/env" +import "proctor/shared/model/metadata/env" type Metadata struct { Name string `json:"name"` diff --git a/shared/model/model.go b/shared/model/model.go new file mode 100644 index 00000000..8b537907 --- /dev/null +++ b/shared/model/model.go @@ -0,0 +1 @@ +package model diff --git a/utility/sort/sort.go b/utility/sort/sort.go index 29f73924..5413fc00 100644 --- a/utility/sort/sort.go +++ b/utility/sort/sort.go @@ -1,7 +1,7 @@ package sort import ( - "proctor/proctord/jobs/metadata" + "proctor/shared/model/metadata" "sort" ) diff --git a/utility/sort/sort_test.go b/utility/sort/sort_test.go index fcb1e951..711bde9c 100644 --- a/utility/sort/sort_test.go +++ b/utility/sort/sort_test.go @@ -2,7 +2,7 @@ package sort import ( "github.com/stretchr/testify/assert" - "proctor/proctord/jobs/metadata" + "proctor/shared/model/metadata" "testing" ) From af51ec0164f975226ecd153c13c617fd66efbd50 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 12:39:32 +0700 Subject: [PATCH 006/268] [Jasoet|Bimo] refactor: move proctor/utility/sort to shared --- cli/command/list/lister.go | 2 +- {utility => shared/utility}/sort/sort.go | 0 {utility => shared/utility}/sort/sort_test.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {utility => shared/utility}/sort/sort.go (100%) rename {utility => shared/utility}/sort/sort_test.go (100%) diff --git a/cli/command/list/lister.go b/cli/command/list/lister.go index 8a87e23e..609dfb5a 100644 --- a/cli/command/list/lister.go +++ b/cli/command/list/lister.go @@ -7,7 +7,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - "proctor/utility/sort" + "proctor/shared/utility/sort" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/utility/sort/sort.go b/shared/utility/sort/sort.go similarity index 100% rename from utility/sort/sort.go rename to shared/utility/sort/sort.go diff --git a/utility/sort/sort_test.go b/shared/utility/sort/sort_test.go similarity index 100% rename from utility/sort/sort_test.go rename to shared/utility/sort/sort_test.go From 59aa5bf8a188678a6cc940ebb3154df3a957a136 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 12:45:32 +0700 Subject: [PATCH 007/268] [Jasoet|Bimo] refactor: Extract Schedule Model to Shared --- daemon/client.go | 40 ++++++++++++------------- daemon/client_mock.go | 16 +++++----- proctord/jobs/schedule/handler.go | 3 +- proctord/jobs/schedule/scheduled_job.go | 22 +++++--------- shared/model/schedule/schedule_job.go | 11 +++++++ 5 files changed, 48 insertions(+), 44 deletions(-) create mode 100644 shared/model/schedule/schedule_job.go diff --git a/daemon/client.go b/daemon/client.go index b31c4559..fc38e4f6 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -21,19 +21,19 @@ import ( "github.com/fatih/color" "github.com/gorilla/websocket" "proctor/config" - proc_metadata "proctor/shared/model/metadata" - "proctor/proctord/jobs/schedule" "proctor/proctord/utility" + procMetadata "proctor/shared/model/metadata" + modelSchedule "proctor/shared/model/schedule" ) type Client interface { - ListProcs() ([]proc_metadata.Metadata, error) + ListProcs() ([]procMetadata.Metadata, error) ExecuteProc(string, map[string]string) (string, error) StreamProcLogs(string) error GetDefinitiveProcExecutionStatus(string) (string, error) ScheduleJob(string, string, string, string, string, map[string]string) (string, error) - ListScheduledProcs() ([]schedule.ScheduledJob, error) - DescribeScheduledProc(string) (schedule.ScheduledJob, error) + ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) + DescribeScheduledProc(string) (modelSchedule.ScheduledJob, error) RemoveScheduledProc(string) error } @@ -130,10 +130,10 @@ func (c *client) loadProctorConfig() error { return nil } -func (c *client) ListProcs() ([]proc_metadata.Metadata, error) { +func (c *client) ListProcs() ([]procMetadata.Metadata, error) { err := c.loadProctorConfig() if err != nil { - return []proc_metadata.Metadata{}, err + return []procMetadata.Metadata{}, err } client := &http.Client{ @@ -146,23 +146,23 @@ func (c *client) ListProcs() ([]proc_metadata.Metadata, error) { resp, err := client.Do(req) if err != nil { - return []proc_metadata.Metadata{}, buildNetworkError(err) + return []procMetadata.Metadata{}, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return []proc_metadata.Metadata{}, buildHTTPError(c, resp) + return []procMetadata.Metadata{}, buildHTTPError(c, resp) } - var procList []proc_metadata.Metadata + var procList []procMetadata.Metadata err = json.NewDecoder(resp.Body).Decode(&procList) return procList, err } -func (c *client) ListScheduledProcs() ([]schedule.ScheduledJob, error) { +func (c *client) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { err := c.loadProctorConfig() if err != nil { - return []schedule.ScheduledJob{}, err + return []modelSchedule.ScheduledJob{}, err } client := &http.Client{ @@ -175,23 +175,23 @@ func (c *client) ListScheduledProcs() ([]schedule.ScheduledJob, error) { resp, err := client.Do(req) if err != nil { - return []schedule.ScheduledJob{}, buildNetworkError(err) + return []modelSchedule.ScheduledJob{}, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return []schedule.ScheduledJob{}, buildHTTPError(c, resp) + return []modelSchedule.ScheduledJob{}, buildHTTPError(c, resp) } - var scheduledProcsList []schedule.ScheduledJob + var scheduledProcsList []modelSchedule.ScheduledJob err = json.NewDecoder(resp.Body).Decode(&scheduledProcsList) return scheduledProcsList, err } -func (c *client) DescribeScheduledProc(jobID string) (schedule.ScheduledJob, error) { +func (c *client) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob, error) { err := c.loadProctorConfig() if err != nil { - return schedule.ScheduledJob{}, err + return modelSchedule.ScheduledJob{}, err } client := &http.Client{ @@ -205,15 +205,15 @@ func (c *client) DescribeScheduledProc(jobID string) (schedule.ScheduledJob, err resp, err := client.Do(req) if err != nil { - return schedule.ScheduledJob{}, buildNetworkError(err) + return modelSchedule.ScheduledJob{}, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return schedule.ScheduledJob{}, buildHTTPError(c, resp) + return modelSchedule.ScheduledJob{}, buildHTTPError(c, resp) } - var scheduledProc schedule.ScheduledJob + var scheduledProc modelSchedule.ScheduledJob err = json.NewDecoder(resp.Body).Decode(&scheduledProc) return scheduledProc, err } diff --git a/daemon/client_mock.go b/daemon/client_mock.go index 8ead6ce8..95329a42 100644 --- a/daemon/client_mock.go +++ b/daemon/client_mock.go @@ -2,22 +2,22 @@ package daemon import ( "github.com/stretchr/testify/mock" - proc_metadata "proctor/shared/model/metadata" - "proctor/proctord/jobs/schedule" + procMetadata "proctor/shared/model/metadata" + modelSchedule "proctor/shared/model/schedule" ) type MockClient struct { mock.Mock } -func (m *MockClient) ListProcs() ([]proc_metadata.Metadata, error) { +func (m *MockClient) ListProcs() ([]procMetadata.Metadata, error) { args := m.Called() - return args.Get(0).([]proc_metadata.Metadata), args.Error(1) + return args.Get(0).([]procMetadata.Metadata), args.Error(1) } -func (m *MockClient) ListScheduledProcs() ([]schedule.ScheduledJob, error) { +func (m *MockClient) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { args := m.Called() - return args.Get(0).([]schedule.ScheduledJob), args.Error(1) + return args.Get(0).([]modelSchedule.ScheduledJob), args.Error(1) } func (m *MockClient) ExecuteProc(name string, procArgs map[string]string) (string, error) { @@ -40,9 +40,9 @@ func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, gr return args.Get(0).(string), args.Error(1) } -func (m *MockClient) DescribeScheduledProc(jobID string) (schedule.ScheduledJob, error) { +func (m *MockClient) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob, error) { args := m.Called(jobID) - return args.Get(0).(schedule.ScheduledJob), args.Error(1) + return args.Get(0).(modelSchedule.ScheduledJob), args.Error(1) } func (m *MockClient) RemoveScheduledProc(jobID string) error { diff --git a/proctord/jobs/schedule/handler.go b/proctord/jobs/schedule/handler.go index 3f72be7b..4effb2e9 100644 --- a/proctord/jobs/schedule/handler.go +++ b/proctord/jobs/schedule/handler.go @@ -15,6 +15,7 @@ import ( "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/utility" + modelSchedule "proctor/shared/model/schedule" ) type scheduler struct { @@ -38,7 +39,7 @@ func NewScheduler(store storage.Store, metadataStore metadata.Store) Scheduler { func (scheduler *scheduler) Schedule() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - var scheduledJob ScheduledJob + var scheduledJob modelSchedule.ScheduledJob err := json.NewDecoder(req.Body).Decode(&scheduledJob) userEmail := req.Header.Get(utility.UserEmailHeaderKey) defer req.Body.Close() diff --git a/proctord/jobs/schedule/scheduled_job.go b/proctord/jobs/schedule/scheduled_job.go index 6b1fd25b..79252d81 100644 --- a/proctord/jobs/schedule/scheduled_job.go +++ b/proctord/jobs/schedule/scheduled_job.go @@ -3,20 +3,12 @@ package schedule import ( "proctor/proctord/storage/postgres" "proctor/proctord/utility" -) -type ScheduledJob struct { - ID string `json:"id"` - Name string `json:"name"` - Args map[string]string `json:"args"` - NotificationEmails string `json:"notification_emails"` - Time string `json:"time"` - Tags string `json:"tags"` - Group string `json:"group_name"` -} + modelSchedule "proctor/shared/model/schedule" +) -func FromStoreToHandler(scheduledJobsStoreFormat []postgres.JobsSchedule) ([]ScheduledJob, error) { - var scheduledJobs []ScheduledJob +func FromStoreToHandler(scheduledJobsStoreFormat []postgres.JobsSchedule) ([]modelSchedule.ScheduledJob, error) { + var scheduledJobs []modelSchedule.ScheduledJob for _, scheduledJobStoreFormat := range scheduledJobsStoreFormat { scheduledJob, err := GetScheduledJob(scheduledJobStoreFormat) if err != nil { @@ -27,12 +19,12 @@ func FromStoreToHandler(scheduledJobsStoreFormat []postgres.JobsSchedule) ([]Sch return scheduledJobs, nil } -func GetScheduledJob(scheduledJobStoreFormat postgres.JobsSchedule) (ScheduledJob, error) { +func GetScheduledJob(scheduledJobStoreFormat postgres.JobsSchedule) (modelSchedule.ScheduledJob, error) { args, err := utility.DeserializeMap(scheduledJobStoreFormat.Args) if err != nil { - return ScheduledJob{}, err + return modelSchedule.ScheduledJob{}, err } - scheduledJob := ScheduledJob{ + scheduledJob := modelSchedule.ScheduledJob{ ID: scheduledJobStoreFormat.ID, Name: scheduledJobStoreFormat.Name, Args: args, diff --git a/shared/model/schedule/schedule_job.go b/shared/model/schedule/schedule_job.go new file mode 100644 index 00000000..153e3769 --- /dev/null +++ b/shared/model/schedule/schedule_job.go @@ -0,0 +1,11 @@ +package schedule + +type ScheduledJob struct { + ID string `json:"id"` + Name string `json:"name"` + Args map[string]string `json:"args"` + NotificationEmails string `json:"notification_emails"` + Time string `json:"time"` + Tags string `json:"tags"` + Group string `json:"group_name"` +} From 97a2314f52ec3f5ea09100231373ee81a7b01092 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 13:04:52 +0700 Subject: [PATCH 008/268] [Jasoet|Bimo] refactor: Extract Proctord Utility to shared --- cli/command/execution/executioner.go | 4 +- cli/command/execution/executioner_test.go | 10 +- config/config.go | 4 +- daemon/client.go | 86 ++++---- daemon/client_mock.go | 6 +- daemon/client_test.go | 188 +++++++++--------- proctord/audit/auditor.go | 2 +- proctord/audit/auditor_test.go | 2 +- proctord/jobs/execution/executioner.go | 5 +- proctord/jobs/execution/executioner_test.go | 4 +- proctord/jobs/execution/handler.go | 2 +- proctord/jobs/execution/handler_test.go | 2 +- proctord/jobs/logs/handler.go | 2 +- proctord/jobs/logs/handler_test.go | 2 +- proctord/jobs/metadata/handler.go | 2 +- proctord/jobs/metadata/handler_test.go | 2 +- proctord/jobs/schedule/handler.go | 2 +- proctord/jobs/schedule/handler_test.go | 2 +- proctord/jobs/schedule/scheduled_job.go | 2 +- proctord/jobs/schedule/worker.go | 7 +- proctord/jobs/schedule/worker_test.go | 2 +- proctord/jobs/secrets/handler.go | 2 +- proctord/jobs/secrets/handler_test.go | 2 +- proctord/kubernetes/client.go | 2 +- proctord/kubernetes/client_mock.go | 2 +- proctord/kubernetes/client_test.go | 2 +- proctord/mail/mailer.go | 2 +- proctord/mail/mailer_test.go | 2 +- .../middleware/validate_client_version.go | 2 +- .../validate_client_version_test.go | 2 +- proctord/storage/store_test.go | 2 +- .../utils.go => shared/constant/constant.go | 45 +---- {proctord => shared}/utility/buffer.go | 0 shared/utility/utils.go | 44 ++++ 34 files changed, 225 insertions(+), 222 deletions(-) rename proctord/utility/utils.go => shared/constant/constant.go (75%) rename {proctord => shared}/utility/buffer.go (100%) create mode 100644 shared/utility/utils.go diff --git a/cli/command/execution/executioner.go b/cli/command/execution/executioner.go index f8fd6ab8..1d38fc48 100644 --- a/cli/command/execution/executioner.go +++ b/cli/command/execution/executioner.go @@ -8,7 +8,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/daemon" - proctordUtility "proctor/proctord/utility" + "proctor/shared/constant" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { @@ -68,7 +68,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in return } - if procExecutionStatus != proctordUtility.JobSucceeded { + if procExecutionStatus != constant.JobSucceeded { printer.Println("Proc execution failed", color.FgRed) osExitFunc(1) return diff --git a/cli/command/execution/executioner_test.go b/cli/command/execution/executioner_test.go index bb17e137..b218c40b 100644 --- a/cli/command/execution/executioner_test.go +++ b/cli/command/execution/executioner_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/daemon" - "proctor/proctord/utility" + "proctor/shared/constant" ) type ExecutionCmdTestSuite struct { @@ -56,7 +56,7 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmd() { s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(utility.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -79,7 +79,7 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForNoProcVariables() { s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(utility.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -103,7 +103,7 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForIncorrectVariableFormat() { s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(utility.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -198,7 +198,7 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(utility.JobFailed, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobFailed, nil).Once() s.mockPrinter.On("Println", "Proc execution failed", color.FgRed).Once() osExitFunc := func(exitCode int) { diff --git a/config/config.go b/config/config.go index bc0e63f1..b4f5ad84 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,7 @@ import ( "time" "github.com/pkg/errors" - "proctor/proctord/utility" + "proctor/shared/constant" "github.com/spf13/viper" ) @@ -73,7 +73,7 @@ func (loader *loader) Load() (ProctorConfig, ConfigError) { proctorHost := viper.GetString(ProctorHost) if proctorHost == "" { - return ProctorConfig{}, ConfigError{error: errors.New("Mandatory Config Missing"), Message: utility.ConfigProctorHostMissingError} + return ProctorConfig{}, ConfigError{error: errors.New("Mandatory Config Missing"), Message: constant.ConfigProctorHostMissingError} } emailId := viper.GetString(EmailId) accessToken := viper.GetString(AccessToken) diff --git a/daemon/client.go b/daemon/client.go index fc38e4f6..8e16ef24 100644 --- a/daemon/client.go +++ b/daemon/client.go @@ -12,6 +12,7 @@ import ( "net/url" "os" "os/signal" + "proctor/shared/constant" "proctor/shared/io" "time" @@ -21,13 +22,12 @@ import ( "github.com/fatih/color" "github.com/gorilla/websocket" "proctor/config" - "proctor/proctord/utility" - procMetadata "proctor/shared/model/metadata" + modelMetadata "proctor/shared/model/metadata" modelSchedule "proctor/shared/model/schedule" ) type Client interface { - ListProcs() ([]procMetadata.Metadata, error) + ListProcs() ([]modelMetadata.Metadata, error) ExecuteProc(string, map[string]string) (string, error) StreamProcLogs(string) error GetDefinitiveProcExecutionStatus(string) (string, error) @@ -93,9 +93,9 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, client := &http.Client{} req, err := http.NewRequest("POST", "http://"+c.proctordHost+"/jobs/schedule", bytes.NewReader(requestBody)) req.Header.Add("Content-Type", "application/json") - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { @@ -130,31 +130,31 @@ func (c *client) loadProctorConfig() error { return nil } -func (c *client) ListProcs() ([]procMetadata.Metadata, error) { +func (c *client) ListProcs() ([]modelMetadata.Metadata, error) { err := c.loadProctorConfig() if err != nil { - return []procMetadata.Metadata{}, err + return []modelMetadata.Metadata{}, err } client := &http.Client{ Timeout: c.connectionTimeoutSecs, } req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/metadata", nil) - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { - return []procMetadata.Metadata{}, buildNetworkError(err) + return []modelMetadata.Metadata{}, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return []procMetadata.Metadata{}, buildHTTPError(c, resp) + return []modelMetadata.Metadata{}, buildHTTPError(c, resp) } - var procList []procMetadata.Metadata + var procList []modelMetadata.Metadata err = json.NewDecoder(resp.Body).Decode(&procList) return procList, err } @@ -169,9 +169,9 @@ func (c *client) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { Timeout: c.connectionTimeoutSecs, } req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/schedule", nil) - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { @@ -199,9 +199,9 @@ func (c *client) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob } url := fmt.Sprintf("http://"+c.proctordHost+"/jobs/schedule/%s", jobID) req, err := http.NewRequest("GET", url, nil) - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { @@ -229,9 +229,9 @@ func (c *client) RemoveScheduledProc(jobID string) error { } url := fmt.Sprintf("http://"+c.proctordHost+"/jobs/schedule/%s", jobID) req, err := http.NewRequest("DELETE", url, nil) - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { @@ -265,9 +265,9 @@ func (c *client) ExecuteProc(name string, args map[string]string) (string, error client := &http.Client{} req, err := http.NewRequest("POST", "http://"+c.proctordHost+"/jobs/execute", bytes.NewReader(requestBody)) req.Header.Add("Content-Type", "application/json") - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { return "", buildNetworkError(err) @@ -304,18 +304,18 @@ func (c *client) StreamProcLogs(name string) error { token := []string{c.accessToken} emailId := []string{c.emailId} clientVersion := []string{c.clientVersion} - headers[utility.AccessTokenHeaderKey] = token - headers[utility.UserEmailHeaderKey] = emailId - headers[utility.ClientVersionHeaderKey] = clientVersion + headers[constant.AccessTokenHeaderKey] = token + headers[constant.UserEmailHeaderKey] = emailId + headers[constant.ClientVersionHeaderKey] = clientVersion wsConn, response, err := websocket.DefaultDialer.Dial(proctodWebsocketURLWithProcName, headers) if err != nil { animation.Stop() if response.StatusCode == http.StatusUnauthorized { if c.emailId == "" || c.accessToken == "" { - return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorMissingConfig) + return fmt.Errorf("%s\n%s", constant.UnauthorizedErrorHeader, constant.UnauthorizedErrorMissingConfig) } - return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorInvalidConfig) + return fmt.Errorf("%s\n%s", constant.UnauthorizedErrorHeader, constant.UnauthorizedErrorInvalidConfig) } return err } @@ -359,9 +359,9 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro } req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/execute/"+procName+"/status", nil) - req.Header.Add(utility.UserEmailHeaderKey, c.emailId) - req.Header.Add(utility.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(utility.ClientVersionHeaderKey, c.clientVersion) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := httpClient.Do(req) if err != nil { @@ -379,7 +379,7 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro } procExecutionStatus := string(body) - if procExecutionStatus == utility.JobSucceeded || procExecutionStatus == utility.JobFailed { + if procExecutionStatus == constant.JobSucceeded || procExecutionStatus == constant.JobFailed { return procExecutionStatus, nil } @@ -390,17 +390,17 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro func buildNetworkError(err error) error { if netError, ok := err.(net.Error); ok && netError.Timeout() { - return fmt.Errorf("%s\n%s\n%s", utility.GenericTimeoutErrorHeader, netError.Error(), utility.GenericTimeoutErrorBody) + return fmt.Errorf("%s\n%s\n%s", constant.GenericTimeoutErrorHeader, netError.Error(), constant.GenericTimeoutErrorBody) } - return fmt.Errorf("%s\n%s", utility.GenericNetworkErrorHeader, err.Error()) + return fmt.Errorf("%s\n%s", constant.GenericNetworkErrorHeader, err.Error()) } func buildHTTPError(c *client, resp *http.Response) error { if resp.StatusCode == http.StatusUnauthorized { if c.emailId == "" || c.accessToken == "" { - return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorMissingConfig) + return fmt.Errorf("%s\n%s", constant.UnauthorizedErrorHeader, constant.UnauthorizedErrorMissingConfig) } - return fmt.Errorf("%s\n%s", utility.UnauthorizedErrorHeader, utility.UnauthorizedErrorInvalidConfig) + return fmt.Errorf("%s\n%s", constant.UnauthorizedErrorHeader, constant.UnauthorizedErrorInvalidConfig) } if resp.StatusCode == http.StatusBadRequest { @@ -408,18 +408,18 @@ func buildHTTPError(c *client, resp *http.Response) error { } if resp.StatusCode == http.StatusNoContent { - return fmt.Errorf(utility.NoScheduledJobsError) + return fmt.Errorf(constant.NoScheduledJobsError) } if resp.StatusCode == http.StatusNotFound { - return fmt.Errorf(utility.JobNotFoundError) + return fmt.Errorf(constant.JobNotFoundError) } if resp.StatusCode == http.StatusForbidden { - return fmt.Errorf(utility.JobForbiddenErrorHeader) + return fmt.Errorf(constant.JobForbiddenErrorHeader) } - return fmt.Errorf("%s\nStatus Code: %d, %s", utility.GenericResponseErrorHeader, resp.StatusCode, http.StatusText(resp.StatusCode)) + return fmt.Errorf("%s\nStatus Code: %d, %s", constant.GenericResponseErrorHeader, resp.StatusCode, http.StatusText(resp.StatusCode)) } func getHttpResponseError(response io_reader.ReadCloser) error { diff --git a/daemon/client_mock.go b/daemon/client_mock.go index 95329a42..f6ea5076 100644 --- a/daemon/client_mock.go +++ b/daemon/client_mock.go @@ -2,7 +2,7 @@ package daemon import ( "github.com/stretchr/testify/mock" - procMetadata "proctor/shared/model/metadata" + modelMetadata "proctor/shared/model/metadata" modelSchedule "proctor/shared/model/schedule" ) @@ -10,9 +10,9 @@ type MockClient struct { mock.Mock } -func (m *MockClient) ListProcs() ([]procMetadata.Metadata, error) { +func (m *MockClient) ListProcs() ([]modelMetadata.Metadata, error) { args := m.Called() - return args.Get(0).([]procMetadata.Metadata), args.Error(1) + return args.Get(0).([]modelMetadata.Metadata), args.Error(1) } func (m *MockClient) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { diff --git a/daemon/client_test.go b/daemon/client_test.go index a3d68529..1d106833 100644 --- a/daemon/client_test.go +++ b/daemon/client_test.go @@ -18,7 +18,7 @@ import ( "proctor/config" proc_metadata "proctor/shared/model/metadata" "proctor/shared/model/metadata/env" - "proctor/proctord/utility" + "proctor/shared/constant" ) type TestConnectionError struct { @@ -78,9 +78,9 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -111,9 +111,9 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -145,9 +145,9 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -178,9 +178,9 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -211,9 +211,9 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedUser() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -243,9 +243,9 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{""}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{""}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -279,9 +279,9 @@ func (s *ClientTestSuite) TestExecuteProc() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -321,9 +321,9 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -360,9 +360,9 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -393,9 +393,9 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -424,9 +424,9 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -456,9 +456,9 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{""}, - utility.AccessTokenHeaderKey: []string{""}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{""}, + constant.AccessTokenHeaderKey: []string{""}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -488,9 +488,9 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -513,9 +513,9 @@ func (s *ClientTestSuite) TestLogStreamForAuthorizedUser() { logStreamAuthorizer := func(t *testing.T) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { upgrader := websocket.Upgrader{} - assert.Equal(t, "proctor@example.com", r.Header.Get(utility.UserEmailHeaderKey)) - assert.Equal(t, "access-token", r.Header.Get(utility.AccessTokenHeaderKey)) - assert.Equal(t, version.ClientVersion, r.Header.Get(utility.ClientVersionHeaderKey)) + assert.Equal(t, "proctor@example.com", r.Header.Get(constant.UserEmailHeaderKey)) + assert.Equal(t, "access-token", r.Header.Get(constant.AccessTokenHeaderKey)) + assert.Equal(t, version.ClientVersion, r.Header.Get(constant.ClientVersionHeaderKey)) conn, _ := upgrader.Upgrade(w, r, nil) defer conn.Close() } @@ -574,7 +574,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( httpmock.Activate() defer httpmock.DeactivateAndReset() - expectedProcExecutionStatus := utility.JobSucceeded + expectedProcExecutionStatus := constant.JobSucceeded responseBody := expectedProcExecutionStatus httpmock.RegisterStubRequest( @@ -586,9 +586,9 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -610,7 +610,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { httpmock.Activate() defer httpmock.DeactivateAndReset() - expectedProcExecutionStatus := utility.JobFailed + expectedProcExecutionStatus := constant.JobFailed responseBody := expectedProcExecutionStatus httpmock.RegisterStubRequest( @@ -622,9 +622,9 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -655,9 +655,9 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -688,9 +688,9 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -712,7 +712,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: expectedRequestsToProctorDCount} - expectedProcExecutionStatus := utility.JobWaiting + expectedProcExecutionStatus := constant.JobWaiting responseBody := expectedProcExecutionStatus httpmock.Activate() @@ -728,9 +728,9 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -764,9 +764,9 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -800,9 +800,9 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -833,9 +833,9 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWhenJobIDNotFound() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -866,9 +866,9 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -900,9 +900,9 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -935,9 +935,9 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenNoJobsScheduled() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -967,9 +967,9 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -1001,9 +1001,9 @@ func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -1034,9 +1034,9 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWithInvalidJobID() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -1067,9 +1067,9 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWhenJobIDNotFound() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) @@ -1100,9 +1100,9 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { }, ).WithHeader( &http.Header{ - utility.UserEmailHeaderKey: []string{"proctor@example.com"}, - utility.AccessTokenHeaderKey: []string{"access-token"}, - utility.ClientVersionHeaderKey: []string{version.ClientVersion}, + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) diff --git a/proctord/audit/auditor.go b/proctord/audit/auditor.go index d065baaa..609b5bc7 100644 --- a/proctord/audit/auditor.go +++ b/proctord/audit/auditor.go @@ -6,7 +6,7 @@ import ( "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type Auditor interface { diff --git a/proctord/audit/auditor_test.go b/proctord/audit/auditor_test.go index 700f83a1..80d6c8ce 100644 --- a/proctord/audit/auditor_test.go +++ b/proctord/audit/auditor_test.go @@ -6,7 +6,7 @@ import ( "proctor/proctord/kubernetes" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) func TestJobsExecutionAuditing(t *testing.T) { diff --git a/proctord/jobs/execution/executioner.go b/proctord/jobs/execution/executioner.go index eef8c4f1..29c83645 100644 --- a/proctord/jobs/execution/executioner.go +++ b/proctord/jobs/execution/executioner.go @@ -8,7 +8,8 @@ import ( "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + "proctor/shared/constant" + "proctor/shared/utility" ) type executioner struct { @@ -52,7 +53,7 @@ func (executioner *executioner) Execute(jobsExecutionAuditLog *postgres.JobsExec return "", errors.New(fmt.Sprintf("Error submitting job to kube: %s. Error: %s", jobName, err.Error())) } jobsExecutionAuditLog.AddExecutionID(jobExecutionID) - jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionSuccess + jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionSuccess return jobExecutionID, nil } diff --git a/proctord/jobs/execution/executioner_test.go b/proctord/jobs/execution/executioner_test.go index 45c3bb0f..ec0a5f10 100644 --- a/proctord/jobs/execution/executioner_test.go +++ b/proctord/jobs/execution/executioner_test.go @@ -12,13 +12,13 @@ import ( "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type ExecutionerTestSuite struct { suite.Suite mockKubeClient kubernetes.MockClient - mockMetadataStore *metadata.MockStore + mockMetadataStore *modelMetadata.MockStore mockSecretsStore *secrets.MockStore testExecutioner Executioner } diff --git a/proctord/jobs/execution/handler.go b/proctord/jobs/execution/handler.go index 04faefab..faa3f4d1 100644 --- a/proctord/jobs/execution/handler.go +++ b/proctord/jobs/execution/handler.go @@ -11,7 +11,7 @@ import ( "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" "time" ) diff --git a/proctord/jobs/execution/handler_test.go b/proctord/jobs/execution/handler_test.go index 2e23f763..bc1f0dd3 100644 --- a/proctord/jobs/execution/handler_test.go +++ b/proctord/jobs/execution/handler_test.go @@ -14,7 +14,7 @@ import ( "net/http/httptest" "proctor/proctord/audit" "proctor/proctord/storage" - "proctor/proctord/utility" + utility "proctor/shared/constant" "testing" ) diff --git a/proctord/jobs/logs/handler.go b/proctord/jobs/logs/handler.go index 50e07b9f..57c1f3cd 100644 --- a/proctord/jobs/logs/handler.go +++ b/proctord/jobs/logs/handler.go @@ -10,7 +10,7 @@ import ( "proctor/proctord/config" "proctor/proctord/kubernetes" _logger "proctor/proctord/logger" - "proctor/proctord/utility" + utility "proctor/shared/constant" "github.com/gorilla/websocket" ) diff --git a/proctord/jobs/logs/handler_test.go b/proctord/jobs/logs/handler_test.go index 549470d6..1b299237 100644 --- a/proctord/jobs/logs/handler_test.go +++ b/proctord/jobs/logs/handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "proctor/proctord/kubernetes" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type LoggerTestSuite struct { diff --git a/proctord/jobs/metadata/handler.go b/proctord/jobs/metadata/handler.go index 30afcfec..0ab7d4c3 100644 --- a/proctord/jobs/metadata/handler.go +++ b/proctord/jobs/metadata/handler.go @@ -6,7 +6,7 @@ import ( "net/http" procMetadata "proctor/shared/model/metadata" "proctor/proctord/logger" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type handler struct { diff --git a/proctord/jobs/metadata/handler_test.go b/proctord/jobs/metadata/handler_test.go index 7c8e6830..7c9d7aae 100644 --- a/proctord/jobs/metadata/handler_test.go +++ b/proctord/jobs/metadata/handler_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type MetadataHandlerTestSuite struct { diff --git a/proctord/jobs/schedule/handler.go b/proctord/jobs/schedule/handler.go index 4effb2e9..e59d8e88 100644 --- a/proctord/jobs/schedule/handler.go +++ b/proctord/jobs/schedule/handler.go @@ -14,7 +14,7 @@ import ( "proctor/proctord/jobs/metadata" "proctor/proctord/logger" "proctor/proctord/storage" - "proctor/proctord/utility" + utility "proctor/shared/constant" modelSchedule "proctor/shared/model/schedule" ) diff --git a/proctord/jobs/schedule/handler_test.go b/proctord/jobs/schedule/handler_test.go index 73597f08..a680807e 100644 --- a/proctord/jobs/schedule/handler_test.go +++ b/proctord/jobs/schedule/handler_test.go @@ -17,7 +17,7 @@ import ( "proctor/proctord/jobs/metadata" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type SchedulerTestSuite struct { diff --git a/proctord/jobs/schedule/scheduled_job.go b/proctord/jobs/schedule/scheduled_job.go index 79252d81..e52df819 100644 --- a/proctord/jobs/schedule/scheduled_job.go +++ b/proctord/jobs/schedule/scheduled_job.go @@ -2,7 +2,7 @@ package schedule import ( "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + "proctor/shared/utility" modelSchedule "proctor/shared/model/schedule" ) diff --git a/proctord/jobs/schedule/worker.go b/proctord/jobs/schedule/worker.go index d8cd4c6f..f0098d39 100644 --- a/proctord/jobs/schedule/worker.go +++ b/proctord/jobs/schedule/worker.go @@ -14,7 +14,8 @@ import ( "proctor/proctord/mail" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + "proctor/shared/constant" + "proctor/shared/utility" ) type worker struct { @@ -58,7 +59,7 @@ func (worker *worker) enableScheduledJobIfItDoesNotExist(scheduledJob postgres.J cronJob := cron.New() err = cronJob.AddFunc(scheduledJob.Time, func() { jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{} - jobsExecutionAuditLog.UserEmail = utility.WorkerEmail + jobsExecutionAuditLog.UserEmail = constant.WorkerEmail jobExecutionID, err := worker.executioner.Execute(jobsExecutionAuditLog, scheduledJob.Name, jobArgs) if err != nil { @@ -66,7 +67,7 @@ func (worker *worker) enableScheduledJobIfItDoesNotExist(scheduledJob postgres.J raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) jobsExecutionAuditLog.Errors = fmt.Sprintf("Error executing job: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionServerError + jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionServerError worker.auditor.JobsExecution(jobsExecutionAuditLog) return } diff --git a/proctord/jobs/schedule/worker_test.go b/proctord/jobs/schedule/worker_test.go index 70cc92b9..077cb375 100644 --- a/proctord/jobs/schedule/worker_test.go +++ b/proctord/jobs/schedule/worker_test.go @@ -17,7 +17,7 @@ import ( "proctor/proctord/mail" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type WorkerTestSuite struct { diff --git a/proctord/jobs/secrets/handler.go b/proctord/jobs/secrets/handler.go index 984524a6..8a42f81a 100644 --- a/proctord/jobs/secrets/handler.go +++ b/proctord/jobs/secrets/handler.go @@ -6,7 +6,7 @@ import ( "net/http" "proctor/proctord/logger" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type handler struct { diff --git a/proctord/jobs/secrets/handler_test.go b/proctord/jobs/secrets/handler_test.go index dd776444..1a193044 100644 --- a/proctord/jobs/secrets/handler_test.go +++ b/proctord/jobs/secrets/handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) type SecretsHandlerTestSuite struct { diff --git a/proctord/kubernetes/client.go b/proctord/kubernetes/client.go index b824929c..d6431cfd 100644 --- a/proctord/kubernetes/client.go +++ b/proctord/kubernetes/client.go @@ -14,7 +14,7 @@ import ( "k8s.io/client-go/kubernetes" "proctor/proctord/config" "proctor/proctord/logger" - "proctor/proctord/utility" + utility "proctor/shared/constant" //Package needed for kubernetes cluster in google cloud _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/clientcmd" diff --git a/proctord/kubernetes/client_mock.go b/proctord/kubernetes/client_mock.go index 25edf517..5a8c85df 100644 --- a/proctord/kubernetes/client_mock.go +++ b/proctord/kubernetes/client_mock.go @@ -4,7 +4,7 @@ import ( "io" "github.com/stretchr/testify/mock" - "proctor/proctord/utility" + "proctor/shared/utility" ) type MockClient struct { diff --git a/proctord/kubernetes/client_test.go b/proctord/kubernetes/client_test.go index 1cc7e372..fbc673bd 100644 --- a/proctord/kubernetes/client_test.go +++ b/proctord/kubernetes/client_test.go @@ -15,7 +15,7 @@ import ( "k8s.io/apimachinery/pkg/watch" batch_v1 "k8s.io/client-go/kubernetes/typed/batch/v1" "proctor/proctord/config" - "proctor/proctord/utility" + utility "proctor/shared/constant" batchV1 "k8s.io/api/batch/v1" fakeclientset "k8s.io/client-go/kubernetes/fake" diff --git a/proctord/mail/mailer.go b/proctord/mail/mailer.go index ed913b7b..81df3430 100644 --- a/proctord/mail/mailer.go +++ b/proctord/mail/mailer.go @@ -4,7 +4,7 @@ import ( "net/smtp" "proctor/proctord/config" - "proctor/proctord/utility" + "proctor/shared/utility" ) type Mailer interface { diff --git a/proctord/mail/mailer_test.go b/proctord/mail/mailer_test.go index bf609647..b76f743c 100644 --- a/proctord/mail/mailer_test.go +++ b/proctord/mail/mailer_test.go @@ -10,7 +10,7 @@ import ( "testing" "proctor/proctord/config" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) func TestSendMail(t *testing.T) { diff --git a/proctord/middleware/validate_client_version.go b/proctord/middleware/validate_client_version.go index 01819ef6..31ab4e12 100644 --- a/proctord/middleware/validate_client_version.go +++ b/proctord/middleware/validate_client_version.go @@ -6,7 +6,7 @@ import ( "net/http" "proctor/proctord/config" "proctor/proctord/logger" - "proctor/proctord/utility" + utility "proctor/shared/constant" ) func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { diff --git a/proctord/middleware/validate_client_version_test.go b/proctord/middleware/validate_client_version_test.go index c7bb0411..80468b00 100644 --- a/proctord/middleware/validate_client_version_test.go +++ b/proctord/middleware/validate_client_version_test.go @@ -6,7 +6,7 @@ import ( "net/http" "net/http/httptest" "os" - "proctor/proctord/utility" + utility "proctor/shared/constant" "testing" ) diff --git a/proctord/storage/store_test.go b/proctord/storage/store_test.go index f714bc60..0c350323 100644 --- a/proctord/storage/store_test.go +++ b/proctord/storage/store_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "proctor/proctord/storage/postgres" - "proctor/proctord/utility" + utility "proctor/shared/constant" "testing" ) diff --git a/proctord/utility/utils.go b/shared/constant/constant.go similarity index 75% rename from proctord/utility/utils.go rename to shared/constant/constant.go index b55f31fb..14476b93 100644 --- a/proctord/utility/utils.go +++ b/shared/constant/constant.go @@ -1,12 +1,4 @@ -package utility - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "strings" -) +package constant const ClientError = "malformed request" const NonExistentProcClientError = "proc name non existent" @@ -57,38 +49,3 @@ const AccessTokenHeaderKey = "Access-Token" const ClientVersionHeaderKey = "Client-Version" const WorkerEmail = "worker@proctor" - -func MergeMaps(mapOne, mapTwo map[string]string) map[string]string { - result := make(map[string]string) - - for k, v := range mapOne { - result[k] = v - } - for k, v := range mapTwo { - result[k] = v - } - return result -} - -func MapToString(someMap map[string]string) string { - b := new(bytes.Buffer) - for key, value := range someMap { - fmt.Fprintf(b, "%s=\"%s\",", key, value) - } - return strings.TrimRight(b.String(), ",") -} - -func DeserializeMap(encodedMap string) (map[string]string, error) { - var mapStringToString map[string]string - if encodedMap == "" { - return mapStringToString, nil - } - - decodedMap, err := base64.StdEncoding.DecodeString(encodedMap) - if err != nil { - return mapStringToString, err - } - - err = json.Unmarshal(decodedMap, &mapStringToString) - return mapStringToString, err -} diff --git a/proctord/utility/buffer.go b/shared/utility/buffer.go similarity index 100% rename from proctord/utility/buffer.go rename to shared/utility/buffer.go diff --git a/shared/utility/utils.go b/shared/utility/utils.go new file mode 100644 index 00000000..c591b1b8 --- /dev/null +++ b/shared/utility/utils.go @@ -0,0 +1,44 @@ +package utility + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "strings" +) + +func MergeMaps(mapOne, mapTwo map[string]string) map[string]string { + result := make(map[string]string) + + for k, v := range mapOne { + result[k] = v + } + for k, v := range mapTwo { + result[k] = v + } + return result +} + +func MapToString(someMap map[string]string) string { + b := new(bytes.Buffer) + for key, value := range someMap { + fmt.Fprintf(b, "%s=\"%s\",", key, value) + } + return strings.TrimRight(b.String(), ",") +} + +func DeserializeMap(encodedMap string) (map[string]string, error) { + var mapStringToString map[string]string + if encodedMap == "" { + return mapStringToString, nil + } + + decodedMap, err := base64.StdEncoding.DecodeString(encodedMap) + if err != nil { + return mapStringToString, err + } + + err = json.Unmarshal(decodedMap, &mapStringToString) + return mapStringToString, err +} From 517807b769a4fc5b2f28c9392393df008f888e25 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 13:10:50 +0700 Subject: [PATCH 009/268] [Jasoet|Bimo] refactor: Move proctor/config and proctor/daemon into cli --- cli/command/config/manager.go | 2 +- cli/command/config/view/view.go | 2 +- cli/command/description/descriptor.go | 2 +- cli/command/description/descriptor_test.go | 2 +- cli/command/execution/executioner.go | 2 +- cli/command/execution/executioner_test.go | 2 +- cli/command/list/lister.go | 2 +- cli/command/list/lister_test.go | 2 +- cli/command/root.go | 2 +- cli/command/root_test.go | 2 +- cli/command/schedule/describe/describe.go | 2 +- .../schedule/describe/describe_test.go | 2 +- cli/command/schedule/list/list.go | 2 +- cli/command/schedule/list/list_test.go | 2 +- cli/command/schedule/remove/remove.go | 2 +- cli/command/schedule/remove/remove_test.go | 2 +- cli/command/schedule/schedule.go | 2 +- cli/command/schedule/schedule_test.go | 2 +- {config => cli/config}/config.go | 0 {config => cli/config}/config_mock.go | 0 {config => cli/config}/config_test.go | 0 {config => cli/config}/data.go | 0 {daemon => cli/daemon}/client.go | 6 +++--- {daemon => cli/daemon}/client_mock.go | 0 {daemon => cli/daemon}/client_test.go | 20 +++++++++---------- exec/cli/cli.go | 4 ++-- 26 files changed, 33 insertions(+), 33 deletions(-) rename {config => cli/config}/config.go (100%) rename {config => cli/config}/config_mock.go (100%) rename {config => cli/config}/config_test.go (100%) rename {config => cli/config}/data.go (100%) rename {daemon => cli/daemon}/client.go (99%) rename {daemon => cli/daemon}/client_mock.go (100%) rename {daemon => cli/daemon}/client_test.go (98%) diff --git a/cli/command/config/manager.go b/cli/command/config/manager.go index 08acaa20..905f2e3c 100644 --- a/cli/command/config/manager.go +++ b/cli/command/config/manager.go @@ -11,7 +11,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - proctorConfig "proctor/config" + proctorConfig "proctor/cli/config" ) func CreateDirIfNotExist(dir string) { diff --git a/cli/command/config/view/view.go b/cli/command/config/view/view.go index 3ca49df5..fb4bc8ab 100644 --- a/cli/command/config/view/view.go +++ b/cli/command/config/view/view.go @@ -8,7 +8,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - proctorConfig "proctor/config" + proctorConfig "proctor/cli/config" "proctor/shared/io" ) diff --git a/cli/command/description/descriptor.go b/cli/command/description/descriptor.go index 557ef9b1..a305efc2 100644 --- a/cli/command/description/descriptor.go +++ b/cli/command/description/descriptor.go @@ -7,7 +7,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" procMetadata "proctor/shared/model/metadata" ) diff --git a/cli/command/description/descriptor_test.go b/cli/command/description/descriptor_test.go index 0d98d453..f498a2de 100644 --- a/cli/command/description/descriptor_test.go +++ b/cli/command/description/descriptor_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "proctor/daemon" + "proctor/cli/daemon" procMetadata "proctor/shared/model/metadata" "proctor/shared/model/metadata/env" diff --git a/cli/command/execution/executioner.go b/cli/command/execution/executioner.go index 1d38fc48..9e18fa55 100644 --- a/cli/command/execution/executioner.go +++ b/cli/command/execution/executioner.go @@ -7,7 +7,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/constant" ) diff --git a/cli/command/execution/executioner_test.go b/cli/command/execution/executioner_test.go index b218c40b..072c7c18 100644 --- a/cli/command/execution/executioner_test.go +++ b/cli/command/execution/executioner_test.go @@ -11,7 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/constant" ) diff --git a/cli/command/list/lister.go b/cli/command/list/lister.go index 609dfb5a..fbc20dbe 100644 --- a/cli/command/list/lister.go +++ b/cli/command/list/lister.go @@ -6,7 +6,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/utility/sort" ) diff --git a/cli/command/list/lister_test.go b/cli/command/list/lister_test.go index f37094fb..4ebd43c2 100644 --- a/cli/command/list/lister_test.go +++ b/cli/command/list/lister_test.go @@ -10,7 +10,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" procMetadata "proctor/shared/model/metadata" ) diff --git a/cli/command/root.go b/cli/command/root.go index 561e5253..cc047ca8 100644 --- a/cli/command/root.go +++ b/cli/command/root.go @@ -17,7 +17,7 @@ import ( scheduleList "proctor/cli/command/schedule/list" "proctor/cli/command/version" "proctor/cli/command/version/github" - "proctor/daemon" + "proctor/cli/daemon" ) var ( diff --git a/cli/command/root_test.go b/cli/command/root_test.go index bfdbbb74..23ab49c4 100644 --- a/cli/command/root_test.go +++ b/cli/command/root_test.go @@ -7,7 +7,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "proctor/cli/command/version/github" - "proctor/daemon" + "proctor/cli/daemon" ) func TestRootCmdUsage(t *testing.T) { diff --git a/cli/command/schedule/describe/describe.go b/cli/command/schedule/describe/describe.go index a9b34c31..357dc8d1 100644 --- a/cli/command/schedule/describe/describe.go +++ b/cli/command/schedule/describe/describe.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" ) diff --git a/cli/command/schedule/describe/describe_test.go b/cli/command/schedule/describe/describe_test.go index 706b23a7..b72b7595 100644 --- a/cli/command/schedule/describe/describe_test.go +++ b/cli/command/schedule/describe/describe_test.go @@ -4,7 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" "testing" ) diff --git a/cli/command/schedule/list/list.go b/cli/command/schedule/list/list.go index afc88d12..2151c467 100644 --- a/cli/command/schedule/list/list.go +++ b/cli/command/schedule/list/list.go @@ -6,7 +6,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/schedule/list/list_test.go b/cli/command/schedule/list/list_test.go index b3a3e4ce..99e21e0d 100644 --- a/cli/command/schedule/list/list_test.go +++ b/cli/command/schedule/list/list_test.go @@ -4,7 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" "testing" ) diff --git a/cli/command/schedule/remove/remove.go b/cli/command/schedule/remove/remove.go index 07115108..44699ef2 100644 --- a/cli/command/schedule/remove/remove.go +++ b/cli/command/schedule/remove/remove.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" ) diff --git a/cli/command/schedule/remove/remove_test.go b/cli/command/schedule/remove/remove_test.go index 33343d24..ab06a3d5 100644 --- a/cli/command/schedule/remove/remove_test.go +++ b/cli/command/schedule/remove/remove_test.go @@ -4,7 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" "testing" ) diff --git a/cli/command/schedule/schedule.go b/cli/command/schedule/schedule.go index acb20d6e..52db3afb 100644 --- a/cli/command/schedule/schedule.go +++ b/cli/command/schedule/schedule.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" "strings" ) diff --git a/cli/command/schedule/schedule_test.go b/cli/command/schedule/schedule_test.go index 477df824..7a795d11 100644 --- a/cli/command/schedule/schedule_test.go +++ b/cli/command/schedule/schedule_test.go @@ -4,7 +4,7 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/daemon" + "proctor/cli/daemon" "proctor/shared/io" "testing" ) diff --git a/config/config.go b/cli/config/config.go similarity index 100% rename from config/config.go rename to cli/config/config.go diff --git a/config/config_mock.go b/cli/config/config_mock.go similarity index 100% rename from config/config_mock.go rename to cli/config/config_mock.go diff --git a/config/config_test.go b/cli/config/config_test.go similarity index 100% rename from config/config_test.go rename to cli/config/config_test.go diff --git a/config/data.go b/cli/config/data.go similarity index 100% rename from config/data.go rename to cli/config/data.go diff --git a/daemon/client.go b/cli/daemon/client.go similarity index 99% rename from daemon/client.go rename to cli/daemon/client.go index 8e16ef24..e5c173ef 100644 --- a/daemon/client.go +++ b/cli/daemon/client.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - io_reader "io" + ioReader "io" "io/ioutil" "net" "net/http" @@ -21,7 +21,7 @@ import ( "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/gorilla/websocket" - "proctor/config" + "proctor/cli/config" modelMetadata "proctor/shared/model/metadata" modelSchedule "proctor/shared/model/schedule" ) @@ -422,7 +422,7 @@ func buildHTTPError(c *client, resp *http.Response) error { return fmt.Errorf("%s\nStatus Code: %d, %s", constant.GenericResponseErrorHeader, resp.StatusCode, http.StatusText(resp.StatusCode)) } -func getHttpResponseError(response io_reader.ReadCloser) error { +func getHttpResponseError(response ioReader.ReadCloser) error { body, _ := ioutil.ReadAll(response) bodyString := string(body) return fmt.Errorf(bodyString) diff --git a/daemon/client_mock.go b/cli/daemon/client_mock.go similarity index 100% rename from daemon/client_mock.go rename to cli/daemon/client_mock.go diff --git a/daemon/client_test.go b/cli/daemon/client_test.go similarity index 98% rename from daemon/client_test.go rename to cli/daemon/client_test.go index 1d106833..07845eca 100644 --- a/daemon/client_test.go +++ b/cli/daemon/client_test.go @@ -15,10 +15,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/thingful/httpmock" - "proctor/config" - proc_metadata "proctor/shared/model/metadata" - "proctor/shared/model/metadata/env" + "proctor/cli/config" "proctor/shared/constant" + modelMetadata "proctor/shared/model/metadata" + "proctor/shared/model/metadata/env" ) type TestConnectionError struct { @@ -60,8 +60,8 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { var args = []env.VarMetadata{env.VarMetadata{Name: "ARG1", Description: "Argument name"}} var secrets = []env.VarMetadata{env.VarMetadata{Name: "SECRET1", Description: "Base64 encoded secret for authentication."}} envVars := env.Vars{Secrets: secrets, Args: args} - var expectedProcList = []proc_metadata.Metadata{ - proc_metadata.Metadata{ + var expectedProcList = []modelMetadata.Metadata{ + modelMetadata.Metadata{ Name: "job-1", Description: "job description", ImageName: "hub.docker.com/job-1:latest", @@ -122,7 +122,7 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { procList, err := s.testClient.ListProcs() - assert.Equal(t, []proc_metadata.Metadata{}, procList) + assert.Equal(t, []modelMetadata.Metadata{}, procList) assert.Error(t, err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) @@ -157,7 +157,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { procList, err := s.testClient.ListProcs() assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/jobs/metadata: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) - assert.Equal(t, []proc_metadata.Metadata{}, procList) + assert.Equal(t, []modelMetadata.Metadata{}, procList) s.mockConfigLoader.AssertExpectations(t) } @@ -190,7 +190,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { procList, err := s.testClient.ListProcs() assert.Equal(t, errors.New("Network Error!!!\nGet http://proctor.example.com/jobs/metadata: Unknown Error"), err) - assert.Equal(t, []proc_metadata.Metadata{}, procList) + assert.Equal(t, []modelMetadata.Metadata{}, procList) s.mockConfigLoader.AssertExpectations(t) } @@ -222,7 +222,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedUser() { procList, err := s.testClient.ListProcs() - assert.Equal(t, []proc_metadata.Metadata{}, procList) + assert.Equal(t, []modelMetadata.Metadata{}, procList) assert.Equal(t, "Unauthorized Access!!!\nPlease check the EMAIL_ID and ACCESS_TOKEN validity in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -253,7 +253,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() procList, err := s.testClient.ListProcs() - assert.Equal(t, []proc_metadata.Metadata{}, procList) + assert.Equal(t, []modelMetadata.Metadata{}, procList) assert.Equal(t, "Unauthorized Access!!!\nEMAIL_ID or ACCESS_TOKEN is not present in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) } diff --git a/exec/cli/cli.go b/exec/cli/cli.go index b5848122..f8d19117 100644 --- a/exec/cli/cli.go +++ b/exec/cli/cli.go @@ -3,8 +3,8 @@ package main import ( "proctor/cli/command" "proctor/cli/command/version/github" - "proctor/config" - "proctor/daemon" + "proctor/cli/config" + "proctor/cli/daemon" "proctor/shared/io" ) From 2e72893ef741b417df65e64c7513f80233c5d764 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 13:45:06 +0700 Subject: [PATCH 010/268] [Jasoet|Bimo] refactor: Fix generate Script --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 72953353..0ad3cbeb 100644 --- a/Makefile +++ b/Makefile @@ -38,7 +38,7 @@ cli: go build -o $(BIN_DIR)/cli ./exec/cli/cli.go generate: - go get github.com/go-bindata/go-bindata + go get -u github.com/go-bindata/go-bindata/... $(GOPATH)/bin/go-bindata -pkg config -o config/data.go data/config_template.yaml db.setup: db.create db.migrate From cfdff8131eb63b1ad741744af057d3a3df3e062f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 17:06:41 +0700 Subject: [PATCH 011/268] [Jasoet|Bimo] refactor: Move main file to particular directory --- Makefile | 4 +- cli/cli.go | 1 - exec/cli/cli.go => cli/main.go | 0 exec/server/server.go | 65 --------------------- proctord/jobs/execution/executioner_test.go | 4 +- proctord/jobs/metadata/handler.go | 2 +- proctord/jobs/metadata/store.go | 2 +- exec/server.go => proctord/main.go | 0 8 files changed, 6 insertions(+), 72 deletions(-) delete mode 100644 cli/cli.go rename exec/cli/cli.go => cli/main.go (100%) delete mode 100644 exec/server/server.go rename exec/server.go => proctord/main.go (100%) diff --git a/Makefile b/Makefile index 0ad3cbeb..4dd1bbdf 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ test: .PHONY: server server: - go build -o $(BIN_DIR)/server ./exec/server/server.go + go build -o $(BIN_DIR)/server ./proctord/main.go .PHONY: start-server start-server: @@ -35,7 +35,7 @@ start-server: .PHONY: cli cli: - go build -o $(BIN_DIR)/cli ./exec/cli/cli.go + go build -o $(BIN_DIR)/cli ./cli/main.go generate: go get -u github.com/go-bindata/go-bindata/... diff --git a/cli/cli.go b/cli/cli.go deleted file mode 100644 index 7f1e458c..00000000 --- a/cli/cli.go +++ /dev/null @@ -1 +0,0 @@ -package cli diff --git a/exec/cli/cli.go b/cli/main.go similarity index 100% rename from exec/cli/cli.go rename to cli/main.go diff --git a/exec/server/server.go b/exec/server/server.go deleted file mode 100644 index 35203bd8..00000000 --- a/exec/server/server.go +++ /dev/null @@ -1,65 +0,0 @@ -package main - -import ( - "os" - - "github.com/getsentry/raven-go" - "github.com/urfave/cli" - - "proctor/proctord/config" - "proctor/proctord/logger" - "proctor/proctord/scheduler" - "proctor/proctord/server" - "proctor/proctord/storage/postgres" -) - -func main() { - logger.Setup() - raven.SetDSN(config.SentryDSN()) - - proctord := cli.NewApp() - proctord.Name = "proctord" - proctord.Usage = "Handle executing jobs and maintaining their configuration" - proctord.Version = "0.2.0" - proctord.Commands = []cli.Command{ - { - Name: "migrate", - Description: "Run database migrations for proctord", - Action: func(c *cli.Context) { - err := postgres.Up() - if err != nil { - panic(err.Error()) - } - logger.Info("Migration successful") - }, - }, - { - Name: "rollback", - Description: "Rollback database migrations by one step for proctord", - Action: func(c *cli.Context) { - err := postgres.DownOneStep() - if err != nil { - panic(err.Error()) - } - logger.Info("Rollback successful") - }, - }, - { - Name: "start", - Aliases: []string{"s"}, - Usage: "starts server", - Action: func(c *cli.Context) error { - return server.Start() - }, - }, - { - Name: "start-scheduler", - Usage: "starts scheduler", - Action: func(c *cli.Context) error { - return scheduler.Start() - }, - }, - } - - proctord.Run(os.Args) -} diff --git a/proctord/jobs/execution/executioner_test.go b/proctord/jobs/execution/executioner_test.go index ec0a5f10..7fe54c4f 100644 --- a/proctord/jobs/execution/executioner_test.go +++ b/proctord/jobs/execution/executioner_test.go @@ -7,12 +7,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - modelMetadata "proctor/shared/model/metadata" - "proctor/shared/model/metadata" "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" utility "proctor/shared/constant" + "proctor/shared/model/metadata" + modelMetadata "proctor/shared/model/metadata" ) type ExecutionerTestSuite struct { diff --git a/proctord/jobs/metadata/handler.go b/proctord/jobs/metadata/handler.go index 0ab7d4c3..bcb6d264 100644 --- a/proctord/jobs/metadata/handler.go +++ b/proctord/jobs/metadata/handler.go @@ -4,9 +4,9 @@ import ( "encoding/json" "github.com/getsentry/raven-go" "net/http" - procMetadata "proctor/shared/model/metadata" "proctor/proctord/logger" utility "proctor/shared/constant" + procMetadata "proctor/shared/model/metadata" ) type handler struct { diff --git a/proctord/jobs/metadata/store.go b/proctord/jobs/metadata/store.go index 4a6c741f..4a7aa648 100644 --- a/proctord/jobs/metadata/store.go +++ b/proctord/jobs/metadata/store.go @@ -2,8 +2,8 @@ package metadata import ( "encoding/json" - modelMetadata "proctor/shared/model/metadata" "proctor/proctord/redis" + modelMetadata "proctor/shared/model/metadata" ) const JobNameKeySuffix = "-metadata" diff --git a/exec/server.go b/proctord/main.go similarity index 100% rename from exec/server.go rename to proctord/main.go From 199c55cd5bfe0acb9788479b1b431fe74082769f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 18:12:49 +0700 Subject: [PATCH 012/268] [Jasoet|Bimo] refactor: move non source code directory to assets --- .travis.yml | 2 +- Makefile | 2 +- {data => assets}/config_template.yaml | 0 {procs => assets/procs_example}/README.md | 0 {procs => assets/procs_example}/say-hello-world/Dockerfile | 0 {procs => assets/procs_example}/say-hello-world/metadata.json | 0 .../procs_example}/say-hello-world/say_hello_world.sh | 0 {scripts => assets/scripts}/proctor.rb.tpl | 0 {scripts => assets/scripts}/proctor_template.go | 0 {scripts => assets/scripts}/release.sh | 0 10 files changed, 2 insertions(+), 2 deletions(-) rename {data => assets}/config_template.yaml (100%) rename {procs => assets/procs_example}/README.md (100%) rename {procs => assets/procs_example}/say-hello-world/Dockerfile (100%) rename {procs => assets/procs_example}/say-hello-world/metadata.json (100%) rename {procs => assets/procs_example}/say-hello-world/say_hello_world.sh (100%) rename {scripts => assets/scripts}/proctor.rb.tpl (100%) rename {scripts => assets/scripts}/proctor_template.go (100%) rename {scripts => assets/scripts}/release.sh (100%) diff --git a/.travis.yml b/.travis.yml index 44daf6b7..c0eebe87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,4 @@ jobs: - make db.setup test-with-race after_success: - - scripts/release.sh + - assets/scripts/release.sh diff --git a/Makefile b/Makefile index 4dd1bbdf..38f2c45e 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ cli: generate: go get -u github.com/go-bindata/go-bindata/... - $(GOPATH)/bin/go-bindata -pkg config -o config/data.go data/config_template.yaml + $(GOPATH)/bin/go-bindata -pkg config -o cli/config/data.go assets/config_template.yaml db.setup: db.create db.migrate diff --git a/data/config_template.yaml b/assets/config_template.yaml similarity index 100% rename from data/config_template.yaml rename to assets/config_template.yaml diff --git a/procs/README.md b/assets/procs_example/README.md similarity index 100% rename from procs/README.md rename to assets/procs_example/README.md diff --git a/procs/say-hello-world/Dockerfile b/assets/procs_example/say-hello-world/Dockerfile similarity index 100% rename from procs/say-hello-world/Dockerfile rename to assets/procs_example/say-hello-world/Dockerfile diff --git a/procs/say-hello-world/metadata.json b/assets/procs_example/say-hello-world/metadata.json similarity index 100% rename from procs/say-hello-world/metadata.json rename to assets/procs_example/say-hello-world/metadata.json diff --git a/procs/say-hello-world/say_hello_world.sh b/assets/procs_example/say-hello-world/say_hello_world.sh similarity index 100% rename from procs/say-hello-world/say_hello_world.sh rename to assets/procs_example/say-hello-world/say_hello_world.sh diff --git a/scripts/proctor.rb.tpl b/assets/scripts/proctor.rb.tpl similarity index 100% rename from scripts/proctor.rb.tpl rename to assets/scripts/proctor.rb.tpl diff --git a/scripts/proctor_template.go b/assets/scripts/proctor_template.go similarity index 100% rename from scripts/proctor_template.go rename to assets/scripts/proctor_template.go diff --git a/scripts/release.sh b/assets/scripts/release.sh similarity index 100% rename from scripts/release.sh rename to assets/scripts/release.sh From 76bccb2db954d69af2e33ff4855c677271a615fb Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 18:13:37 +0700 Subject: [PATCH 013/268] [Jasoet|Bimo] refactor: fix import error for test codes --- proctord/jobs/execution/executioner_test.go | 8 +-- proctord/jobs/logs/handler_test.go | 5 +- proctord/jobs/metadata/handler_test.go | 21 ++++---- proctord/jobs/metadata/store_test.go | 13 ++--- proctord/jobs/schedule/handler_test.go | 60 +++++++++++---------- proctord/mail/mailer_test.go | 2 +- 6 files changed, 57 insertions(+), 52 deletions(-) diff --git a/proctord/jobs/execution/executioner_test.go b/proctord/jobs/execution/executioner_test.go index 7fe54c4f..9cdaa585 100644 --- a/proctord/jobs/execution/executioner_test.go +++ b/proctord/jobs/execution/executioner_test.go @@ -7,25 +7,25 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + jobsMetadata "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/secrets" "proctor/proctord/kubernetes" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" "proctor/shared/model/metadata" - modelMetadata "proctor/shared/model/metadata" + "proctor/shared/utility" ) type ExecutionerTestSuite struct { suite.Suite mockKubeClient kubernetes.MockClient - mockMetadataStore *modelMetadata.MockStore + mockMetadataStore *jobsMetadata.MockStore mockSecretsStore *secrets.MockStore testExecutioner Executioner } func (suite *ExecutionerTestSuite) SetupTest() { suite.mockKubeClient = kubernetes.MockClient{} - suite.mockMetadataStore = &metadata.MockStore{} + suite.mockMetadataStore = &jobsMetadata.MockStore{} suite.mockSecretsStore = &secrets.MockStore{} suite.testExecutioner = NewExecutioner(&suite.mockKubeClient, suite.mockMetadataStore, suite.mockSecretsStore) } diff --git a/proctord/jobs/logs/handler_test.go b/proctord/jobs/logs/handler_test.go index 1b299237..259a251c 100644 --- a/proctord/jobs/logs/handler_test.go +++ b/proctord/jobs/logs/handler_test.go @@ -12,7 +12,8 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "proctor/proctord/kubernetes" - utility "proctor/shared/constant" + "proctor/shared/constant" + "proctor/shared/utility" ) type LoggerTestSuite struct { @@ -95,7 +96,7 @@ func (suite *LoggerTestSuite) TestLoggerStreamConnectionUpgradeFailure() { suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, "Bad Request\n"+utility.ClientError, responseRecorder.Body.String()) + assert.Equal(t, "Bad Request\n"+constant.ClientError, responseRecorder.Body.String()) } func (suite *LoggerTestSuite) TestLoggerStreamForNoJobName() { diff --git a/proctord/jobs/metadata/handler_test.go b/proctord/jobs/metadata/handler_test.go index 7c9d7aae..2bd4c97e 100644 --- a/proctord/jobs/metadata/handler_test.go +++ b/proctord/jobs/metadata/handler_test.go @@ -14,7 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - utility "proctor/shared/constant" + "proctor/shared/constant" + modelMetadata "proctor/shared/model/metadata" ) type MetadataHandlerTestSuite struct { @@ -51,7 +52,7 @@ func (s *MetadataHandlerTestSuite) TestSuccessfulMetadataSubmission() { Secrets: secrets, Args: args, } - metadata := Metadata{ + metadata := modelMetadata.Metadata{ Name: "run-sample", Description: "This is a hello world script", ImageName: "proctor-jobs-run-sample", @@ -62,7 +63,7 @@ func (s *MetadataHandlerTestSuite) TestSuccessfulMetadataSubmission() { Organization: "Test Org", } - jobsMetadata := []Metadata{metadata} + jobsMetadata := []modelMetadata.Metadata{metadata} metadataSubmissionRequestBody, err := json.Marshal(jobsMetadata) assert.NoError(t, err) @@ -90,15 +91,15 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionMalformedRequest() { s.mockStore.AssertNotCalled(t, "CreateOrUpdateJobMetadata", mock.Anything) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, utility.ClientError, responseRecorder.Body.String()) + assert.Equal(t, constant.ClientError, responseRecorder.Body.String()) } func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionForStoreFailure() { t := s.T() - metadata := Metadata{} + metadata := modelMetadata.Metadata{} - jobMetadata := []Metadata{metadata} + jobMetadata := []modelMetadata.Metadata{metadata} metadataSubmissionRequestBody, err := json.Marshal(jobMetadata) assert.NoError(t, err) @@ -112,7 +113,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionForStoreFailure() { s.mockStore.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, utility.ServerError, responseRecorder.Body.String()) + assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) } func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { @@ -121,7 +122,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { req := httptest.NewRequest("GET", "/jobs/metadata", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() - jobsMetadata := []Metadata{} + jobsMetadata := []modelMetadata.Metadata{} s.mockStore.On("GetAllJobsMetadata").Return(jobsMetadata, nil).Once() s.testMetadataHandler.HandleBulkDisplay()(responseRecorder, req) @@ -141,7 +142,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { req := httptest.NewRequest("GET", "/jobs/metadata", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() - jobsMetadata := []Metadata{} + jobsMetadata := []modelMetadata.Metadata{} s.mockStore.On("GetAllJobsMetadata").Return(jobsMetadata, errors.New("error")).Once() s.testMetadataHandler.HandleBulkDisplay()(responseRecorder, req) @@ -149,7 +150,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { s.mockStore.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, utility.ServerError, responseRecorder.Body.String()) + assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) } func TestMetadataHandlerTestSuite(t *testing.T) { diff --git a/proctord/jobs/metadata/store_test.go b/proctord/jobs/metadata/store_test.go index b108fc77..203f7cdd 100644 --- a/proctord/jobs/metadata/store_test.go +++ b/proctord/jobs/metadata/store_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "proctor/proctord/redis" + modelMetadata "proctor/shared/model/metadata" ) type MetadataStoreTestSuite struct { @@ -26,7 +27,7 @@ func (s *MetadataStoreTestSuite) SetupTest() { func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadata() { t := s.T() - metadata := Metadata{ + metadata := modelMetadata.Metadata{ Name: "any-name", ImageName: "any-image-name", Description: "any-description", @@ -48,7 +49,7 @@ func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadata() { func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadataForRedisClientFailure() { t := s.T() - metadata := Metadata{} + metadata := modelMetadata.Metadata{} expectedError := errors.New("any-error") s.mockRedisClient.On("SET", mock.Anything, mock.Anything).Return(expectedError).Once() @@ -61,7 +62,7 @@ func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadataForRedisClientFail func (s *MetadataStoreTestSuite) TestGetAllJobsMetadata() { t := s.T() - metadata1 := Metadata{ + metadata1 := modelMetadata.Metadata{ Name: "job1", ImageName: "job1-image-name", Description: "desc1", @@ -69,7 +70,7 @@ func (s *MetadataStoreTestSuite) TestGetAllJobsMetadata() { Contributors: "Test User Date: Tue, 18 Jun 2019 18:15:42 +0700 Subject: [PATCH 014/268] [Jasoet|Bimo] refactor: Cleanup cli codes --- cli/config/data.go | 4 ++-- cli/daemon/client_test.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/config/data.go b/cli/config/data.go index 7f433388..ba202b8b 100644 --- a/cli/config/data.go +++ b/cli/config/data.go @@ -184,8 +184,8 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "data": &bintree{nil, map[string]*bintree{ - "config_template.yaml": &bintree{dataConfig_templateYaml, map[string]*bintree{}}, + "data": {nil, map[string]*bintree{ + "config_template.yaml": {dataConfig_templateYaml, map[string]*bintree{}}, }}, }} diff --git a/cli/daemon/client_test.go b/cli/daemon/client_test.go index 07845eca..e58721dd 100644 --- a/cli/daemon/client_test.go +++ b/cli/daemon/client_test.go @@ -57,11 +57,11 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { defer httpmock.DeactivateAndReset() body := `[ { "name": "job-1", "description": "job description", "image_name": "hub.docker.com/job-1:latest", "env_vars": { "secrets": [ { "name": "SECRET1", "description": "Base64 encoded secret for authentication." } ], "args": [ { "name": "ARG1", "description": "Argument name" } ] } } ]` - var args = []env.VarMetadata{env.VarMetadata{Name: "ARG1", Description: "Argument name"}} - var secrets = []env.VarMetadata{env.VarMetadata{Name: "SECRET1", Description: "Base64 encoded secret for authentication."}} + var args = []env.VarMetadata{{Name: "ARG1", Description: "Argument name"}} + var secrets = []env.VarMetadata{{Name: "SECRET1", Description: "Base64 encoded secret for authentication."}} envVars := env.Vars{Secrets: secrets, Args: args} var expectedProcList = []modelMetadata.Metadata{ - modelMetadata.Metadata{ + { Name: "job-1", Description: "job description", ImageName: "hub.docker.com/job-1:latest", From 43f537222d4f4bb92c0835ea36d8ba08243fff88 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 18 Jun 2019 18:17:21 +0700 Subject: [PATCH 015/268] [Jasoet|Bimo] refactor: cleanup proctord --- proctord/jobs/metadata/handler_test.go | 4 ++-- proctord/jobs/schedule/handler_test.go | 4 ++-- proctord/jobs/schedule/worker.go | 2 +- proctord/jobs/schedule/worker_test.go | 8 ++++---- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/proctord/jobs/metadata/handler_test.go b/proctord/jobs/metadata/handler_test.go index 2bd4c97e..6f971580 100644 --- a/proctord/jobs/metadata/handler_test.go +++ b/proctord/jobs/metadata/handler_test.go @@ -37,13 +37,13 @@ func (s *MetadataHandlerTestSuite) TestSuccessfulMetadataSubmission() { t := s.T() secrets := []env.VarMetadata{ - env.VarMetadata{ + { Name: "SAMPLE_SECRET", Description: "description of secret", }, } args := []env.VarMetadata{ - env.VarMetadata{ + { Name: "SAMPLE_ARG", Description: "description of arg", }, diff --git a/proctord/jobs/schedule/handler_test.go b/proctord/jobs/schedule/handler_test.go index 8237f46f..d368b514 100644 --- a/proctord/jobs/schedule/handler_test.go +++ b/proctord/jobs/schedule/handler_test.go @@ -294,7 +294,7 @@ func (s *SchedulerTestSuite) TestGetScheduledJobs() { responseRecorder := httptest.NewRecorder() scheduledJobsStoreFormat := []postgres.JobsSchedule{ - postgres.JobsSchedule{ + { ID: "some-id", }, } @@ -350,7 +350,7 @@ func (s *SchedulerTestSuite) TestGetScheduledJobByID() { jobID := "some-id" scheduledJobsStoreFormat := []postgres.JobsSchedule{ - postgres.JobsSchedule{ + { ID: jobID, }, } diff --git a/proctord/jobs/schedule/worker.go b/proctord/jobs/schedule/worker.go index f0098d39..21466bec 100644 --- a/proctord/jobs/schedule/worker.go +++ b/proctord/jobs/schedule/worker.go @@ -122,7 +122,7 @@ func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Sig } } case <-signalsChan: - for id, _ := range worker.inMemoryScheduledJobs { + for id := range worker.inMemoryScheduledJobs { worker.disableScheduledJobIfItExists(id) } //TODO: wait for all active executions to complete diff --git a/proctord/jobs/schedule/worker_test.go b/proctord/jobs/schedule/worker_test.go index 077cb375..8acf2840 100644 --- a/proctord/jobs/schedule/worker_test.go +++ b/proctord/jobs/schedule/worker_test.go @@ -49,7 +49,7 @@ func (suite *WorkerTestSuite) TestCronEnablingForScheduledJobs() { disabledJob := "some-job-two" notificationEmails := "foo@bar.com,goo@bar.com" scheduledJobs := []postgres.JobsSchedule{ - postgres.JobsSchedule{ + { ID: "some-uuid-one", Enabled: true, Time: "*/1 * * * * *", @@ -57,7 +57,7 @@ func (suite *WorkerTestSuite) TestCronEnablingForScheduledJobs() { Args: base64.StdEncoding.EncodeToString(jsonEncodedJobArgs), NotificationEmails: notificationEmails, }, - postgres.JobsSchedule{ + { ID: "some-uuid-two", Enabled: false, Time: "*/1 * * * * *", @@ -109,7 +109,7 @@ func (suite *WorkerTestSuite) TestCronForDisablingEnabledScheduledJobs() { jobName := "some-job-one" notificationEmails := "foo@bar.com,goo@bar.com" scheduledJobs := []postgres.JobsSchedule{ - postgres.JobsSchedule{ + { ID: "some-uuid-one", Enabled: true, Time: "*/1 * * * * *", @@ -120,7 +120,7 @@ func (suite *WorkerTestSuite) TestCronForDisablingEnabledScheduledJobs() { } disabledScheduledJobs := []postgres.JobsSchedule{ - postgres.JobsSchedule{ + { ID: "some-uuid-one", Enabled: false, Time: "*/1 * * * * *", From 90462ea8436132bf3f590b84f548e4b8ff9618f8 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 20 Jun 2019 12:26:26 +0700 Subject: [PATCH 016/268] [Jasoet|Bimo] refactor: cleanup unreported error --- proctord/audit/auditor.go | 6 +- proctord/audit/auditor_test.go | 6 +- proctord/config/config.go | 2 +- proctord/config/config_test.go | 62 ++++---- proctord/jobs/execution/handler.go | 24 +-- proctord/jobs/execution/handler_test.go | 48 +++--- proctord/jobs/logs/handler.go | 4 +- proctord/jobs/logs/handler_test.go | 2 +- proctord/jobs/metadata/handler.go | 14 +- proctord/jobs/schedule/handler.go | 60 +++---- proctord/jobs/schedule/handler_test.go | 150 +++++++++--------- proctord/jobs/schedule/worker_test.go | 6 +- proctord/jobs/secrets/handler.go | 6 +- proctord/jobs/secrets/handler_test.go | 6 +- proctord/kubernetes/client.go | 12 +- proctord/kubernetes/client_test.go | 2 +- proctord/main.go | 4 +- .../middleware/validate_client_version.go | 6 +- .../validate_client_version_test.go | 12 +- proctord/redis/client_test.go | 2 +- proctord/scheduler/scheduler.go | 6 +- proctord/server/api.go | 2 +- proctord/server/router.go | 6 +- proctord/storage/store_test.go | 6 +- shared/utility/utils.go | 2 +- 25 files changed, 228 insertions(+), 228 deletions(-) diff --git a/proctord/audit/auditor.go b/proctord/audit/auditor.go index 609b5bc7..515e3c98 100644 --- a/proctord/audit/auditor.go +++ b/proctord/audit/auditor.go @@ -6,7 +6,7 @@ import ( "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" + "proctor/shared/constant" ) type Auditor interface { @@ -30,9 +30,9 @@ func New(store storage.Store, kubeClient kubernetes.Client) Auditor { func (auditor *auditor) JobsExecutionAndStatus(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog) { auditor.JobsExecution(jobsExecutionAuditLog) - if jobsExecutionAuditLog.JobSubmissionStatus == utility.JobSubmissionSuccess && + if jobsExecutionAuditLog.JobSubmissionStatus == constant.JobSubmissionSuccess && jobsExecutionAuditLog.ExecutionID.Valid { - auditor.JobsExecutionStatus(jobsExecutionAuditLog.ExecutionID.String) + _, _ = auditor.JobsExecutionStatus(jobsExecutionAuditLog.ExecutionID.String) } } diff --git a/proctord/audit/auditor_test.go b/proctord/audit/auditor_test.go index 80d6c8ce..9924af9b 100644 --- a/proctord/audit/auditor_test.go +++ b/proctord/audit/auditor_test.go @@ -6,7 +6,7 @@ import ( "proctor/proctord/kubernetes" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" + "proctor/shared/constant" ) func TestJobsExecutionAuditing(t *testing.T) { @@ -36,7 +36,7 @@ func TestAuditJobsExecutionStatusAuditing(t *testing.T) { mockKubeClient.On("JobExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) mockStore.On("UpdateJobsExecutionAuditLog", jobExecutionID, jobExecutionStatus).Return(nil).Once() - testAuditor.JobsExecutionStatus(jobExecutionID) + _,_ = testAuditor.JobsExecutionStatus(jobExecutionID) mockStore.AssertExpectations(t) mockKubeClient.AssertExpectations(t) @@ -52,7 +52,7 @@ func TestAuditJobsExecutionAndStatusAuditing(t *testing.T) { jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ JobName: "any-job-name", ExecutionID: postgres.StringToSQLString(jobExecutionID), - JobSubmissionStatus: utility.JobSubmissionSuccess, + JobSubmissionStatus: constant.JobSubmissionSuccess, } mockStore.On("AuditJobsExecution", jobsExecutionAuditLog).Return(nil).Once() diff --git a/proctord/config/config.go b/proctord/config/config.go index 96384d45..9097591b 100644 --- a/proctord/config/config.go +++ b/proctord/config/config.go @@ -135,7 +135,7 @@ func JobPodAnnotations() map[string]string { err := json.Unmarshal(jsonStr, &annotations) if err != nil { - fmt.Errorf(err.Error(), "Invalid value for key PROCTOR_JOB_POD_ANNOTATIONS") + _ = fmt.Errorf(err.Error(), "Invalid value for key PROCTOR_JOB_POD_ANNOTATIONS") } return annotations diff --git a/proctord/config/config_test.go b/proctord/config/config_test.go index 122ccb80..52542b81 100644 --- a/proctord/config/config_test.go +++ b/proctord/config/config_test.go @@ -9,7 +9,7 @@ import ( ) func TestEnvironment(t *testing.T) { - os.Setenv("PROCTOR_KUBE_CONFIG", "in-cluster") + _ = os.Setenv("PROCTOR_KUBE_CONFIG", "in-cluster") viper.AutomaticEnv() @@ -17,7 +17,7 @@ func TestEnvironment(t *testing.T) { } func TestLogLevel(t *testing.T) { - os.Setenv("PROCTOR_LOG_LEVEL", "debug") + _ = os.Setenv("PROCTOR_LOG_LEVEL", "debug") viper.AutomaticEnv() @@ -25,7 +25,7 @@ func TestLogLevel(t *testing.T) { } func TestAppPort(t *testing.T) { - os.Setenv("PROCTOR_APP_PORT", "3000") + _ = os.Setenv("PROCTOR_APP_PORT", "3000") viper.AutomaticEnv() @@ -33,7 +33,7 @@ func TestAppPort(t *testing.T) { } func TestDefaultNamespace(t *testing.T) { - os.Setenv("PROCTOR_DEFAULT_NAMESPACE", "default") + _ = os.Setenv("PROCTOR_DEFAULT_NAMESPACE", "default") viper.AutomaticEnv() @@ -41,7 +41,7 @@ func TestDefaultNamespace(t *testing.T) { } func TestRedisAddress(t *testing.T) { - os.Setenv("PROCTOR_REDIS_ADDRESS", "localhost:6379") + _ = os.Setenv("PROCTOR_REDIS_ADDRESS", "localhost:6379") viper.AutomaticEnv() @@ -49,7 +49,7 @@ func TestRedisAddress(t *testing.T) { } func TestKubeClusterHostName(t *testing.T) { - os.Setenv("PROCTOR_KUBE_CLUSTER_HOST_NAME", "somekube.io") + _ = os.Setenv("PROCTOR_KUBE_CLUSTER_HOST_NAME", "somekube.io") viper.AutomaticEnv() @@ -57,7 +57,7 @@ func TestKubeClusterHostName(t *testing.T) { } func TestKubeCACertEncoded(t *testing.T) { - os.Setenv("PROCTOR_KUBE_CA_CERT_ENCODED", "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K") + _ = os.Setenv("PROCTOR_KUBE_CA_CERT_ENCODED", "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K") viper.AutomaticEnv() @@ -65,7 +65,7 @@ func TestKubeCACertEncoded(t *testing.T) { } func TestKubeBasicAuthEncoded(t *testing.T) { - os.Setenv("PROCTOR_KUBE_BASIC_AUTH_ENCODED", "YWRtaW46cGFzc3dvcmQK") + _ = os.Setenv("PROCTOR_KUBE_BASIC_AUTH_ENCODED", "YWRtaW46cGFzc3dvcmQK") viper.AutomaticEnv() @@ -73,7 +73,7 @@ func TestKubeBasicAuthEncoded(t *testing.T) { } func TestRedisMaxActiveConnections(t *testing.T) { - os.Setenv("PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS", "50") + _ = os.Setenv("PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS", "50") viper.AutomaticEnv() @@ -81,7 +81,7 @@ func TestRedisMaxActiveConnections(t *testing.T) { } func TestLogsStreamReadBufferSize(t *testing.T) { - os.Setenv("PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE", "140") + _ = os.Setenv("PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE", "140") viper.AutomaticEnv() @@ -89,7 +89,7 @@ func TestLogsStreamReadBufferSize(t *testing.T) { } func TestLogsStreamWriteBufferSize(t *testing.T) { - os.Setenv("PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE", "4096") + _ = os.Setenv("PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE", "4096") viper.AutomaticEnv() @@ -97,7 +97,7 @@ func TestLogsStreamWriteBufferSize(t *testing.T) { } func TestKubeJobActiveDeadlineSeconds(t *testing.T) { - os.Setenv("PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS", "900") + _ = os.Setenv("PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS", "900") viper.AutomaticEnv() @@ -106,7 +106,7 @@ func TestKubeJobActiveDeadlineSeconds(t *testing.T) { } func TestKubeJobRetries(t *testing.T) { - os.Setenv("PROCTOR_KUBE_JOB_RETRIES", "0") + _ = os.Setenv("PROCTOR_KUBE_JOB_RETRIES", "0") viper.AutomaticEnv() @@ -115,7 +115,7 @@ func TestKubeJobRetries(t *testing.T) { } func TestPostgresUser(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_USER", "postgres") + _ = os.Setenv("PROCTOR_POSTGRES_USER", "postgres") viper.AutomaticEnv() @@ -123,7 +123,7 @@ func TestPostgresUser(t *testing.T) { } func TestPostgresPassword(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_PASSWORD", "ipsum-lorem") + _ = os.Setenv("PROCTOR_POSTGRES_PASSWORD", "ipsum-lorem") viper.AutomaticEnv() @@ -131,7 +131,7 @@ func TestPostgresPassword(t *testing.T) { } func TestPostgresHost(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_HOST", "localhost") + _ = os.Setenv("PROCTOR_POSTGRES_HOST", "localhost") viper.AutomaticEnv() @@ -139,7 +139,7 @@ func TestPostgresHost(t *testing.T) { } func TestPostgresPort(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_PORT", "5432") + _ = os.Setenv("PROCTOR_POSTGRES_PORT", "5432") viper.AutomaticEnv() @@ -147,7 +147,7 @@ func TestPostgresPort(t *testing.T) { } func TestPostgresDatabase(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_DATABASE", "proctord_development") + _ = os.Setenv("PROCTOR_POSTGRES_DATABASE", "proctord_development") viper.AutomaticEnv() @@ -155,7 +155,7 @@ func TestPostgresDatabase(t *testing.T) { } func TestPostgresMaxConnections(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_MAX_CONNECTIONS", "50") + _ = os.Setenv("PROCTOR_POSTGRES_MAX_CONNECTIONS", "50") viper.AutomaticEnv() @@ -163,7 +163,7 @@ func TestPostgresMaxConnections(t *testing.T) { } func TestPostgresConnectionMaxLifetime(t *testing.T) { - os.Setenv("PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME", "30") + _ = os.Setenv("PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME", "30") viper.AutomaticEnv() @@ -171,7 +171,7 @@ func TestPostgresConnectionMaxLifetime(t *testing.T) { } func TestNewRelicAppName(t *testing.T) { - os.Setenv("PROCTOR_NEW_RELIC_APP_NAME", "PROCTORD") + _ = os.Setenv("PROCTOR_NEW_RELIC_APP_NAME", "PROCTORD") viper.AutomaticEnv() @@ -179,7 +179,7 @@ func TestNewRelicAppName(t *testing.T) { } func TestNewRelicLicenceKey(t *testing.T) { - os.Setenv("PROCTOR_NEW_RELIC_LICENCE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") + _ = os.Setenv("PROCTOR_NEW_RELIC_LICENCE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") viper.AutomaticEnv() @@ -187,7 +187,7 @@ func TestNewRelicLicenceKey(t *testing.T) { } func TestMinClientVersion(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") + _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") viper.AutomaticEnv() @@ -195,7 +195,7 @@ func TestMinClientVersion(t *testing.T) { } func TestScheduledJobsFetchIntervalInMins(t *testing.T) { - os.Setenv("PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS", "5") + _ = os.Setenv("PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS", "5") viper.AutomaticEnv() @@ -203,7 +203,7 @@ func TestScheduledJobsFetchIntervalInMins(t *testing.T) { } func TestMailUsername(t *testing.T) { - os.Setenv("PROCTOR_MAIL_USERNAME", "foo@bar.com") + _ = os.Setenv("PROCTOR_MAIL_USERNAME", "foo@bar.com") viper.AutomaticEnv() @@ -211,7 +211,7 @@ func TestMailUsername(t *testing.T) { } func TestMailPassword(t *testing.T) { - os.Setenv("PROCTOR_MAIL_PASSWORD", "password") + _ = os.Setenv("PROCTOR_MAIL_PASSWORD", "password") viper.AutomaticEnv() @@ -219,7 +219,7 @@ func TestMailPassword(t *testing.T) { } func TestMailServerHost(t *testing.T) { - os.Setenv("PROCTOR_MAIL_SERVER_HOST", "127.0.0.1") + _ = os.Setenv("PROCTOR_MAIL_SERVER_HOST", "127.0.0.1") viper.AutomaticEnv() @@ -227,7 +227,7 @@ func TestMailServerHost(t *testing.T) { } func TestMailServerPort(t *testing.T) { - os.Setenv("PROCTOR_MAIL_SERVER_PORT", "123") + _ = os.Setenv("PROCTOR_MAIL_SERVER_PORT", "123") viper.AutomaticEnv() @@ -235,7 +235,7 @@ func TestMailServerPort(t *testing.T) { } func TestJobPodAnnotations(t *testing.T) { - os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") viper.AutomaticEnv() @@ -243,7 +243,7 @@ func TestJobPodAnnotations(t *testing.T) { } func TestSentryDSN(t *testing.T) { - os.Setenv("PROCTOR_SENTRY_DSN", "domain") + _ = os.Setenv("PROCTOR_SENTRY_DSN", "domain") viper.AutomaticEnv() @@ -251,7 +251,7 @@ func TestSentryDSN(t *testing.T) { } func TestDocsPath(t *testing.T) { - os.Setenv("PROCTOR_DOCS_PATH", "path1") + _ = os.Setenv("PROCTOR_DOCS_PATH", "path1") viper.AutomaticEnv() diff --git a/proctord/jobs/execution/handler.go b/proctord/jobs/execution/handler.go index faa3f4d1..4860bcef 100644 --- a/proctord/jobs/execution/handler.go +++ b/proctord/jobs/execution/handler.go @@ -11,7 +11,7 @@ import ( "proctor/proctord/logger" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" + "proctor/shared/constant" "time" ) @@ -41,7 +41,7 @@ func (handler *executionHandler) Status() http.HandlerFunc { jobExecutionStatus, err := handler.store.GetJobExecutionStatus(jobExecutionID) if err != nil { w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) logger.Error(fmt.Sprintf("Error getting job status for job_id: %s", jobExecutionID), err.Error()) raven.CaptureError(err, map[string]string{"job_id": jobExecutionID}) @@ -53,7 +53,7 @@ func (handler *executionHandler) Status() http.HandlerFunc { return } - fmt.Fprintf(w, jobExecutionStatus) + _, _ = fmt.Fprintf(w, jobExecutionStatus) } } @@ -63,7 +63,7 @@ func (handler *executionHandler) Handle() http.HandlerFunc { JobExecutionStatus: "WAITING", } - userEmail := req.Header.Get(utility.UserEmailHeaderKey) + userEmail := req.Header.Get(constant.UserEmailHeaderKey) jobsExecutionAuditLog.UserEmail = userEmail var job Job @@ -74,10 +74,10 @@ func (handler *executionHandler) Handle() http.HandlerFunc { raven.CaptureError(err, map[string]string{"user_email": userEmail}) jobsExecutionAuditLog.Errors = fmt.Sprintf("Error parsing request body: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionClientError + jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionClientError w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.ClientError)) + _, _ = w.Write([]byte(constant.ClientError)) go handler.auditor.JobsExecutionAndStatus(jobsExecutionAuditLog) return @@ -88,17 +88,17 @@ func (handler *executionHandler) Handle() http.HandlerFunc { raven.CaptureError(err, map[string]string{"user_email": userEmail, "job_name": job.Name}) jobsExecutionAuditLog.Errors = fmt.Sprintf("Error executing job: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = utility.JobSubmissionServerError + jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionServerError go handler.auditor.JobsExecutionAndStatus(jobsExecutionAuditLog) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } w.WriteHeader(http.StatusCreated) - w.Write([]byte(fmt.Sprintf("{ \"name\":\"%s\" }", jobExecutionID))) + _, _ = w.Write([]byte(fmt.Sprintf("{ \"name\":\"%s\" }", jobExecutionID))) remoteCallerURL := job.CallbackURL go handler.postJobExecute(jobsExecutionAuditLog, remoteCallerURL, jobExecutionID) @@ -114,16 +114,16 @@ func (handler *executionHandler) postJobExecute(jobsExecutionAuditLog *postgres. } func (handler *executionHandler) sendStatusToCaller(remoteCallerURL, jobExecutionID string) { - status := utility.JobWaiting + status := constant.JobWaiting for { jobExecutionStatus, _ := handler.store.GetJobExecutionStatus(jobExecutionID) if jobExecutionStatus == "" { - status = utility.JobNotFound + status = constant.JobNotFound break } - if jobExecutionStatus == utility.JobSucceeded || jobExecutionStatus == utility.JobFailed { + if jobExecutionStatus == constant.JobSucceeded || jobExecutionStatus == constant.JobFailed { status = jobExecutionStatus break } diff --git a/proctord/jobs/execution/handler_test.go b/proctord/jobs/execution/handler_test.go index bc1f0dd3..037fbf17 100644 --- a/proctord/jobs/execution/handler_test.go +++ b/proctord/jobs/execution/handler_test.go @@ -14,7 +14,7 @@ import ( "net/http/httptest" "proctor/proctord/audit" "proctor/proctord/storage" - utility "proctor/shared/constant" + "proctor/shared/constant" "testing" ) @@ -59,7 +59,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandler() { w.WriteHeader(http.StatusOK) assert.Equal(t, jobExecutionID, value["name"]) - assert.Equal(t, utility.JobSucceeded, value["status"]) + assert.Equal(t, constant.JobSucceeded, value["status"]) statusChan <- true })) @@ -78,7 +78,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandler() { assert.NoError(t, err) req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) - req.Header.Set(utility.UserEmailHeaderKey, userEmail) + req.Header.Set(constant.UserEmailHeaderKey, userEmail) responseRecorder := httptest.NewRecorder() suite.mockExecutioner.On("Execute", mock.Anything, job.Name, job.Args).Return(jobExecutionID, nil).Once() @@ -88,7 +88,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandler() { suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( func(args mock.Arguments) { auditingChan <- true }, ) - suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(utility.JobSucceeded, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(constant.JobSucceeded, nil).Once() suite.testExecutionHandler.Handle()(responseRecorder, req) @@ -121,7 +121,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandlerWithout assert.NoError(t, err) req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) - req.Header.Set(utility.UserEmailHeaderKey, userEmail) + req.Header.Set(constant.UserEmailHeaderKey, userEmail) responseRecorder := httptest.NewRecorder() suite.mockExecutioner.On("Execute", mock.Anything, job.Name, job.Args).Return(jobExecutionID, nil).Once() @@ -131,7 +131,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandlerWithout suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( func(args mock.Arguments) { auditingChan <- true }, ) - suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(utility.JobSucceeded, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(constant.JobSucceeded, nil).Once() suite.testExecutionHandler.Handle()(responseRecorder, req) @@ -161,7 +161,7 @@ func (suite *ExecutionHandlerTestSuite) TestJobExecutionOnMalformedRequest() { suite.mockAuditor.AssertExpectations(t) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, utility.ClientError, responseRecorder.Body.String()) + assert.Equal(t, constant.ClientError, responseRecorder.Body.String()) } func (suite *ExecutionHandlerTestSuite) TestJobExecutionServerFailure() { @@ -195,7 +195,7 @@ func (suite *ExecutionHandlerTestSuite) TestJobExecutionServerFailure() { suite.mockExecutioner.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, utility.ServerError, responseRecorder.Body.String()) + assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) } func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200OnSuccess() { @@ -205,7 +205,7 @@ func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200OnSuccess() url := fmt.Sprintf("%s/jobs/execute/%s/status", suite.TestServer.URL, jobName) - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobSucceeded, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobSucceeded, nil).Once() req, _ := http.NewRequest("GET", url, nil) @@ -214,9 +214,9 @@ func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200OnSuccess() assert.Equal(suite.T(), http.StatusOK, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) jobStatus := buf.String() - assert.Equal(suite.T(), utility.JobSucceeded, jobStatus) + assert.Equal(suite.T(), constant.JobSucceeded, jobStatus) } func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn404IfJobStatusIsNotFound() { @@ -251,9 +251,9 @@ func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200IfJobStatusI assert.Equal(suite.T(), http.StatusOK, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) jobStatus := buf.String() - assert.Equal(suite.T(), utility.JobWaiting, jobStatus) + assert.Equal(suite.T(), constant.JobWaiting, jobStatus) } func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn500OnError() { @@ -287,14 +287,14 @@ func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnSuccess() { w.WriteHeader(http.StatusOK) assert.Equal(t, jobName, value["name"]) - assert.Equal(t, utility.JobSucceeded, value["status"]) + assert.Equal(t, constant.JobSucceeded, value["status"]) })) defer ts.Close() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobSucceeded, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobSucceeded, nil).Once() remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) @@ -317,14 +317,14 @@ func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnFailure() { w.WriteHeader(http.StatusOK) assert.Equal(t, jobName, value["name"]) - assert.Equal(t, utility.JobFailed, value["status"]) + assert.Equal(t, constant.JobFailed, value["status"]) })) defer ts.Close() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(utility.JobFailed, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() + suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobFailed, nil).Once() remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) @@ -347,7 +347,7 @@ func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnJobNotFound() { w.WriteHeader(http.StatusOK) assert.Equal(t, jobName, value["name"]) - assert.Equal(t, utility.JobNotFound, value["status"]) + assert.Equal(t, constant.JobNotFound, value["status"]) })) defer ts.Close() diff --git a/proctord/jobs/logs/handler.go b/proctord/jobs/logs/handler.go index 57c1f3cd..65db0523 100644 --- a/proctord/jobs/logs/handler.go +++ b/proctord/jobs/logs/handler.go @@ -10,7 +10,7 @@ import ( "proctor/proctord/config" "proctor/proctord/kubernetes" _logger "proctor/proctord/logger" - utility "proctor/shared/constant" + "proctor/shared/constant" "github.com/gorilla/websocket" ) @@ -51,7 +51,7 @@ func (l *logger) Stream() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.ClientError)) + _, _ = w.Write([]byte(constant.ClientError)) return } defer conn.Close() diff --git a/proctord/jobs/logs/handler_test.go b/proctord/jobs/logs/handler_test.go index 259a251c..a926dff5 100644 --- a/proctord/jobs/logs/handler_test.go +++ b/proctord/jobs/logs/handler_test.go @@ -61,7 +61,7 @@ func (suite *LoggerTestSuite) TestLoggerStream() { defer s.Close() buffer := utility.NewBuffer() - buffer.Write([]byte("first line\nsecond line\n")) + _, _ = buffer.Write([]byte("first line\nsecond line\n")) suite.mockKubeClient.On("StreamJobLogs", "sample").Return(buffer, nil).Once() c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) diff --git a/proctord/jobs/metadata/handler.go b/proctord/jobs/metadata/handler.go index bcb6d264..63f351b2 100644 --- a/proctord/jobs/metadata/handler.go +++ b/proctord/jobs/metadata/handler.go @@ -5,8 +5,8 @@ import ( "github.com/getsentry/raven-go" "net/http" "proctor/proctord/logger" - utility "proctor/shared/constant" - procMetadata "proctor/shared/model/metadata" + "proctor/shared/constant" + modelMetadata "proctor/shared/model/metadata" ) type handler struct { @@ -26,14 +26,14 @@ func NewHandler(store Store) Handler { func (handler *handler) HandleSubmission() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - var jobMetadata []procMetadata.Metadata + var jobMetadata []modelMetadata.Metadata err := json.NewDecoder(req.Body).Decode(&jobMetadata) defer req.Body.Close() if err != nil { logger.Error("Error parsing request body", err.Error()) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.ClientError)) + _, _ = w.Write([]byte(constant.ClientError)) return } @@ -44,7 +44,7 @@ func (handler *handler) HandleSubmission() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + w.Write([]byte(constant.ServerError)) return } } @@ -62,7 +62,7 @@ func (handler *handler) HandleBulkDisplay() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + w.Write([]byte(constant.ServerError)) return } @@ -72,7 +72,7 @@ func (handler *handler) HandleBulkDisplay() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + w.Write([]byte(constant.ServerError)) return } diff --git a/proctord/jobs/schedule/handler.go b/proctord/jobs/schedule/handler.go index e59d8e88..39812417 100644 --- a/proctord/jobs/schedule/handler.go +++ b/proctord/jobs/schedule/handler.go @@ -14,7 +14,7 @@ import ( "proctor/proctord/jobs/metadata" "proctor/proctord/logger" "proctor/proctord/storage" - utility "proctor/shared/constant" + "proctor/shared/constant" modelSchedule "proctor/shared/model/schedule" ) @@ -41,14 +41,14 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { var scheduledJob modelSchedule.ScheduledJob err := json.NewDecoder(req.Body).Decode(&scheduledJob) - userEmail := req.Header.Get(utility.UserEmailHeaderKey) + userEmail := req.Header.Get(constant.UserEmailHeaderKey) defer req.Body.Close() if err != nil { logger.Error("Error parsing request body for scheduling jobs: ", err.Error()) raven.CaptureError(err, nil) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.ClientError)) + _, _ = w.Write([]byte(constant.ClientError)) return } @@ -56,7 +56,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { if scheduledJob.Tags == "" { logger.Error("Tag(s) are missing") w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.InvalidTagError)) + _, _ = w.Write([]byte(constant.InvalidTagError)) return } @@ -65,7 +65,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { logger.Error(fmt.Sprintf("Client provided invalid cron expression: %s ", scheduledJob.Tags), scheduledJob.Name, scheduledJob.Time) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.InvalidCronExpressionClientError)) + _, _ = w.Write([]byte(constant.InvalidCronExpressionClientError)) return } @@ -76,7 +76,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { if err != nil { logger.Error(fmt.Sprintf("Client provided invalid email address: %s: ", scheduledJob.Tags), scheduledJob.Name, notificationEmail) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.InvalidEmailIdClientError)) + _, _ = w.Write([]byte(constant.InvalidEmailIdClientError)) return } } @@ -84,7 +84,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { if scheduledJob.Group == "" { logger.Error(fmt.Sprintf("Group Name is missing %s: ", scheduledJob.Tags), scheduledJob.Name) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.GroupNameMissingError)) + _, _ = w.Write([]byte(constant.GroupNameMissingError)) return } @@ -94,13 +94,13 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", scheduledJob.Tags), scheduledJob.Name) w.WriteHeader(http.StatusNotFound) - w.Write([]byte(utility.NonExistentProcClientError)) + _, _ = w.Write([]byte(constant.NonExistentProcClientError)) } else { logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", scheduledJob.Tags), scheduledJob.Name, err.Error()) raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) } return @@ -114,7 +114,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) w.WriteHeader(http.StatusConflict) - w.Write([]byte(utility.DuplicateJobNameArgsClientError)) + _, _ = w.Write([]byte(constant.DuplicateJobNameArgsClientError)) return } else { @@ -122,7 +122,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } @@ -134,13 +134,13 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } w.WriteHeader(http.StatusCreated) - w.Write(responseBody) + _, _ = w.Write(responseBody) return } } @@ -153,12 +153,12 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } if len(scheduledJobsStoreFormat) == 0 { - logger.Error(utility.NoScheduledJobsError, nil) + logger.Error(constant.NoScheduledJobsError, nil) w.WriteHeader(http.StatusNoContent) return @@ -170,7 +170,7 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } @@ -180,11 +180,11 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } - w.Write(scheduledJobsJson) + _, _ = w.Write(scheduledJobsJson) } } @@ -196,22 +196,22 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("Invalid Job ID")) + _, _ = w.Write([]byte("Invalid Job ID")) return } logger.Error("Error fetching scheduled job", err.Error()) raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } if len(scheduledJob) == 0 { - logger.Error(utility.JobNotFoundError, nil) + logger.Error(constant.JobNotFoundError, nil) w.WriteHeader(http.StatusNotFound) - w.Write([]byte(utility.JobNotFoundError)) + _, _ = w.Write([]byte(constant.JobNotFoundError)) return } @@ -221,7 +221,7 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } @@ -231,11 +231,11 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } - w.Write(scheduledJobJson) + _, _ = w.Write(scheduledJobJson) } } @@ -248,26 +248,26 @@ func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc { logger.Error(err.Error()) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte("Invalid Job ID")) + _, _ = w.Write([]byte("Invalid Job ID")) return } logger.Error("Error fetching scheduled job", err.Error()) raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } if removedJobsCount == 0 { - logger.Error(utility.JobNotFoundError, nil) + logger.Error(constant.JobNotFoundError, nil) w.WriteHeader(http.StatusNotFound) - w.Write([]byte(utility.JobNotFoundError)) + _, _ = w.Write([]byte(constant.JobNotFoundError)) return } w.WriteHeader(http.StatusOK) - w.Write([]byte(fmt.Sprintf("Successfully unscheduled Job ID: %s", jobID))) + _, _ = w.Write([]byte(fmt.Sprintf("Successfully unscheduled Job ID: %s", jobID))) } } diff --git a/proctord/jobs/schedule/handler_test.go b/proctord/jobs/schedule/handler_test.go index d368b514..cbca9704 100644 --- a/proctord/jobs/schedule/handler_test.go +++ b/proctord/jobs/schedule/handler_test.go @@ -287,8 +287,8 @@ func (suite *SchedulerTestSuite) TestErrorPersistingScheduledJob() { assert.Equal(t, constant.ServerError, string(responseBody)) } -func (s *SchedulerTestSuite) TestGetScheduledJobs() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobs() { + t := suite.T() req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() @@ -298,11 +298,11 @@ func (s *SchedulerTestSuite) TestGetScheduledJobs() { ID: "some-id", }, } - s.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() + suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() - s.testScheduler.GetScheduledJobs()(responseRecorder, req) + suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) assert.Equal(t, http.StatusOK, responseRecorder.Code) @@ -312,41 +312,41 @@ func (s *SchedulerTestSuite) TestGetScheduledJobs() { assert.Equal(t, scheduledJobsStoreFormat[0].ID, scheduledJobs[0].ID) } -func (s *SchedulerTestSuite) TestGetScheduledJobsWhenNoJobsFound() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobsWhenNoJobsFound() { + t := suite.T() req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() - scheduledJobsStoreFormat := []postgres.JobsSchedule{} - s.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() + var scheduledJobsStoreFormat []postgres.JobsSchedule + suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() - s.testScheduler.GetScheduledJobs()(responseRecorder, req) + suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) assert.Equal(t, http.StatusNoContent, responseRecorder.Code) } -func (s *SchedulerTestSuite) TestGetScheduledJobsFailure() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobsFailure() { + t := suite.T() req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() - scheduledJobs := []postgres.JobsSchedule{} - s.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobs, errors.New("error")).Once() + var scheduledJobs []postgres.JobsSchedule + suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobs, errors.New("error")).Once() - s.testScheduler.GetScheduledJobs()(responseRecorder, req) + suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) } -func (s *SchedulerTestSuite) TestGetScheduledJobByID() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobByID() { + t := suite.T() jobID := "some-id" scheduledJobsStoreFormat := []postgres.JobsSchedule{ @@ -354,12 +354,12 @@ func (s *SchedulerTestSuite) TestGetScheduledJobByID() { ID: jobID, }, } - s.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() + suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusOK, response.StatusCode) @@ -369,167 +369,167 @@ func (s *SchedulerTestSuite) TestGetScheduledJobByID() { assert.NoError(t, err) assert.Equal(t, jobID, scheduledJob.ID) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestGetScheduledJobByIDOnInvalidJobID() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInvalidJobID() { + t := suite.T() jobID := "invalid-job-id" - scheduledJobsStoreFormat := []postgres.JobsSchedule{} + var scheduledJobsStoreFormat []postgres.JobsSchedule - s.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() + suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%suite\"", jobID))).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Invalid Job ID", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestGetScheduledJobByIDOnInternalServerError() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInternalServerError() { + t := suite.T() jobID := "job-id" - scheduledJobsStoreFormat := []postgres.JobsSchedule{} + var scheduledJobsStoreFormat []postgres.JobsSchedule - s.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New("some-error")).Once() + suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New("some-error")).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusInternalServerError, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Something went wrong", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestGetScheduledJobByIDOnJobIDNotFound() { - t := s.T() +func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnJobIDNotFound() { + t := suite.T() jobID := "absent-job-id" - scheduledJobsStoreFormat := []postgres.JobsSchedule{} + var scheduledJobsStoreFormat []postgres.JobsSchedule - s.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() + suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusNotFound, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Job not found", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestRemoveScheduledJobByID() { - t := s.T() +func (suite *SchedulerTestSuite) TestRemoveScheduledJobByID() { + t := suite.T() jobID := "some-id" - s.mockStore.On("RemoveScheduledJob", jobID).Return(int64(1), nil).Once() + suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(1), nil).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusOK, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Successfully unscheduled Job ID: some-id", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInvalidJobID() { - t := s.T() +func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInvalidJobID() { + t := suite.T() jobID := "invalid-job-id" - s.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() + suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%suite\"", jobID))).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusBadRequest, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Invalid Job ID", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInternalServerError() { - t := s.T() +func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInternalServerError() { + t := suite.T() jobID := "job-id" - s.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New("some-error")).Once() + suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New("some-error")).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusInternalServerError, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Something went wrong", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } -func (s *SchedulerTestSuite) TestRemoveScheduledJobByIDOnJobIDNotFound() { - t := s.T() +func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnJobIDNotFound() { + t := suite.T() jobID := "absent-job-id" - s.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), nil).Once() + suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), nil).Once() - url := fmt.Sprintf("%s/jobs/schedule/%s", s.TestServer.URL, jobID) + url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) - response, err := s.Client.Do(req) + response, err := suite.Client.Do(req) assert.NoError(t, err) assert.Equal(t, http.StatusNotFound, response.StatusCode) buf := new(bytes.Buffer) - buf.ReadFrom(response.Body) + _, _ = buf.ReadFrom(response.Body) responseBody := buf.String() assert.Equal(t, "Job not found", responseBody) - s.mockStore.AssertExpectations(t) + suite.mockStore.AssertExpectations(t) } func TestScheduleTestSuite(t *testing.T) { diff --git a/proctord/jobs/schedule/worker_test.go b/proctord/jobs/schedule/worker_test.go index 8acf2840..b0ad6492 100644 --- a/proctord/jobs/schedule/worker_test.go +++ b/proctord/jobs/schedule/worker_test.go @@ -17,7 +17,7 @@ import ( "proctor/proctord/mail" "proctor/proctord/storage" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" + "proctor/shared/constant" ) type WorkerTestSuite struct { @@ -76,7 +76,7 @@ func (suite *WorkerTestSuite) TestCronEnablingForScheduledJobs() { jobExecutionID := "job-execution-id" suite.mockExecutioner.On("Execute", mock.Anything, enabledJob, jobArgs).Return(jobExecutionID, nil) - jobExecutionStatus := utility.JobSucceeded + jobExecutionStatus := constant.JobSucceeded suite.mockAuditor.On("JobsExecution", mock.Anything).Return() suite.mockAuditor.On("JobsExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) @@ -144,7 +144,7 @@ func (suite *WorkerTestSuite) TestCronForDisablingEnabledScheduledJobs() { suite.mockExecutioner.On("Execute", mock.Anything, jobName, jobArgs).Return(jobExecutionID, nil) suite.mockAuditor.On("JobsExecution", mock.Anything).Return() - jobExecutionStatus := utility.JobSucceeded + jobExecutionStatus := constant.JobSucceeded suite.mockAuditor.On("JobsExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) expectedRecipients := strings.Split(notificationEmails, ",") diff --git a/proctord/jobs/secrets/handler.go b/proctord/jobs/secrets/handler.go index 8a42f81a..625659d3 100644 --- a/proctord/jobs/secrets/handler.go +++ b/proctord/jobs/secrets/handler.go @@ -6,7 +6,7 @@ import ( "net/http" "proctor/proctord/logger" - utility "proctor/shared/constant" + "proctor/shared/constant" ) type handler struct { @@ -32,7 +32,7 @@ func (handler *handler) HandleSubmission() http.HandlerFunc { logger.Error("Error parsing request body", err.Error()) w.WriteHeader(http.StatusBadRequest) - w.Write([]byte(utility.ClientError)) + _, _ = w.Write([]byte(constant.ClientError)) return } @@ -42,7 +42,7 @@ func (handler *handler) HandleSubmission() http.HandlerFunc { raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(utility.ServerError)) + _, _ = w.Write([]byte(constant.ServerError)) return } diff --git a/proctord/jobs/secrets/handler_test.go b/proctord/jobs/secrets/handler_test.go index 1a193044..1694a6d7 100644 --- a/proctord/jobs/secrets/handler_test.go +++ b/proctord/jobs/secrets/handler_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - utility "proctor/shared/constant" + "proctor/shared/constant" ) type SecretsHandlerTestSuite struct { @@ -63,7 +63,7 @@ func (suite *SecretsHandlerTestSuite) TestSecretsUpdationSecretsMalformedData() suite.mockSecretsStore.AssertNotCalled(t, "CreateOrUpdateJobSecret", mock.Anything) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, utility.ClientError, responseRecorder.Body.String()) + assert.Equal(t, constant.ClientError, responseRecorder.Body.String()) } func (suite *SecretsHandlerTestSuite) TestSecretsUpdationSecretsStoreFailure() { @@ -87,7 +87,7 @@ func (suite *SecretsHandlerTestSuite) TestSecretsUpdationSecretsStoreFailure() { suite.mockSecretsStore.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, utility.ServerError, responseRecorder.Body.String()) + assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) } func TestSecretsHandlerTestSuite(t *testing.T) { diff --git a/proctord/kubernetes/client.go b/proctord/kubernetes/client.go index d6431cfd..f7ab60ca 100644 --- a/proctord/kubernetes/client.go +++ b/proctord/kubernetes/client.go @@ -14,7 +14,7 @@ import ( "k8s.io/client-go/kubernetes" "proctor/proctord/config" "proctor/proctord/logger" - utility "proctor/shared/constant" + "proctor/shared/constant" //Package needed for kubernetes cluster in google cloud _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" "k8s.io/client-go/tools/clientcmd" @@ -217,7 +217,7 @@ func (client *client) JobExecutionStatus(jobExecutionID string) (string, error) watchJob, err := kubernetesJobs.Watch(listOptions) if err != nil { - return utility.JobFailed, err + return constant.JobFailed, err } resultChan := watchJob.ResultChan() @@ -227,18 +227,18 @@ func (client *client) JobExecutionStatus(jobExecutionID string) (string, error) for event = range resultChan { if event.Type == watch.Error { - return utility.JobExecutionStatusFetchError, nil + return constant.JobExecutionStatusFetchError, nil } jobEvent = event.Object.(*batch_v1.Job) if jobEvent.Status.Succeeded >= int32(1) { - return utility.JobSucceeded, nil + return constant.JobSucceeded, nil } else if jobEvent.Status.Failed >= int32(1) { - return utility.JobFailed, nil + return constant.JobFailed, nil } } - return utility.NoDefinitiveJobExecutionStatusFound, nil + return constant.NoDefinitiveJobExecutionStatusFound, nil } func (client *client) getLogsStreamReaderFor(podName string) (io.ReadCloser, error) { diff --git a/proctord/kubernetes/client_test.go b/proctord/kubernetes/client_test.go index fbc673bd..86a58bd5 100644 --- a/proctord/kubernetes/client_test.go +++ b/proctord/kubernetes/client_test.go @@ -69,7 +69,7 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) TestJobExecution() { t := suite.T() - os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} sampleImageName := "img1" diff --git a/proctord/main.go b/proctord/main.go index 35203bd8..75354c45 100644 --- a/proctord/main.go +++ b/proctord/main.go @@ -15,7 +15,7 @@ import ( func main() { logger.Setup() - raven.SetDSN(config.SentryDSN()) + _ = raven.SetDSN(config.SentryDSN()) proctord := cli.NewApp() proctord.Name = "proctord" @@ -61,5 +61,5 @@ func main() { }, } - proctord.Run(os.Args) + _ = proctord.Run(os.Args) } diff --git a/proctord/middleware/validate_client_version.go b/proctord/middleware/validate_client_version.go index 31ab4e12..831080c9 100644 --- a/proctord/middleware/validate_client_version.go +++ b/proctord/middleware/validate_client_version.go @@ -6,13 +6,13 @@ import ( "net/http" "proctor/proctord/config" "proctor/proctord/logger" - utility "proctor/shared/constant" + "proctor/shared/constant" ) func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - requestHeaderClientVersion := r.Header.Get(utility.ClientVersionHeaderKey) + requestHeaderClientVersion := r.Header.Get(constant.ClientVersionHeaderKey) if requestHeaderClientVersion != "" { clientVersion, err := version.NewVersion(requestHeaderClientVersion) @@ -27,7 +27,7 @@ func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { if clientVersion.LessThan(minClientVersion) { w.WriteHeader(400) - w.Write([]byte(fmt.Sprintf(utility.ClientOutdatedErrorMessage, clientVersion))) + _, _ = w.Write([]byte(fmt.Sprintf(constant.ClientOutdatedErrorMessage, clientVersion))) return } next.ServeHTTP(w, r) diff --git a/proctord/middleware/validate_client_version_test.go b/proctord/middleware/validate_client_version_test.go index 80468b00..82a2ee82 100644 --- a/proctord/middleware/validate_client_version_test.go +++ b/proctord/middleware/validate_client_version_test.go @@ -6,7 +6,7 @@ import ( "net/http" "net/http/httptest" "os" - utility "proctor/shared/constant" + "proctor/shared/constant" "testing" ) @@ -18,7 +18,7 @@ func getTestHandler() http.HandlerFunc { func TestValidClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") + _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -26,7 +26,7 @@ func TestValidClientVersionHttpHeader(t *testing.T) { client := &http.Client{} req, _ := http.NewRequest("GET", ts.URL+"/jobs/metadata", nil) - req.Header.Add(utility.ClientVersionHeaderKey, "0.2.0") + req.Header.Add(constant.ClientVersionHeaderKey, "0.2.0") resp, _ := client.Do(req) defer resp.Body.Close() @@ -36,7 +36,7 @@ func TestValidClientVersionHttpHeader(t *testing.T) { func TestEmptyClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") + _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -53,7 +53,7 @@ func TestEmptyClientVersionHttpHeader(t *testing.T) { func TestInvalidClientVersionHttpHeader(t *testing.T) { - os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.3.0") + _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.3.0") ts := httptest.NewServer(ValidateClientVersion(getTestHandler())) defer ts.Close() @@ -61,7 +61,7 @@ func TestInvalidClientVersionHttpHeader(t *testing.T) { client := &http.Client{} req, _ := http.NewRequest("GET", ts.URL+"/jobs/metadata", nil) - req.Header.Add(utility.ClientVersionHeaderKey, "0.1.0") + req.Header.Add(constant.ClientVersionHeaderKey, "0.1.0") resp, _ := client.Do(req) defer resp.Body.Close() diff --git a/proctord/redis/client_test.go b/proctord/redis/client_test.go index 77f7d495..692ce29f 100644 --- a/proctord/redis/client_test.go +++ b/proctord/redis/client_test.go @@ -99,7 +99,7 @@ func (s *RedisClientTestSuite) TestMGET() { } func (s *RedisClientTestSuite) TearDownSuite() { - s.testRedisConn.Close() + _ = s.testRedisConn.Close() } func TestRedisClientTestSuite(t *testing.T) { diff --git a/proctord/scheduler/scheduler.go b/proctord/scheduler/scheduler.go index 0b0218f5..a99b51b6 100644 --- a/proctord/scheduler/scheduler.go +++ b/proctord/scheduler/scheduler.go @@ -7,7 +7,7 @@ import ( "proctor/proctord/audit" "proctor/proctord/config" - http_client "proctor/proctord/http" + httpClient "proctor/proctord/http" "proctor/proctord/jobs/execution" "proctor/proctord/jobs/metadata" "proctor/proctord/jobs/schedule" @@ -29,7 +29,7 @@ func Start() error { metadataStore := metadata.NewStore(redisClient) secretsStore := secrets.NewStore(redisClient) - httpClient, err := http_client.NewClient() + httpClient, err := httpClient.NewClient() if err != nil { return err } @@ -48,6 +48,6 @@ func Start() error { signalsChan := make(chan os.Signal, 1) worker.Run(ticker.C, signalsChan) - postgresClient.Close() + _ = postgresClient.Close() return nil } diff --git a/proctord/server/api.go b/proctord/server/api.go index c56a518c..52645762 100644 --- a/proctord/server/api.go +++ b/proctord/server/api.go @@ -29,7 +29,7 @@ func Start() error { graceful.Run(appPort, 2*time.Second, server) - postgresClient.Close() + _ = postgresClient.Close() logger.Info("Stopped server gracefully") return nil } diff --git a/proctord/server/router.go b/proctord/server/router.go index 994d7643..8af73525 100644 --- a/proctord/server/router.go +++ b/proctord/server/router.go @@ -7,7 +7,7 @@ import ( "proctor/proctord/audit" "proctor/proctord/config" "proctor/proctord/docs" - http_client "proctor/proctord/http" + httpClient "proctor/proctord/http" "proctor/proctord/jobs/execution" "proctor/proctord/jobs/logs" "proctor/proctord/jobs/metadata" @@ -35,7 +35,7 @@ func NewRouter() (*mux.Router, error) { metadataStore := metadata.NewStore(redisClient) secretsStore := secrets.NewStore(redisClient) - httpClient, err := http_client.NewClient() + httpClient, err := httpClient.NewClient() if err != nil { return router, err } @@ -52,7 +52,7 @@ func NewRouter() (*mux.Router, error) { scheduledJobsHandler := schedule.NewScheduler(store, metadataStore) router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { - fmt.Fprintf(w, "pong") + _, _ = fmt.Fprintf(w, "pong") }) router.HandleFunc("/docs", docs.APIDocHandler) diff --git a/proctord/storage/store_test.go b/proctord/storage/store_test.go index 0c350323..5d6ad869 100644 --- a/proctord/storage/store_test.go +++ b/proctord/storage/store_test.go @@ -8,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "proctor/proctord/storage/postgres" - utility "proctor/shared/constant" + "proctor/shared/constant" "testing" ) @@ -102,7 +102,7 @@ func TestGetJobsStatusWhenJobIsPresent(t *testing.T) { Run(func(args mock.Arguments) { jobsExecutionAuditLogResult := args.Get(0).(*[]postgres.JobsExecutionAuditLog) *jobsExecutionAuditLogResult = append(*jobsExecutionAuditLogResult, postgres.JobsExecutionAuditLog{ - JobExecutionStatus: utility.JobSucceeded, + JobExecutionStatus: constant.JobSucceeded, }) }). Once() @@ -110,7 +110,7 @@ func TestGetJobsStatusWhenJobIsPresent(t *testing.T) { status, err := testStore.GetJobExecutionStatus(jobName) assert.NoError(t, err) - assert.Equal(t, utility.JobSucceeded, status) + assert.Equal(t, constant.JobSucceeded, status) mockPostgresClient.AssertExpectations(t) } diff --git a/shared/utility/utils.go b/shared/utility/utils.go index c591b1b8..69e94911 100644 --- a/shared/utility/utils.go +++ b/shared/utility/utils.go @@ -23,7 +23,7 @@ func MergeMaps(mapOne, mapTwo map[string]string) map[string]string { func MapToString(someMap map[string]string) string { b := new(bytes.Buffer) for key, value := range someMap { - fmt.Fprintf(b, "%s=\"%s\",", key, value) + _, _ = fmt.Fprintf(b, "%s=\"%s\",", key, value) } return strings.TrimRight(b.String(), ",") } From d8db15f4bec48d040ab541bd5acd4cb50e00d224 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 20 Jun 2019 14:26:27 +0700 Subject: [PATCH 017/268] [Jasoet|Bimo] refactor: fix typo on string template --- proctord/jobs/schedule/handler_test.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/proctord/jobs/schedule/handler_test.go b/proctord/jobs/schedule/handler_test.go index cbca9704..a1dd1a3d 100644 --- a/proctord/jobs/schedule/handler_test.go +++ b/proctord/jobs/schedule/handler_test.go @@ -356,7 +356,7 @@ func (suite *SchedulerTestSuite) TestGetScheduledJobByID() { } suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) response, err := suite.Client.Do(req) @@ -378,9 +378,9 @@ func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInvalidJobID() { var scheduledJobsStoreFormat []postgres.JobsSchedule - suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%suite\"", jobID))).Once() + suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) response, err := suite.Client.Do(req) @@ -404,7 +404,7 @@ func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInternalServerError() suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New("some-error")).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) response, err := suite.Client.Do(req) @@ -428,7 +428,7 @@ func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnJobIDNotFound() { suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("GET", url, nil) response, err := suite.Client.Do(req) @@ -450,7 +450,7 @@ func (suite *SchedulerTestSuite) TestRemoveScheduledJobByID() { suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(1), nil).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) response, err := suite.Client.Do(req) @@ -470,9 +470,9 @@ func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInvalidJobID() { t := suite.T() jobID := "invalid-job-id" - suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%suite\"", jobID))).Once() + suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) response, err := suite.Client.Do(req) @@ -494,7 +494,7 @@ func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInternalServerError suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New("some-error")).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) response, err := suite.Client.Do(req) @@ -516,7 +516,7 @@ func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnJobIDNotFound() { suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), nil).Once() - url := fmt.Sprintf("%suite/jobs/schedule/%suite", suite.TestServer.URL, jobID) + url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) req, _ := http.NewRequest("DELETE", url, nil) response, err := suite.Client.Do(req) From 973c9b4baae55e5a1e576ee7b95349ef903e0e23 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 8 Jul 2019 18:10:51 +0530 Subject: [PATCH 018/268] [Jasoet|Bimozx] Refactor based on https://github.com/golang-standards/project-layout --- .travis.yml | 2 +- AUTHORS.md | 2 + CONTRIBUTING.md | 4 +- Makefile | 6 +- README.md | 8 +- {doc => assets/img}/proctor-logo.png | Bin {doc => assets/img}/proctor_describe.gif | Bin {doc => assets/img}/proctor_execute.gif | Bin {doc => assets/img}/schedule.gif | Bin {doc => assets/img}/schedule_describe.gif | Bin {doc => assets/img}/schedule_list.gif | Bin {doc => assets/img}/schedule_remove.gif | Bin {cli => cmd/cli}/main.go | 10 +- {proctord => cmd/proctord}/main.go | 13 +- internal/app/cli/cli.go | 1 + .../app/cli}/command/config/manager.go | 30 +- .../app/cli}/command/config/view/view.go | 6 +- .../cli}/command/description/descriptor.go | 8 +- .../command/description/descriptor_test.go | 12 +- .../app/cli}/command/execution/executioner.go | 6 +- .../command/execution/executioner_test.go | 6 +- .../app/cli}/command/list/lister.go | 6 +- .../app/cli}/command/list/lister_test.go | 14 +- {cli => internal/app/cli}/command/root.go | 26 +- .../app/cli}/command/root_test.go | 6 +- .../command/schedule/describe/describe.go | 4 +- .../schedule/describe/describe_test.go | 4 +- .../app/cli}/command/schedule/list/list.go | 4 +- .../cli}/command/schedule/list/list_test.go | 4 +- .../cli}/command/schedule/remove/remove.go | 4 +- .../command/schedule/remove/remove_test.go | 4 +- .../app/cli}/command/schedule/schedule.go | 4 +- .../cli}/command/schedule/schedule_test.go | 4 +- .../app/cli}/command/version/github/client.go | 0 .../command/version/github/client_mock.go | 0 .../app/cli}/command/version/version.go | 4 +- .../app/cli}/command/version/version_test.go | 10 +- {cli => internal/app/cli}/config/config.go | 2 +- .../app/cli}/config/config_mock.go | 0 .../app/cli}/config/config_test.go | 0 {cli => internal/app/cli}/config/data.go | 0 .../app/cli}/config_template.yaml | 0 {cli => internal/app/cli}/daemon/client.go | 13 +- .../app/cli}/daemon/client_mock.go | 4 +- .../app/cli}/daemon/client_test.go | 13 +- .../app/proctord}/audit/auditor.go | 10 +- .../app/proctord}/audit/auditor_mock.go | 2 +- .../app/proctord}/audit/auditor_test.go | 8 +- .../app/proctord}/config/config.go | 0 .../app/proctord}/config/config_test.go | 0 .../app/proctord}/docs/css/print.css | 0 .../app/proctord}/docs/css/reset.css | 0 .../app/proctord}/docs/css/screen.css | 0 .../app/proctord}/docs/css/style.css | 0 .../app/proctord}/docs/css/typography.css | 0 .../proctord}/docs/fonts/DroidSans-Bold.ttf | Bin .../app/proctord}/docs/fonts/DroidSans.ttf | Bin .../app/proctord}/docs/handler.go | 0 .../app/proctord}/docs/images/collapse.gif | Bin .../app/proctord}/docs/images/expand.gif | Bin .../proctord}/docs/images/explorer_icons.png | Bin .../proctord}/docs/images/favicon-16x16.png | Bin .../proctord}/docs/images/favicon-32x32.png | Bin .../app/proctord}/docs/images/favicon.ico | Bin .../app/proctord}/docs/images/logo_small.png | Bin .../proctord}/docs/images/pet_store_api.png | Bin .../app/proctord}/docs/images/throbber.gif | Bin .../app/proctord}/docs/images/wordnik_api.png | Bin .../app/proctord}/docs/index.html | 0 .../app/proctord}/docs/lang/ca.js | 0 .../app/proctord}/docs/lang/el.js | 0 .../app/proctord}/docs/lang/en.js | 0 .../app/proctord}/docs/lang/es.js | 0 .../app/proctord}/docs/lang/fr.js | 0 .../app/proctord}/docs/lang/geo.js | 0 .../app/proctord}/docs/lang/it.js | 0 .../app/proctord}/docs/lang/ja.js | 0 .../app/proctord}/docs/lang/ko-kr.js | 0 .../app/proctord}/docs/lang/pl.js | 0 .../app/proctord}/docs/lang/pt.js | 0 .../app/proctord}/docs/lang/ru.js | 0 .../app/proctord}/docs/lang/tr.js | 0 .../app/proctord}/docs/lang/translator.js | 0 .../app/proctord}/docs/lang/zh-cn.js | 0 .../app/proctord}/docs/lib/backbone-min.js | 0 .../app/proctord}/docs/lib/es5-shim.js | 0 .../proctord}/docs/lib/handlebars-4.0.5.js | 0 .../docs/lib/highlight.9.1.0.pack.js | 0 .../docs/lib/highlight.9.1.0.pack_extended.js | 0 .../proctord}/docs/lib/jquery-1.8.0.min.js | 0 .../proctord}/docs/lib/jquery.ba-bbq.min.js | 0 .../proctord}/docs/lib/jquery.slideto.min.js | 0 .../proctord}/docs/lib/jquery.wiggle.min.js | 0 .../app/proctord}/docs/lib/js-yaml.min.js | 0 .../app/proctord}/docs/lib/jsoneditor.min.js | 0 .../app/proctord}/docs/lib/lodash.min.js | 0 .../app/proctord}/docs/lib/marked.js | 0 .../docs/lib/object-assign-pollyfill.js | 0 .../proctord}/docs/lib/sanitize-html.min.js | 0 .../app/proctord}/docs/lib/swagger-oauth.js | 0 .../app/proctord}/docs/o2c.html | 0 .../app/proctord}/docs/swagger-ui.js | 330 +++++++++--------- .../app/proctord}/docs/swagger-ui.min.js | 0 .../app/proctord}/docs/swagger.yml | 0 .../app/proctord}/http/client.go | 3 +- .../app/proctord}/http/client_test.go | 2 +- .../app/proctord}/instrumentation/newrelic.go | 5 +- .../instrumentation/newrelic_stub.go | 0 .../proctord}/jobs/execution/executioner.go | 18 +- .../jobs/execution/executioner_mock.go | 2 +- .../jobs/execution/executioner_test.go | 28 +- .../app/proctord}/jobs/execution/handler.go | 10 +- .../proctord}/jobs/execution/handler_test.go | 6 +- .../app/proctord}/jobs/execution/job.go | 0 .../app/proctord}/jobs/logs/handler.go | 8 +- .../app/proctord}/jobs/logs/handler_test.go | 6 +- .../app/proctord}/jobs/metadata/handler.go | 8 +- .../proctord}/jobs/metadata/handler_test.go | 6 +- .../app/proctord}/jobs/metadata/store.go | 4 +- .../app/proctord}/jobs/metadata/store_mock.go | 2 +- .../app/proctord}/jobs/metadata/store_test.go | 4 +- .../app/proctord}/jobs/schedule/handler.go | 14 +- .../proctord}/jobs/schedule/handler_test.go | 16 +- .../proctord}/jobs/schedule/scheduled_job.go | 7 +- .../app/proctord}/jobs/schedule/worker.go | 16 +- .../proctord}/jobs/schedule/worker_test.go | 12 +- .../app/proctord}/jobs/secrets/handler.go | 4 +- .../proctord}/jobs/secrets/handler_test.go | 2 +- .../app/proctord}/jobs/secrets/secret.go | 0 .../app/proctord}/jobs/secrets/store.go | 3 +- .../app/proctord}/jobs/secrets/store_mock.go | 0 .../app/proctord}/jobs/secrets/store_test.go | 2 +- .../app/proctord}/kubernetes/client.go | 6 +- .../app/proctord}/kubernetes/client_mock.go | 2 +- .../app/proctord}/kubernetes/client_test.go | 9 +- .../app/proctord}/kubernetes/utils.go | 5 +- .../app/proctord}/logger/logrus.go | 3 +- .../app/proctord}/mail/mailer.go | 4 +- .../app/proctord}/mail/mailer_mock.go | 0 .../app/proctord}/mail/mailer_test.go | 4 +- .../middleware/validate_client_version.go | 6 +- .../validate_client_version_test.go | 2 +- .../app/proctord}/redis/client.go | 0 .../app/proctord}/redis/client_mock.go | 0 .../app/proctord}/redis/client_test.go | 0 .../app/proctord}/redis/pool.go | 3 +- .../app/proctord}/scheduler/scheduler.go | 27 +- .../app/proctord}/server/api.go | 7 +- .../app/proctord}/server/router.go | 30 +- .../app/proctord}/storage/postgres/client.go | 4 +- .../proctord}/storage/postgres/client_mock.go | 0 .../proctord}/storage/postgres/client_test.go | 8 +- .../proctord}/storage/postgres/migrations.go | 4 +- .../app/proctord}/storage/postgres/schema.go | 3 +- .../app/proctord}/storage/postgres/utils.go | 0 .../proctord}/storage/postgres/utils_test.go | 0 .../app/proctord}/storage/store.go | 5 +- .../app/proctord}/storage/store_mock.go | 2 +- .../app/proctord}/storage/store_test.go | 6 +- {shared => internal/pkg}/constant/constant.go | 0 {shared => internal/pkg}/io/printer.go | 0 {shared => internal/pkg}/io/printer_mock.go | 0 .../pkg}/model/metadata/env/vars.go | 0 internal/pkg/model/metadata/metadata.go | 16 + {shared => internal/pkg}/model/model.go | 0 .../pkg}/model/schedule/schedule_job.go | 0 internal/pkg/pkg.go | 1 + {shared => internal/pkg}/utility/buffer.go | 0 {shared => internal/pkg}/utility/sort/sort.go | 2 +- .../pkg}/utility/sort/sort_test.go | 2 +- {shared => internal/pkg}/utility/utils.go | 0 {assets/scripts => scripts}/proctor.rb.tpl | 6 +- .../scripts => scripts}/proctor_template.go | 0 {assets/scripts => scripts}/release.sh | 8 +- shared/model/metadata/metadata.go | 14 - shared/shared.go | 1 - {assets => test}/procs_example/README.md | 0 .../procs_example/say-hello-world/Dockerfile | 0 .../say-hello-world/metadata.json | 0 .../say-hello-world/say_hello_world.sh | 0 180 files changed, 481 insertions(+), 491 deletions(-) rename {doc => assets/img}/proctor-logo.png (100%) rename {doc => assets/img}/proctor_describe.gif (100%) rename {doc => assets/img}/proctor_execute.gif (100%) rename {doc => assets/img}/schedule.gif (100%) rename {doc => assets/img}/schedule_describe.gif (100%) rename {doc => assets/img}/schedule_list.gif (100%) rename {doc => assets/img}/schedule_remove.gif (100%) rename {cli => cmd/cli}/main.go (59%) rename {proctord => cmd/proctord}/main.go (85%) create mode 100644 internal/app/cli/cli.go rename {cli => internal/app/cli}/command/config/manager.go (72%) rename {cli => internal/app/cli}/command/config/view/view.go (87%) rename {cli => internal/app/cli}/command/description/descriptor.go (92%) rename {cli => internal/app/cli}/command/description/descriptor_test.go (94%) rename {cli => internal/app/cli}/command/execution/executioner.go (95%) rename {cli => internal/app/cli}/command/execution/executioner_test.go (98%) rename {cli => internal/app/cli}/command/list/lister.go (89%) rename {cli => internal/app/cli}/command/list/lister_test.go (85%) rename {cli => internal/app/cli}/command/root.go (76%) rename {cli => internal/app/cli}/command/root_test.go (90%) rename {cli => internal/app/cli}/command/schedule/describe/describe.go (95%) rename {cli => internal/app/cli}/command/schedule/describe/describe_test.go (94%) rename {cli => internal/app/cli}/command/schedule/list/list.go (93%) rename {cli => internal/app/cli}/command/schedule/list/list_test.go (93%) rename {cli => internal/app/cli}/command/schedule/remove/remove.go (91%) rename {cli => internal/app/cli}/command/schedule/remove/remove_test.go (94%) rename {cli => internal/app/cli}/command/schedule/schedule.go (97%) rename {cli => internal/app/cli}/command/schedule/schedule_test.go (94%) rename {cli => internal/app/cli}/command/version/github/client.go (100%) rename {cli => internal/app/cli}/command/version/github/client_mock.go (100%) rename {cli => internal/app/cli}/command/version/version.go (91%) rename {cli => internal/app/cli}/command/version/version_test.go (89%) rename {cli => internal/app/cli}/config/config.go (98%) rename {cli => internal/app/cli}/config/config_mock.go (100%) rename {cli => internal/app/cli}/config/config_test.go (100%) rename {cli => internal/app/cli}/config/data.go (100%) rename {assets => internal/app/cli}/config_template.yaml (100%) rename {cli => internal/app/cli}/daemon/client.go (98%) rename {cli => internal/app/cli}/daemon/client_mock.go (93%) rename {cli => internal/app/cli}/daemon/client_test.go (99%) rename {proctord => internal/app/proctord}/audit/auditor.go (88%) rename {proctord => internal/app/proctord}/audit/auditor_mock.go (91%) rename {proctord => internal/app/proctord}/audit/auditor_test.go (92%) rename {proctord => internal/app/proctord}/config/config.go (100%) rename {proctord => internal/app/proctord}/config/config_test.go (100%) rename {proctord => internal/app/proctord}/docs/css/print.css (100%) rename {proctord => internal/app/proctord}/docs/css/reset.css (100%) rename {proctord => internal/app/proctord}/docs/css/screen.css (100%) rename {proctord => internal/app/proctord}/docs/css/style.css (100%) rename {proctord => internal/app/proctord}/docs/css/typography.css (100%) rename {proctord => internal/app/proctord}/docs/fonts/DroidSans-Bold.ttf (100%) rename {proctord => internal/app/proctord}/docs/fonts/DroidSans.ttf (100%) rename {proctord => internal/app/proctord}/docs/handler.go (100%) rename {proctord => internal/app/proctord}/docs/images/collapse.gif (100%) rename {proctord => internal/app/proctord}/docs/images/expand.gif (100%) rename {proctord => internal/app/proctord}/docs/images/explorer_icons.png (100%) rename {proctord => internal/app/proctord}/docs/images/favicon-16x16.png (100%) rename {proctord => internal/app/proctord}/docs/images/favicon-32x32.png (100%) rename {proctord => internal/app/proctord}/docs/images/favicon.ico (100%) rename {proctord => internal/app/proctord}/docs/images/logo_small.png (100%) rename {proctord => internal/app/proctord}/docs/images/pet_store_api.png (100%) rename {proctord => internal/app/proctord}/docs/images/throbber.gif (100%) rename {proctord => internal/app/proctord}/docs/images/wordnik_api.png (100%) rename {proctord => internal/app/proctord}/docs/index.html (100%) rename {proctord => internal/app/proctord}/docs/lang/ca.js (100%) rename {proctord => internal/app/proctord}/docs/lang/el.js (100%) rename {proctord => internal/app/proctord}/docs/lang/en.js (100%) rename {proctord => internal/app/proctord}/docs/lang/es.js (100%) rename {proctord => internal/app/proctord}/docs/lang/fr.js (100%) rename {proctord => internal/app/proctord}/docs/lang/geo.js (100%) rename {proctord => internal/app/proctord}/docs/lang/it.js (100%) rename {proctord => internal/app/proctord}/docs/lang/ja.js (100%) rename {proctord => internal/app/proctord}/docs/lang/ko-kr.js (100%) rename {proctord => internal/app/proctord}/docs/lang/pl.js (100%) rename {proctord => internal/app/proctord}/docs/lang/pt.js (100%) rename {proctord => internal/app/proctord}/docs/lang/ru.js (100%) rename {proctord => internal/app/proctord}/docs/lang/tr.js (100%) rename {proctord => internal/app/proctord}/docs/lang/translator.js (100%) rename {proctord => internal/app/proctord}/docs/lang/zh-cn.js (100%) rename {proctord => internal/app/proctord}/docs/lib/backbone-min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/es5-shim.js (100%) rename {proctord => internal/app/proctord}/docs/lib/handlebars-4.0.5.js (100%) rename {proctord => internal/app/proctord}/docs/lib/highlight.9.1.0.pack.js (100%) rename {proctord => internal/app/proctord}/docs/lib/highlight.9.1.0.pack_extended.js (100%) rename {proctord => internal/app/proctord}/docs/lib/jquery-1.8.0.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/jquery.ba-bbq.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/jquery.slideto.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/jquery.wiggle.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/js-yaml.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/jsoneditor.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/lodash.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/marked.js (100%) rename {proctord => internal/app/proctord}/docs/lib/object-assign-pollyfill.js (100%) rename {proctord => internal/app/proctord}/docs/lib/sanitize-html.min.js (100%) rename {proctord => internal/app/proctord}/docs/lib/swagger-oauth.js (100%) rename {proctord => internal/app/proctord}/docs/o2c.html (100%) rename {proctord => internal/app/proctord}/docs/swagger-ui.js (98%) rename {proctord => internal/app/proctord}/docs/swagger-ui.min.js (100%) rename {proctord => internal/app/proctord}/docs/swagger.yml (100%) rename {proctord => internal/app/proctord}/http/client.go (92%) rename {proctord => internal/app/proctord}/http/client_test.go (93%) rename {proctord => internal/app/proctord}/instrumentation/newrelic.go (87%) rename {proctord => internal/app/proctord}/instrumentation/newrelic_stub.go (100%) rename {proctord => internal/app/proctord}/jobs/execution/executioner.go (79%) rename {proctord => internal/app/proctord}/jobs/execution/executioner_mock.go (87%) rename {proctord => internal/app/proctord}/jobs/execution/executioner_test.go (81%) rename {proctord => internal/app/proctord}/jobs/execution/handler.go (95%) rename {proctord => internal/app/proctord}/jobs/execution/handler_test.go (99%) rename {proctord => internal/app/proctord}/jobs/execution/job.go (100%) rename {proctord => internal/app/proctord}/jobs/logs/handler.go (93%) rename {proctord => internal/app/proctord}/jobs/logs/handler_test.go (97%) rename {proctord => internal/app/proctord}/jobs/metadata/handler.go (91%) rename {proctord => internal/app/proctord}/jobs/metadata/handler_test.go (97%) rename {proctord => internal/app/proctord}/jobs/metadata/store.go (95%) rename {proctord => internal/app/proctord}/jobs/metadata/store_mock.go (91%) rename {proctord => internal/app/proctord}/jobs/metadata/store_test.go (97%) rename {proctord => internal/app/proctord}/jobs/schedule/handler.go (95%) rename {proctord => internal/app/proctord}/jobs/schedule/handler_test.go (97%) rename {proctord => internal/app/proctord}/jobs/schedule/scheduled_job.go (89%) rename {proctord => internal/app/proctord}/jobs/schedule/worker.go (93%) rename {proctord => internal/app/proctord}/jobs/schedule/worker_test.go (95%) rename {proctord => internal/app/proctord}/jobs/secrets/handler.go (93%) rename {proctord => internal/app/proctord}/jobs/secrets/handler_test.go (98%) rename {proctord => internal/app/proctord}/jobs/secrets/secret.go (100%) rename {proctord => internal/app/proctord}/jobs/secrets/store.go (96%) rename {proctord => internal/app/proctord}/jobs/secrets/store_mock.go (100%) rename {proctord => internal/app/proctord}/jobs/secrets/store_test.go (98%) rename {proctord => internal/app/proctord}/kubernetes/client.go (98%) rename {proctord => internal/app/proctord}/kubernetes/client_mock.go (94%) rename {proctord => internal/app/proctord}/kubernetes/client_test.go (98%) rename {proctord => internal/app/proctord}/kubernetes/utils.go (86%) rename {proctord => internal/app/proctord}/logger/logrus.go (95%) rename {proctord => internal/app/proctord}/mail/mailer.go (94%) rename {proctord => internal/app/proctord}/mail/mailer_mock.go (100%) rename {proctord => internal/app/proctord}/mail/mailer_test.go (97%) rename {proctord => internal/app/proctord}/middleware/validate_client_version.go (89%) rename {proctord => internal/app/proctord}/middleware/validate_client_version_test.go (98%) rename {proctord => internal/app/proctord}/redis/client.go (100%) rename {proctord => internal/app/proctord}/redis/client_mock.go (100%) rename {proctord => internal/app/proctord}/redis/client_test.go (100%) rename {proctord => internal/app/proctord}/redis/pool.go (94%) rename {proctord => internal/app/proctord}/scheduler/scheduler.go (62%) rename {proctord => internal/app/proctord}/server/api.go (80%) rename {proctord => internal/app/proctord}/server/router.go (81%) rename {proctord => internal/app/proctord}/storage/postgres/client.go (94%) rename {proctord => internal/app/proctord}/storage/postgres/client_mock.go (100%) rename {proctord => internal/app/proctord}/storage/postgres/client_test.go (90%) rename {proctord => internal/app/proctord}/storage/postgres/migrations.go (91%) rename {proctord => internal/app/proctord}/storage/postgres/schema.go (97%) rename {proctord => internal/app/proctord}/storage/postgres/utils.go (100%) rename {proctord => internal/app/proctord}/storage/postgres/utils_test.go (100%) rename {proctord => internal/app/proctord}/storage/store.go (98%) rename {proctord => internal/app/proctord}/storage/store_mock.go (96%) rename {proctord => internal/app/proctord}/storage/store_test.go (98%) rename {shared => internal/pkg}/constant/constant.go (100%) rename {shared => internal/pkg}/io/printer.go (100%) rename {shared => internal/pkg}/io/printer_mock.go (100%) rename {shared => internal/pkg}/model/metadata/env/vars.go (100%) create mode 100644 internal/pkg/model/metadata/metadata.go rename {shared => internal/pkg}/model/model.go (100%) rename {shared => internal/pkg}/model/schedule/schedule_job.go (100%) create mode 100644 internal/pkg/pkg.go rename {shared => internal/pkg}/utility/buffer.go (100%) rename {shared => internal/pkg}/utility/sort/sort.go (81%) rename {shared => internal/pkg}/utility/sort/sort_test.go (91%) rename {shared => internal/pkg}/utility/utils.go (100%) rename {assets/scripts => scripts}/proctor.rb.tpl (50%) rename {assets/scripts => scripts}/proctor_template.go (100%) rename {assets/scripts => scripts}/release.sh (75%) delete mode 100644 shared/model/metadata/metadata.go delete mode 100644 shared/shared.go rename {assets => test}/procs_example/README.md (100%) rename {assets => test}/procs_example/say-hello-world/Dockerfile (100%) rename {assets => test}/procs_example/say-hello-world/metadata.json (100%) rename {assets => test}/procs_example/say-hello-world/say_hello_world.sh (100%) diff --git a/.travis.yml b/.travis.yml index c0eebe87..44daf6b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,4 +21,4 @@ jobs: - make db.setup test-with-race after_success: - - assets/scripts/release.sh + - scripts/release.sh diff --git a/AUTHORS.md b/AUTHORS.md index a7a37732..f9a1fb58 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,4 +1,6 @@ # Proctor Team * [Akshat](//github.com/olttwa) +* [Jasoet](//github.com/jasoet) +* [Bimo Horizon](//github.com/bimozx) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6b9e998c..3be328d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ Proctor - Contributing -Proctor `github.com/gojektech/proctor` is an open-source project. +Proctor `github.com/gojek/proctor` is an open-source project. It is licensed using the [Apache License 2.0][1]. We appreciate pull requests; here are our guidelines: @@ -36,7 +36,7 @@ Much Thanks! ❤❤❤ GO-JEK Tech [1]: http://www.apache.org/licenses/LICENSE-2.0 -[2]: https://github.com/gojektech/proctor/issues +[2]: https://github.com/gojek/proctor/issues [3]: https://golang.org/doc/effective_go.html [4]: http://gun.io/blog/how-to-github-fork-branch-and-pull-request [5]: https://chris.beams.io/posts/git-commit/ diff --git a/Makefile b/Makefile index 38f2c45e..b9131564 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ test: .PHONY: server server: - go build -o $(BIN_DIR)/server ./proctord/main.go + go build -o $(BIN_DIR)/server ./cmd/proctord/main.go .PHONY: start-server start-server: @@ -35,11 +35,11 @@ start-server: .PHONY: cli cli: - go build -o $(BIN_DIR)/cli ./cli/main.go + go build -o $(BIN_DIR)/cli ./cmd/cli/main.go generate: go get -u github.com/go-bindata/go-bindata/... - $(GOPATH)/bin/go-bindata -pkg config -o cli/config/data.go assets/config_template.yaml + $(GOPATH)/bin/go-bindata -pkg config -o internal/app/cli/config/data.go internal/app/cli/config_template.yaml db.setup: db.create db.migrate diff --git a/README.md b/README.md index 2b5f067f..7c18a731 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Proctor -

+

- Build Status - - + Build Status + +

## Description diff --git a/doc/proctor-logo.png b/assets/img/proctor-logo.png similarity index 100% rename from doc/proctor-logo.png rename to assets/img/proctor-logo.png diff --git a/doc/proctor_describe.gif b/assets/img/proctor_describe.gif similarity index 100% rename from doc/proctor_describe.gif rename to assets/img/proctor_describe.gif diff --git a/doc/proctor_execute.gif b/assets/img/proctor_execute.gif similarity index 100% rename from doc/proctor_execute.gif rename to assets/img/proctor_execute.gif diff --git a/doc/schedule.gif b/assets/img/schedule.gif similarity index 100% rename from doc/schedule.gif rename to assets/img/schedule.gif diff --git a/doc/schedule_describe.gif b/assets/img/schedule_describe.gif similarity index 100% rename from doc/schedule_describe.gif rename to assets/img/schedule_describe.gif diff --git a/doc/schedule_list.gif b/assets/img/schedule_list.gif similarity index 100% rename from doc/schedule_list.gif rename to assets/img/schedule_list.gif diff --git a/doc/schedule_remove.gif b/assets/img/schedule_remove.gif similarity index 100% rename from doc/schedule_remove.gif rename to assets/img/schedule_remove.gif diff --git a/cli/main.go b/cmd/cli/main.go similarity index 59% rename from cli/main.go rename to cmd/cli/main.go index f8d19117..a889e403 100644 --- a/cli/main.go +++ b/cmd/cli/main.go @@ -1,11 +1,11 @@ package main import ( - "proctor/cli/command" - "proctor/cli/command/version/github" - "proctor/cli/config" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/command" + "proctor/internal/app/cli/command/version/github" + "proctor/internal/app/cli/config" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" ) func main() { diff --git a/proctord/main.go b/cmd/proctord/main.go similarity index 85% rename from proctord/main.go rename to cmd/proctord/main.go index 75354c45..779691f0 100644 --- a/proctord/main.go +++ b/cmd/proctord/main.go @@ -1,16 +1,15 @@ package main import ( + "github.com/getsentry/raven-go" "os" + "proctor/internal/app/proctord/config" + "proctor/internal/app/proctord/logger" + "proctor/internal/app/proctord/scheduler" + "proctor/internal/app/proctord/server" + "proctor/internal/app/proctord/storage/postgres" - "github.com/getsentry/raven-go" "github.com/urfave/cli" - - "proctor/proctord/config" - "proctor/proctord/logger" - "proctor/proctord/scheduler" - "proctor/proctord/server" - "proctor/proctord/storage/postgres" ) func main() { diff --git a/internal/app/cli/cli.go b/internal/app/cli/cli.go new file mode 100644 index 00000000..7f1e458c --- /dev/null +++ b/internal/app/cli/cli.go @@ -0,0 +1 @@ +package cli diff --git a/cli/command/config/manager.go b/internal/app/cli/command/config/manager.go similarity index 72% rename from cli/command/config/manager.go rename to internal/app/cli/command/config/manager.go index 905f2e3c..21986a1e 100644 --- a/cli/command/config/manager.go +++ b/internal/app/cli/command/config/manager.go @@ -6,12 +6,12 @@ import ( "io/ioutil" "os" "path/filepath" - "proctor/shared/io" + "proctor/internal/app/cli/config" + "proctor/internal/pkg/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" - proctorConfig "proctor/cli/config" ) func CreateDirIfNotExist(dir string) { @@ -28,11 +28,11 @@ func NewCmd(printer io.Printer) *cobra.Command { Use: "config", Short: "Configure proctor client", Long: "This command helps configure client with proctord host, email id and access token", - Example: fmt.Sprintf("proctor config %s=example.proctor.com %s=example@proctor.com %s=XXXXX", proctorConfig.ProctorHost, proctorConfig.EmailId, proctorConfig.AccessToken), + Example: fmt.Sprintf("proctor config %s=example.proctor.com %s=example@proctor.com %s=XXXXX", config.ProctorHost, config.EmailId, config.AccessToken), Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { - configFile := filepath.Join(proctorConfig.ConfigFileDir(), "proctor.yaml") + configFile := filepath.Join(config.ConfigFileDir(), "proctor.yaml") if _, err := os.Stat(configFile); err == nil { printer.Println("[Warning] This will overwrite current config:", color.FgYellow) existingProctorConfig, err := ioutil.ReadFile(configFile) @@ -58,7 +58,7 @@ func NewCmd(printer io.Printer) *cobra.Command { } } - CreateDirIfNotExist(proctorConfig.ConfigFileDir()) + CreateDirIfNotExist(config.ConfigFileDir()) var configFileContent string for _, v := range args { arg := strings.Split(v, "=") @@ -69,16 +69,16 @@ func NewCmd(printer io.Printer) *cobra.Command { } switch arg[0] { - case proctorConfig.ProctorHost: - configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ProctorHost, arg[1]) - case proctorConfig.EmailId: - configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.EmailId, arg[1]) - case proctorConfig.AccessToken: - configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.AccessToken, arg[1]) - case proctorConfig.ConnectionTimeoutSecs: - configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ConnectionTimeoutSecs, arg[1]) - case proctorConfig.ProcExecutionStatusPollCount: - configFileContent += fmt.Sprintf("%s: %s\n", proctorConfig.ProcExecutionStatusPollCount, arg[1]) + case config.ProctorHost: + configFileContent += fmt.Sprintf("%s: %s\n", config.ProctorHost, arg[1]) + case config.EmailId: + configFileContent += fmt.Sprintf("%s: %s\n", config.EmailId, arg[1]) + case config.AccessToken: + configFileContent += fmt.Sprintf("%s: %s\n", config.AccessToken, arg[1]) + case config.ConnectionTimeoutSecs: + configFileContent += fmt.Sprintf("%s: %s\n", config.ConnectionTimeoutSecs, arg[1]) + case config.ProcExecutionStatusPollCount: + configFileContent += fmt.Sprintf("%s: %s\n", config.ProcExecutionStatusPollCount, arg[1]) default: printer.Println(fmt.Sprintf("Proctor doesn't support config key: %s", arg[0]), color.FgYellow) } diff --git a/cli/command/config/view/view.go b/internal/app/cli/command/config/view/view.go similarity index 87% rename from cli/command/config/view/view.go rename to internal/app/cli/command/config/view/view.go index fb4bc8ab..4b37d0e4 100644 --- a/cli/command/config/view/view.go +++ b/internal/app/cli/command/config/view/view.go @@ -5,11 +5,11 @@ import ( "io/ioutil" "os" "path/filepath" + "proctor/internal/app/cli/config" "github.com/fatih/color" "github.com/spf13/cobra" - proctorConfig "proctor/cli/config" - "proctor/shared/io" + "proctor/internal/pkg/io" ) func NewCmd(printer io.Printer) *cobra.Command { @@ -20,7 +20,7 @@ func NewCmd(printer io.Printer) *cobra.Command { Example: fmt.Sprintf("proctor config show"), Run: func(cmd *cobra.Command, args []string) { - configFile := filepath.Join(proctorConfig.ConfigFileDir(), "proctor.yaml") + configFile := filepath.Join(config.ConfigFileDir(), "proctor.yaml") if _, err := os.Stat(configFile); os.IsNotExist(err) { printer.Println(fmt.Sprintf("Client Config is absent: %s", configFile), color.FgRed) printer.Println(fmt.Sprintf("Setup config using `proctor config PROCTOR_HOST=some.host ...`"), color.FgRed) diff --git a/cli/command/description/descriptor.go b/internal/app/cli/command/description/descriptor.go similarity index 92% rename from cli/command/description/descriptor.go rename to internal/app/cli/command/description/descriptor.go index a305efc2..7fc563ec 100644 --- a/cli/command/description/descriptor.go +++ b/internal/app/cli/command/description/descriptor.go @@ -2,13 +2,13 @@ package description import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - procMetadata "proctor/shared/model/metadata" + modelMetadata "proctor/internal/pkg/model/metadata" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { @@ -31,7 +31,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { } userProvidedProcName := args[0] - desiredProc := procMetadata.Metadata{} + desiredProc := modelMetadata.Metadata{} for _, proc := range procList { if userProvidedProcName == proc.Name { desiredProc = proc diff --git a/cli/command/description/descriptor_test.go b/internal/app/cli/command/description/descriptor_test.go similarity index 94% rename from cli/command/description/descriptor_test.go rename to internal/app/cli/command/description/descriptor_test.go index f498a2de..fbc4af16 100644 --- a/cli/command/description/descriptor_test.go +++ b/internal/app/cli/command/description/descriptor_test.go @@ -3,13 +3,13 @@ package description import ( "errors" "fmt" - "proctor/shared/io" + daemon2 "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "strings" "testing" - "proctor/cli/daemon" - procMetadata "proctor/shared/model/metadata" - "proctor/shared/model/metadata/env" + procMetadata "proctor/internal/pkg/model/metadata" + "proctor/internal/pkg/model/metadata/env" "github.com/fatih/color" "github.com/spf13/cobra" @@ -20,13 +20,13 @@ import ( type DescribeCmdTestSuite struct { suite.Suite mockPrinter *io.MockPrinter - mockProctorDClient *daemon.MockClient + mockProctorDClient *daemon2.MockClient testDescribeCmd *cobra.Command } func (s *DescribeCmdTestSuite) SetupTest() { s.mockPrinter = &io.MockPrinter{} - s.mockProctorDClient = &daemon.MockClient{} + s.mockProctorDClient = &daemon2.MockClient{} s.testDescribeCmd = NewCmd(s.mockPrinter, s.mockProctorDClient) } diff --git a/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go similarity index 95% rename from cli/command/execution/executioner.go rename to internal/app/cli/command/execution/executioner.go index 9e18fa55..513d16d3 100644 --- a/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -2,13 +2,13 @@ package execution import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - "proctor/shared/constant" + "proctor/internal/pkg/constant" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { diff --git a/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go similarity index 98% rename from cli/command/execution/executioner_test.go rename to internal/app/cli/command/execution/executioner_test.go index 072c7c18..1eb096e8 100644 --- a/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -4,15 +4,15 @@ import ( "errors" "fmt" "github.com/stretchr/testify/mock" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - "proctor/shared/constant" + "proctor/internal/pkg/constant" ) type ExecutionCmdTestSuite struct { diff --git a/cli/command/list/lister.go b/internal/app/cli/command/list/lister.go similarity index 89% rename from cli/command/list/lister.go rename to internal/app/cli/command/list/lister.go index fbc20dbe..533cc1a6 100644 --- a/cli/command/list/lister.go +++ b/internal/app/cli/command/list/lister.go @@ -2,12 +2,12 @@ package list import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - "proctor/shared/utility/sort" + "proctor/internal/pkg/utility/sort" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/list/lister_test.go b/internal/app/cli/command/list/lister_test.go similarity index 85% rename from cli/command/list/lister_test.go rename to internal/app/cli/command/list/lister_test.go index 4ebd43c2..14d51df5 100644 --- a/cli/command/list/lister_test.go +++ b/internal/app/cli/command/list/lister_test.go @@ -3,15 +3,15 @@ package list import ( "errors" "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - procMetadata "proctor/shared/model/metadata" + modelMetadata "proctor/internal/pkg/model/metadata" ) type ListCmdTestSuite struct { @@ -38,15 +38,15 @@ func (s *ListCmdTestSuite) TestListCmdHelp() { } func (s *ListCmdTestSuite) TestListCmdRun() { - procOne := procMetadata.Metadata{ + procOne := modelMetadata.Metadata{ Name: "one", Description: "proc one description", } - procTwo := procMetadata.Metadata{ + procTwo := modelMetadata.Metadata{ Name: "two", Description: "proc two description", } - procList := []procMetadata.Metadata{procOne, procTwo} + procList := []modelMetadata.Metadata{procOne, procTwo} s.mockProctorDClient.On("ListProcs").Return(procList, nil).Once() @@ -61,7 +61,7 @@ func (s *ListCmdTestSuite) TestListCmdRun() { } func (s *ListCmdTestSuite) TestListCmdRunProctorDClientFailure() { - s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, errors.New("Error!!!\nUnknown Error.")).Once() + s.mockProctorDClient.On("ListProcs").Return([]modelMetadata.Metadata{}, errors.New("Error!!!\nUnknown Error.")).Once() s.mockPrinter.On("Println", "Error!!!\nUnknown Error.", color.FgRed).Once() s.testListCmd.Run(&cobra.Command{}, []string{}) diff --git a/cli/command/root.go b/internal/app/cli/command/root.go similarity index 76% rename from cli/command/root.go rename to internal/app/cli/command/root.go index cc047ca8..0401b325 100644 --- a/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -3,21 +3,21 @@ package command import ( "fmt" "os" - "proctor/cli/command/schedule/remove" - "proctor/shared/io" + "proctor/internal/app/cli/command/config" + "proctor/internal/app/cli/command/config/view" + "proctor/internal/app/cli/command/description" + "proctor/internal/app/cli/command/execution" + "proctor/internal/app/cli/command/list" + "proctor/internal/app/cli/command/schedule" + scheduleDescribe "proctor/internal/app/cli/command/schedule/describe" + scheduleList "proctor/internal/app/cli/command/schedule/list" + "proctor/internal/app/cli/command/schedule/remove" + "proctor/internal/app/cli/command/version" + "proctor/internal/app/cli/command/version/github" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "github.com/spf13/cobra" - "proctor/cli/command/config" - "proctor/cli/command/config/view" - "proctor/cli/command/description" - "proctor/cli/command/execution" - "proctor/cli/command/list" - "proctor/cli/command/schedule" - scheduleDescribe "proctor/cli/command/schedule/describe" - scheduleList "proctor/cli/command/schedule/list" - "proctor/cli/command/version" - "proctor/cli/command/version/github" - "proctor/cli/daemon" ) var ( diff --git a/cli/command/root_test.go b/internal/app/cli/command/root_test.go similarity index 90% rename from cli/command/root_test.go rename to internal/app/cli/command/root_test.go index 23ab49c4..1fb8f835 100644 --- a/cli/command/root_test.go +++ b/internal/app/cli/command/root_test.go @@ -1,13 +1,13 @@ package command import ( - "proctor/shared/io" + "proctor/internal/app/cli/command/version/github" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - "proctor/cli/command/version/github" - "proctor/cli/daemon" ) func TestRootCmdUsage(t *testing.T) { diff --git a/cli/command/schedule/describe/describe.go b/internal/app/cli/command/schedule/describe/describe.go similarity index 95% rename from cli/command/schedule/describe/describe.go rename to internal/app/cli/command/schedule/describe/describe.go index 357dc8d1..d9a81522 100644 --- a/cli/command/schedule/describe/describe.go +++ b/internal/app/cli/command/schedule/describe/describe.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/schedule/describe/describe_test.go b/internal/app/cli/command/schedule/describe/describe_test.go similarity index 94% rename from cli/command/schedule/describe/describe_test.go rename to internal/app/cli/command/schedule/describe/describe_test.go index b72b7595..80246000 100644 --- a/cli/command/schedule/describe/describe_test.go +++ b/internal/app/cli/command/schedule/describe/describe_test.go @@ -4,8 +4,8 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" ) diff --git a/cli/command/schedule/list/list.go b/internal/app/cli/command/schedule/list/list.go similarity index 93% rename from cli/command/schedule/list/list.go rename to internal/app/cli/command/schedule/list/list.go index 2151c467..5587efac 100644 --- a/cli/command/schedule/list/list.go +++ b/internal/app/cli/command/schedule/list/list.go @@ -2,11 +2,11 @@ package list import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/schedule/list/list_test.go b/internal/app/cli/command/schedule/list/list_test.go similarity index 93% rename from cli/command/schedule/list/list_test.go rename to internal/app/cli/command/schedule/list/list_test.go index 99e21e0d..0ebbd9dc 100644 --- a/cli/command/schedule/list/list_test.go +++ b/internal/app/cli/command/schedule/list/list_test.go @@ -4,8 +4,8 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" ) diff --git a/cli/command/schedule/remove/remove.go b/internal/app/cli/command/schedule/remove/remove.go similarity index 91% rename from cli/command/schedule/remove/remove.go rename to internal/app/cli/command/schedule/remove/remove.go index 44699ef2..f13c1fbc 100644 --- a/cli/command/schedule/remove/remove.go +++ b/internal/app/cli/command/schedule/remove/remove.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/cli/command/schedule/remove/remove_test.go b/internal/app/cli/command/schedule/remove/remove_test.go similarity index 94% rename from cli/command/schedule/remove/remove_test.go rename to internal/app/cli/command/schedule/remove/remove_test.go index ab06a3d5..8176df85 100644 --- a/cli/command/schedule/remove/remove_test.go +++ b/internal/app/cli/command/schedule/remove/remove_test.go @@ -4,8 +4,8 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" ) diff --git a/cli/command/schedule/schedule.go b/internal/app/cli/command/schedule/schedule.go similarity index 97% rename from cli/command/schedule/schedule.go rename to internal/app/cli/command/schedule/schedule.go index 52db3afb..036f0037 100644 --- a/cli/command/schedule/schedule.go +++ b/internal/app/cli/command/schedule/schedule.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "strings" ) diff --git a/cli/command/schedule/schedule_test.go b/internal/app/cli/command/schedule/schedule_test.go similarity index 94% rename from cli/command/schedule/schedule_test.go rename to internal/app/cli/command/schedule/schedule_test.go index 7a795d11..018d5015 100644 --- a/cli/command/schedule/schedule_test.go +++ b/internal/app/cli/command/schedule/schedule_test.go @@ -4,8 +4,8 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/cli/daemon" - "proctor/shared/io" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" "testing" ) diff --git a/cli/command/version/github/client.go b/internal/app/cli/command/version/github/client.go similarity index 100% rename from cli/command/version/github/client.go rename to internal/app/cli/command/version/github/client.go diff --git a/cli/command/version/github/client_mock.go b/internal/app/cli/command/version/github/client_mock.go similarity index 100% rename from cli/command/version/github/client_mock.go rename to internal/app/cli/command/version/github/client_mock.go diff --git a/cli/command/version/version.go b/internal/app/cli/command/version/version.go similarity index 91% rename from cli/command/version/version.go rename to internal/app/cli/command/version/version.go index f5d78e1a..19d2ab62 100644 --- a/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -2,11 +2,11 @@ package version import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/command/version/github" + "proctor/internal/pkg/io" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/cli/command/version/github" ) const ClientVersion = "v0.6.0" diff --git a/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go similarity index 89% rename from cli/command/version/version_test.go rename to internal/app/cli/command/version/version_test.go index 66ef7dbc..524a6789 100644 --- a/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -2,17 +2,17 @@ package version import ( "fmt" - "proctor/shared/io" + "proctor/internal/app/cli/command/version/github" + "proctor/internal/pkg/io" "testing" "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" - gh "proctor/cli/command/version/github" ) func TestVersionCmdUsage(t *testing.T) { - githubClient := &gh.MockClient{} + githubClient := &github.MockClient{} versionCmd := NewCmd(&io.MockPrinter{}, githubClient) assert.Equal(t, "version", versionCmd.Use) assert.Equal(t, "Print version of Proctor command-line tool", versionCmd.Short) @@ -21,7 +21,7 @@ func TestVersionCmdUsage(t *testing.T) { func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} - githubClient := &gh.MockClient{} + githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) version := "v0.6.0" @@ -35,7 +35,7 @@ func TestLatestVersionCmd(t *testing.T) { func TestOldVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} - githubClient := &gh.MockClient{} + githubClient := &github.MockClient{} version := "v1000.0.0" versionCmd := NewCmd(mockPrinter, githubClient) diff --git a/cli/config/config.go b/internal/app/cli/config/config.go similarity index 98% rename from cli/config/config.go rename to internal/app/cli/config/config.go index b4f5ad84..fd70c929 100644 --- a/cli/config/config.go +++ b/internal/app/cli/config/config.go @@ -6,7 +6,7 @@ import ( "time" "github.com/pkg/errors" - "proctor/shared/constant" + "proctor/internal/pkg/constant" "github.com/spf13/viper" ) diff --git a/cli/config/config_mock.go b/internal/app/cli/config/config_mock.go similarity index 100% rename from cli/config/config_mock.go rename to internal/app/cli/config/config_mock.go diff --git a/cli/config/config_test.go b/internal/app/cli/config/config_test.go similarity index 100% rename from cli/config/config_test.go rename to internal/app/cli/config/config_test.go diff --git a/cli/config/data.go b/internal/app/cli/config/data.go similarity index 100% rename from cli/config/data.go rename to internal/app/cli/config/data.go diff --git a/assets/config_template.yaml b/internal/app/cli/config_template.yaml similarity index 100% rename from assets/config_template.yaml rename to internal/app/cli/config_template.yaml diff --git a/cli/daemon/client.go b/internal/app/cli/daemon/client.go similarity index 98% rename from cli/daemon/client.go rename to internal/app/cli/daemon/client.go index e5c173ef..77cabacb 100644 --- a/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -12,18 +12,17 @@ import ( "net/url" "os" "os/signal" - "proctor/shared/constant" - "proctor/shared/io" + "proctor/internal/app/cli/command/version" + "proctor/internal/app/cli/config" + "proctor/internal/pkg/constant" + "proctor/internal/pkg/io" "time" - "proctor/cli/command/version" - "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/gorilla/websocket" - "proctor/cli/config" - modelMetadata "proctor/shared/model/metadata" - modelSchedule "proctor/shared/model/schedule" + modelMetadata "proctor/internal/pkg/model/metadata" + modelSchedule "proctor/internal/pkg/model/schedule" ) type Client interface { diff --git a/cli/daemon/client_mock.go b/internal/app/cli/daemon/client_mock.go similarity index 93% rename from cli/daemon/client_mock.go rename to internal/app/cli/daemon/client_mock.go index f6ea5076..d9be1356 100644 --- a/cli/daemon/client_mock.go +++ b/internal/app/cli/daemon/client_mock.go @@ -2,8 +2,8 @@ package daemon import ( "github.com/stretchr/testify/mock" - modelMetadata "proctor/shared/model/metadata" - modelSchedule "proctor/shared/model/schedule" + modelMetadata "proctor/internal/pkg/model/metadata" + modelSchedule "proctor/internal/pkg/model/schedule" ) type MockClient struct { diff --git a/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go similarity index 99% rename from cli/daemon/client_test.go rename to internal/app/cli/daemon/client_test.go index e58721dd..567b49d4 100644 --- a/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -5,20 +5,19 @@ import ( "fmt" "net/http" "net/http/httptest" - "proctor/shared/io" + "proctor/internal/app/cli/command/version" + "proctor/internal/app/cli/config" + "proctor/internal/pkg/io" "strings" "testing" - "proctor/cli/command/version" - "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/thingful/httpmock" - "proctor/cli/config" - "proctor/shared/constant" - modelMetadata "proctor/shared/model/metadata" - "proctor/shared/model/metadata/env" + "proctor/internal/pkg/constant" + modelMetadata "proctor/internal/pkg/model/metadata" + "proctor/internal/pkg/model/metadata/env" ) type TestConnectionError struct { diff --git a/proctord/audit/auditor.go b/internal/app/proctord/audit/auditor.go similarity index 88% rename from proctord/audit/auditor.go rename to internal/app/proctord/audit/auditor.go index 515e3c98..dd7a2311 100644 --- a/proctord/audit/auditor.go +++ b/internal/app/proctord/audit/auditor.go @@ -2,11 +2,11 @@ package audit import ( "github.com/getsentry/raven-go" - "proctor/proctord/kubernetes" - "proctor/proctord/logger" - "proctor/proctord/storage" - "proctor/proctord/storage/postgres" - "proctor/shared/constant" + "proctor/internal/app/proctord/kubernetes" + "proctor/internal/app/proctord/logger" + "proctor/internal/app/proctord/storage" + "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/pkg/constant" ) type Auditor interface { diff --git a/proctord/audit/auditor_mock.go b/internal/app/proctord/audit/auditor_mock.go similarity index 91% rename from proctord/audit/auditor_mock.go rename to internal/app/proctord/audit/auditor_mock.go index 296d98f1..69cc507b 100644 --- a/proctord/audit/auditor_mock.go +++ b/internal/app/proctord/audit/auditor_mock.go @@ -2,7 +2,7 @@ package audit import ( "github.com/stretchr/testify/mock" - "proctor/proctord/storage/postgres" + "proctor/internal/app/proctord/storage/postgres" ) type MockAuditor struct { diff --git a/proctord/audit/auditor_test.go b/internal/app/proctord/audit/auditor_test.go similarity index 92% rename from proctord/audit/auditor_test.go rename to internal/app/proctord/audit/auditor_test.go index 9924af9b..9e1848f5 100644 --- a/proctord/audit/auditor_test.go +++ b/internal/app/proctord/audit/auditor_test.go @@ -1,12 +1,12 @@ package audit import ( + "proctor/internal/app/proctord/kubernetes" + "proctor/internal/app/proctord/storage" + "proctor/internal/app/proctord/storage/postgres" "testing" - "proctor/proctord/kubernetes" - "proctor/proctord/storage" - "proctor/proctord/storage/postgres" - "proctor/shared/constant" + "proctor/internal/pkg/constant" ) func TestJobsExecutionAuditing(t *testing.T) { diff --git a/proctord/config/config.go b/internal/app/proctord/config/config.go similarity index 100% rename from proctord/config/config.go rename to internal/app/proctord/config/config.go diff --git a/proctord/config/config_test.go b/internal/app/proctord/config/config_test.go similarity index 100% rename from proctord/config/config_test.go rename to internal/app/proctord/config/config_test.go diff --git a/proctord/docs/css/print.css b/internal/app/proctord/docs/css/print.css similarity index 100% rename from proctord/docs/css/print.css rename to internal/app/proctord/docs/css/print.css diff --git a/proctord/docs/css/reset.css b/internal/app/proctord/docs/css/reset.css similarity index 100% rename from proctord/docs/css/reset.css rename to internal/app/proctord/docs/css/reset.css diff --git a/proctord/docs/css/screen.css b/internal/app/proctord/docs/css/screen.css similarity index 100% rename from proctord/docs/css/screen.css rename to internal/app/proctord/docs/css/screen.css diff --git a/proctord/docs/css/style.css b/internal/app/proctord/docs/css/style.css similarity index 100% rename from proctord/docs/css/style.css rename to internal/app/proctord/docs/css/style.css diff --git a/proctord/docs/css/typography.css b/internal/app/proctord/docs/css/typography.css similarity index 100% rename from proctord/docs/css/typography.css rename to internal/app/proctord/docs/css/typography.css diff --git a/proctord/docs/fonts/DroidSans-Bold.ttf b/internal/app/proctord/docs/fonts/DroidSans-Bold.ttf similarity index 100% rename from proctord/docs/fonts/DroidSans-Bold.ttf rename to internal/app/proctord/docs/fonts/DroidSans-Bold.ttf diff --git a/proctord/docs/fonts/DroidSans.ttf b/internal/app/proctord/docs/fonts/DroidSans.ttf similarity index 100% rename from proctord/docs/fonts/DroidSans.ttf rename to internal/app/proctord/docs/fonts/DroidSans.ttf diff --git a/proctord/docs/handler.go b/internal/app/proctord/docs/handler.go similarity index 100% rename from proctord/docs/handler.go rename to internal/app/proctord/docs/handler.go diff --git a/proctord/docs/images/collapse.gif b/internal/app/proctord/docs/images/collapse.gif similarity index 100% rename from proctord/docs/images/collapse.gif rename to internal/app/proctord/docs/images/collapse.gif diff --git a/proctord/docs/images/expand.gif b/internal/app/proctord/docs/images/expand.gif similarity index 100% rename from proctord/docs/images/expand.gif rename to internal/app/proctord/docs/images/expand.gif diff --git a/proctord/docs/images/explorer_icons.png b/internal/app/proctord/docs/images/explorer_icons.png similarity index 100% rename from proctord/docs/images/explorer_icons.png rename to internal/app/proctord/docs/images/explorer_icons.png diff --git a/proctord/docs/images/favicon-16x16.png b/internal/app/proctord/docs/images/favicon-16x16.png similarity index 100% rename from proctord/docs/images/favicon-16x16.png rename to internal/app/proctord/docs/images/favicon-16x16.png diff --git a/proctord/docs/images/favicon-32x32.png b/internal/app/proctord/docs/images/favicon-32x32.png similarity index 100% rename from proctord/docs/images/favicon-32x32.png rename to internal/app/proctord/docs/images/favicon-32x32.png diff --git a/proctord/docs/images/favicon.ico b/internal/app/proctord/docs/images/favicon.ico similarity index 100% rename from proctord/docs/images/favicon.ico rename to internal/app/proctord/docs/images/favicon.ico diff --git a/proctord/docs/images/logo_small.png b/internal/app/proctord/docs/images/logo_small.png similarity index 100% rename from proctord/docs/images/logo_small.png rename to internal/app/proctord/docs/images/logo_small.png diff --git a/proctord/docs/images/pet_store_api.png b/internal/app/proctord/docs/images/pet_store_api.png similarity index 100% rename from proctord/docs/images/pet_store_api.png rename to internal/app/proctord/docs/images/pet_store_api.png diff --git a/proctord/docs/images/throbber.gif b/internal/app/proctord/docs/images/throbber.gif similarity index 100% rename from proctord/docs/images/throbber.gif rename to internal/app/proctord/docs/images/throbber.gif diff --git a/proctord/docs/images/wordnik_api.png b/internal/app/proctord/docs/images/wordnik_api.png similarity index 100% rename from proctord/docs/images/wordnik_api.png rename to internal/app/proctord/docs/images/wordnik_api.png diff --git a/proctord/docs/index.html b/internal/app/proctord/docs/index.html similarity index 100% rename from proctord/docs/index.html rename to internal/app/proctord/docs/index.html diff --git a/proctord/docs/lang/ca.js b/internal/app/proctord/docs/lang/ca.js similarity index 100% rename from proctord/docs/lang/ca.js rename to internal/app/proctord/docs/lang/ca.js diff --git a/proctord/docs/lang/el.js b/internal/app/proctord/docs/lang/el.js similarity index 100% rename from proctord/docs/lang/el.js rename to internal/app/proctord/docs/lang/el.js diff --git a/proctord/docs/lang/en.js b/internal/app/proctord/docs/lang/en.js similarity index 100% rename from proctord/docs/lang/en.js rename to internal/app/proctord/docs/lang/en.js diff --git a/proctord/docs/lang/es.js b/internal/app/proctord/docs/lang/es.js similarity index 100% rename from proctord/docs/lang/es.js rename to internal/app/proctord/docs/lang/es.js diff --git a/proctord/docs/lang/fr.js b/internal/app/proctord/docs/lang/fr.js similarity index 100% rename from proctord/docs/lang/fr.js rename to internal/app/proctord/docs/lang/fr.js diff --git a/proctord/docs/lang/geo.js b/internal/app/proctord/docs/lang/geo.js similarity index 100% rename from proctord/docs/lang/geo.js rename to internal/app/proctord/docs/lang/geo.js diff --git a/proctord/docs/lang/it.js b/internal/app/proctord/docs/lang/it.js similarity index 100% rename from proctord/docs/lang/it.js rename to internal/app/proctord/docs/lang/it.js diff --git a/proctord/docs/lang/ja.js b/internal/app/proctord/docs/lang/ja.js similarity index 100% rename from proctord/docs/lang/ja.js rename to internal/app/proctord/docs/lang/ja.js diff --git a/proctord/docs/lang/ko-kr.js b/internal/app/proctord/docs/lang/ko-kr.js similarity index 100% rename from proctord/docs/lang/ko-kr.js rename to internal/app/proctord/docs/lang/ko-kr.js diff --git a/proctord/docs/lang/pl.js b/internal/app/proctord/docs/lang/pl.js similarity index 100% rename from proctord/docs/lang/pl.js rename to internal/app/proctord/docs/lang/pl.js diff --git a/proctord/docs/lang/pt.js b/internal/app/proctord/docs/lang/pt.js similarity index 100% rename from proctord/docs/lang/pt.js rename to internal/app/proctord/docs/lang/pt.js diff --git a/proctord/docs/lang/ru.js b/internal/app/proctord/docs/lang/ru.js similarity index 100% rename from proctord/docs/lang/ru.js rename to internal/app/proctord/docs/lang/ru.js diff --git a/proctord/docs/lang/tr.js b/internal/app/proctord/docs/lang/tr.js similarity index 100% rename from proctord/docs/lang/tr.js rename to internal/app/proctord/docs/lang/tr.js diff --git a/proctord/docs/lang/translator.js b/internal/app/proctord/docs/lang/translator.js similarity index 100% rename from proctord/docs/lang/translator.js rename to internal/app/proctord/docs/lang/translator.js diff --git a/proctord/docs/lang/zh-cn.js b/internal/app/proctord/docs/lang/zh-cn.js similarity index 100% rename from proctord/docs/lang/zh-cn.js rename to internal/app/proctord/docs/lang/zh-cn.js diff --git a/proctord/docs/lib/backbone-min.js b/internal/app/proctord/docs/lib/backbone-min.js similarity index 100% rename from proctord/docs/lib/backbone-min.js rename to internal/app/proctord/docs/lib/backbone-min.js diff --git a/proctord/docs/lib/es5-shim.js b/internal/app/proctord/docs/lib/es5-shim.js similarity index 100% rename from proctord/docs/lib/es5-shim.js rename to internal/app/proctord/docs/lib/es5-shim.js diff --git a/proctord/docs/lib/handlebars-4.0.5.js b/internal/app/proctord/docs/lib/handlebars-4.0.5.js similarity index 100% rename from proctord/docs/lib/handlebars-4.0.5.js rename to internal/app/proctord/docs/lib/handlebars-4.0.5.js diff --git a/proctord/docs/lib/highlight.9.1.0.pack.js b/internal/app/proctord/docs/lib/highlight.9.1.0.pack.js similarity index 100% rename from proctord/docs/lib/highlight.9.1.0.pack.js rename to internal/app/proctord/docs/lib/highlight.9.1.0.pack.js diff --git a/proctord/docs/lib/highlight.9.1.0.pack_extended.js b/internal/app/proctord/docs/lib/highlight.9.1.0.pack_extended.js similarity index 100% rename from proctord/docs/lib/highlight.9.1.0.pack_extended.js rename to internal/app/proctord/docs/lib/highlight.9.1.0.pack_extended.js diff --git a/proctord/docs/lib/jquery-1.8.0.min.js b/internal/app/proctord/docs/lib/jquery-1.8.0.min.js similarity index 100% rename from proctord/docs/lib/jquery-1.8.0.min.js rename to internal/app/proctord/docs/lib/jquery-1.8.0.min.js diff --git a/proctord/docs/lib/jquery.ba-bbq.min.js b/internal/app/proctord/docs/lib/jquery.ba-bbq.min.js similarity index 100% rename from proctord/docs/lib/jquery.ba-bbq.min.js rename to internal/app/proctord/docs/lib/jquery.ba-bbq.min.js diff --git a/proctord/docs/lib/jquery.slideto.min.js b/internal/app/proctord/docs/lib/jquery.slideto.min.js similarity index 100% rename from proctord/docs/lib/jquery.slideto.min.js rename to internal/app/proctord/docs/lib/jquery.slideto.min.js diff --git a/proctord/docs/lib/jquery.wiggle.min.js b/internal/app/proctord/docs/lib/jquery.wiggle.min.js similarity index 100% rename from proctord/docs/lib/jquery.wiggle.min.js rename to internal/app/proctord/docs/lib/jquery.wiggle.min.js diff --git a/proctord/docs/lib/js-yaml.min.js b/internal/app/proctord/docs/lib/js-yaml.min.js similarity index 100% rename from proctord/docs/lib/js-yaml.min.js rename to internal/app/proctord/docs/lib/js-yaml.min.js diff --git a/proctord/docs/lib/jsoneditor.min.js b/internal/app/proctord/docs/lib/jsoneditor.min.js similarity index 100% rename from proctord/docs/lib/jsoneditor.min.js rename to internal/app/proctord/docs/lib/jsoneditor.min.js diff --git a/proctord/docs/lib/lodash.min.js b/internal/app/proctord/docs/lib/lodash.min.js similarity index 100% rename from proctord/docs/lib/lodash.min.js rename to internal/app/proctord/docs/lib/lodash.min.js diff --git a/proctord/docs/lib/marked.js b/internal/app/proctord/docs/lib/marked.js similarity index 100% rename from proctord/docs/lib/marked.js rename to internal/app/proctord/docs/lib/marked.js diff --git a/proctord/docs/lib/object-assign-pollyfill.js b/internal/app/proctord/docs/lib/object-assign-pollyfill.js similarity index 100% rename from proctord/docs/lib/object-assign-pollyfill.js rename to internal/app/proctord/docs/lib/object-assign-pollyfill.js diff --git a/proctord/docs/lib/sanitize-html.min.js b/internal/app/proctord/docs/lib/sanitize-html.min.js similarity index 100% rename from proctord/docs/lib/sanitize-html.min.js rename to internal/app/proctord/docs/lib/sanitize-html.min.js diff --git a/proctord/docs/lib/swagger-oauth.js b/internal/app/proctord/docs/lib/swagger-oauth.js similarity index 100% rename from proctord/docs/lib/swagger-oauth.js rename to internal/app/proctord/docs/lib/swagger-oauth.js diff --git a/proctord/docs/o2c.html b/internal/app/proctord/docs/o2c.html similarity index 100% rename from proctord/docs/o2c.html rename to internal/app/proctord/docs/o2c.html diff --git a/proctord/docs/swagger-ui.js b/internal/app/proctord/docs/swagger-ui.js similarity index 98% rename from proctord/docs/swagger-ui.js rename to internal/app/proctord/docs/swagger-ui.js index cd008ef9..d5780c21 100644 --- a/proctord/docs/swagger-ui.js +++ b/internal/app/proctord/docs/swagger-ui.js @@ -3181,8 +3181,8 @@ Handlebars.registerHelper('escape', function (value) { (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.sanitizeHtml=f()}})(function(){var define,module,exports;return function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o=0){globRegex.push(quoteRegexp(name).replace(/\\\*/g,".*"))}else{allowedAttributesMap[tag].push(name)}});allowedAttributesGlobMap[tag]=new RegExp("^("+globRegex.join("|")+")$")})}var allowedClassesMap={};each(options.allowedClasses,function(classes,tag){if(allowedAttributesMap){if(!has(allowedAttributesMap,tag)){allowedAttributesMap[tag]=[]}allowedAttributesMap[tag].push("class")}allowedClassesMap[tag]=classes});var transformTagsMap={};var transformTagsAll;each(options.transformTags,function(transform,tag){var transFun;if(typeof transform==="function"){transFun=transform}else if(typeof transform==="string"){transFun=sanitizeHtml.simpleTransform(transform)}if(tag==="*"){transformTagsAll=transFun}else{transformTagsMap[tag]=transFun}});var depth=0;var stack=[];var skipMap={};var transformMap={};var skipText=false;var skipTextDepth=0;var parser=new htmlparser.Parser({onopentag:function(name,attribs){if(skipText){skipTextDepth++;return}var frame=new Frame(name,attribs);stack.push(frame);var skip=false;var hasText=frame.text?true:false;var transformedTag;if(has(transformTagsMap,name)){transformedTag=transformTagsMap[name](name,attribs);frame.attribs=attribs=transformedTag.attribs;if(transformedTag.text!==undefined){frame.innerText=transformedTag.text}if(name!==transformedTag.tagName){frame.name=name=transformedTag.tagName;transformMap[depth]=transformedTag.tagName}}if(transformTagsAll){transformedTag=transformTagsAll(name,attribs);frame.attribs=attribs=transformedTag.attribs;if(name!==transformedTag.tagName){frame.name=name=transformedTag.tagName;transformMap[depth]=transformedTag.tagName}}if(options.allowedTags&&options.allowedTags.indexOf(name)===-1){skip=true;if(nonTextTagsArray.indexOf(name)!==-1){skipText=true;skipTextDepth=1}skipMap[depth]=true}depth++;if(skip){return}result+="<"+name;if(!allowedAttributesMap||has(allowedAttributesMap,name)||allowedAttributesMap["*"]){each(attribs,function(value,a){if(!allowedAttributesMap||has(allowedAttributesMap,name)&&allowedAttributesMap[name].indexOf(a)!==-1||allowedAttributesMap["*"]&&allowedAttributesMap["*"].indexOf(a)!==-1||has(allowedAttributesGlobMap,name)&&allowedAttributesGlobMap[name].test(a)||allowedAttributesGlobMap["*"]&&allowedAttributesGlobMap["*"].test(a)){if(a==="href"||a==="src"){if(naughtyHref(name,value)){delete frame.attribs[a];return}}if(a==="class"){value=filterClasses(value,allowedClassesMap[name]);if(!value.length){delete frame.attribs[a];return}}result+=" "+a;if(value.length){result+='="'+escapeHtml(value)+'"'}}else{delete frame.attribs[a]}})}if(options.selfClosing.indexOf(name)!==-1){result+=" />"}else{result+=">";if(frame.innerText&&!hasText&&!options.textFilter){result+=frame.innerText}}},ontext:function(text){if(skipText){return}var lastFrame=stack[stack.length-1];var tag;if(lastFrame){tag=lastFrame.tag;text=lastFrame.innerText!==undefined?lastFrame.innerText:text}if(tag==="script"||tag==="style"){result+=text}else{var escaped=escapeHtml(text);if(options.textFilter){result+=options.textFilter(escaped)}else{result+=escaped}}if(stack.length){var frame=stack[stack.length-1];frame.text+=text}},onclosetag:function(name){if(skipText){skipTextDepth--;if(!skipTextDepth){skipText=false}else{return}}var frame=stack.pop();if(!frame){return}skipText=false;depth--;if(skipMap[depth]){delete skipMap[depth];frame.updateParentNodeText();return}if(transformMap[depth]){name=transformMap[depth];delete transformMap[depth]}if(options.exclusiveFilter&&options.exclusiveFilter(frame)){result=result.substr(0,frame.tagPosition);return}frame.updateParentNodeText();if(options.selfClosing.indexOf(name)!==-1){return}result+=""}},options.parser);parser.write(html);parser.end();return result;function escapeHtml(s){if(typeof s!=="string"){s=s+""}return s.replace(/\&/g,"&").replace(//g,">").replace(/\"/g,""")}function naughtyHref(name,href){href=href.replace(/[\x00-\x20]+/g,"");href=href.replace(/<\!\-\-.*?\-\-\>/g,"");var matches=href.match(/^([a-zA-Z]+)\:/);if(!matches){return false}var scheme=matches[1].toLowerCase();if(has(options.allowedSchemesByTag,name)){return options.allowedSchemesByTag[name].indexOf(scheme)===-1}return!options.allowedSchemes||options.allowedSchemes.indexOf(scheme)===-1}function filterClasses(classes,allowed){if(!allowed){return classes}classes=classes.split(/\s+/);return classes.filter(function(clss){return allowed.indexOf(clss)!==-1}).join(" ")}}var htmlParserDefaults={decodeEntities:true};sanitizeHtml.defaults={allowedTags:["h3","h4","h5","h6","blockquote","p","a","ul","ol","nl","li","b","i","strong","em","strike","code","hr","br","div","table","thead","caption","tbody","tr","th","td","pre"],allowedAttributes:{a:["href","name","target"],img:["src"]},selfClosing:["img","br","hr","area","base","basefont","input","link","meta"],allowedSchemes:["http","https","ftp","mailto"],allowedSchemesByTag:{}};sanitizeHtml.simpleTransform=function(newTagName,newAttribs,merge){merge=merge===undefined?true:merge;newAttribs=newAttribs||{};return function(tagName,attribs){var attrib;if(merge){for(attrib in newAttribs){attribs[attrib]=newAttribs[attrib]}}else{attribs=newAttribs}return{tagName:newTagName,attribs:attribs}}}},{htmlparser2:36,"regexp-quote":54,xtend:58}],2:[function(require,module,exports){"use strict";exports.toByteArray=toByteArray;exports.fromByteArray=fromByteArray;var lookup=[];var revLookup=[];var Arr=typeof Uint8Array!=="undefined"?Uint8Array:Array;function init(){var code="ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";for(var i=0,len=code.length;i0){throw new Error("Invalid string. Length must be a multiple of 4")}placeHolders=b64[len-2]==="="?2:b64[len-1]==="="?1:0;arr=new Arr(len*3/4-placeHolders);l=placeHolders>0?len-4:len;var L=0;for(i=0,j=0;i>16&255;arr[L++]=tmp>>8&255;arr[L++]=tmp&255}if(placeHolders===2){tmp=revLookup[b64.charCodeAt(i)]<<2|revLookup[b64.charCodeAt(i+1)]>>4;arr[L++]=tmp&255}else if(placeHolders===1){tmp=revLookup[b64.charCodeAt(i)]<<10|revLookup[b64.charCodeAt(i+1)]<<4|revLookup[b64.charCodeAt(i+2)]>>2;arr[L++]=tmp>>8&255;arr[L++]=tmp&255}return arr}function tripletToBase64(num){return lookup[num>>18&63]+lookup[num>>12&63]+lookup[num>>6&63]+lookup[num&63]}function encodeChunk(uint8,start,end){var tmp;var output=[];for(var i=start;ilen2?len2:i+maxChunkLength))}if(extraBytes===1){tmp=uint8[len-1];output+=lookup[tmp>>2];output+=lookup[tmp<<4&63];output+="=="}else if(extraBytes===2){tmp=(uint8[len-2]<<8)+uint8[len-1];output+=lookup[tmp>>10];output+=lookup[tmp>>4&63];output+=lookup[tmp<<2&63];output+="="}parts.push(output);return parts.join("")}},{}],3:[function(require,module,exports){},{}],4:[function(require,module,exports){(function(global){"use strict";var buffer=require("buffer");var Buffer=buffer.Buffer;var SlowBuffer=buffer.SlowBuffer;var MAX_LEN=buffer.kMaxLength||2147483647;exports.alloc=function alloc(size,fill,encoding){if(typeof Buffer.alloc==="function"){return Buffer.alloc(size,fill,encoding)}if(typeof encoding==="number"){throw new TypeError("encoding must not be number")}if(typeof size!=="number"){throw new TypeError("size must be a number")}if(size>MAX_LEN){throw new RangeError("size is too large")}var enc=encoding;var _fill=fill;if(_fill===undefined){enc=undefined;_fill=0}var buf=new Buffer(size);if(typeof _fill==="string"){var fillBuf=new Buffer(_fill,enc);var flen=fillBuf.length;var i=-1;while(++iMAX_LEN){throw new RangeError("size is too large")}return new Buffer(size)};exports.from=function from(value,encodingOrOffset,length){if(typeof Buffer.from==="function"&&(!global.Uint8Array||Uint8Array.from!==Buffer.from)){return Buffer.from(value,encodingOrOffset,length)}if(typeof value==="number"){throw new TypeError('"value" argument must not be a number')}if(typeof value==="string"){return new Buffer(value,encodingOrOffset)}if(typeof ArrayBuffer!=="undefined"&&value instanceof ArrayBuffer){var offset=encodingOrOffset;if(arguments.length===1){return new Buffer(value)}if(typeof offset==="undefined"){offset=0}var len=length;if(typeof len==="undefined"){len=value.byteLength-offset}if(offset>=value.byteLength){throw new RangeError("'offset' is out of bounds")}if(len>value.byteLength-offset){throw new RangeError("'length' is out of bounds")}return new Buffer(value.slice(offset,offset+len))}if(Buffer.isBuffer(value)){var out=new Buffer(value.length);value.copy(out,0,0,value.length);return out}if(value){if(Array.isArray(value)||typeof ArrayBuffer!=="undefined"&&value.buffer instanceof ArrayBuffer||"length"in value){return new Buffer(value)}if(value.type==="Buffer"&&Array.isArray(value.data)){return new Buffer(value.data)}}throw new TypeError("First argument must be a string, Buffer, "+"ArrayBuffer, Array, or array-like object.")};exports.allocUnsafeSlow=function allocUnsafeSlow(size){if(typeof Buffer.allocUnsafeSlow==="function"){return Buffer.allocUnsafeSlow(size)}if(typeof size!=="number"){throw new TypeError("size must be a number")}if(size>=MAX_LEN){throw new RangeError("size is too large")}return new SlowBuffer(size)}}).call(this,typeof global!=="undefined"?global:typeof self!=="undefined"?self:typeof window!=="undefined"?window:{})},{buffer:5}],5:[function(require,module,exports){(function(global){"use strict";var base64=require("base64-js");var ieee754=require("ieee754");var isArray=require("isarray");exports.Buffer=Buffer;exports.SlowBuffer=SlowBuffer;exports.INSPECT_MAX_BYTES=50;Buffer.TYPED_ARRAY_SUPPORT=global.TYPED_ARRAY_SUPPORT!==undefined?global.TYPED_ARRAY_SUPPORT:typedArraySupport();exports.kMaxLength=kMaxLength();function typedArraySupport(){try{var arr=new Uint8Array(1);arr.__proto__={__proto__:Uint8Array.prototype,foo:function(){return 42}};return arr.foo()===42&&typeof arr.subarray==="function"&&arr.subarray(1,1).byteLength===0}catch(e){return false}}function kMaxLength(){return Buffer.TYPED_ARRAY_SUPPORT?2147483647:1073741823}function createBuffer(that,length){if(kMaxLength()=kMaxLength()){throw new RangeError("Attempt to allocate Buffer larger than maximum "+"size: 0x"+kMaxLength().toString(16)+" bytes")}return length|0}function SlowBuffer(length){if(+length!=length){length=0}return Buffer.alloc(+length)}Buffer.isBuffer=function isBuffer(b){return!!(b!=null&&b._isBuffer)};Buffer.compare=function compare(a,b){if(!Buffer.isBuffer(a)||!Buffer.isBuffer(b)){throw new TypeError("Arguments must be Buffers")}if(a===b)return 0;var x=a.length;var y=b.length;for(var i=0,len=Math.min(x,y);i>>1;case"base64":return base64ToBytes(string).length;default:if(loweredCase)return utf8ToBytes(string).length;encoding=(""+encoding).toLowerCase();loweredCase=true}}}Buffer.byteLength=byteLength;function slowToString(encoding,start,end){var loweredCase=false;if(start===undefined||start<0){start=0}if(start>this.length){return""}if(end===undefined||end>this.length){end=this.length}if(end<=0){return""}end>>>=0;start>>>=0;if(end<=start){return""}if(!encoding)encoding="utf8";while(true){switch(encoding){case"hex":return hexSlice(this,start,end);case"utf8":case"utf-8":return utf8Slice(this,start,end);case"ascii":return asciiSlice(this,start,end);case"latin1":case"binary":return latin1Slice(this,start,end);case"base64":return base64Slice(this,start,end);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return utf16leSlice(this,start,end);default:if(loweredCase)throw new TypeError("Unknown encoding: "+encoding);encoding=(encoding+"").toLowerCase();loweredCase=true}}}Buffer.prototype._isBuffer=true;function swap(b,n,m){var i=b[n];b[n]=b[m];b[m]=i}Buffer.prototype.swap16=function swap16(){var len=this.length;if(len%2!==0){throw new RangeError("Buffer size must be a multiple of 16-bits")}for(var i=0;i0){str=this.toString("hex",0,max).match(/.{2}/g).join(" ");if(this.length>max)str+=" ... "}return""};Buffer.prototype.compare=function compare(target,start,end,thisStart,thisEnd){if(!Buffer.isBuffer(target)){throw new TypeError("Argument must be a Buffer")}if(start===undefined){start=0}if(end===undefined){end=target?target.length:0}if(thisStart===undefined){thisStart=0}if(thisEnd===undefined){thisEnd=this.length}if(start<0||end>target.length||thisStart<0||thisEnd>this.length){throw new RangeError("out of range index")}if(thisStart>=thisEnd&&start>=end){return 0}if(thisStart>=thisEnd){return-1}if(start>=end){return 1}start>>>=0;end>>>=0;thisStart>>>=0;thisEnd>>>=0;if(this===target)return 0;var x=thisEnd-thisStart;var y=end-start;var len=Math.min(x,y);var thisCopy=this.slice(thisStart,thisEnd);var targetCopy=target.slice(start,end);for(var i=0;i2147483647){byteOffset=2147483647}else if(byteOffset<-2147483648){byteOffset=-2147483648}byteOffset=+byteOffset;if(isNaN(byteOffset)){byteOffset=dir?0:buffer.length-1}if(byteOffset<0)byteOffset=buffer.length+byteOffset;if(byteOffset>=buffer.length){if(dir)return-1;else byteOffset=buffer.length-1}else if(byteOffset<0){if(dir)byteOffset=0;else return-1}if(typeof val==="string"){val=Buffer.from(val,encoding)}if(Buffer.isBuffer(val)){if(val.length===0){return-1}return arrayIndexOf(buffer,val,byteOffset,encoding,dir)}else if(typeof val==="number"){val=val&255;if(Buffer.TYPED_ARRAY_SUPPORT&&typeof Uint8Array.prototype.indexOf==="function"){if(dir){return Uint8Array.prototype.indexOf.call(buffer,val,byteOffset)}else{return Uint8Array.prototype.lastIndexOf.call(buffer,val,byteOffset)}}return arrayIndexOf(buffer,[val],byteOffset,encoding,dir)}throw new TypeError("val must be string, number or Buffer")}function arrayIndexOf(arr,val,byteOffset,encoding,dir){var indexSize=1;var arrLength=arr.length;var valLength=val.length;if(encoding!==undefined){encoding=String(encoding).toLowerCase();if(encoding==="ucs2"||encoding==="ucs-2"||encoding==="utf16le"||encoding==="utf-16le"){if(arr.length<2||val.length<2){return-1}indexSize=2;arrLength/=2;valLength/=2;byteOffset/=2}}function read(buf,i){if(indexSize===1){return buf[i]}else{return buf.readUInt16BE(i*indexSize)}}var i;if(dir){var foundIndex=-1;for(i=byteOffset;iarrLength)byteOffset=arrLength-valLength;for(i=byteOffset;i>=0;i--){var found=true;for(var j=0;jremaining){length=remaining}}var strLen=string.length;if(strLen%2!==0)throw new TypeError("Invalid hex string");if(length>strLen/2){length=strLen/2}for(var i=0;iremaining)length=remaining;if(string.length>0&&(length<0||offset<0)||offset>this.length){throw new RangeError("Attempt to write outside buffer bounds")}if(!encoding)encoding="utf8";var loweredCase=false;for(;;){switch(encoding){case"hex":return hexWrite(this,string,offset,length);case"utf8":case"utf-8":return utf8Write(this,string,offset,length);case"ascii":return asciiWrite(this,string,offset,length);case"latin1":case"binary":return latin1Write(this,string,offset,length);case"base64":return base64Write(this,string,offset,length);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return ucs2Write(this,string,offset,length);default:if(loweredCase)throw new TypeError("Unknown encoding: "+encoding);encoding=(""+encoding).toLowerCase();loweredCase=true}}};Buffer.prototype.toJSON=function toJSON(){return{type:"Buffer",data:Array.prototype.slice.call(this._arr||this,0)}};function base64Slice(buf,start,end){if(start===0&&end===buf.length){return base64.fromByteArray(buf)}else{return base64.fromByteArray(buf.slice(start,end))}}function utf8Slice(buf,start,end){end=Math.min(buf.length,end);var res=[];var i=start;while(i239?4:firstByte>223?3:firstByte>191?2:1;if(i+bytesPerSequence<=end){var secondByte,thirdByte,fourthByte,tempCodePoint;switch(bytesPerSequence){case 1:if(firstByte<128){codePoint=firstByte}break;case 2:secondByte=buf[i+1];if((secondByte&192)===128){tempCodePoint=(firstByte&31)<<6|secondByte&63;if(tempCodePoint>127){codePoint=tempCodePoint}}break;case 3:secondByte=buf[i+1];thirdByte=buf[i+2];if((secondByte&192)===128&&(thirdByte&192)===128){tempCodePoint=(firstByte&15)<<12|(secondByte&63)<<6|thirdByte&63;if(tempCodePoint>2047&&(tempCodePoint<55296||tempCodePoint>57343)){codePoint=tempCodePoint}}break;case 4:secondByte=buf[i+1];thirdByte=buf[i+2];fourthByte=buf[i+3];if((secondByte&192)===128&&(thirdByte&192)===128&&(fourthByte&192)===128){tempCodePoint=(firstByte&15)<<18|(secondByte&63)<<12|(thirdByte&63)<<6|fourthByte&63;if(tempCodePoint>65535&&tempCodePoint<1114112){codePoint=tempCodePoint}}}}if(codePoint===null){codePoint=65533;bytesPerSequence=1}else if(codePoint>65535){codePoint-=65536;res.push(codePoint>>>10&1023|55296);codePoint=56320|codePoint&1023}res.push(codePoint);i+=bytesPerSequence}return decodeCodePointsArray(res)}var MAX_ARGUMENTS_LENGTH=4096;function decodeCodePointsArray(codePoints){var len=codePoints.length;if(len<=MAX_ARGUMENTS_LENGTH){return String.fromCharCode.apply(String,codePoints)}var res="";var i=0;while(ilen)end=len;var out="";for(var i=start;ilen){start=len}if(end<0){end+=len;if(end<0)end=0}else if(end>len){end=len}if(endlength)throw new RangeError("Trying to access beyond buffer length")}Buffer.prototype.readUIntLE=function readUIntLE(offset,byteLength,noAssert){offset=offset|0;byteLength=byteLength|0;if(!noAssert)checkOffset(offset,byteLength,this.length);var val=this[offset];var mul=1;var i=0;while(++i0&&(mul*=256)){val+=this[offset+--byteLength]*mul}return val};Buffer.prototype.readUInt8=function readUInt8(offset,noAssert){if(!noAssert)checkOffset(offset,1,this.length);return this[offset]};Buffer.prototype.readUInt16LE=function readUInt16LE(offset,noAssert){if(!noAssert)checkOffset(offset,2,this.length);return this[offset]|this[offset+1]<<8};Buffer.prototype.readUInt16BE=function readUInt16BE(offset,noAssert){if(!noAssert)checkOffset(offset,2,this.length);return this[offset]<<8|this[offset+1]};Buffer.prototype.readUInt32LE=function readUInt32LE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return(this[offset]|this[offset+1]<<8|this[offset+2]<<16)+this[offset+3]*16777216};Buffer.prototype.readUInt32BE=function readUInt32BE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return this[offset]*16777216+(this[offset+1]<<16|this[offset+2]<<8|this[offset+3])};Buffer.prototype.readIntLE=function readIntLE(offset,byteLength,noAssert){offset=offset|0;byteLength=byteLength|0;if(!noAssert)checkOffset(offset,byteLength,this.length);var val=this[offset];var mul=1;var i=0;while(++i=mul)val-=Math.pow(2,8*byteLength);return val};Buffer.prototype.readIntBE=function readIntBE(offset,byteLength,noAssert){offset=offset|0;byteLength=byteLength|0;if(!noAssert)checkOffset(offset,byteLength,this.length); var i=byteLength;var mul=1;var val=this[offset+--i];while(i>0&&(mul*=256)){val+=this[offset+--i]*mul}mul*=128;if(val>=mul)val-=Math.pow(2,8*byteLength);return val};Buffer.prototype.readInt8=function readInt8(offset,noAssert){if(!noAssert)checkOffset(offset,1,this.length);if(!(this[offset]&128))return this[offset];return(255-this[offset]+1)*-1};Buffer.prototype.readInt16LE=function readInt16LE(offset,noAssert){if(!noAssert)checkOffset(offset,2,this.length);var val=this[offset]|this[offset+1]<<8;return val&32768?val|4294901760:val};Buffer.prototype.readInt16BE=function readInt16BE(offset,noAssert){if(!noAssert)checkOffset(offset,2,this.length);var val=this[offset+1]|this[offset]<<8;return val&32768?val|4294901760:val};Buffer.prototype.readInt32LE=function readInt32LE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return this[offset]|this[offset+1]<<8|this[offset+2]<<16|this[offset+3]<<24};Buffer.prototype.readInt32BE=function readInt32BE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return this[offset]<<24|this[offset+1]<<16|this[offset+2]<<8|this[offset+3]};Buffer.prototype.readFloatLE=function readFloatLE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return ieee754.read(this,offset,true,23,4)};Buffer.prototype.readFloatBE=function readFloatBE(offset,noAssert){if(!noAssert)checkOffset(offset,4,this.length);return ieee754.read(this,offset,false,23,4)};Buffer.prototype.readDoubleLE=function readDoubleLE(offset,noAssert){if(!noAssert)checkOffset(offset,8,this.length);return ieee754.read(this,offset,true,52,8)};Buffer.prototype.readDoubleBE=function readDoubleBE(offset,noAssert){if(!noAssert)checkOffset(offset,8,this.length);return ieee754.read(this,offset,false,52,8)};function checkInt(buf,value,offset,ext,max,min){if(!Buffer.isBuffer(buf))throw new TypeError('"buffer" argument must be a Buffer instance');if(value>max||valuebuf.length)throw new RangeError("Index out of range")}Buffer.prototype.writeUIntLE=function writeUIntLE(value,offset,byteLength,noAssert){value=+value;offset=offset|0;byteLength=byteLength|0;if(!noAssert){var maxBytes=Math.pow(2,8*byteLength)-1;checkInt(this,value,offset,byteLength,maxBytes,0)}var mul=1;var i=0;this[offset]=value&255;while(++i=0&&(mul*=256)){this[offset+i]=value/mul&255}return offset+byteLength};Buffer.prototype.writeUInt8=function writeUInt8(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,1,255,0);if(!Buffer.TYPED_ARRAY_SUPPORT)value=Math.floor(value);this[offset]=value&255;return offset+1};function objectWriteUInt16(buf,value,offset,littleEndian){if(value<0)value=65535+value+1;for(var i=0,j=Math.min(buf.length-offset,2);i>>(littleEndian?i:1-i)*8}}Buffer.prototype.writeUInt16LE=function writeUInt16LE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,2,65535,0);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value&255;this[offset+1]=value>>>8}else{objectWriteUInt16(this,value,offset,true)}return offset+2};Buffer.prototype.writeUInt16BE=function writeUInt16BE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,2,65535,0);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value>>>8;this[offset+1]=value&255}else{objectWriteUInt16(this,value,offset,false)}return offset+2};function objectWriteUInt32(buf,value,offset,littleEndian){if(value<0)value=4294967295+value+1;for(var i=0,j=Math.min(buf.length-offset,4);i>>(littleEndian?i:3-i)*8&255}}Buffer.prototype.writeUInt32LE=function writeUInt32LE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,4,4294967295,0);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset+3]=value>>>24;this[offset+2]=value>>>16;this[offset+1]=value>>>8;this[offset]=value&255}else{objectWriteUInt32(this,value,offset,true)}return offset+4};Buffer.prototype.writeUInt32BE=function writeUInt32BE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,4,4294967295,0);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value>>>24;this[offset+1]=value>>>16;this[offset+2]=value>>>8;this[offset+3]=value&255}else{objectWriteUInt32(this,value,offset,false)}return offset+4};Buffer.prototype.writeIntLE=function writeIntLE(value,offset,byteLength,noAssert){value=+value;offset=offset|0;if(!noAssert){var limit=Math.pow(2,8*byteLength-1);checkInt(this,value,offset,byteLength,limit-1,-limit)}var i=0;var mul=1;var sub=0;this[offset]=value&255;while(++i>0)-sub&255}return offset+byteLength};Buffer.prototype.writeIntBE=function writeIntBE(value,offset,byteLength,noAssert){value=+value;offset=offset|0;if(!noAssert){var limit=Math.pow(2,8*byteLength-1);checkInt(this,value,offset,byteLength,limit-1,-limit)}var i=byteLength-1;var mul=1;var sub=0;this[offset+i]=value&255;while(--i>=0&&(mul*=256)){if(value<0&&sub===0&&this[offset+i+1]!==0){sub=1}this[offset+i]=(value/mul>>0)-sub&255}return offset+byteLength};Buffer.prototype.writeInt8=function writeInt8(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,1,127,-128);if(!Buffer.TYPED_ARRAY_SUPPORT)value=Math.floor(value);if(value<0)value=255+value+1;this[offset]=value&255;return offset+1};Buffer.prototype.writeInt16LE=function writeInt16LE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,2,32767,-32768);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value&255;this[offset+1]=value>>>8}else{objectWriteUInt16(this,value,offset,true)}return offset+2};Buffer.prototype.writeInt16BE=function writeInt16BE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,2,32767,-32768);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value>>>8;this[offset+1]=value&255}else{objectWriteUInt16(this,value,offset,false)}return offset+2};Buffer.prototype.writeInt32LE=function writeInt32LE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,4,2147483647,-2147483648);if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value&255;this[offset+1]=value>>>8;this[offset+2]=value>>>16;this[offset+3]=value>>>24}else{objectWriteUInt32(this,value,offset,true)}return offset+4};Buffer.prototype.writeInt32BE=function writeInt32BE(value,offset,noAssert){value=+value;offset=offset|0;if(!noAssert)checkInt(this,value,offset,4,2147483647,-2147483648);if(value<0)value=4294967295+value+1;if(Buffer.TYPED_ARRAY_SUPPORT){this[offset]=value>>>24;this[offset+1]=value>>>16;this[offset+2]=value>>>8;this[offset+3]=value&255}else{objectWriteUInt32(this,value,offset,false)}return offset+4};function checkIEEE754(buf,value,offset,ext,max,min){if(offset+ext>buf.length)throw new RangeError("Index out of range");if(offset<0)throw new RangeError("Index out of range")}function writeFloat(buf,value,offset,littleEndian,noAssert){if(!noAssert){checkIEEE754(buf,value,offset,4,3.4028234663852886e38,-3.4028234663852886e38)}ieee754.write(buf,value,offset,littleEndian,23,4);return offset+4}Buffer.prototype.writeFloatLE=function writeFloatLE(value,offset,noAssert){return writeFloat(this,value,offset,true,noAssert)};Buffer.prototype.writeFloatBE=function writeFloatBE(value,offset,noAssert){return writeFloat(this,value,offset,false,noAssert)};function writeDouble(buf,value,offset,littleEndian,noAssert){if(!noAssert){checkIEEE754(buf,value,offset,8,1.7976931348623157e308,-1.7976931348623157e308)}ieee754.write(buf,value,offset,littleEndian,52,8);return offset+8}Buffer.prototype.writeDoubleLE=function writeDoubleLE(value,offset,noAssert){return writeDouble(this,value,offset,true,noAssert)};Buffer.prototype.writeDoubleBE=function writeDoubleBE(value,offset,noAssert){return writeDouble(this,value,offset,false,noAssert)};Buffer.prototype.copy=function copy(target,targetStart,start,end){if(!start)start=0;if(!end&&end!==0)end=this.length;if(targetStart>=target.length)targetStart=target.length;if(!targetStart)targetStart=0;if(end>0&&end=this.length)throw new RangeError("sourceStart out of bounds");if(end<0)throw new RangeError("sourceEnd out of bounds");if(end>this.length)end=this.length;if(target.length-targetStart=0;--i){target[i+targetStart]=this[i+start]}}else if(len<1e3||!Buffer.TYPED_ARRAY_SUPPORT){for(i=0;i>>0;end=end===undefined?this.length:end>>>0;if(!val)val=0;var i;if(typeof val==="number"){for(i=start;i55295&&codePoint<57344){if(!leadSurrogate){if(codePoint>56319){if((units-=3)>-1)bytes.push(239,191,189);continue}else if(i+1===length){if((units-=3)>-1)bytes.push(239,191,189);continue}leadSurrogate=codePoint;continue}if(codePoint<56320){if((units-=3)>-1)bytes.push(239,191,189);leadSurrogate=codePoint;continue}codePoint=(leadSurrogate-55296<<10|codePoint-56320)+65536}else if(leadSurrogate){if((units-=3)>-1)bytes.push(239,191,189)}leadSurrogate=null;if(codePoint<128){if((units-=1)<0)break;bytes.push(codePoint)}else if(codePoint<2048){if((units-=2)<0)break;bytes.push(codePoint>>6|192,codePoint&63|128)}else if(codePoint<65536){if((units-=3)<0)break;bytes.push(codePoint>>12|224,codePoint>>6&63|128,codePoint&63|128)}else if(codePoint<1114112){if((units-=4)<0)break;bytes.push(codePoint>>18|240,codePoint>>12&63|128,codePoint>>6&63|128,codePoint&63|128)}else{throw new Error("Invalid code point")}}return bytes}function asciiToBytes(str){var byteArray=[];for(var i=0;i>8;lo=c%256;byteArray.push(lo);byteArray.push(hi)}return byteArray}function base64ToBytes(str){return base64.toByteArray(base64clean(str))}function blitBuffer(src,dst,offset,length){for(var i=0;i=dst.length||i>=src.length)break;dst[i+offset]=src[i]}return i}function isnan(val){return val!==val}}).call(this,typeof global!=="undefined"?global:typeof self!=="undefined"?self:typeof window!=="undefined"?window:{})},{"base64-js":2,ieee754:37,isarray:40}],6:[function(require,module,exports){(function(Buffer){function isArray(arg){if(Array.isArray){return Array.isArray(arg)}return objectToString(arg)==="[object Array]"}exports.isArray=isArray;function isBoolean(arg){return typeof arg==="boolean"}exports.isBoolean=isBoolean;function isNull(arg){return arg===null}exports.isNull=isNull;function isNullOrUndefined(arg){return arg==null}exports.isNullOrUndefined=isNullOrUndefined;function isNumber(arg){return typeof arg==="number"}exports.isNumber=isNumber;function isString(arg){return typeof arg==="string"}exports.isString=isString;function isSymbol(arg){return typeof arg==="symbol"}exports.isSymbol=isSymbol;function isUndefined(arg){return arg===void 0}exports.isUndefined=isUndefined;function isRegExp(re){return objectToString(re)==="[object RegExp]"}exports.isRegExp=isRegExp;function isObject(arg){return typeof arg==="object"&&arg!==null}exports.isObject=isObject;function isDate(d){return objectToString(d)==="[object Date]"}exports.isDate=isDate;function isError(e){return objectToString(e)==="[object Error]"||e instanceof Error}exports.isError=isError;function isFunction(arg){return typeof arg==="function"}exports.isFunction=isFunction;function isPrimitive(arg){return arg===null||typeof arg==="boolean"||typeof arg==="number"||typeof arg==="string"||typeof arg==="symbol"||typeof arg==="undefined"}exports.isPrimitive=isPrimitive;exports.isBuffer=Buffer.isBuffer;function objectToString(o){return Object.prototype.toString.call(o)}}).call(this,{isBuffer:require("../../is-buffer/index.js")})},{"../../is-buffer/index.js":39}],7:[function(require,module,exports){var ElementType=require("domelementtype");var entities=require("entities");var booleanAttributes={__proto__:null,allowfullscreen:true,async:true,autofocus:true,autoplay:true,checked:true,controls:true,default:true,defer:true,disabled:true,hidden:true,ismap:true,loop:true,multiple:true,muted:true,open:true,readonly:true,required:true,reversed:true,scoped:true,seamless:true,selected:true,typemustmatch:true};var unencodedElements={__proto__:null,style:true,script:true,xmp:true,iframe:true,noembed:true,noframes:true,plaintext:true,noscript:true};function formatAttrs(attributes,opts){if(!attributes)return;var output="",value;for(var key in attributes){value=attributes[key];if(output){output+=" "}if(!value&&booleanAttributes[key]){output+=key}else{output+=key+'="'+(opts.decodeEntities?entities.encodeXML(value):value)+'"'}}return output}var singleTag={__proto__:null,area:true,base:true,basefont:true,br:true,col:true,command:true,embed:true,frame:true,hr:true,img:true,input:true,isindex:true,keygen:true,link:true,meta:true,param:true,source:true,track:true,wbr:true};var render=module.exports=function(dom,opts){if(!Array.isArray(dom)&&!dom.cheerio)dom=[dom];opts=opts||{};var output="";for(var i=0;i"}else{tag+=">";if(elem.children){tag+=render(elem.children,opts)}if(!singleTag[elem.name]||opts.xmlMode){tag+=""}}return tag}function renderDirective(elem){return"<"+elem.data+">"}function renderText(elem,opts){var data=elem.data||"";if(opts.decodeEntities&&!(elem.parent&&elem.parent.name in unencodedElements)){data=entities.encodeXML(data)}return data}function renderCdata(elem){return""}function renderComment(elem){return""}},{domelementtype:8,entities:20}],8:[function(require,module,exports){module.exports={Text:"text",Directive:"directive",Comment:"comment",Script:"script",Style:"style",Tag:"tag",CDATA:"cdata",isTag:function(elem){return elem.type==="tag"||elem.type==="script"||elem.type==="style"}}},{}],9:[function(require,module,exports){module.exports={Text:"text",Directive:"directive",Comment:"comment",Script:"script",Style:"style",Tag:"tag",CDATA:"cdata",Doctype:"doctype",isTag:function(elem){return elem.type==="tag"||elem.type==="script"||elem.type==="style"}}},{}],10:[function(require,module,exports){var ElementType=require("domelementtype");var re_whitespace=/\s+/g;var NodePrototype=require("./lib/node");var ElementPrototype=require("./lib/element");function DomHandler(callback,options,elementCB){if(typeof callback==="object"){elementCB=options;options=callback;callback=null}else if(typeof options==="function"){elementCB=options;options=defaultOpts}this._callback=callback;this._options=options||defaultOpts;this._elementCB=elementCB;this.dom=[];this._done=false;this._tagStack=[];this._parser=this._parser||null}var defaultOpts={normalizeWhitespace:false,withStartIndices:false};DomHandler.prototype.onparserinit=function(parser){this._parser=parser};DomHandler.prototype.onreset=function(){DomHandler.call(this,this._callback,this._options,this._elementCB)};DomHandler.prototype.onend=function(){if(this._done)return;this._done=true;this._parser=null;this._handleCallback(null)};DomHandler.prototype._handleCallback=DomHandler.prototype.onerror=function(error){if(typeof this._callback==="function"){this._callback(error,this.dom)}else{if(error)throw error}};DomHandler.prototype.onclosetag=function(){var elem=this._tagStack.pop();if(this._elementCB)this._elementCB(elem)};DomHandler.prototype._addDomElement=function(element){var parent=this._tagStack[this._tagStack.length-1];var siblings=parent?parent.children:this.dom;var previousSibling=siblings[siblings.length-1];element.next=null;if(this._options.withStartIndices){element.startIndex=this._parser.startIndex}if(this._options.withDomLvl1){element.__proto__=element.type==="tag"?ElementPrototype:NodePrototype}if(previousSibling){element.prev=previousSibling;previousSibling.next=element}else{element.prev=null}siblings.push(element);element.parent=parent||null};DomHandler.prototype.onopentag=function(name,attribs){var element={type:name==="script"?ElementType.Script:name==="style"?ElementType.Style:ElementType.Tag,name:name,attribs:attribs,children:[]};this._addDomElement(element);this._tagStack.push(element)};DomHandler.prototype.ontext=function(data){var normalize=this._options.normalizeWhitespace||this._options.ignoreWhitespace;var lastTag;if(!this._tagStack.length&&this.dom.length&&(lastTag=this.dom[this.dom.length-1]).type===ElementType.Text){if(normalize){lastTag.data=(lastTag.data+data).replace(re_whitespace," ")}else{lastTag.data+=data}}else{if(this._tagStack.length&&(lastTag=this._tagStack[this._tagStack.length-1])&&(lastTag=lastTag.children[lastTag.children.length-1])&&lastTag.type===ElementType.Text){if(normalize){lastTag.data=(lastTag.data+data).replace(re_whitespace," ")}else{lastTag.data+=data}}else{if(normalize){data=data.replace(re_whitespace," ")}this._addDomElement({data:data,type:ElementType.Text})}}};DomHandler.prototype.oncomment=function(data){var lastTag=this._tagStack[this._tagStack.length-1];if(lastTag&&lastTag.type===ElementType.Comment){lastTag.data+=data;return}var element={data:data,type:ElementType.Comment};this._addDomElement(element);this._tagStack.push(element)};DomHandler.prototype.oncdatastart=function(){var element={children:[{data:"",type:ElementType.Text}],type:ElementType.CDATA};this._addDomElement(element);this._tagStack.push(element)};DomHandler.prototype.oncommentend=DomHandler.prototype.oncdataend=function(){this._tagStack.pop()};DomHandler.prototype.onprocessinginstruction=function(name,data){this._addDomElement({name:name,data:data,type:ElementType.Directive})};module.exports=DomHandler},{"./lib/element":11,"./lib/node":12,domelementtype:9}],11:[function(require,module,exports){var NodePrototype=require("./node");var ElementPrototype=module.exports=Object.create(NodePrototype);var domLvl1={tagName:"name"};Object.keys(domLvl1).forEach(function(key){var shorthand=domLvl1[key];Object.defineProperty(ElementPrototype,key,{get:function(){return this[shorthand]||null},set:function(val){this[shorthand]=val;return val}})})},{"./node":12}],12:[function(require,module,exports){var NodePrototype=module.exports={get firstChild(){var children=this.children;return children&&children[0]||null},get lastChild(){var children=this.children;return children&&children[children.length-1]||null},get nodeType(){return nodeTypes[this.type]||nodeTypes.element}};var domLvl1={tagName:"name",childNodes:"children",parentNode:"parent",previousSibling:"prev",nextSibling:"next",nodeValue:"data"};var nodeTypes={element:1,text:3,cdata:4,comment:8};Object.keys(domLvl1).forEach(function(key){var shorthand=domLvl1[key];Object.defineProperty(NodePrototype,key,{get:function(){return this[shorthand]||null},set:function(val){this[shorthand]=val;return val}})})},{}],13:[function(require,module,exports){var DomUtils=module.exports;[require("./lib/stringify"),require("./lib/traversal"),require("./lib/manipulation"),require("./lib/querying"),require("./lib/legacy"),require("./lib/helpers")].forEach(function(ext){Object.keys(ext).forEach(function(key){DomUtils[key]=ext[key].bind(DomUtils)})})},{"./lib/helpers":14,"./lib/legacy":15,"./lib/manipulation":16,"./lib/querying":17,"./lib/stringify":18,"./lib/traversal":19}],14:[function(require,module,exports){exports.removeSubsets=function(nodes){var idx=nodes.length,node,ancestor,replace;while(--idx>-1){node=ancestor=nodes[idx];nodes[idx]=null;replace=true;while(ancestor){if(nodes.indexOf(ancestor)>-1){replace=false;nodes.splice(idx,1);break}ancestor=ancestor.parent}if(replace){nodes[idx]=node}}return nodes};var POSITION={DISCONNECTED:1,PRECEDING:2,FOLLOWING:4,CONTAINS:8,CONTAINED_BY:16};var comparePos=exports.compareDocumentPosition=function(nodeA,nodeB){var aParents=[];var bParents=[];var current,sharedParent,siblings,aSibling,bSibling,idx;if(nodeA===nodeB){return 0}current=nodeA;while(current){aParents.unshift(current);current=current.parent}current=nodeB;while(current){bParents.unshift(current);current=current.parent}idx=0;while(aParents[idx]===bParents[idx]){idx++}if(idx===0){return POSITION.DISCONNECTED}sharedParent=aParents[idx-1];siblings=sharedParent.children;aSibling=aParents[idx];bSibling=bParents[idx];if(siblings.indexOf(aSibling)>siblings.indexOf(bSibling)){if(sharedParent===nodeB){return POSITION.FOLLOWING|POSITION.CONTAINED_BY}return POSITION.FOLLOWING}else{if(sharedParent===nodeA){return POSITION.PRECEDING|POSITION.CONTAINS}return POSITION.PRECEDING}};exports.uniqueSort=function(nodes){var idx=nodes.length,node,position;nodes=nodes.slice();while(--idx>-1){node=nodes[idx];position=nodes.indexOf(node);if(position>-1&&position0){childs=find(test,childs,recurse,limit);result=result.concat(childs);limit-=childs.length;if(limit<=0)break}}return result}function findOneChild(test,elems){for(var i=0,l=elems.length;i0){elem=findOne(test,elems[i].children)}}return elem}function existsOne(test,elems){for(var i=0,l=elems.length;i0&&existsOne(test,elems[i].children))){return true}}return false}function findAll(test,elems){var result=[];for(var i=0,j=elems.length;i0){result=result.concat(findAll(test,elems[i].children))}}return result}},{domelementtype:9}],18:[function(require,module,exports){var ElementType=require("domelementtype"),getOuterHTML=require("dom-serializer"),isTag=ElementType.isTag;module.exports={getInnerHTML:getInnerHTML,getOuterHTML:getOuterHTML,getText:getText};function getInnerHTML(elem,opts){return elem.children?elem.children.map(function(elem){return getOuterHTML(elem,opts)}).join(""):""}function getText(elem){if(Array.isArray(elem))return elem.map(getText).join("");if(isTag(elem)||elem.type===ElementType.CDATA)return getText(elem.children);if(elem.type===ElementType.Text)return elem.data;return""}},{"dom-serializer":7,domelementtype:9}],19:[function(require,module,exports){var getChildren=exports.getChildren=function(elem){return elem.children};var getParent=exports.getParent=function(elem){return elem.parent};exports.getSiblings=function(elem){var parent=getParent(elem);return parent?getChildren(parent):[elem]};exports.getAttributeValue=function(elem,name){return elem.attribs&&elem.attribs[name]};exports.hasAttrib=function(elem,name){return!!elem.attribs&&hasOwnProperty.call(elem.attribs,name)};exports.getName=function(elem){return elem.name}},{}],20:[function(require,module,exports){var encode=require("./lib/encode.js"),decode=require("./lib/decode.js");exports.decode=function(data,level){return(!level||level<=0?decode.XML:decode.HTML)(data)};exports.decodeStrict=function(data,level){return(!level||level<=0?decode.XML:decode.HTMLStrict)(data)};exports.encode=function(data,level){return(!level||level<=0?encode.XML:encode.HTML)(data)};exports.encodeXML=encode.XML;exports.encodeHTML4=exports.encodeHTML5=exports.encodeHTML=encode.HTML;exports.decodeXML=exports.decodeXMLStrict=decode.XML;exports.decodeHTML4=exports.decodeHTML5=exports.decodeHTML=decode.HTML;exports.decodeHTML4Strict=exports.decodeHTML5Strict=exports.decodeHTMLStrict=decode.HTMLStrict;exports.escape=encode.escape},{"./lib/decode.js":21,"./lib/encode.js":23}],21:[function(require,module,exports){var entityMap=require("../maps/entities.json"),legacyMap=require("../maps/legacy.json"),xmlMap=require("../maps/xml.json"),decodeCodePoint=require("./decode_codepoint.js");var decodeXMLStrict=getStrictDecoder(xmlMap),decodeHTMLStrict=getStrictDecoder(entityMap);function getStrictDecoder(map){var keys=Object.keys(map).join("|"),replace=getReplacer(map);keys+="|#[xX][\\da-fA-F]+|#\\d+";var re=new RegExp("&(?:"+keys+");","g");return function(str){return String(str).replace(re,replace)}}var decodeHTML=function(){var legacy=Object.keys(legacyMap).sort(sorter);var keys=Object.keys(entityMap).sort(sorter);for(var i=0,j=0;i=55296&&codePoint<=57343||codePoint>1114111){return"�"}if(codePoint in decodeMap){codePoint=decodeMap[codePoint]}var output="";if(codePoint>65535){codePoint-=65536;output+=String.fromCharCode(codePoint>>>10&1023|55296);codePoint=56320|codePoint&1023}output+=String.fromCharCode(codePoint);return output}},{"../maps/decode.json":24}],23:[function(require,module,exports){var inverseXML=getInverseObj(require("../maps/xml.json")),xmlReplacer=getInverseReplacer(inverseXML);exports.XML=getInverse(inverseXML,xmlReplacer);var inverseHTML=getInverseObj(require("../maps/entities.json")),htmlReplacer=getInverseReplacer(inverseHTML);exports.HTML=getInverse(inverseHTML,htmlReplacer);function getInverseObj(obj){return Object.keys(obj).sort().reduce(function(inverse,name){inverse[obj[name]]="&"+name+";";return inverse},{})}function getInverseReplacer(inverse){var single=[],multiple=[];Object.keys(inverse).forEach(function(k){if(k.length===1){single.push("\\"+k)}else{multiple.push(k)}});multiple.unshift("["+single.join("")+"]");return new RegExp(multiple.join("|"),"g")}var re_nonASCII=/[^\0-\x7F]/g,re_astralSymbols=/[\uD800-\uDBFF][\uDC00-\uDFFF]/g;function singleCharReplacer(c){return"&#x"+c.charCodeAt(0).toString(16).toUpperCase()+";"}function astralReplacer(c){var high=c.charCodeAt(0);var low=c.charCodeAt(1);var codePoint=(high-55296)*1024+low-56320+65536;return"&#x"+codePoint.toString(16).toUpperCase()+";"}function getInverse(inverse,re){function func(name){return inverse[name]}return function(data){return data.replace(re,func).replace(re_astralSymbols,astralReplacer).replace(re_nonASCII,singleCharReplacer)}}var re_xmlChars=getInverseReplacer(inverseXML);function escapeXML(data){return data.replace(re_xmlChars,singleCharReplacer).replace(re_astralSymbols,astralReplacer).replace(re_nonASCII,singleCharReplacer)}exports.escape=escapeXML},{"../maps/entities.json":25,"../maps/xml.json":27}],24:[function(require,module,exports){module.exports={0:65533,128:8364,130:8218,131:402,132:8222,133:8230,134:8224,135:8225,136:710,137:8240,138:352,139:8249,140:338,142:381,145:8216,146:8217,147:8220,148:8221,149:8226,150:8211,151:8212,152:732,153:8482,154:353,155:8250,156:339,158:382,159:376}},{}],25:[function(require,module,exports){module.exports={Aacute:"Á",aacute:"á",Abreve:"Ă",abreve:"ă",ac:"∾",acd:"∿",acE:"∾̳",Acirc:"Â",acirc:"â",acute:"´",Acy:"А",acy:"а",AElig:"Æ",aelig:"æ",af:"⁡",Afr:"𝔄",afr:"𝔞",Agrave:"À",agrave:"à",alefsym:"ℵ",aleph:"ℵ",Alpha:"Α",alpha:"α",Amacr:"Ā",amacr:"ā",amalg:"⨿",amp:"&",AMP:"&",andand:"⩕",And:"⩓",and:"∧",andd:"⩜",andslope:"⩘",andv:"⩚",ang:"∠",ange:"⦤",angle:"∠",angmsdaa:"⦨",angmsdab:"⦩",angmsdac:"⦪",angmsdad:"⦫",angmsdae:"⦬",angmsdaf:"⦭",angmsdag:"⦮",angmsdah:"⦯",angmsd:"∡",angrt:"∟",angrtvb:"⊾",angrtvbd:"⦝",angsph:"∢",angst:"Å",angzarr:"⍼",Aogon:"Ą",aogon:"ą",Aopf:"𝔸",aopf:"𝕒",apacir:"⩯",ap:"≈",apE:"⩰",ape:"≊",apid:"≋",apos:"'",ApplyFunction:"⁡",approx:"≈",approxeq:"≊",Aring:"Å",aring:"å",Ascr:"𝒜",ascr:"𝒶",Assign:"≔",ast:"*",asymp:"≈",asympeq:"≍",Atilde:"Ã",atilde:"ã",Auml:"Ä",auml:"ä",awconint:"∳",awint:"⨑",backcong:"≌",backepsilon:"϶",backprime:"‵",backsim:"∽",backsimeq:"⋍",Backslash:"∖",Barv:"⫧",barvee:"⊽",barwed:"⌅",Barwed:"⌆",barwedge:"⌅",bbrk:"⎵",bbrktbrk:"⎶",bcong:"≌",Bcy:"Б",bcy:"б",bdquo:"„",becaus:"∵",because:"∵",Because:"∵",bemptyv:"⦰",bepsi:"϶",bernou:"ℬ",Bernoullis:"ℬ",Beta:"Β",beta:"β",beth:"ℶ",between:"≬",Bfr:"𝔅",bfr:"𝔟",bigcap:"⋂",bigcirc:"◯",bigcup:"⋃",bigodot:"⨀",bigoplus:"⨁",bigotimes:"⨂",bigsqcup:"⨆",bigstar:"★",bigtriangledown:"▽",bigtriangleup:"△",biguplus:"⨄",bigvee:"⋁",bigwedge:"⋀",bkarow:"⤍",blacklozenge:"⧫",blacksquare:"▪",blacktriangle:"▴",blacktriangledown:"▾",blacktriangleleft:"◂",blacktriangleright:"▸",blank:"␣",blk12:"▒",blk14:"░",blk34:"▓",block:"█",bne:"=⃥",bnequiv:"≡⃥",bNot:"⫭",bnot:"⌐",Bopf:"𝔹",bopf:"𝕓",bot:"⊥",bottom:"⊥",bowtie:"⋈",boxbox:"⧉",boxdl:"┐",boxdL:"╕",boxDl:"╖",boxDL:"╗",boxdr:"┌",boxdR:"╒",boxDr:"╓",boxDR:"╔",boxh:"─",boxH:"═",boxhd:"┬",boxHd:"╤",boxhD:"╥",boxHD:"╦",boxhu:"┴",boxHu:"╧",boxhU:"╨",boxHU:"╩",boxminus:"⊟",boxplus:"⊞",boxtimes:"⊠",boxul:"┘",boxuL:"╛",boxUl:"╜",boxUL:"╝",boxur:"└",boxuR:"╘",boxUr:"╙",boxUR:"╚",boxv:"│",boxV:"║",boxvh:"┼",boxvH:"╪",boxVh:"╫",boxVH:"╬",boxvl:"┤",boxvL:"╡",boxVl:"╢",boxVL:"╣",boxvr:"├",boxvR:"╞",boxVr:"╟",boxVR:"╠",bprime:"‵",breve:"˘",Breve:"˘",brvbar:"¦",bscr:"𝒷",Bscr:"ℬ",bsemi:"⁏",bsim:"∽",bsime:"⋍",bsolb:"⧅",bsol:"\\",bsolhsub:"⟈",bull:"•",bullet:"•",bump:"≎",bumpE:"⪮",bumpe:"≏",Bumpeq:"≎",bumpeq:"≏",Cacute:"Ć",cacute:"ć",capand:"⩄",capbrcup:"⩉",capcap:"⩋",cap:"∩",Cap:"⋒",capcup:"⩇",capdot:"⩀",CapitalDifferentialD:"ⅅ",caps:"∩︀",caret:"⁁",caron:"ˇ",Cayleys:"ℭ",ccaps:"⩍",Ccaron:"Č",ccaron:"č",Ccedil:"Ç",ccedil:"ç",Ccirc:"Ĉ",ccirc:"ĉ",Cconint:"∰",ccups:"⩌",ccupssm:"⩐",Cdot:"Ċ",cdot:"ċ",cedil:"¸",Cedilla:"¸",cemptyv:"⦲",cent:"¢",centerdot:"·",CenterDot:"·",cfr:"𝔠",Cfr:"ℭ",CHcy:"Ч",chcy:"ч",check:"✓",checkmark:"✓",Chi:"Χ",chi:"χ",circ:"ˆ",circeq:"≗",circlearrowleft:"↺",circlearrowright:"↻",circledast:"⊛",circledcirc:"⊚",circleddash:"⊝",CircleDot:"⊙",circledR:"®",circledS:"Ⓢ",CircleMinus:"⊖",CirclePlus:"⊕",CircleTimes:"⊗",cir:"○",cirE:"⧃",cire:"≗",cirfnint:"⨐",cirmid:"⫯",cirscir:"⧂",ClockwiseContourIntegral:"∲",CloseCurlyDoubleQuote:"”",CloseCurlyQuote:"’",clubs:"♣",clubsuit:"♣",colon:":",Colon:"∷",Colone:"⩴",colone:"≔",coloneq:"≔",comma:",",commat:"@",comp:"∁",compfn:"∘",complement:"∁",complexes:"ℂ",cong:"≅",congdot:"⩭",Congruent:"≡",conint:"∮",Conint:"∯",ContourIntegral:"∮",copf:"𝕔",Copf:"ℂ",coprod:"∐",Coproduct:"∐",copy:"©",COPY:"©",copysr:"℗",CounterClockwiseContourIntegral:"∳",crarr:"↵",cross:"✗",Cross:"⨯",Cscr:"𝒞",cscr:"𝒸",csub:"⫏",csube:"⫑",csup:"⫐",csupe:"⫒",ctdot:"⋯",cudarrl:"⤸",cudarrr:"⤵",cuepr:"⋞",cuesc:"⋟",cularr:"↶",cularrp:"⤽",cupbrcap:"⩈",cupcap:"⩆",CupCap:"≍",cup:"∪",Cup:"⋓",cupcup:"⩊",cupdot:"⊍",cupor:"⩅",cups:"∪︀",curarr:"↷",curarrm:"⤼",curlyeqprec:"⋞",curlyeqsucc:"⋟",curlyvee:"⋎",curlywedge:"⋏",curren:"¤",curvearrowleft:"↶",curvearrowright:"↷",cuvee:"⋎",cuwed:"⋏",cwconint:"∲",cwint:"∱",cylcty:"⌭",dagger:"†",Dagger:"‡",daleth:"ℸ",darr:"↓",Darr:"↡",dArr:"⇓",dash:"‐",Dashv:"⫤",dashv:"⊣",dbkarow:"⤏",dblac:"˝",Dcaron:"Ď",dcaron:"ď",Dcy:"Д",dcy:"д",ddagger:"‡",ddarr:"⇊",DD:"ⅅ",dd:"ⅆ",DDotrahd:"⤑",ddotseq:"⩷",deg:"°",Del:"∇",Delta:"Δ",delta:"δ",demptyv:"⦱",dfisht:"⥿",Dfr:"𝔇",dfr:"𝔡",dHar:"⥥",dharl:"⇃",dharr:"⇂",DiacriticalAcute:"´",DiacriticalDot:"˙",DiacriticalDoubleAcute:"˝",DiacriticalGrave:"`",DiacriticalTilde:"˜",diam:"⋄",diamond:"⋄",Diamond:"⋄",diamondsuit:"♦",diams:"♦",die:"¨",DifferentialD:"ⅆ",digamma:"ϝ",disin:"⋲",div:"÷",divide:"÷",divideontimes:"⋇",divonx:"⋇",DJcy:"Ђ",djcy:"ђ",dlcorn:"⌞",dlcrop:"⌍",dollar:"$",Dopf:"𝔻",dopf:"𝕕",Dot:"¨",dot:"˙",DotDot:"⃜",doteq:"≐",doteqdot:"≑",DotEqual:"≐",dotminus:"∸",dotplus:"∔",dotsquare:"⊡",doublebarwedge:"⌆",DoubleContourIntegral:"∯",DoubleDot:"¨",DoubleDownArrow:"⇓",DoubleLeftArrow:"⇐",DoubleLeftRightArrow:"⇔",DoubleLeftTee:"⫤",DoubleLongLeftArrow:"⟸",DoubleLongLeftRightArrow:"⟺",DoubleLongRightArrow:"⟹",DoubleRightArrow:"⇒",DoubleRightTee:"⊨",DoubleUpArrow:"⇑",DoubleUpDownArrow:"⇕",DoubleVerticalBar:"∥",DownArrowBar:"⤓",downarrow:"↓",DownArrow:"↓",Downarrow:"⇓",DownArrowUpArrow:"⇵",DownBreve:"̑",downdownarrows:"⇊",downharpoonleft:"⇃",downharpoonright:"⇂",DownLeftRightVector:"⥐",DownLeftTeeVector:"⥞",DownLeftVectorBar:"⥖",DownLeftVector:"↽",DownRightTeeVector:"⥟",DownRightVectorBar:"⥗",DownRightVector:"⇁",DownTeeArrow:"↧",DownTee:"⊤",drbkarow:"⤐",drcorn:"⌟",drcrop:"⌌",Dscr:"𝒟",dscr:"𝒹",DScy:"Ѕ",dscy:"ѕ",dsol:"⧶",Dstrok:"Đ",dstrok:"đ",dtdot:"⋱",dtri:"▿",dtrif:"▾",duarr:"⇵",duhar:"⥯",dwangle:"⦦",DZcy:"Џ",dzcy:"џ",dzigrarr:"⟿",Eacute:"É",eacute:"é",easter:"⩮",Ecaron:"Ě",ecaron:"ě",Ecirc:"Ê",ecirc:"ê",ecir:"≖",ecolon:"≕",Ecy:"Э",ecy:"э",eDDot:"⩷",Edot:"Ė",edot:"ė",eDot:"≑",ee:"ⅇ",efDot:"≒",Efr:"𝔈",efr:"𝔢",eg:"⪚",Egrave:"È",egrave:"è",egs:"⪖",egsdot:"⪘",el:"⪙",Element:"∈",elinters:"⏧",ell:"ℓ",els:"⪕",elsdot:"⪗",Emacr:"Ē",emacr:"ē",empty:"∅",emptyset:"∅",EmptySmallSquare:"◻",emptyv:"∅",EmptyVerySmallSquare:"▫",emsp13:" ",emsp14:" ",emsp:" ",ENG:"Ŋ",eng:"ŋ",ensp:" ",Eogon:"Ę",eogon:"ę",Eopf:"𝔼",eopf:"𝕖",epar:"⋕",eparsl:"⧣",eplus:"⩱",epsi:"ε",Epsilon:"Ε",epsilon:"ε",epsiv:"ϵ",eqcirc:"≖",eqcolon:"≕",eqsim:"≂",eqslantgtr:"⪖",eqslantless:"⪕",Equal:"⩵",equals:"=",EqualTilde:"≂",equest:"≟",Equilibrium:"⇌",equiv:"≡",equivDD:"⩸",eqvparsl:"⧥",erarr:"⥱",erDot:"≓",escr:"ℯ",Escr:"ℰ",esdot:"≐",Esim:"⩳",esim:"≂",Eta:"Η",eta:"η",ETH:"Ð",eth:"ð",Euml:"Ë",euml:"ë",euro:"€",excl:"!",exist:"∃",Exists:"∃",expectation:"ℰ",exponentiale:"ⅇ",ExponentialE:"ⅇ",fallingdotseq:"≒",Fcy:"Ф",fcy:"ф",female:"♀",ffilig:"ffi",fflig:"ff",ffllig:"ffl",Ffr:"𝔉",ffr:"𝔣",filig:"fi",FilledSmallSquare:"◼",FilledVerySmallSquare:"▪",fjlig:"fj",flat:"♭",fllig:"fl",fltns:"▱",fnof:"ƒ",Fopf:"𝔽",fopf:"𝕗",forall:"∀",ForAll:"∀",fork:"⋔",forkv:"⫙",Fouriertrf:"ℱ",fpartint:"⨍",frac12:"½",frac13:"⅓",frac14:"¼",frac15:"⅕",frac16:"⅙",frac18:"⅛",frac23:"⅔",frac25:"⅖",frac34:"¾",frac35:"⅗",frac38:"⅜",frac45:"⅘",frac56:"⅚",frac58:"⅝",frac78:"⅞",frasl:"⁄",frown:"⌢",fscr:"𝒻",Fscr:"ℱ",gacute:"ǵ",Gamma:"Γ",gamma:"γ",Gammad:"Ϝ",gammad:"ϝ",gap:"⪆",Gbreve:"Ğ",gbreve:"ğ",Gcedil:"Ģ",Gcirc:"Ĝ",gcirc:"ĝ",Gcy:"Г",gcy:"г",Gdot:"Ġ",gdot:"ġ",ge:"≥",gE:"≧",gEl:"⪌",gel:"⋛",geq:"≥",geqq:"≧",geqslant:"⩾",gescc:"⪩",ges:"⩾",gesdot:"⪀",gesdoto:"⪂",gesdotol:"⪄",gesl:"⋛︀",gesles:"⪔",Gfr:"𝔊",gfr:"𝔤",gg:"≫",Gg:"⋙",ggg:"⋙",gimel:"ℷ",GJcy:"Ѓ",gjcy:"ѓ",gla:"⪥",gl:"≷",glE:"⪒",glj:"⪤",gnap:"⪊",gnapprox:"⪊",gne:"⪈",gnE:"≩",gneq:"⪈",gneqq:"≩",gnsim:"⋧",Gopf:"𝔾",gopf:"𝕘",grave:"`",GreaterEqual:"≥",GreaterEqualLess:"⋛",GreaterFullEqual:"≧",GreaterGreater:"⪢",GreaterLess:"≷",GreaterSlantEqual:"⩾",GreaterTilde:"≳",Gscr:"𝒢",gscr:"ℊ",gsim:"≳",gsime:"⪎",gsiml:"⪐",gtcc:"⪧",gtcir:"⩺",gt:">",GT:">",Gt:"≫",gtdot:"⋗",gtlPar:"⦕",gtquest:"⩼",gtrapprox:"⪆",gtrarr:"⥸",gtrdot:"⋗",gtreqless:"⋛",gtreqqless:"⪌",gtrless:"≷",gtrsim:"≳",gvertneqq:"≩︀",gvnE:"≩︀",Hacek:"ˇ",hairsp:" ",half:"½",hamilt:"ℋ",HARDcy:"Ъ",hardcy:"ъ",harrcir:"⥈",harr:"↔",hArr:"⇔",harrw:"↭",Hat:"^",hbar:"ℏ",Hcirc:"Ĥ",hcirc:"ĥ",hearts:"♥",heartsuit:"♥",hellip:"…",hercon:"⊹",hfr:"𝔥",Hfr:"ℌ",HilbertSpace:"ℋ",hksearow:"⤥",hkswarow:"⤦",hoarr:"⇿",homtht:"∻",hookleftarrow:"↩",hookrightarrow:"↪",hopf:"𝕙",Hopf:"ℍ",horbar:"―",HorizontalLine:"─",hscr:"𝒽",Hscr:"ℋ",hslash:"ℏ",Hstrok:"Ħ",hstrok:"ħ",HumpDownHump:"≎",HumpEqual:"≏",hybull:"⁃",hyphen:"‐",Iacute:"Í",iacute:"í",ic:"⁣",Icirc:"Î",icirc:"î",Icy:"И",icy:"и",Idot:"İ",IEcy:"Е",iecy:"е",iexcl:"¡",iff:"⇔",ifr:"𝔦",Ifr:"ℑ",Igrave:"Ì",igrave:"ì",ii:"ⅈ",iiiint:"⨌",iiint:"∭",iinfin:"⧜",iiota:"℩",IJlig:"IJ",ijlig:"ij",Imacr:"Ī",imacr:"ī",image:"ℑ",ImaginaryI:"ⅈ",imagline:"ℐ",imagpart:"ℑ",imath:"ı",Im:"ℑ",imof:"⊷",imped:"Ƶ",Implies:"⇒",incare:"℅",in:"∈",infin:"∞",infintie:"⧝",inodot:"ı",intcal:"⊺",int:"∫",Int:"∬",integers:"ℤ",Integral:"∫",intercal:"⊺",Intersection:"⋂",intlarhk:"⨗",intprod:"⨼",InvisibleComma:"⁣",InvisibleTimes:"⁢",IOcy:"Ё",iocy:"ё",Iogon:"Į",iogon:"į",Iopf:"𝕀",iopf:"𝕚",Iota:"Ι",iota:"ι",iprod:"⨼",iquest:"¿",iscr:"𝒾",Iscr:"ℐ",isin:"∈",isindot:"⋵",isinE:"⋹",isins:"⋴",isinsv:"⋳",isinv:"∈",it:"⁢",Itilde:"Ĩ",itilde:"ĩ",Iukcy:"І",iukcy:"і",Iuml:"Ï",iuml:"ï",Jcirc:"Ĵ",jcirc:"ĵ",Jcy:"Й",jcy:"й",Jfr:"𝔍",jfr:"𝔧",jmath:"ȷ",Jopf:"𝕁",jopf:"𝕛",Jscr:"𝒥",jscr:"𝒿",Jsercy:"Ј",jsercy:"ј",Jukcy:"Є",jukcy:"є",Kappa:"Κ",kappa:"κ",kappav:"ϰ",Kcedil:"Ķ",kcedil:"ķ",Kcy:"К",kcy:"к",Kfr:"𝔎",kfr:"𝔨",kgreen:"ĸ",KHcy:"Х",khcy:"х",KJcy:"Ќ",kjcy:"ќ",Kopf:"𝕂",kopf:"𝕜",Kscr:"𝒦",kscr:"𝓀",lAarr:"⇚",Lacute:"Ĺ",lacute:"ĺ",laemptyv:"⦴",lagran:"ℒ",Lambda:"Λ",lambda:"λ",lang:"⟨",Lang:"⟪",langd:"⦑",langle:"⟨",lap:"⪅",Laplacetrf:"ℒ",laquo:"«",larrb:"⇤",larrbfs:"⤟",larr:"←",Larr:"↞",lArr:"⇐",larrfs:"⤝",larrhk:"↩",larrlp:"↫",larrpl:"⤹",larrsim:"⥳",larrtl:"↢",latail:"⤙",lAtail:"⤛",lat:"⪫",late:"⪭",lates:"⪭︀",lbarr:"⤌",lBarr:"⤎",lbbrk:"❲",lbrace:"{",lbrack:"[",lbrke:"⦋",lbrksld:"⦏",lbrkslu:"⦍",Lcaron:"Ľ",lcaron:"ľ",Lcedil:"Ļ",lcedil:"ļ",lceil:"⌈",lcub:"{",Lcy:"Л",lcy:"л",ldca:"⤶",ldquo:"“",ldquor:"„",ldrdhar:"⥧",ldrushar:"⥋",ldsh:"↲",le:"≤",lE:"≦",LeftAngleBracket:"⟨",LeftArrowBar:"⇤",leftarrow:"←",LeftArrow:"←",Leftarrow:"⇐",LeftArrowRightArrow:"⇆",leftarrowtail:"↢",LeftCeiling:"⌈",LeftDoubleBracket:"⟦",LeftDownTeeVector:"⥡",LeftDownVectorBar:"⥙",LeftDownVector:"⇃",LeftFloor:"⌊",leftharpoondown:"↽",leftharpoonup:"↼",leftleftarrows:"⇇",leftrightarrow:"↔",LeftRightArrow:"↔",Leftrightarrow:"⇔",leftrightarrows:"⇆",leftrightharpoons:"⇋",leftrightsquigarrow:"↭",LeftRightVector:"⥎",LeftTeeArrow:"↤",LeftTee:"⊣",LeftTeeVector:"⥚",leftthreetimes:"⋋",LeftTriangleBar:"⧏",LeftTriangle:"⊲",LeftTriangleEqual:"⊴",LeftUpDownVector:"⥑",LeftUpTeeVector:"⥠",LeftUpVectorBar:"⥘",LeftUpVector:"↿",LeftVectorBar:"⥒",LeftVector:"↼",lEg:"⪋",leg:"⋚",leq:"≤",leqq:"≦",leqslant:"⩽",lescc:"⪨",les:"⩽",lesdot:"⩿",lesdoto:"⪁",lesdotor:"⪃",lesg:"⋚︀",lesges:"⪓",lessapprox:"⪅",lessdot:"⋖",lesseqgtr:"⋚",lesseqqgtr:"⪋",LessEqualGreater:"⋚",LessFullEqual:"≦",LessGreater:"≶",lessgtr:"≶",LessLess:"⪡",lesssim:"≲",LessSlantEqual:"⩽",LessTilde:"≲",lfisht:"⥼",lfloor:"⌊",Lfr:"𝔏",lfr:"𝔩",lg:"≶",lgE:"⪑",lHar:"⥢",lhard:"↽",lharu:"↼",lharul:"⥪",lhblk:"▄",LJcy:"Љ",ljcy:"љ",llarr:"⇇",ll:"≪",Ll:"⋘",llcorner:"⌞",Lleftarrow:"⇚",llhard:"⥫",lltri:"◺",Lmidot:"Ŀ",lmidot:"ŀ",lmoustache:"⎰",lmoust:"⎰",lnap:"⪉",lnapprox:"⪉",lne:"⪇",lnE:"≨",lneq:"⪇",lneqq:"≨",lnsim:"⋦",loang:"⟬",loarr:"⇽",lobrk:"⟦",longleftarrow:"⟵",LongLeftArrow:"⟵",Longleftarrow:"⟸",longleftrightarrow:"⟷",LongLeftRightArrow:"⟷",Longleftrightarrow:"⟺",longmapsto:"⟼",longrightarrow:"⟶",LongRightArrow:"⟶",Longrightarrow:"⟹",looparrowleft:"↫",looparrowright:"↬",lopar:"⦅",Lopf:"𝕃",lopf:"𝕝",loplus:"⨭",lotimes:"⨴",lowast:"∗",lowbar:"_",LowerLeftArrow:"↙",LowerRightArrow:"↘",loz:"◊",lozenge:"◊",lozf:"⧫",lpar:"(",lparlt:"⦓",lrarr:"⇆",lrcorner:"⌟",lrhar:"⇋",lrhard:"⥭",lrm:"‎",lrtri:"⊿",lsaquo:"‹",lscr:"𝓁",Lscr:"ℒ",lsh:"↰",Lsh:"↰",lsim:"≲",lsime:"⪍",lsimg:"⪏",lsqb:"[",lsquo:"‘",lsquor:"‚",Lstrok:"Ł",lstrok:"ł",ltcc:"⪦",ltcir:"⩹",lt:"<",LT:"<",Lt:"≪",ltdot:"⋖",lthree:"⋋",ltimes:"⋉",ltlarr:"⥶",ltquest:"⩻",ltri:"◃",ltrie:"⊴",ltrif:"◂",ltrPar:"⦖",lurdshar:"⥊",luruhar:"⥦",lvertneqq:"≨︀",lvnE:"≨︀",macr:"¯",male:"♂",malt:"✠",maltese:"✠",Map:"⤅",map:"↦",mapsto:"↦",mapstodown:"↧",mapstoleft:"↤",mapstoup:"↥",marker:"▮",mcomma:"⨩",Mcy:"М",mcy:"м",mdash:"—",mDDot:"∺",measuredangle:"∡",MediumSpace:" ",Mellintrf:"ℳ",Mfr:"𝔐",mfr:"𝔪",mho:"℧",micro:"µ",midast:"*",midcir:"⫰",mid:"∣",middot:"·",minusb:"⊟",minus:"−",minusd:"∸",minusdu:"⨪",MinusPlus:"∓",mlcp:"⫛",mldr:"…",mnplus:"∓",models:"⊧",Mopf:"𝕄",mopf:"𝕞",mp:"∓",mscr:"𝓂",Mscr:"ℳ",mstpos:"∾",Mu:"Μ",mu:"μ",multimap:"⊸",mumap:"⊸",nabla:"∇",Nacute:"Ń",nacute:"ń",nang:"∠⃒",nap:"≉",napE:"⩰̸",napid:"≋̸",napos:"ʼn",napprox:"≉",natural:"♮",naturals:"ℕ",natur:"♮",nbsp:" ",nbump:"≎̸",nbumpe:"≏̸",ncap:"⩃",Ncaron:"Ň",ncaron:"ň",Ncedil:"Ņ",ncedil:"ņ",ncong:"≇",ncongdot:"⩭̸",ncup:"⩂",Ncy:"Н",ncy:"н",ndash:"–",nearhk:"⤤",nearr:"↗",neArr:"⇗",nearrow:"↗",ne:"≠",nedot:"≐̸",NegativeMediumSpace:"​",NegativeThickSpace:"​",NegativeThinSpace:"​",NegativeVeryThinSpace:"​",nequiv:"≢",nesear:"⤨",nesim:"≂̸",NestedGreaterGreater:"≫",NestedLessLess:"≪",NewLine:"\n",nexist:"∄",nexists:"∄",Nfr:"𝔑",nfr:"𝔫",ngE:"≧̸",nge:"≱",ngeq:"≱",ngeqq:"≧̸",ngeqslant:"⩾̸",nges:"⩾̸",nGg:"⋙̸",ngsim:"≵",nGt:"≫⃒",ngt:"≯",ngtr:"≯",nGtv:"≫̸",nharr:"↮",nhArr:"⇎",nhpar:"⫲",ni:"∋",nis:"⋼",nisd:"⋺",niv:"∋",NJcy:"Њ",njcy:"њ",nlarr:"↚",nlArr:"⇍",nldr:"‥",nlE:"≦̸",nle:"≰",nleftarrow:"↚",nLeftarrow:"⇍",nleftrightarrow:"↮",nLeftrightarrow:"⇎",nleq:"≰",nleqq:"≦̸",nleqslant:"⩽̸",nles:"⩽̸",nless:"≮",nLl:"⋘̸",nlsim:"≴",nLt:"≪⃒",nlt:"≮",nltri:"⋪",nltrie:"⋬",nLtv:"≪̸",nmid:"∤",NoBreak:"⁠",NonBreakingSpace:" ",nopf:"𝕟",Nopf:"ℕ",Not:"⫬",not:"¬",NotCongruent:"≢",NotCupCap:"≭",NotDoubleVerticalBar:"∦",NotElement:"∉",NotEqual:"≠",NotEqualTilde:"≂̸",NotExists:"∄",NotGreater:"≯",NotGreaterEqual:"≱",NotGreaterFullEqual:"≧̸",NotGreaterGreater:"≫̸",NotGreaterLess:"≹",NotGreaterSlantEqual:"⩾̸",NotGreaterTilde:"≵",NotHumpDownHump:"≎̸",NotHumpEqual:"≏̸",notin:"∉",notindot:"⋵̸",notinE:"⋹̸",notinva:"∉",notinvb:"⋷",notinvc:"⋶",NotLeftTriangleBar:"⧏̸",NotLeftTriangle:"⋪",NotLeftTriangleEqual:"⋬",NotLess:"≮",NotLessEqual:"≰",NotLessGreater:"≸",NotLessLess:"≪̸",NotLessSlantEqual:"⩽̸",NotLessTilde:"≴",NotNestedGreaterGreater:"⪢̸",NotNestedLessLess:"⪡̸",notni:"∌",notniva:"∌",notnivb:"⋾",notnivc:"⋽",NotPrecedes:"⊀",NotPrecedesEqual:"⪯̸",NotPrecedesSlantEqual:"⋠",NotReverseElement:"∌",NotRightTriangleBar:"⧐̸",NotRightTriangle:"⋫",NotRightTriangleEqual:"⋭",NotSquareSubset:"⊏̸",NotSquareSubsetEqual:"⋢",NotSquareSuperset:"⊐̸",NotSquareSupersetEqual:"⋣",NotSubset:"⊂⃒",NotSubsetEqual:"⊈",NotSucceeds:"⊁",NotSucceedsEqual:"⪰̸",NotSucceedsSlantEqual:"⋡",NotSucceedsTilde:"≿̸",NotSuperset:"⊃⃒",NotSupersetEqual:"⊉",NotTilde:"≁",NotTildeEqual:"≄",NotTildeFullEqual:"≇",NotTildeTilde:"≉",NotVerticalBar:"∤",nparallel:"∦",npar:"∦",nparsl:"⫽⃥",npart:"∂̸",npolint:"⨔",npr:"⊀",nprcue:"⋠",nprec:"⊀",npreceq:"⪯̸",npre:"⪯̸",nrarrc:"⤳̸",nrarr:"↛",nrArr:"⇏",nrarrw:"↝̸",nrightarrow:"↛",nRightarrow:"⇏",nrtri:"⋫",nrtrie:"⋭",nsc:"⊁",nsccue:"⋡",nsce:"⪰̸",Nscr:"𝒩",nscr:"𝓃",nshortmid:"∤",nshortparallel:"∦",nsim:"≁",nsime:"≄",nsimeq:"≄",nsmid:"∤",nspar:"∦",nsqsube:"⋢",nsqsupe:"⋣",nsub:"⊄",nsubE:"⫅̸",nsube:"⊈",nsubset:"⊂⃒",nsubseteq:"⊈",nsubseteqq:"⫅̸",nsucc:"⊁",nsucceq:"⪰̸",nsup:"⊅",nsupE:"⫆̸",nsupe:"⊉",nsupset:"⊃⃒",nsupseteq:"⊉",nsupseteqq:"⫆̸",ntgl:"≹",Ntilde:"Ñ",ntilde:"ñ",ntlg:"≸",ntriangleleft:"⋪",ntrianglelefteq:"⋬",ntriangleright:"⋫",ntrianglerighteq:"⋭",Nu:"Ν",nu:"ν",num:"#",numero:"№",numsp:" ",nvap:"≍⃒",nvdash:"⊬",nvDash:"⊭",nVdash:"⊮",nVDash:"⊯",nvge:"≥⃒",nvgt:">⃒",nvHarr:"⤄",nvinfin:"⧞",nvlArr:"⤂",nvle:"≤⃒",nvlt:"<⃒",nvltrie:"⊴⃒",nvrArr:"⤃",nvrtrie:"⊵⃒",nvsim:"∼⃒",nwarhk:"⤣",nwarr:"↖",nwArr:"⇖",nwarrow:"↖",nwnear:"⤧",Oacute:"Ó",oacute:"ó",oast:"⊛",Ocirc:"Ô",ocirc:"ô",ocir:"⊚",Ocy:"О",ocy:"о",odash:"⊝",Odblac:"Ő",odblac:"ő",odiv:"⨸",odot:"⊙",odsold:"⦼",OElig:"Œ",oelig:"œ",ofcir:"⦿",Ofr:"𝔒",ofr:"𝔬",ogon:"˛",Ograve:"Ò",ograve:"ò",ogt:"⧁",ohbar:"⦵",ohm:"Ω",oint:"∮",olarr:"↺",olcir:"⦾",olcross:"⦻",oline:"‾",olt:"⧀",Omacr:"Ō",omacr:"ō",Omega:"Ω",omega:"ω",Omicron:"Ο",omicron:"ο",omid:"⦶",ominus:"⊖",Oopf:"𝕆",oopf:"𝕠",opar:"⦷",OpenCurlyDoubleQuote:"“",OpenCurlyQuote:"‘",operp:"⦹",oplus:"⊕",orarr:"↻",Or:"⩔",or:"∨",ord:"⩝",order:"ℴ",orderof:"ℴ",ordf:"ª",ordm:"º",origof:"⊶",oror:"⩖",orslope:"⩗",orv:"⩛",oS:"Ⓢ",Oscr:"𝒪",oscr:"ℴ",Oslash:"Ø",oslash:"ø",osol:"⊘",Otilde:"Õ",otilde:"õ",otimesas:"⨶",Otimes:"⨷",otimes:"⊗",Ouml:"Ö",ouml:"ö",ovbar:"⌽",OverBar:"‾",OverBrace:"⏞",OverBracket:"⎴",OverParenthesis:"⏜",para:"¶",parallel:"∥",par:"∥",parsim:"⫳",parsl:"⫽",part:"∂",PartialD:"∂",Pcy:"П",pcy:"п",percnt:"%",period:".",permil:"‰",perp:"⊥",pertenk:"‱",Pfr:"𝔓",pfr:"𝔭",Phi:"Φ",phi:"φ",phiv:"ϕ",phmmat:"ℳ",phone:"☎",Pi:"Π",pi:"π",pitchfork:"⋔",piv:"ϖ",planck:"ℏ",planckh:"ℎ",plankv:"ℏ",plusacir:"⨣",plusb:"⊞",pluscir:"⨢",plus:"+",plusdo:"∔",plusdu:"⨥",pluse:"⩲",PlusMinus:"±",plusmn:"±",plussim:"⨦",plustwo:"⨧",pm:"±",Poincareplane:"ℌ",pointint:"⨕",popf:"𝕡",Popf:"ℙ",pound:"£",prap:"⪷",Pr:"⪻",pr:"≺",prcue:"≼",precapprox:"⪷",prec:"≺",preccurlyeq:"≼",Precedes:"≺",PrecedesEqual:"⪯",PrecedesSlantEqual:"≼",PrecedesTilde:"≾",preceq:"⪯",precnapprox:"⪹",precneqq:"⪵",precnsim:"⋨",pre:"⪯",prE:"⪳",precsim:"≾",prime:"′",Prime:"″",primes:"ℙ",prnap:"⪹",prnE:"⪵",prnsim:"⋨",prod:"∏",Product:"∏",profalar:"⌮",profline:"⌒",profsurf:"⌓",prop:"∝",Proportional:"∝",Proportion:"∷",propto:"∝",prsim:"≾",prurel:"⊰",Pscr:"𝒫",pscr:"𝓅",Psi:"Ψ",psi:"ψ",puncsp:" ",Qfr:"𝔔",qfr:"𝔮",qint:"⨌",qopf:"𝕢",Qopf:"ℚ",qprime:"⁗",Qscr:"𝒬",qscr:"𝓆",quaternions:"ℍ",quatint:"⨖",quest:"?",questeq:"≟",quot:'"',QUOT:'"',rAarr:"⇛",race:"∽̱",Racute:"Ŕ",racute:"ŕ",radic:"√",raemptyv:"⦳",rang:"⟩",Rang:"⟫",rangd:"⦒",range:"⦥",rangle:"⟩",raquo:"»",rarrap:"⥵",rarrb:"⇥",rarrbfs:"⤠",rarrc:"⤳",rarr:"→",Rarr:"↠",rArr:"⇒",rarrfs:"⤞",rarrhk:"↪",rarrlp:"↬",rarrpl:"⥅",rarrsim:"⥴",Rarrtl:"⤖",rarrtl:"↣",rarrw:"↝",ratail:"⤚",rAtail:"⤜",ratio:"∶",rationals:"ℚ",rbarr:"⤍",rBarr:"⤏",RBarr:"⤐",rbbrk:"❳",rbrace:"}",rbrack:"]",rbrke:"⦌",rbrksld:"⦎",rbrkslu:"⦐",Rcaron:"Ř",rcaron:"ř",Rcedil:"Ŗ",rcedil:"ŗ",rceil:"⌉",rcub:"}",Rcy:"Р",rcy:"р",rdca:"⤷",rdldhar:"⥩",rdquo:"”",rdquor:"”",rdsh:"↳",real:"ℜ",realine:"ℛ",realpart:"ℜ",reals:"ℝ",Re:"ℜ",rect:"▭",reg:"®",REG:"®",ReverseElement:"∋",ReverseEquilibrium:"⇋",ReverseUpEquilibrium:"⥯",rfisht:"⥽",rfloor:"⌋",rfr:"𝔯",Rfr:"ℜ",rHar:"⥤",rhard:"⇁",rharu:"⇀",rharul:"⥬",Rho:"Ρ",rho:"ρ",rhov:"ϱ",RightAngleBracket:"⟩",RightArrowBar:"⇥",rightarrow:"→",RightArrow:"→",Rightarrow:"⇒",RightArrowLeftArrow:"⇄",rightarrowtail:"↣",RightCeiling:"⌉",RightDoubleBracket:"⟧",RightDownTeeVector:"⥝",RightDownVectorBar:"⥕",RightDownVector:"⇂",RightFloor:"⌋",rightharpoondown:"⇁",rightharpoonup:"⇀",rightleftarrows:"⇄",rightleftharpoons:"⇌",rightrightarrows:"⇉",rightsquigarrow:"↝",RightTeeArrow:"↦",RightTee:"⊢",RightTeeVector:"⥛",rightthreetimes:"⋌",RightTriangleBar:"⧐",RightTriangle:"⊳",RightTriangleEqual:"⊵",RightUpDownVector:"⥏",RightUpTeeVector:"⥜",RightUpVectorBar:"⥔",RightUpVector:"↾",RightVectorBar:"⥓",RightVector:"⇀",ring:"˚",risingdotseq:"≓",rlarr:"⇄",rlhar:"⇌",rlm:"‏",rmoustache:"⎱",rmoust:"⎱",rnmid:"⫮",roang:"⟭",roarr:"⇾",robrk:"⟧",ropar:"⦆",ropf:"𝕣",Ropf:"ℝ",roplus:"⨮",rotimes:"⨵",RoundImplies:"⥰",rpar:")",rpargt:"⦔",rppolint:"⨒",rrarr:"⇉",Rrightarrow:"⇛",rsaquo:"›",rscr:"𝓇",Rscr:"ℛ",rsh:"↱",Rsh:"↱",rsqb:"]",rsquo:"’",rsquor:"’",rthree:"⋌",rtimes:"⋊",rtri:"▹",rtrie:"⊵",rtrif:"▸",rtriltri:"⧎",RuleDelayed:"⧴",ruluhar:"⥨",rx:"℞",Sacute:"Ś",sacute:"ś",sbquo:"‚",scap:"⪸",Scaron:"Š",scaron:"š",Sc:"⪼",sc:"≻",sccue:"≽",sce:"⪰",scE:"⪴",Scedil:"Ş",scedil:"ş",Scirc:"Ŝ",scirc:"ŝ",scnap:"⪺",scnE:"⪶",scnsim:"⋩",scpolint:"⨓",scsim:"≿",Scy:"С",scy:"с",sdotb:"⊡",sdot:"⋅",sdote:"⩦",searhk:"⤥",searr:"↘",seArr:"⇘",searrow:"↘",sect:"§",semi:";",seswar:"⤩",setminus:"∖",setmn:"∖",sext:"✶",Sfr:"𝔖",sfr:"𝔰",sfrown:"⌢",sharp:"♯",SHCHcy:"Щ",shchcy:"щ",SHcy:"Ш",shcy:"ш",ShortDownArrow:"↓",ShortLeftArrow:"←",shortmid:"∣",shortparallel:"∥",ShortRightArrow:"→",ShortUpArrow:"↑",shy:"­",Sigma:"Σ",sigma:"σ",sigmaf:"ς",sigmav:"ς",sim:"∼",simdot:"⩪",sime:"≃",simeq:"≃",simg:"⪞",simgE:"⪠",siml:"⪝",simlE:"⪟",simne:"≆",simplus:"⨤",simrarr:"⥲",slarr:"←",SmallCircle:"∘",smallsetminus:"∖",smashp:"⨳",smeparsl:"⧤",smid:"∣",smile:"⌣",smt:"⪪",smte:"⪬",smtes:"⪬︀",SOFTcy:"Ь",softcy:"ь",solbar:"⌿",solb:"⧄",sol:"/",Sopf:"𝕊",sopf:"𝕤",spades:"♠",spadesuit:"♠",spar:"∥",sqcap:"⊓",sqcaps:"⊓︀",sqcup:"⊔",sqcups:"⊔︀",Sqrt:"√",sqsub:"⊏",sqsube:"⊑",sqsubset:"⊏",sqsubseteq:"⊑",sqsup:"⊐",sqsupe:"⊒",sqsupset:"⊐",sqsupseteq:"⊒",square:"□",Square:"□",SquareIntersection:"⊓",SquareSubset:"⊏",SquareSubsetEqual:"⊑",SquareSuperset:"⊐",SquareSupersetEqual:"⊒",SquareUnion:"⊔",squarf:"▪",squ:"□",squf:"▪",srarr:"→",Sscr:"𝒮",sscr:"𝓈",ssetmn:"∖",ssmile:"⌣",sstarf:"⋆",Star:"⋆",star:"☆",starf:"★",straightepsilon:"ϵ",straightphi:"ϕ",strns:"¯",sub:"⊂",Sub:"⋐",subdot:"⪽",subE:"⫅",sube:"⊆",subedot:"⫃",submult:"⫁",subnE:"⫋",subne:"⊊",subplus:"⪿",subrarr:"⥹",subset:"⊂",Subset:"⋐",subseteq:"⊆",subseteqq:"⫅",SubsetEqual:"⊆",subsetneq:"⊊",subsetneqq:"⫋",subsim:"⫇",subsub:"⫕",subsup:"⫓",succapprox:"⪸",succ:"≻",succcurlyeq:"≽",Succeeds:"≻",SucceedsEqual:"⪰",SucceedsSlantEqual:"≽",SucceedsTilde:"≿",succeq:"⪰",succnapprox:"⪺",succneqq:"⪶",succnsim:"⋩",succsim:"≿",SuchThat:"∋",sum:"∑",Sum:"∑",sung:"♪",sup1:"¹",sup2:"²",sup3:"³",sup:"⊃",Sup:"⋑",supdot:"⪾",supdsub:"⫘",supE:"⫆",supe:"⊇",supedot:"⫄",Superset:"⊃",SupersetEqual:"⊇",suphsol:"⟉",suphsub:"⫗",suplarr:"⥻",supmult:"⫂",supnE:"⫌",supne:"⊋",supplus:"⫀",supset:"⊃",Supset:"⋑",supseteq:"⊇",supseteqq:"⫆",supsetneq:"⊋",supsetneqq:"⫌",supsim:"⫈",supsub:"⫔",supsup:"⫖",swarhk:"⤦",swarr:"↙",swArr:"⇙",swarrow:"↙",swnwar:"⤪",szlig:"ß",Tab:"\t",target:"⌖",Tau:"Τ",tau:"τ",tbrk:"⎴",Tcaron:"Ť",tcaron:"ť",Tcedil:"Ţ",tcedil:"ţ",Tcy:"Т",tcy:"т",tdot:"⃛",telrec:"⌕",Tfr:"𝔗",tfr:"𝔱",there4:"∴",therefore:"∴",Therefore:"∴",Theta:"Θ",theta:"θ",thetasym:"ϑ",thetav:"ϑ",thickapprox:"≈",thicksim:"∼",ThickSpace:"  ",ThinSpace:" ",thinsp:" ",thkap:"≈",thksim:"∼",THORN:"Þ",thorn:"þ",tilde:"˜",Tilde:"∼",TildeEqual:"≃",TildeFullEqual:"≅",TildeTilde:"≈",timesbar:"⨱",timesb:"⊠",times:"×",timesd:"⨰",tint:"∭",toea:"⤨",topbot:"⌶",topcir:"⫱",top:"⊤",Topf:"𝕋",topf:"𝕥",topfork:"⫚",tosa:"⤩",tprime:"‴",trade:"™",TRADE:"™",triangle:"▵",triangledown:"▿",triangleleft:"◃",trianglelefteq:"⊴",triangleq:"≜",triangleright:"▹",trianglerighteq:"⊵",tridot:"◬",trie:"≜",triminus:"⨺",TripleDot:"⃛",triplus:"⨹",trisb:"⧍",tritime:"⨻",trpezium:"⏢",Tscr:"𝒯",tscr:"𝓉",TScy:"Ц",tscy:"ц",TSHcy:"Ћ",tshcy:"ћ",Tstrok:"Ŧ",tstrok:"ŧ",twixt:"≬",twoheadleftarrow:"↞",twoheadrightarrow:"↠",Uacute:"Ú",uacute:"ú",uarr:"↑",Uarr:"↟",uArr:"⇑",Uarrocir:"⥉",Ubrcy:"Ў",ubrcy:"ў",Ubreve:"Ŭ",ubreve:"ŭ",Ucirc:"Û",ucirc:"û",Ucy:"У",ucy:"у",udarr:"⇅",Udblac:"Ű",udblac:"ű",udhar:"⥮",ufisht:"⥾",Ufr:"𝔘",ufr:"𝔲",Ugrave:"Ù",ugrave:"ù",uHar:"⥣",uharl:"↿",uharr:"↾",uhblk:"▀",ulcorn:"⌜",ulcorner:"⌜",ulcrop:"⌏",ultri:"◸",Umacr:"Ū",umacr:"ū",uml:"¨",UnderBar:"_",UnderBrace:"⏟",UnderBracket:"⎵",UnderParenthesis:"⏝",Union:"⋃",UnionPlus:"⊎",Uogon:"Ų",uogon:"ų",Uopf:"𝕌",uopf:"𝕦",UpArrowBar:"⤒",uparrow:"↑",UpArrow:"↑",Uparrow:"⇑",UpArrowDownArrow:"⇅",updownarrow:"↕",UpDownArrow:"↕",Updownarrow:"⇕",UpEquilibrium:"⥮",upharpoonleft:"↿",upharpoonright:"↾",uplus:"⊎",UpperLeftArrow:"↖",UpperRightArrow:"↗",upsi:"υ",Upsi:"ϒ",upsih:"ϒ",Upsilon:"Υ",upsilon:"υ",UpTeeArrow:"↥",UpTee:"⊥",upuparrows:"⇈",urcorn:"⌝",urcorner:"⌝",urcrop:"⌎",Uring:"Ů",uring:"ů",urtri:"◹",Uscr:"𝒰",uscr:"𝓊",utdot:"⋰",Utilde:"Ũ",utilde:"ũ",utri:"▵",utrif:"▴",uuarr:"⇈",Uuml:"Ü",uuml:"ü",uwangle:"⦧",vangrt:"⦜",varepsilon:"ϵ",varkappa:"ϰ",varnothing:"∅",varphi:"ϕ",varpi:"ϖ",varpropto:"∝",varr:"↕",vArr:"⇕",varrho:"ϱ",varsigma:"ς",varsubsetneq:"⊊︀",varsubsetneqq:"⫋︀",varsupsetneq:"⊋︀",varsupsetneqq:"⫌︀",vartheta:"ϑ",vartriangleleft:"⊲",vartriangleright:"⊳",vBar:"⫨",Vbar:"⫫",vBarv:"⫩",Vcy:"В",vcy:"в",vdash:"⊢",vDash:"⊨",Vdash:"⊩",VDash:"⊫",Vdashl:"⫦",veebar:"⊻",vee:"∨",Vee:"⋁",veeeq:"≚",vellip:"⋮",verbar:"|",Verbar:"‖",vert:"|",Vert:"‖",VerticalBar:"∣",VerticalLine:"|",VerticalSeparator:"❘",VerticalTilde:"≀",VeryThinSpace:" ",Vfr:"𝔙",vfr:"𝔳",vltri:"⊲",vnsub:"⊂⃒",vnsup:"⊃⃒",Vopf:"𝕍",vopf:"𝕧",vprop:"∝",vrtri:"⊳",Vscr:"𝒱",vscr:"𝓋",vsubnE:"⫋︀",vsubne:"⊊︀",vsupnE:"⫌︀",vsupne:"⊋︀",Vvdash:"⊪",vzigzag:"⦚",Wcirc:"Ŵ",wcirc:"ŵ",wedbar:"⩟",wedge:"∧",Wedge:"⋀",wedgeq:"≙",weierp:"℘",Wfr:"𝔚",wfr:"𝔴",Wopf:"𝕎",wopf:"𝕨",wp:"℘",wr:"≀",wreath:"≀",Wscr:"𝒲",wscr:"𝓌",xcap:"⋂",xcirc:"◯",xcup:"⋃",xdtri:"▽",Xfr:"𝔛",xfr:"𝔵",xharr:"⟷",xhArr:"⟺",Xi:"Ξ",xi:"ξ",xlarr:"⟵",xlArr:"⟸",xmap:"⟼",xnis:"⋻",xodot:"⨀",Xopf:"𝕏",xopf:"𝕩",xoplus:"⨁",xotime:"⨂",xrarr:"⟶",xrArr:"⟹",Xscr:"𝒳",xscr:"𝓍",xsqcup:"⨆",xuplus:"⨄",xutri:"△",xvee:"⋁",xwedge:"⋀",Yacute:"Ý",yacute:"ý",YAcy:"Я",yacy:"я",Ycirc:"Ŷ",ycirc:"ŷ",Ycy:"Ы",ycy:"ы",yen:"¥",Yfr:"𝔜",yfr:"𝔶",YIcy:"Ї",yicy:"ї",Yopf:"𝕐",yopf:"𝕪",Yscr:"𝒴",yscr:"𝓎",YUcy:"Ю",yucy:"ю",yuml:"ÿ",Yuml:"Ÿ",Zacute:"Ź",zacute:"ź",Zcaron:"Ž",zcaron:"ž",Zcy:"З",zcy:"з",Zdot:"Ż",zdot:"ż",zeetrf:"ℨ",ZeroWidthSpace:"​",Zeta:"Ζ",zeta:"ζ",zfr:"𝔷",Zfr:"ℨ",ZHcy:"Ж",zhcy:"ж",zigrarr:"⇝",zopf:"𝕫",Zopf:"ℤ",Zscr:"𝒵",zscr:"𝓏",zwj:"‍",zwnj:"‌"}},{}],26:[function(require,module,exports){module.exports={Aacute:"Á",aacute:"á",Acirc:"Â",acirc:"â",acute:"´",AElig:"Æ",aelig:"æ",Agrave:"À",agrave:"à",amp:"&",AMP:"&",Aring:"Å",aring:"å",Atilde:"Ã",atilde:"ã",Auml:"Ä",auml:"ä",brvbar:"¦",Ccedil:"Ç",ccedil:"ç",cedil:"¸",cent:"¢",copy:"©",COPY:"©",curren:"¤",deg:"°",divide:"÷",Eacute:"É",eacute:"é",Ecirc:"Ê",ecirc:"ê",Egrave:"È",egrave:"è",ETH:"Ð",eth:"ð",Euml:"Ë",euml:"ë",frac12:"½",frac14:"¼",frac34:"¾",gt:">",GT:">",Iacute:"Í",iacute:"í",Icirc:"Î",icirc:"î",iexcl:"¡",Igrave:"Ì",igrave:"ì",iquest:"¿",Iuml:"Ï",iuml:"ï",laquo:"«",lt:"<",LT:"<",macr:"¯",micro:"µ",middot:"·",nbsp:" ",not:"¬",Ntilde:"Ñ",ntilde:"ñ",Oacute:"Ó",oacute:"ó",Ocirc:"Ô",ocirc:"ô",Ograve:"Ò",ograve:"ò",ordf:"ª",ordm:"º",Oslash:"Ø",oslash:"ø",Otilde:"Õ",otilde:"õ",Ouml:"Ö",ouml:"ö",para:"¶",plusmn:"±",pound:"£",quot:'"',QUOT:'"',raquo:"»",reg:"®",REG:"®",sect:"§",shy:"­",sup1:"¹",sup2:"²",sup3:"³",szlig:"ß",THORN:"Þ",thorn:"þ",times:"×",Uacute:"Ú",uacute:"ú",Ucirc:"Û",ucirc:"û",Ugrave:"Ù",ugrave:"ù",uml:"¨",Uuml:"Ü",uuml:"ü",Yacute:"Ý",yacute:"ý",yen:"¥",yuml:"ÿ"}},{}],27:[function(require,module,exports){module.exports={amp:"&",apos:"'",gt:">",lt:"<",quot:'"'}},{}],28:[function(require,module,exports){function EventEmitter(){this._events=this._events||{};this._maxListeners=this._maxListeners||undefined}module.exports=EventEmitter;EventEmitter.EventEmitter=EventEmitter;EventEmitter.prototype._events=undefined;EventEmitter.prototype._maxListeners=undefined;EventEmitter.defaultMaxListeners=10;EventEmitter.prototype.setMaxListeners=function(n){if(!isNumber(n)||n<0||isNaN(n))throw TypeError("n must be a positive number");this._maxListeners=n;return this};EventEmitter.prototype.emit=function(type){var er,handler,len,args,i,listeners;if(!this._events)this._events={};if(type==="error"){if(!this._events.error||isObject(this._events.error)&&!this._events.error.length){er=arguments[1];if(er instanceof Error){throw er}else{var err=new Error('Uncaught, unspecified "error" event. ('+er+")");err.context=er;throw err}}}handler=this._events[type];if(isUndefined(handler))return false;if(isFunction(handler)){switch(arguments.length){case 1:handler.call(this);break;case 2:handler.call(this,arguments[1]);break;case 3:handler.call(this,arguments[1],arguments[2]);break;default:args=Array.prototype.slice.call(arguments,1);handler.apply(this,args)}}else if(isObject(handler)){args=Array.prototype.slice.call(arguments,1);listeners=handler.slice();len=listeners.length;for(i=0;i0&&this._events[type].length>m){this._events[type].warned=true;console.error("(node) warning: possible EventEmitter memory "+"leak detected. %d listeners added. "+"Use emitter.setMaxListeners() to increase limit.",this._events[type].length);if(typeof console.trace==="function"){console.trace()}}}return this};EventEmitter.prototype.on=EventEmitter.prototype.addListener;EventEmitter.prototype.once=function(type,listener){if(!isFunction(listener))throw TypeError("listener must be a function");var fired=false;function g(){this.removeListener(type,g);if(!fired){fired=true;listener.apply(this,arguments)}}g.listener=listener;this.on(type,g);return this};EventEmitter.prototype.removeListener=function(type,listener){var list,position,length,i;if(!isFunction(listener))throw TypeError("listener must be a function");if(!this._events||!this._events[type])return this;list=this._events[type];length=list.length;position=-1;if(list===listener||isFunction(list.listener)&&list.listener===listener){delete this._events[type];if(this._events.removeListener)this.emit("removeListener",type,listener)}else if(isObject(list)){for(i=length;i-- >0;){if(list[i]===listener||list[i].listener&&list[i].listener===listener){position=i;break}}if(position<0)return this;if(list.length===1){list.length=0;delete this._events[type]}else{list.splice(position,1); -}if(this._events.removeListener)this.emit("removeListener",type,listener)}return this};EventEmitter.prototype.removeAllListeners=function(type){var key,listeners;if(!this._events)return this;if(!this._events.removeListener){if(arguments.length===0)this._events={};else if(this._events[type])delete this._events[type];return this}if(arguments.length===0){for(key in this._events){if(key==="removeListener")continue;this.removeAllListeners(key)}this.removeAllListeners("removeListener");this._events={};return this}listeners=this._events[type];if(isFunction(listeners)){this.removeListener(type,listeners)}else if(listeners){while(listeners.length)this.removeListener(type,listeners[listeners.length-1])}delete this._events[type];return this};EventEmitter.prototype.listeners=function(type){var ret;if(!this._events||!this._events[type])ret=[];else if(isFunction(this._events[type]))ret=[this._events[type]];else ret=this._events[type].slice();return ret};EventEmitter.prototype.listenerCount=function(type){if(this._events){var evlistener=this._events[type];if(isFunction(evlistener))return 1;else if(evlistener)return evlistener.length}return 0};EventEmitter.listenerCount=function(emitter,type){return emitter.listenerCount(type)};function isFunction(arg){return typeof arg==="function"}function isNumber(arg){return typeof arg==="number"}function isObject(arg){return typeof arg==="object"&&arg!==null}function isUndefined(arg){return arg===void 0}},{}],29:[function(require,module,exports){module.exports=CollectingHandler;function CollectingHandler(cbs){this._cbs=cbs||{};this.events=[]}var EVENTS=require("./").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){name="on"+name;CollectingHandler.prototype[name]=function(){this.events.push([name]);if(this._cbs[name])this._cbs[name]()}}else if(EVENTS[name]===1){name="on"+name;CollectingHandler.prototype[name]=function(a){this.events.push([name,a]);if(this._cbs[name])this._cbs[name](a)}}else if(EVENTS[name]===2){name="on"+name;CollectingHandler.prototype[name]=function(a,b){this.events.push([name,a,b]);if(this._cbs[name])this._cbs[name](a,b)}}else{throw Error("wrong number of arguments")}});CollectingHandler.prototype.onreset=function(){this.events=[];if(this._cbs.onreset)this._cbs.onreset()};CollectingHandler.prototype.restart=function(){if(this._cbs.onreset)this._cbs.onreset();for(var i=0,len=this.events.length;i0;this._cbs.onclosetag(this._stack[--i]));}if(this._cbs.onend)this._cbs.onend()};Parser.prototype.reset=function(){if(this._cbs.onreset)this._cbs.onreset();this._tokenizer.reset();this._tagname="";this._attribname="";this._attribs=null;this._stack=[];if(this._cbs.onparserinit)this._cbs.onparserinit(this)};Parser.prototype.parseComplete=function(data){this.reset();this.end(data)};Parser.prototype.write=function(chunk){this._tokenizer.write(chunk)};Parser.prototype.end=function(chunk){this._tokenizer.end(chunk)};Parser.prototype.pause=function(){this._tokenizer.pause()};Parser.prototype.resume=function(){this._tokenizer.resume()};Parser.prototype.parseChunk=Parser.prototype.write;Parser.prototype.done=Parser.prototype.end;module.exports=Parser},{"./Tokenizer.js":34,events:28,inherits:38}],32:[function(require,module,exports){module.exports=ProxyHandler;function ProxyHandler(cbs){this._cbs=cbs||{}}var EVENTS=require("./").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){name="on"+name;ProxyHandler.prototype[name]=function(){if(this._cbs[name])this._cbs[name]()}}else if(EVENTS[name]===1){name="on"+name;ProxyHandler.prototype[name]=function(a){if(this._cbs[name])this._cbs[name](a)}}else if(EVENTS[name]===2){name="on"+name;ProxyHandler.prototype[name]=function(a,b){if(this._cbs[name])this._cbs[name](a,b)}}else{throw Error("wrong number of arguments")}})},{"./":36}],33:[function(require,module,exports){module.exports=Stream;var Parser=require("./WritableStream.js");function Stream(options){Parser.call(this,new Cbs(this),options)}require("inherits")(Stream,Parser);Stream.prototype.readable=true;function Cbs(scope){this.scope=scope}var EVENTS=require("../").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){Cbs.prototype["on"+name]=function(){this.scope.emit(name)}}else if(EVENTS[name]===1){Cbs.prototype["on"+name]=function(a){this.scope.emit(name,a)}}else if(EVENTS[name]===2){Cbs.prototype["on"+name]=function(a,b){this.scope.emit(name,a,b)}}else{throw Error("wrong number of arguments!")}})},{"../":36,"./WritableStream.js":35,inherits:38}],34:[function(require,module,exports){module.exports=Tokenizer;var decodeCodePoint=require("entities/lib/decode_codepoint.js"),entityMap=require("entities/maps/entities.json"),legacyMap=require("entities/maps/legacy.json"),xmlMap=require("entities/maps/xml.json"),i=0,TEXT=i++,BEFORE_TAG_NAME=i++,IN_TAG_NAME=i++,IN_SELF_CLOSING_TAG=i++,BEFORE_CLOSING_TAG_NAME=i++,IN_CLOSING_TAG_NAME=i++,AFTER_CLOSING_TAG_NAME=i++,BEFORE_ATTRIBUTE_NAME=i++,IN_ATTRIBUTE_NAME=i++,AFTER_ATTRIBUTE_NAME=i++,BEFORE_ATTRIBUTE_VALUE=i++,IN_ATTRIBUTE_VALUE_DQ=i++,IN_ATTRIBUTE_VALUE_SQ=i++,IN_ATTRIBUTE_VALUE_NQ=i++,BEFORE_DECLARATION=i++,IN_DECLARATION=i++,IN_PROCESSING_INSTRUCTION=i++,BEFORE_COMMENT=i++,IN_COMMENT=i++,AFTER_COMMENT_1=i++,AFTER_COMMENT_2=i++,BEFORE_CDATA_1=i++,BEFORE_CDATA_2=i++,BEFORE_CDATA_3=i++,BEFORE_CDATA_4=i++,BEFORE_CDATA_5=i++,BEFORE_CDATA_6=i++,IN_CDATA=i++,AFTER_CDATA_1=i++,AFTER_CDATA_2=i++,BEFORE_SPECIAL=i++,BEFORE_SPECIAL_END=i++,BEFORE_SCRIPT_1=i++,BEFORE_SCRIPT_2=i++,BEFORE_SCRIPT_3=i++,BEFORE_SCRIPT_4=i++,BEFORE_SCRIPT_5=i++,AFTER_SCRIPT_1=i++,AFTER_SCRIPT_2=i++,AFTER_SCRIPT_3=i++,AFTER_SCRIPT_4=i++,AFTER_SCRIPT_5=i++,BEFORE_STYLE_1=i++,BEFORE_STYLE_2=i++,BEFORE_STYLE_3=i++,BEFORE_STYLE_4=i++,AFTER_STYLE_1=i++,AFTER_STYLE_2=i++,AFTER_STYLE_3=i++,AFTER_STYLE_4=i++,BEFORE_ENTITY=i++,BEFORE_NUMERIC_ENTITY=i++,IN_NAMED_ENTITY=i++,IN_NUMERIC_ENTITY=i++,IN_HEX_ENTITY=i++,j=0,SPECIAL_NONE=j++,SPECIAL_SCRIPT=j++,SPECIAL_STYLE=j++;function whitespace(c){return c===" "||c==="\n"||c==="\t"||c==="\f"||c==="\r"}function characterState(char,SUCCESS){return function(c){if(c===char)this._state=SUCCESS}}function ifElseState(upper,SUCCESS,FAILURE){var lower=upper.toLowerCase();if(upper===lower){return function(c){if(c===lower){this._state=SUCCESS}else{this._state=FAILURE;this._index--}}}else{return function(c){if(c===lower||c===upper){this._state=SUCCESS}else{this._state=FAILURE;this._index--}}}}function consumeSpecialNameChar(upper,NEXT_STATE){var lower=upper.toLowerCase();return function(c){if(c===lower||c===upper){this._state=NEXT_STATE}else{this._state=IN_TAG_NAME;this._index--}}}function Tokenizer(options,cbs){this._state=TEXT;this._buffer="";this._sectionStart=0;this._index=0;this._bufferOffset=0;this._baseState=TEXT;this._special=SPECIAL_NONE;this._cbs=cbs;this._running=true;this._ended=false;this._xmlMode=!!(options&&options.xmlMode);this._decodeEntities=!!(options&&options.decodeEntities)}Tokenizer.prototype._stateText=function(c){if(c==="<"){if(this._index>this._sectionStart){this._cbs.ontext(this._getSection())}this._state=BEFORE_TAG_NAME;this._sectionStart=this._index}else if(this._decodeEntities&&this._special===SPECIAL_NONE&&c==="&"){if(this._index>this._sectionStart){this._cbs.ontext(this._getSection())}this._baseState=TEXT;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeTagName=function(c){if(c==="/"){this._state=BEFORE_CLOSING_TAG_NAME}else if(c==="<"){this._cbs.ontext(this._getSection());this._sectionStart=this._index}else if(c===">"||this._special!==SPECIAL_NONE||whitespace(c)){this._state=TEXT}else if(c==="!"){this._state=BEFORE_DECLARATION;this._sectionStart=this._index+1}else if(c==="?"){this._state=IN_PROCESSING_INSTRUCTION;this._sectionStart=this._index+1}else{this._state=!this._xmlMode&&(c==="s"||c==="S")?BEFORE_SPECIAL:IN_TAG_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInTagName=function(c){if(c==="/"||c===">"||whitespace(c)){this._emitToken("onopentagname");this._state=BEFORE_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateBeforeCloseingTagName=function(c){if(whitespace(c));else if(c===">"){this._state=TEXT}else if(this._special!==SPECIAL_NONE){if(c==="s"||c==="S"){this._state=BEFORE_SPECIAL_END}else{this._state=TEXT;this._index--}}else{this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInCloseingTagName=function(c){if(c===">"||whitespace(c)){this._emitToken("onclosetag");this._state=AFTER_CLOSING_TAG_NAME;this._index--}};Tokenizer.prototype._stateAfterCloseingTagName=function(c){if(c===">"){this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateBeforeAttributeName=function(c){if(c===">"){this._cbs.onopentagend();this._state=TEXT;this._sectionStart=this._index+1}else if(c==="/"){this._state=IN_SELF_CLOSING_TAG}else if(!whitespace(c)){this._state=IN_ATTRIBUTE_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInSelfClosingTag=function(c){if(c===">"){this._cbs.onselfclosingtag();this._state=TEXT;this._sectionStart=this._index+1}else if(!whitespace(c)){this._state=BEFORE_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateInAttributeName=function(c){if(c==="="||c==="/"||c===">"||whitespace(c)){this._cbs.onattribname(this._getSection());this._sectionStart=-1;this._state=AFTER_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateAfterAttributeName=function(c){if(c==="="){this._state=BEFORE_ATTRIBUTE_VALUE}else if(c==="/"||c===">"){this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME;this._index--}else if(!whitespace(c)){this._cbs.onattribend();this._state=IN_ATTRIBUTE_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeAttributeValue=function(c){if(c==='"'){this._state=IN_ATTRIBUTE_VALUE_DQ;this._sectionStart=this._index+1}else if(c==="'"){this._state=IN_ATTRIBUTE_VALUE_SQ;this._sectionStart=this._index+1}else if(!whitespace(c)){this._state=IN_ATTRIBUTE_VALUE_NQ;this._sectionStart=this._index;this._index--}};Tokenizer.prototype._stateInAttributeValueDoubleQuotes=function(c){if(c==='"'){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateInAttributeValueSingleQuotes=function(c){if(c==="'"){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateInAttributeValueNoQuotes=function(c){if(whitespace(c)||c===">"){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME;this._index--}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeDeclaration=function(c){this._state=c==="["?BEFORE_CDATA_1:c==="-"?BEFORE_COMMENT:IN_DECLARATION};Tokenizer.prototype._stateInDeclaration=function(c){if(c===">"){this._cbs.ondeclaration(this._getSection());this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateInProcessingInstruction=function(c){if(c===">"){this._cbs.onprocessinginstruction(this._getSection());this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateBeforeComment=function(c){if(c==="-"){this._state=IN_COMMENT;this._sectionStart=this._index+1}else{this._state=IN_DECLARATION}};Tokenizer.prototype._stateInComment=function(c){if(c==="-")this._state=AFTER_COMMENT_1};Tokenizer.prototype._stateAfterComment1=function(c){if(c==="-"){this._state=AFTER_COMMENT_2}else{this._state=IN_COMMENT}};Tokenizer.prototype._stateAfterComment2=function(c){if(c===">"){this._cbs.oncomment(this._buffer.substring(this._sectionStart,this._index-2));this._state=TEXT;this._sectionStart=this._index+1}else if(c!=="-"){this._state=IN_COMMENT}};Tokenizer.prototype._stateBeforeCdata1=ifElseState("C",BEFORE_CDATA_2,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata2=ifElseState("D",BEFORE_CDATA_3,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata3=ifElseState("A",BEFORE_CDATA_4,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata4=ifElseState("T",BEFORE_CDATA_5,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata5=ifElseState("A",BEFORE_CDATA_6,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata6=function(c){if(c==="["){this._state=IN_CDATA;this._sectionStart=this._index+1}else{this._state=IN_DECLARATION;this._index--}};Tokenizer.prototype._stateInCdata=function(c){if(c==="]")this._state=AFTER_CDATA_1};Tokenizer.prototype._stateAfterCdata1=characterState("]",AFTER_CDATA_2);Tokenizer.prototype._stateAfterCdata2=function(c){if(c===">"){this._cbs.oncdata(this._buffer.substring(this._sectionStart,this._index-2));this._state=TEXT;this._sectionStart=this._index+1}else if(c!=="]"){this._state=IN_CDATA}};Tokenizer.prototype._stateBeforeSpecial=function(c){if(c==="c"||c==="C"){this._state=BEFORE_SCRIPT_1}else if(c==="t"||c==="T"){this._state=BEFORE_STYLE_1}else{this._state=IN_TAG_NAME;this._index--}};Tokenizer.prototype._stateBeforeSpecialEnd=function(c){if(this._special===SPECIAL_SCRIPT&&(c==="c"||c==="C")){this._state=AFTER_SCRIPT_1}else if(this._special===SPECIAL_STYLE&&(c==="t"||c==="T")){this._state=AFTER_STYLE_1}else this._state=TEXT};Tokenizer.prototype._stateBeforeScript1=consumeSpecialNameChar("R",BEFORE_SCRIPT_2);Tokenizer.prototype._stateBeforeScript2=consumeSpecialNameChar("I",BEFORE_SCRIPT_3);Tokenizer.prototype._stateBeforeScript3=consumeSpecialNameChar("P",BEFORE_SCRIPT_4);Tokenizer.prototype._stateBeforeScript4=consumeSpecialNameChar("T",BEFORE_SCRIPT_5);Tokenizer.prototype._stateBeforeScript5=function(c){if(c==="/"||c===">"||whitespace(c)){this._special=SPECIAL_SCRIPT}this._state=IN_TAG_NAME;this._index--};Tokenizer.prototype._stateAfterScript1=ifElseState("R",AFTER_SCRIPT_2,TEXT);Tokenizer.prototype._stateAfterScript2=ifElseState("I",AFTER_SCRIPT_3,TEXT);Tokenizer.prototype._stateAfterScript3=ifElseState("P",AFTER_SCRIPT_4,TEXT);Tokenizer.prototype._stateAfterScript4=ifElseState("T",AFTER_SCRIPT_5,TEXT);Tokenizer.prototype._stateAfterScript5=function(c){if(c===">"||whitespace(c)){this._special=SPECIAL_NONE;this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index-6;this._index--}else this._state=TEXT};Tokenizer.prototype._stateBeforeStyle1=consumeSpecialNameChar("Y",BEFORE_STYLE_2);Tokenizer.prototype._stateBeforeStyle2=consumeSpecialNameChar("L",BEFORE_STYLE_3);Tokenizer.prototype._stateBeforeStyle3=consumeSpecialNameChar("E",BEFORE_STYLE_4);Tokenizer.prototype._stateBeforeStyle4=function(c){if(c==="/"||c===">"||whitespace(c)){this._special=SPECIAL_STYLE}this._state=IN_TAG_NAME;this._index--};Tokenizer.prototype._stateAfterStyle1=ifElseState("Y",AFTER_STYLE_2,TEXT);Tokenizer.prototype._stateAfterStyle2=ifElseState("L",AFTER_STYLE_3,TEXT);Tokenizer.prototype._stateAfterStyle3=ifElseState("E",AFTER_STYLE_4,TEXT);Tokenizer.prototype._stateAfterStyle4=function(c){if(c===">"||whitespace(c)){this._special=SPECIAL_NONE;this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index-5;this._index--}else this._state=TEXT};Tokenizer.prototype._stateBeforeEntity=ifElseState("#",BEFORE_NUMERIC_ENTITY,IN_NAMED_ENTITY);Tokenizer.prototype._stateBeforeNumericEntity=ifElseState("X",IN_HEX_ENTITY,IN_NUMERIC_ENTITY);Tokenizer.prototype._parseNamedEntityStrict=function(){if(this._sectionStart+16)limit=6;while(limit>=2){var entity=this._buffer.substr(start,limit);if(legacyMap.hasOwnProperty(entity)){this._emitPartial(legacyMap[entity]);this._sectionStart+=limit+1;return}else{limit--}}};Tokenizer.prototype._stateInNamedEntity=function(c){if(c===";"){this._parseNamedEntityStrict();if(this._sectionStart+1"z")&&(c<"A"||c>"Z")&&(c<"0"||c>"9")){if(this._xmlMode);else if(this._sectionStart+1===this._index);else if(this._baseState!==TEXT){if(c!=="="){this._parseNamedEntityStrict()}}else{this._parseLegacyEntity()}this._state=this._baseState;this._index--}};Tokenizer.prototype._decodeNumericEntity=function(offset,base){var sectionStart=this._sectionStart+offset;if(sectionStart!==this._index){var entity=this._buffer.substring(sectionStart,this._index);var parsed=parseInt(entity,base);this._emitPartial(decodeCodePoint(parsed));this._sectionStart=this._index}else{this._sectionStart--}this._state=this._baseState};Tokenizer.prototype._stateInNumericEntity=function(c){if(c===";"){this._decodeNumericEntity(2,10);this._sectionStart++}else if(c<"0"||c>"9"){if(!this._xmlMode){this._decodeNumericEntity(2,10)}else{this._state=this._baseState}this._index--}};Tokenizer.prototype._stateInHexEntity=function(c){if(c===";"){this._decodeNumericEntity(3,16);this._sectionStart++}else if((c<"a"||c>"f")&&(c<"A"||c>"F")&&(c<"0"||c>"9")){if(!this._xmlMode){this._decodeNumericEntity(3,16)}else{this._state=this._baseState}this._index--}};Tokenizer.prototype._cleanup=function(){if(this._sectionStart<0){this._buffer="";this._index=0;this._bufferOffset+=this._index}else if(this._running){if(this._state===TEXT){if(this._sectionStart!==this._index){this._cbs.ontext(this._buffer.substr(this._sectionStart))}this._buffer="";this._bufferOffset+=this._index;this._index=0}else if(this._sectionStart===this._index){this._buffer="";this._bufferOffset+=this._index;this._index=0}else{this._buffer=this._buffer.substr(this._sectionStart);this._index-=this._sectionStart;this._bufferOffset+=this._sectionStart}this._sectionStart=0}};Tokenizer.prototype.write=function(chunk){if(this._ended)this._cbs.onerror(Error(".write() after done!"));this._buffer+=chunk;this._parse()};Tokenizer.prototype._parse=function(){while(this._index>1;var nBits=-7;var i=isLE?nBytes-1:0;var d=isLE?-1:1;var s=buffer[offset+i];i+=d;e=s&(1<<-nBits)-1;s>>=-nBits;nBits+=eLen;for(;nBits>0;e=e*256+buffer[offset+i],i+=d,nBits-=8){}m=e&(1<<-nBits)-1;e>>=-nBits;nBits+=mLen;for(;nBits>0;m=m*256+buffer[offset+i],i+=d,nBits-=8){}if(e===0){e=1-eBias}else if(e===eMax){return m?NaN:(s?-1:1)*Infinity}else{m=m+Math.pow(2,mLen);e=e-eBias}return(s?-1:1)*m*Math.pow(2,e-mLen)};exports.write=function(buffer,value,offset,isLE,mLen,nBytes){var e,m,c;var eLen=nBytes*8-mLen-1;var eMax=(1<>1;var rt=mLen===23?Math.pow(2,-24)-Math.pow(2,-77):0;var i=isLE?0:nBytes-1;var d=isLE?1:-1;var s=value<0||value===0&&1/value<0?1:0;value=Math.abs(value);if(isNaN(value)||value===Infinity){m=isNaN(value)?1:0;e=eMax}else{e=Math.floor(Math.log(value)/Math.LN2);if(value*(c=Math.pow(2,-e))<1){e--;c*=2}if(e+eBias>=1){value+=rt/c}else{value+=rt*Math.pow(2,1-eBias)}if(value*c>=2){e++;c/=2}if(e+eBias>=eMax){m=0;e=eMax}else if(e+eBias>=1){m=(value*c-1)*Math.pow(2,mLen);e=e+eBias}else{m=value*Math.pow(2,eBias-1)*Math.pow(2,mLen);e=0}}for(;mLen>=8;buffer[offset+i]=m&255,i+=d,m/=256,mLen-=8){}e=e<0;buffer[offset+i]=e&255,i+=d,e/=256,eLen-=8){}buffer[offset+i-d]|=s*128}},{}],38:[function(require,module,exports){if(typeof Object.create==="function"){module.exports=function inherits(ctor,superCtor){ctor.super_=superCtor;ctor.prototype=Object.create(superCtor.prototype,{constructor:{value:ctor,enumerable:false,writable:true,configurable:true}})}}else{module.exports=function inherits(ctor,superCtor){ctor.super_=superCtor;var TempCtor=function(){};TempCtor.prototype=superCtor.prototype;ctor.prototype=new TempCtor;ctor.prototype.constructor=ctor}}},{}],39:[function(require,module,exports){module.exports=function(obj){return obj!=null&&(isBuffer(obj)||isSlowBuffer(obj)||!!obj._isBuffer)};function isBuffer(obj){return!!obj.constructor&&typeof obj.constructor.isBuffer==="function"&&obj.constructor.isBuffer(obj)}function isSlowBuffer(obj){return typeof obj.readFloatLE==="function"&&typeof obj.slice==="function"&&isBuffer(obj.slice(0,0))}},{}],40:[function(require,module,exports){var toString={}.toString;module.exports=Array.isArray||function(arr){return toString.call(arr)=="[object Array]"}},{}],41:[function(require,module,exports){(function(process){"use strict";if(!process.version||process.version.indexOf("v0.")===0||process.version.indexOf("v1.")===0&&process.version.indexOf("v1.8.")!==0){module.exports=nextTick}else{module.exports=process.nextTick}function nextTick(fn,arg1,arg2,arg3){if(typeof fn!=="function"){throw new TypeError('"callback" argument must be a function')}var len=arguments.length;var args,i;switch(len){case 0:case 1:return process.nextTick(fn);case 2:return process.nextTick(function afterTickOne(){fn.call(null,arg1)});case 3:return process.nextTick(function afterTickTwo(){fn.call(null,arg1,arg2)});case 4:return process.nextTick(function afterTickThree(){fn.call(null,arg1,arg2,arg3)});default:args=new Array(len-1);i=0;while(i1){for(var i=1;i0){if(state.ended&&!addToFront){var e=new Error("stream.push() after EOF");stream.emit("error",e)}else if(state.endEmitted&&addToFront){var _e=new Error("stream.unshift() after end event");stream.emit("error",_e)}else{var skipAdd;if(state.decoder&&!addToFront&&!encoding){chunk=state.decoder.write(chunk);skipAdd=!state.objectMode&&chunk.length===0}if(!addToFront)state.reading=false;if(!skipAdd){if(state.flowing&&state.length===0&&!state.sync){stream.emit("data",chunk);stream.read(0)}else{state.length+=state.objectMode?1:chunk.length;if(addToFront)state.buffer.unshift(chunk);else state.buffer.push(chunk);if(state.needReadable)emitReadable(stream)}}maybeReadMore(stream,state)}}else if(!addToFront){state.reading=false}return needMoreData(state)}function needMoreData(state){return!state.ended&&(state.needReadable||state.length=MAX_HWM){n=MAX_HWM}else{n--;n|=n>>>1;n|=n>>>2;n|=n>>>4;n|=n>>>8;n|=n>>>16;n++}return n}function howMuchToRead(n,state){if(n<=0||state.length===0&&state.ended)return 0;if(state.objectMode)return 1;if(n!==n){if(state.flowing&&state.length)return state.buffer.head.data.length;else return state.length}if(n>state.highWaterMark)state.highWaterMark=computeNewHighWaterMark(n);if(n<=state.length)return n;if(!state.ended){state.needReadable=true;return 0}return state.length}Readable.prototype.read=function(n){debug("read",n);n=parseInt(n,10);var state=this._readableState;var nOrig=n;if(n!==0)state.emittedReadable=false;if(n===0&&state.needReadable&&(state.length>=state.highWaterMark||state.ended)){debug("read: emitReadable",state.length,state.ended);if(state.length===0&&state.ended)endReadable(this);else emitReadable(this);return null}n=howMuchToRead(n,state);if(n===0&&state.ended){if(state.length===0)endReadable(this);return null}var doRead=state.needReadable;debug("need readable",doRead);if(state.length===0||state.length-n0)ret=fromList(n,state);else ret=null;if(ret===null){state.needReadable=true;n=0}else{state.length-=n}if(state.length===0){if(!state.ended)state.needReadable=true;if(nOrig!==n&&state.ended)endReadable(this)}if(ret!==null)this.emit("data",ret);return ret};function chunkInvalid(state,chunk){var er=null;if(!Buffer.isBuffer(chunk)&&typeof chunk!=="string"&&chunk!==null&&chunk!==undefined&&!state.objectMode){er=new TypeError("Invalid non-string/buffer chunk")}return er}function onEofChunk(stream,state){if(state.ended)return;if(state.decoder){var chunk=state.decoder.end();if(chunk&&chunk.length){state.buffer.push(chunk);state.length+=state.objectMode?1:chunk.length}}state.ended=true;emitReadable(stream)}function emitReadable(stream){var state=stream._readableState;state.needReadable=false;if(!state.emittedReadable){debug("emitReadable",state.flowing);state.emittedReadable=true;if(state.sync)processNextTick(emitReadable_,stream);else emitReadable_(stream)}}function emitReadable_(stream){debug("emit readable");stream.emit("readable");flow(stream)}function maybeReadMore(stream,state){if(!state.readingMore){state.readingMore=true;processNextTick(maybeReadMore_,stream,state)}}function maybeReadMore_(stream,state){var len=state.length;while(!state.reading&&!state.flowing&&!state.ended&&state.length1&&indexOf(state.pipes,dest)!==-1)&&!cleanedUp){debug("false write response, pause",src._readableState.awaitDrain);src._readableState.awaitDrain++;increasedAwaitDrain=true}src.pause()}}function onerror(er){debug("onerror",er);unpipe();dest.removeListener("error",onerror);if(EElistenerCount(dest,"error")===0)dest.emit("error",er)}prependListener(dest,"error",onerror);function onclose(){dest.removeListener("finish",onfinish);unpipe()}dest.once("close",onclose);function onfinish(){debug("onfinish");dest.removeListener("close",onclose);unpipe()}dest.once("finish",onfinish);function unpipe(){debug("unpipe");src.unpipe(dest)}dest.emit("pipe",src);if(!state.flowing){debug("pipe resume");src.resume()}return dest};function pipeOnDrain(src){return function(){var state=src._readableState;debug("pipeOnDrain",state.awaitDrain);if(state.awaitDrain)state.awaitDrain--;if(state.awaitDrain===0&&EElistenerCount(src,"data")){state.flowing=true;flow(src)}}}Readable.prototype.unpipe=function(dest){var state=this._readableState;if(state.pipesCount===0)return this;if(state.pipesCount===1){if(dest&&dest!==state.pipes)return this;if(!dest)dest=state.pipes;state.pipes=null;state.pipesCount=0;state.flowing=false;if(dest)dest.emit("unpipe",this);return this}if(!dest){var dests=state.pipes;var len=state.pipesCount;state.pipes=null;state.pipesCount=0;state.flowing=false;for(var _i=0;_i=state.length){if(state.decoder)ret=state.buffer.join("");else if(state.buffer.length===1)ret=state.buffer.head.data;else ret=state.buffer.concat(state.length);state.buffer.clear()}else{ret=fromListPartial(n,state.buffer,state.decoder)}return ret}function fromListPartial(n,list,hasStrings){var ret;if(nstr.length?str.length:n;if(nb===str.length)ret+=str;else ret+=str.slice(0,n);n-=nb;if(n===0){if(nb===str.length){++c;if(p.next)list.head=p.next;else list.head=list.tail=null}else{list.head=p;p.data=str.slice(nb)}break}++c}list.length-=c;return ret}function copyFromBuffer(n,list){var ret=bufferShim.allocUnsafe(n);var p=list.head;var c=1;p.data.copy(ret);n-=p.data.length;while(p=p.next){var buf=p.data;var nb=n>buf.length?buf.length:n;buf.copy(ret,ret.length-n,0,nb);n-=nb;if(n===0){if(nb===buf.length){++c;if(p.next)list.head=p.next;else list.head=list.tail=null}else{list.head=p;p.data=buf.slice(nb)}break}++c}list.length-=c;return ret}function endReadable(stream){var state=stream._readableState;if(state.length>0)throw new Error('"endReadable()" called on non-empty stream');if(!state.endEmitted){state.ended=true;processNextTick(endReadableNT,state,stream)}}function endReadableNT(state,stream){if(!state.endEmitted&&state.length===0){state.endEmitted=true;stream.readable=false;stream.emit("end")}}function forEach(xs,f){for(var i=0,l=xs.length;i-1?setImmediate:processNextTick;Writable.WritableState=WritableState;var util=require("core-util-is");util.inherits=require("inherits");var internalUtil={deprecate:require("util-deprecate")};var Stream;(function(){try{Stream=require("st"+"ream")}catch(_){}finally{if(!Stream)Stream=require("events").EventEmitter}})();var Buffer=require("buffer").Buffer;var bufferShim=require("buffer-shims");util.inherits(Writable,Stream);function nop(){}function WriteReq(chunk,encoding,cb){this.chunk=chunk;this.encoding=encoding;this.callback=cb;this.next=null}var Duplex;function WritableState(options,stream){Duplex=Duplex||require("./_stream_duplex");options=options||{};this.objectMode=!!options.objectMode;if(stream instanceof Duplex)this.objectMode=this.objectMode||!!options.writableObjectMode;var hwm=options.highWaterMark;var defaultHwm=this.objectMode?16:16*1024;this.highWaterMark=hwm||hwm===0?hwm:defaultHwm;this.highWaterMark=~~this.highWaterMark;this.needDrain=false;this.ending=false;this.ended=false;this.finished=false;var noDecode=options.decodeStrings===false;this.decodeStrings=!noDecode;this.defaultEncoding=options.defaultEncoding||"utf8";this.length=0;this.writing=false;this.corked=0;this.sync=true;this.bufferProcessing=false;this.onwrite=function(er){onwrite(stream,er)};this.writecb=null;this.writelen=0;this.bufferedRequest=null;this.lastBufferedRequest=null;this.pendingcb=0;this.prefinished=false;this.errorEmitted=false;this.bufferedRequestCount=0;this.corkedRequestsFree=new CorkedRequest(this)}WritableState.prototype.getBuffer=function writableStateGetBuffer(){var current=this.bufferedRequest;var out=[];while(current){out.push(current);current=current.next}return out};(function(){try{Object.defineProperty(WritableState.prototype,"buffer",{get:internalUtil.deprecate(function(){return this.getBuffer()},"_writableState.buffer is deprecated. Use _writableState.getBuffer "+"instead.")})}catch(_){}})();var Duplex;function Writable(options){Duplex=Duplex||require("./_stream_duplex");if(!(this instanceof Writable)&&!(this instanceof Duplex))return new Writable(options);this._writableState=new WritableState(options,this);this.writable=true;if(options){if(typeof options.write==="function")this._write=options.write;if(typeof options.writev==="function")this._writev=options.writev}Stream.call(this)}Writable.prototype.pipe=function(){this.emit("error",new Error("Cannot pipe, not readable"))};function writeAfterEnd(stream,cb){var er=new Error("write after end");stream.emit("error",er);processNextTick(cb,er)}function validChunk(stream,state,chunk,cb){var valid=true;var er=false;if(chunk===null){er=new TypeError("May not write null values to stream")}else if(!Buffer.isBuffer(chunk)&&typeof chunk!=="string"&&chunk!==undefined&&!state.objectMode){er=new TypeError("Invalid non-string/buffer chunk")}if(er){stream.emit("error",er);processNextTick(cb,er);valid=false}return valid}Writable.prototype.write=function(chunk,encoding,cb){var state=this._writableState;var ret=false;if(typeof encoding==="function"){cb=encoding;encoding=null}if(Buffer.isBuffer(chunk))encoding="buffer";else if(!encoding)encoding=state.defaultEncoding;if(typeof cb!=="function")cb=nop;if(state.ended)writeAfterEnd(this,cb);else if(validChunk(this,state,chunk,cb)){ +}if(this._events.removeListener)this.emit("removeListener",type,listener)}return this};EventEmitter.prototype.removeAllListeners=function(type){var key,listeners;if(!this._events)return this;if(!this._events.removeListener){if(arguments.length===0)this._events={};else if(this._events[type])delete this._events[type];return this}if(arguments.length===0){for(key in this._events){if(key==="removeListener")continue;this.removeAllListeners(key)}this.removeAllListeners("removeListener");this._events={};return this}listeners=this._events[type];if(isFunction(listeners)){this.removeListener(type,listeners)}else if(listeners){while(listeners.length)this.removeListener(type,listeners[listeners.length-1])}delete this._events[type];return this};EventEmitter.prototype.listeners=function(type){var ret;if(!this._events||!this._events[type])ret=[];else if(isFunction(this._events[type]))ret=[this._events[type]];else ret=this._events[type].slice();return ret};EventEmitter.prototype.listenerCount=function(type){if(this._events){var evlistener=this._events[type];if(isFunction(evlistener))return 1;else if(evlistener)return evlistener.length}return 0};EventEmitter.listenerCount=function(emitter,type){return emitter.listenerCount(type)};function isFunction(arg){return typeof arg==="function"}function isNumber(arg){return typeof arg==="number"}function isObject(arg){return typeof arg==="object"&&arg!==null}function isUndefined(arg){return arg===void 0}},{}],29:[function(require,module,exports){module.exports=CollectingHandler;function CollectingHandler(cbs){this._cbs=cbs||{};this.events=[]}var EVENTS=require("./").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){name="on"+name;CollectingHandler.prototype[name]=function(){this.events.push([name]);if(this._cbs[name])this._cbs[name]()}}else if(EVENTS[name]===1){name="on"+name;CollectingHandler.prototype[name]=function(a){this.events.push([name,a]);if(this._cbs[name])this._cbs[name](a)}}else if(EVENTS[name]===2){name="on"+name;CollectingHandler.prototype[name]=function(a, b){this.events.push([name,a,b]);if(this._cbs[name])this._cbs[name](a,b)}}else{throw Error("wrong number of arguments")}});CollectingHandler.prototype.onreset=function(){this.events=[];if(this._cbs.onreset)this._cbs.onreset()};CollectingHandler.prototype.restart=function(){if(this._cbs.onreset)this._cbs.onreset();for(var i=0,len=this.events.length;i0;this._cbs.onclosetag(this._stack[--i]));}if(this._cbs.onend)this._cbs.onend()};Parser.prototype.reset=function(){if(this._cbs.onreset)this._cbs.onreset();this._tokenizer.reset();this._tagname="";this._attribname="";this._attribs=null;this._stack=[];if(this._cbs.onparserinit)this._cbs.onparserinit(this)};Parser.prototype.parseComplete=function(data){this.reset();this.end(data)};Parser.prototype.write=function(chunk){this._tokenizer.write(chunk)};Parser.prototype.end=function(chunk){this._tokenizer.end(chunk)};Parser.prototype.pause=function(){this._tokenizer.pause()};Parser.prototype.resume=function(){this._tokenizer.resume()};Parser.prototype.parseChunk=Parser.prototype.write;Parser.prototype.done=Parser.prototype.end;module.exports=Parser},{"./Tokenizer.js":34,events:28,inherits:38}],32:[function(require, module, exports){module.exports=ProxyHandler;function ProxyHandler(cbs){this._cbs=cbs||{}}var EVENTS=require("./").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){name="on"+name;ProxyHandler.prototype[name]=function(){if(this._cbs[name])this._cbs[name]()}}else if(EVENTS[name]===1){name="on"+name;ProxyHandler.prototype[name]=function(a){if(this._cbs[name])this._cbs[name](a)}}else if(EVENTS[name]===2){name="on"+name;ProxyHandler.prototype[name]=function(a, b){if(this._cbs[name])this._cbs[name](a,b)}}else{throw Error("wrong number of arguments")}})},{"./":36}],33:[function(require, module, exports){module.exports=Stream;var Parser=require("./WritableStream.js");function Stream(options){Parser.call(this,new Cbs(this),options)}require("inherits")(Stream,Parser);Stream.prototype.readable=true;function Cbs(scope){this.scope=scope}var EVENTS=require("./.").EVENTS;Object.keys(EVENTS).forEach(function(name){if(EVENTS[name]===0){Cbs.prototype["on"+name]=function(){this.scope.emit(name)}}else if(EVENTS[name]===1){Cbs.prototype["on"+name]=function(a){this.scope.emit(name,a)}}else if(EVENTS[name]===2){Cbs.prototype["on"+name]=function(a, b){this.scope.emit(name,a,b)}}else{throw Error("wrong number of arguments!")}})},{"../":36,"./WritableStream.js":35,inherits:38}],34:[function(require, module, exports){module.exports=Tokenizer;var decodeCodePoint=require("entities/lib/decode_codepoint.js"),entityMap=require("entities/maps/entities.json"),legacyMap=require("entities/maps/legacy.json"),xmlMap=require("entities/maps/xml.json"),i=0,TEXT=i++,BEFORE_TAG_NAME=i++,IN_TAG_NAME=i++,IN_SELF_CLOSING_TAG=i++,BEFORE_CLOSING_TAG_NAME=i++,IN_CLOSING_TAG_NAME=i++,AFTER_CLOSING_TAG_NAME=i++,BEFORE_ATTRIBUTE_NAME=i++,IN_ATTRIBUTE_NAME=i++,AFTER_ATTRIBUTE_NAME=i++,BEFORE_ATTRIBUTE_VALUE=i++,IN_ATTRIBUTE_VALUE_DQ=i++,IN_ATTRIBUTE_VALUE_SQ=i++,IN_ATTRIBUTE_VALUE_NQ=i++,BEFORE_DECLARATION=i++,IN_DECLARATION=i++,IN_PROCESSING_INSTRUCTION=i++,BEFORE_COMMENT=i++,IN_COMMENT=i++,AFTER_COMMENT_1=i++,AFTER_COMMENT_2=i++,BEFORE_CDATA_1=i++,BEFORE_CDATA_2=i++,BEFORE_CDATA_3=i++,BEFORE_CDATA_4=i++,BEFORE_CDATA_5=i++,BEFORE_CDATA_6=i++,IN_CDATA=i++,AFTER_CDATA_1=i++,AFTER_CDATA_2=i++,BEFORE_SPECIAL=i++,BEFORE_SPECIAL_END=i++,BEFORE_SCRIPT_1=i++,BEFORE_SCRIPT_2=i++,BEFORE_SCRIPT_3=i++,BEFORE_SCRIPT_4=i++,BEFORE_SCRIPT_5=i++,AFTER_SCRIPT_1=i++,AFTER_SCRIPT_2=i++,AFTER_SCRIPT_3=i++,AFTER_SCRIPT_4=i++,AFTER_SCRIPT_5=i++,BEFORE_STYLE_1=i++,BEFORE_STYLE_2=i++,BEFORE_STYLE_3=i++,BEFORE_STYLE_4=i++,AFTER_STYLE_1=i++,AFTER_STYLE_2=i++,AFTER_STYLE_3=i++,AFTER_STYLE_4=i++,BEFORE_ENTITY=i++,BEFORE_NUMERIC_ENTITY=i++,IN_NAMED_ENTITY=i++,IN_NUMERIC_ENTITY=i++,IN_HEX_ENTITY=i++,j=0,SPECIAL_NONE=j++,SPECIAL_SCRIPT=j++,SPECIAL_STYLE=j++;function whitespace(c){return c===" "||c==="\n"||c==="\t"||c==="\f"||c==="\r"}function characterState(char,SUCCESS){return function(c){if(c===char)this._state=SUCCESS}}function ifElseState(upper,SUCCESS,FAILURE){var lower=upper.toLowerCase();if(upper===lower){return function(c){if(c===lower){this._state=SUCCESS}else{this._state=FAILURE;this._index--}}}else{return function(c){if(c===lower||c===upper){this._state=SUCCESS}else{this._state=FAILURE;this._index--}}}}function consumeSpecialNameChar(upper,NEXT_STATE){var lower=upper.toLowerCase();return function(c){if(c===lower||c===upper){this._state=NEXT_STATE}else{this._state=IN_TAG_NAME;this._index--}}}function Tokenizer(options,cbs){this._state=TEXT;this._buffer="";this._sectionStart=0;this._index=0;this._bufferOffset=0;this._baseState=TEXT;this._special=SPECIAL_NONE;this._cbs=cbs;this._running=true;this._ended=false;this._xmlMode=!!(options&&options.xmlMode);this._decodeEntities=!!(options&&options.decodeEntities)}Tokenizer.prototype._stateText=function(c){if(c==="<"){if(this._index>this._sectionStart){this._cbs.ontext(this._getSection())}this._state=BEFORE_TAG_NAME;this._sectionStart=this._index}else if(this._decodeEntities&&this._special===SPECIAL_NONE&&c==="&"){if(this._index>this._sectionStart){this._cbs.ontext(this._getSection())}this._baseState=TEXT;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeTagName=function(c){if(c==="/"){this._state=BEFORE_CLOSING_TAG_NAME}else if(c==="<"){this._cbs.ontext(this._getSection());this._sectionStart=this._index}else if(c===">"||this._special!==SPECIAL_NONE||whitespace(c)){this._state=TEXT}else if(c==="!"){this._state=BEFORE_DECLARATION;this._sectionStart=this._index+1}else if(c==="?"){this._state=IN_PROCESSING_INSTRUCTION;this._sectionStart=this._index+1}else{this._state=!this._xmlMode&&(c==="s"||c==="S")?BEFORE_SPECIAL:IN_TAG_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInTagName=function(c){if(c==="/"||c===">"||whitespace(c)){this._emitToken("onopentagname");this._state=BEFORE_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateBeforeCloseingTagName=function(c){if(whitespace(c));else if(c===">"){this._state=TEXT}else if(this._special!==SPECIAL_NONE){if(c==="s"||c==="S"){this._state=BEFORE_SPECIAL_END}else{this._state=TEXT;this._index--}}else{this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInCloseingTagName=function(c){if(c===">"||whitespace(c)){this._emitToken("onclosetag");this._state=AFTER_CLOSING_TAG_NAME;this._index--}};Tokenizer.prototype._stateAfterCloseingTagName=function(c){if(c===">"){this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateBeforeAttributeName=function(c){if(c===">"){this._cbs.onopentagend();this._state=TEXT;this._sectionStart=this._index+1}else if(c==="/"){this._state=IN_SELF_CLOSING_TAG}else if(!whitespace(c)){this._state=IN_ATTRIBUTE_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateInSelfClosingTag=function(c){if(c===">"){this._cbs.onselfclosingtag();this._state=TEXT;this._sectionStart=this._index+1}else if(!whitespace(c)){this._state=BEFORE_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateInAttributeName=function(c){if(c==="="||c==="/"||c===">"||whitespace(c)){this._cbs.onattribname(this._getSection());this._sectionStart=-1;this._state=AFTER_ATTRIBUTE_NAME;this._index--}};Tokenizer.prototype._stateAfterAttributeName=function(c){if(c==="="){this._state=BEFORE_ATTRIBUTE_VALUE}else if(c==="/"||c===">"){this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME;this._index--}else if(!whitespace(c)){this._cbs.onattribend();this._state=IN_ATTRIBUTE_NAME;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeAttributeValue=function(c){if(c==='"'){this._state=IN_ATTRIBUTE_VALUE_DQ;this._sectionStart=this._index+1}else if(c==="'"){this._state=IN_ATTRIBUTE_VALUE_SQ;this._sectionStart=this._index+1}else if(!whitespace(c)){this._state=IN_ATTRIBUTE_VALUE_NQ;this._sectionStart=this._index;this._index--}};Tokenizer.prototype._stateInAttributeValueDoubleQuotes=function(c){if(c==='"'){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateInAttributeValueSingleQuotes=function(c){if(c==="'"){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateInAttributeValueNoQuotes=function(c){if(whitespace(c)||c===">"){this._emitToken("onattribdata");this._cbs.onattribend();this._state=BEFORE_ATTRIBUTE_NAME;this._index--}else if(this._decodeEntities&&c==="&"){this._emitToken("onattribdata");this._baseState=this._state;this._state=BEFORE_ENTITY;this._sectionStart=this._index}};Tokenizer.prototype._stateBeforeDeclaration=function(c){this._state=c==="["?BEFORE_CDATA_1:c==="-"?BEFORE_COMMENT:IN_DECLARATION};Tokenizer.prototype._stateInDeclaration=function(c){if(c===">"){this._cbs.ondeclaration(this._getSection());this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateInProcessingInstruction=function(c){if(c===">"){this._cbs.onprocessinginstruction(this._getSection());this._state=TEXT;this._sectionStart=this._index+1}};Tokenizer.prototype._stateBeforeComment=function(c){if(c==="-"){this._state=IN_COMMENT;this._sectionStart=this._index+1}else{this._state=IN_DECLARATION}};Tokenizer.prototype._stateInComment=function(c){if(c==="-")this._state=AFTER_COMMENT_1};Tokenizer.prototype._stateAfterComment1=function(c){if(c==="-"){this._state=AFTER_COMMENT_2}else{this._state=IN_COMMENT}};Tokenizer.prototype._stateAfterComment2=function(c){if(c===">"){this._cbs.oncomment(this._buffer.substring(this._sectionStart,this._index-2));this._state=TEXT;this._sectionStart=this._index+1}else if(c!=="-"){this._state=IN_COMMENT}};Tokenizer.prototype._stateBeforeCdata1=ifElseState("C",BEFORE_CDATA_2,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata2=ifElseState("D",BEFORE_CDATA_3,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata3=ifElseState("A",BEFORE_CDATA_4,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata4=ifElseState("T",BEFORE_CDATA_5,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata5=ifElseState("A",BEFORE_CDATA_6,IN_DECLARATION);Tokenizer.prototype._stateBeforeCdata6=function(c){if(c==="["){this._state=IN_CDATA;this._sectionStart=this._index+1}else{this._state=IN_DECLARATION;this._index--}};Tokenizer.prototype._stateInCdata=function(c){if(c==="]")this._state=AFTER_CDATA_1};Tokenizer.prototype._stateAfterCdata1=characterState("]",AFTER_CDATA_2);Tokenizer.prototype._stateAfterCdata2=function(c){if(c===">"){this._cbs.oncdata(this._buffer.substring(this._sectionStart,this._index-2));this._state=TEXT;this._sectionStart=this._index+1}else if(c!=="]"){this._state=IN_CDATA}};Tokenizer.prototype._stateBeforeSpecial=function(c){if(c==="c"||c==="C"){this._state=BEFORE_SCRIPT_1}else if(c==="t"||c==="T"){this._state=BEFORE_STYLE_1}else{this._state=IN_TAG_NAME;this._index--}};Tokenizer.prototype._stateBeforeSpecialEnd=function(c){if(this._special===SPECIAL_SCRIPT&&(c==="c"||c==="C")){this._state=AFTER_SCRIPT_1}else if(this._special===SPECIAL_STYLE&&(c==="t"||c==="T")){this._state=AFTER_STYLE_1}else this._state=TEXT};Tokenizer.prototype._stateBeforeScript1=consumeSpecialNameChar("R",BEFORE_SCRIPT_2);Tokenizer.prototype._stateBeforeScript2=consumeSpecialNameChar("I",BEFORE_SCRIPT_3);Tokenizer.prototype._stateBeforeScript3=consumeSpecialNameChar("P",BEFORE_SCRIPT_4);Tokenizer.prototype._stateBeforeScript4=consumeSpecialNameChar("T",BEFORE_SCRIPT_5);Tokenizer.prototype._stateBeforeScript5=function(c){if(c==="/"||c===">"||whitespace(c)){this._special=SPECIAL_SCRIPT}this._state=IN_TAG_NAME;this._index--};Tokenizer.prototype._stateAfterScript1=ifElseState("R",AFTER_SCRIPT_2,TEXT);Tokenizer.prototype._stateAfterScript2=ifElseState("I",AFTER_SCRIPT_3,TEXT);Tokenizer.prototype._stateAfterScript3=ifElseState("P",AFTER_SCRIPT_4,TEXT);Tokenizer.prototype._stateAfterScript4=ifElseState("T",AFTER_SCRIPT_5,TEXT);Tokenizer.prototype._stateAfterScript5=function(c){if(c===">"||whitespace(c)){this._special=SPECIAL_NONE;this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index-6;this._index--}else this._state=TEXT};Tokenizer.prototype._stateBeforeStyle1=consumeSpecialNameChar("Y",BEFORE_STYLE_2);Tokenizer.prototype._stateBeforeStyle2=consumeSpecialNameChar("L",BEFORE_STYLE_3);Tokenizer.prototype._stateBeforeStyle3=consumeSpecialNameChar("E",BEFORE_STYLE_4);Tokenizer.prototype._stateBeforeStyle4=function(c){if(c==="/"||c===">"||whitespace(c)){this._special=SPECIAL_STYLE}this._state=IN_TAG_NAME;this._index--};Tokenizer.prototype._stateAfterStyle1=ifElseState("Y",AFTER_STYLE_2,TEXT);Tokenizer.prototype._stateAfterStyle2=ifElseState("L",AFTER_STYLE_3,TEXT);Tokenizer.prototype._stateAfterStyle3=ifElseState("E",AFTER_STYLE_4,TEXT);Tokenizer.prototype._stateAfterStyle4=function(c){if(c===">"||whitespace(c)){this._special=SPECIAL_NONE;this._state=IN_CLOSING_TAG_NAME;this._sectionStart=this._index-5;this._index--}else this._state=TEXT};Tokenizer.prototype._stateBeforeEntity=ifElseState("#",BEFORE_NUMERIC_ENTITY,IN_NAMED_ENTITY);Tokenizer.prototype._stateBeforeNumericEntity=ifElseState("X",IN_HEX_ENTITY,IN_NUMERIC_ENTITY);Tokenizer.prototype._parseNamedEntityStrict=function(){if(this._sectionStart+16)limit=6;while(limit>=2){var entity=this._buffer.substr(start,limit);if(legacyMap.hasOwnProperty(entity)){this._emitPartial(legacyMap[entity]);this._sectionStart+=limit+1;return}else{limit--}}};Tokenizer.prototype._stateInNamedEntity=function(c){if(c===";"){this._parseNamedEntityStrict();if(this._sectionStart+1"z")&&(c<"A"||c>"Z")&&(c<"0"||c>"9")){if(this._xmlMode);else if(this._sectionStart+1===this._index);else if(this._baseState!==TEXT){if(c!=="="){this._parseNamedEntityStrict()}}else{this._parseLegacyEntity()}this._state=this._baseState;this._index--}};Tokenizer.prototype._decodeNumericEntity=function(offset,base){var sectionStart=this._sectionStart+offset;if(sectionStart!==this._index){var entity=this._buffer.substring(sectionStart,this._index);var parsed=parseInt(entity,base);this._emitPartial(decodeCodePoint(parsed));this._sectionStart=this._index}else{this._sectionStart--}this._state=this._baseState};Tokenizer.prototype._stateInNumericEntity=function(c){if(c===";"){this._decodeNumericEntity(2,10);this._sectionStart++}else if(c<"0"||c>"9"){if(!this._xmlMode){this._decodeNumericEntity(2,10)}else{this._state=this._baseState}this._index--}};Tokenizer.prototype._stateInHexEntity=function(c){if(c===";"){this._decodeNumericEntity(3,16);this._sectionStart++}else if((c<"a"||c>"f")&&(c<"A"||c>"F")&&(c<"0"||c>"9")){if(!this._xmlMode){this._decodeNumericEntity(3,16)}else{this._state=this._baseState}this._index--}};Tokenizer.prototype._cleanup=function(){if(this._sectionStart<0){this._buffer="";this._index=0;this._bufferOffset+=this._index}else if(this._running){if(this._state===TEXT){if(this._sectionStart!==this._index){this._cbs.ontext(this._buffer.substr(this._sectionStart))}this._buffer="";this._bufferOffset+=this._index;this._index=0}else if(this._sectionStart===this._index){this._buffer="";this._bufferOffset+=this._index;this._index=0}else{this._buffer=this._buffer.substr(this._sectionStart);this._index-=this._sectionStart;this._bufferOffset+=this._sectionStart}this._sectionStart=0}};Tokenizer.prototype.write=function(chunk){if(this._ended)this._cbs.onerror(Error(".write() after done!"));this._buffer+=chunk;this._parse()};Tokenizer.prototype._parse=function(){while(this._index>1;var nBits=-7;var i=isLE?nBytes-1:0;var d=isLE?-1:1;var s=buffer[offset+i];i+=d;e=s&(1<<-nBits)-1;s>>=-nBits;nBits+=eLen;for(;nBits>0;e=e*256+buffer[offset+i],i+=d,nBits-=8){}m=e&(1<<-nBits)-1;e>>=-nBits;nBits+=mLen;for(;nBits>0;m=m*256+buffer[offset+i],i+=d,nBits-=8){}if(e===0){e=1-eBias}else if(e===eMax){return m?NaN:(s?-1:1)*Infinity}else{m=m+Math.pow(2,mLen);e=e-eBias}return(s?-1:1)*m*Math.pow(2,e-mLen)};exports.write=function(buffer,value,offset,isLE,mLen,nBytes){var e,m,c;var eLen=nBytes*8-mLen-1;var eMax=(1<>1;var rt=mLen===23?Math.pow(2,-24)-Math.pow(2,-77):0;var i=isLE?0:nBytes-1;var d=isLE?1:-1;var s=value<0||value===0&&1/value<0?1:0;value=Math.abs(value);if(isNaN(value)||value===Infinity){m=isNaN(value)?1:0;e=eMax}else{e=Math.floor(Math.log(value)/Math.LN2);if(value*(c=Math.pow(2,-e))<1){e--;c*=2}if(e+eBias>=1){value+=rt/c}else{value+=rt*Math.pow(2,1-eBias)}if(value*c>=2){e++;c/=2}if(e+eBias>=eMax){m=0;e=eMax}else if(e+eBias>=1){m=(value*c-1)*Math.pow(2,mLen);e=e+eBias}else{m=value*Math.pow(2,eBias-1)*Math.pow(2,mLen);e=0}}for(;mLen>=8;buffer[offset+i]=m&255,i+=d,m/=256,mLen-=8){}e=e<0;buffer[offset+i]=e&255,i+=d,e/=256,eLen-=8){}buffer[offset+i-d]|=s*128}},{}],38:[function(require,module,exports){if(typeof Object.create==="function"){module.exports=function inherits(ctor,superCtor){ctor.super_=superCtor;ctor.prototype=Object.create(superCtor.prototype,{constructor:{value:ctor,enumerable:false,writable:true,configurable:true}})}}else{module.exports=function inherits(ctor,superCtor){ctor.super_=superCtor;var TempCtor=function(){};TempCtor.prototype=superCtor.prototype;ctor.prototype=new TempCtor;ctor.prototype.constructor=ctor}}},{}],39:[function(require,module,exports){module.exports=function(obj){return obj!=null&&(isBuffer(obj)||isSlowBuffer(obj)||!!obj._isBuffer)};function isBuffer(obj){return!!obj.constructor&&typeof obj.constructor.isBuffer==="function"&&obj.constructor.isBuffer(obj)}function isSlowBuffer(obj){return typeof obj.readFloatLE==="function"&&typeof obj.slice==="function"&&isBuffer(obj.slice(0,0))}},{}],40:[function(require,module,exports){var toString={}.toString;module.exports=Array.isArray||function(arr){return toString.call(arr)=="[object Array]"}},{}],41:[function(require,module,exports){(function(process){"use strict";if(!process.version||process.version.indexOf("v0.")===0||process.version.indexOf("v1.")===0&&process.version.indexOf("v1.8.")!==0){module.exports=nextTick}else{module.exports=process.nextTick}function nextTick(fn,arg1,arg2,arg3){if(typeof fn!=="function"){throw new TypeError('"callback" argument must be a function')}var len=arguments.length;var args,i;switch(len){case 0:case 1:return process.nextTick(fn);case 2:return process.nextTick(function afterTickOne(){fn.call(null,arg1)});case 3:return process.nextTick(function afterTickTwo(){fn.call(null,arg1,arg2)});case 4:return process.nextTick(function afterTickThree(){fn.call(null,arg1,arg2,arg3)});default:args=new Array(len-1);i=0;while(i1){for(var i=1;i0){if(state.ended&&!addToFront){var e=new Error("stream.push() after EOF");stream.emit("error",e)}else if(state.endEmitted&&addToFront){var _e=new Error("stream.unshift() after end event");stream.emit("error",_e)}else{var skipAdd;if(state.decoder&&!addToFront&&!encoding){chunk=state.decoder.write(chunk);skipAdd=!state.objectMode&&chunk.length===0}if(!addToFront)state.reading=false;if(!skipAdd){if(state.flowing&&state.length===0&&!state.sync){stream.emit("data",chunk);stream.read(0)}else{state.length+=state.objectMode?1:chunk.length;if(addToFront)state.buffer.unshift(chunk);else state.buffer.push(chunk);if(state.needReadable)emitReadable(stream)}}maybeReadMore(stream,state)}}else if(!addToFront){state.reading=false}return needMoreData(state)}function needMoreData(state){return!state.ended&&(state.needReadable||state.length=MAX_HWM){n=MAX_HWM}else{n--;n|=n>>>1;n|=n>>>2;n|=n>>>4;n|=n>>>8;n|=n>>>16;n++}return n}function howMuchToRead(n, state){if(n<=0||state.length===0&&state.ended)return 0;if(state.objectMode)return 1;if(n!==n){if(state.flowing&&state.length)return state.buffer.head.data.length;else return state.length}if(n>state.highWaterMark)state.highWaterMark=computeNewHighWaterMark(n);if(n<=state.length)return n;if(!state.ended){state.needReadable=true;return 0}return state.length}Readable.prototype.read=function(n){debug("read",n);n=parseInt(n,10);var state=this._readableState;var nOrig=n;if(n!==0)state.emittedReadable=false;if(n===0&&state.needReadable&&(state.length>=state.highWaterMark||state.ended)){debug("read: emitReadable",state.length,state.ended);if(state.length===0&&state.ended)endReadable(this);else emitReadable(this);return null}n=howMuchToRead(n,state);if(n===0&&state.ended){if(state.length===0)endReadable(this);return null}var doRead=state.needReadable;debug("need readable",doRead);if(state.length===0||state.length-n0)ret=fromList(n,state);else ret=null;if(ret===null){state.needReadable=true;n=0}else{state.length-=n}if(state.length===0){if(!state.ended)state.needReadable=true;if(nOrig!==n&&state.ended)endReadable(this)}if(ret!==null)this.emit("data",ret);return ret};function chunkInvalid(state,chunk){var er=null;if(!Buffer.isBuffer(chunk)&&typeof chunk!=="string"&&chunk!==null&&chunk!==undefined&&!state.objectMode){er=new TypeError("Invalid non-string/buffer chunk")}return er}function onEofChunk(stream,state){if(state.ended)return;if(state.decoder){var chunk=state.decoder.end();if(chunk&&chunk.length){state.buffer.push(chunk);state.length+=state.objectMode?1:chunk.length}}state.ended=true;emitReadable(stream)}function emitReadable(stream){var state=stream._readableState;state.needReadable=false;if(!state.emittedReadable){debug("emitReadable",state.flowing);state.emittedReadable=true;if(state.sync)processNextTick(emitReadable_,stream);else emitReadable_(stream)}}function emitReadable_(stream){debug("emit readable");stream.emit("readable");flow(stream)}function maybeReadMore(stream,state){if(!state.readingMore){state.readingMore=true;processNextTick(maybeReadMore_,stream,state)}}function maybeReadMore_(stream,state){var len=state.length;while(!state.reading&&!state.flowing&&!state.ended&&state.length1&&indexOf(state.pipes,dest)!==-1)&&!cleanedUp){debug("false write response, pause",src._readableState.awaitDrain);src._readableState.awaitDrain++;increasedAwaitDrain=true}src.pause()}}function onerror(er){debug("onerror",er);unpipe();dest.removeListener("error",onerror);if(EElistenerCount(dest,"error")===0)dest.emit("error",er)}prependListener(dest,"error",onerror);function onclose(){dest.removeListener("finish",onfinish);unpipe()}dest.once("close",onclose);function onfinish(){debug("onfinish");dest.removeListener("close",onclose);unpipe()}dest.once("finish",onfinish);function unpipe(){debug("unpipe");src.unpipe(dest)}dest.emit("pipe",src);if(!state.flowing){debug("pipe resume");src.resume()}return dest};function pipeOnDrain(src){return function(){var state=src._readableState;debug("pipeOnDrain",state.awaitDrain);if(state.awaitDrain)state.awaitDrain--;if(state.awaitDrain===0&&EElistenerCount(src,"data")){state.flowing=true;flow(src)}}}Readable.prototype.unpipe=function(dest){var state=this._readableState;if(state.pipesCount===0)return this;if(state.pipesCount===1){if(dest&&dest!==state.pipes)return this;if(!dest)dest=state.pipes;state.pipes=null;state.pipesCount=0;state.flowing=false;if(dest)dest.emit("unpipe",this);return this}if(!dest){var dests=state.pipes;var len=state.pipesCount;state.pipes=null;state.pipesCount=0;state.flowing=false;for(var _i=0;_i=state.length){if(state.decoder)ret=state.buffer.join("");else if(state.buffer.length===1)ret=state.buffer.head.data;else ret=state.buffer.concat(state.length);state.buffer.clear()}else{ret=fromListPartial(n,state.buffer,state.decoder)}return ret}function fromListPartial(n,list,hasStrings){var ret;if(nstr.length?str.length:n;if(nb===str.length)ret+=str;else ret+=str.slice(0,n);n-=nb;if(n===0){if(nb===str.length){++c;if(p.next)list.head=p.next;else list.head=list.tail=null}else{list.head=p;p.data=str.slice(nb)}break}++c}list.length-=c;return ret}function copyFromBuffer(n,list){var ret=bufferShim.allocUnsafe(n);var p=list.head;var c=1;p.data.copy(ret);n-=p.data.length;while(p=p.next){var buf=p.data;var nb=n>buf.length?buf.length:n;buf.copy(ret,ret.length-n,0,nb);n-=nb;if(n===0){if(nb===buf.length){++c;if(p.next)list.head=p.next;else list.head=list.tail=null}else{list.head=p;p.data=buf.slice(nb)}break}++c}list.length-=c;return ret}function endReadable(stream){var state=stream._readableState;if(state.length>0)throw new Error('"endReadable()" called on non-empty stream');if(!state.endEmitted){state.ended=true;processNextTick(endReadableNT,state,stream)}}function endReadableNT(state,stream){if(!state.endEmitted&&state.length===0){state.endEmitted=true;stream.readable=false;stream.emit("end")}}function forEach(xs,f){for(var i=0,l=xs.length;i-1?setImmediate:processNextTick;Writable.WritableState=WritableState;var util=require("core-util-is");util.inherits=require("inherits");var internalUtil={deprecate:require("util-deprecate")};var Stream;(function(){try{Stream=require("st"+"ream")}catch(_){}finally{if(!Stream)Stream=require("events").EventEmitter}})();var Buffer=require("buffer").Buffer;var bufferShim=require("buffer-shims");util.inherits(Writable,Stream);function nop(){}function WriteReq(chunk,encoding,cb){this.chunk=chunk;this.encoding=encoding;this.callback=cb;this.next=null}var Duplex;function WritableState(options,stream){Duplex=Duplex||require("./_stream_duplex");options=options||{};this.objectMode=!!options.objectMode;if(stream instanceof Duplex)this.objectMode=this.objectMode||!!options.writableObjectMode;var hwm=options.highWaterMark;var defaultHwm=this.objectMode?16:16*1024;this.highWaterMark=hwm||hwm===0?hwm:defaultHwm;this.highWaterMark=~~this.highWaterMark;this.needDrain=false;this.ending=false;this.ended=false;this.finished=false;var noDecode=options.decodeStrings===false;this.decodeStrings=!noDecode;this.defaultEncoding=options.defaultEncoding||"utf8";this.length=0;this.writing=false;this.corked=0;this.sync=true;this.bufferProcessing=false;this.onwrite=function(er){onwrite(stream,er)};this.writecb=null;this.writelen=0;this.bufferedRequest=null;this.lastBufferedRequest=null;this.pendingcb=0;this.prefinished=false;this.errorEmitted=false;this.bufferedRequestCount=0;this.corkedRequestsFree=new CorkedRequest(this)}WritableState.prototype.getBuffer=function writableStateGetBuffer(){var current=this.bufferedRequest;var out=[];while(current){out.push(current);current=current.next}return out};(function(){try{Object.defineProperty(WritableState.prototype,"buffer",{get:internalUtil.deprecate(function(){return this.getBuffer()},"_writableState.buffer is deprecated. Use _writableState.getBuffer "+"instead.")})}catch(_){}})();var Duplex;function Writable(options){Duplex=Duplex||require("./_stream_duplex");if(!(this instanceof Writable)&&!(this instanceof Duplex))return new Writable(options);this._writableState=new WritableState(options,this);this.writable=true;if(options){if(typeof options.write==="function")this._write=options.write;if(typeof options.writev==="function")this._writev=options.writev}Stream.call(this)}Writable.prototype.pipe=function(){this.emit("error",new Error("Cannot pipe, not readable"))};function writeAfterEnd(stream,cb){var er=new Error("write after end");stream.emit("error",er);processNextTick(cb,er)}function validChunk(stream,state,chunk,cb){var valid=true;var er=false;if(chunk===null){er=new TypeError("May not write null values to stream")}else if(!Buffer.isBuffer(chunk)&&typeof chunk!=="string"&&chunk!==undefined&&!state.objectMode){er=new TypeError("Invalid non-string/buffer chunk")}if(er){stream.emit("error",er);processNextTick(cb,er);valid=false}return valid}Writable.prototype.write=function(chunk,encoding,cb){var state=this._writableState;var ret=false;if(typeof encoding==="function"){cb=encoding;encoding=null}if(Buffer.isBuffer(chunk))encoding="buffer";else if(!encoding)encoding=state.defaultEncoding;if(typeof cb!=="function")cb=nop;if(state.ended)writeAfterEnd(this,cb);else if(validChunk(this,state,chunk,cb)){ state.pendingcb++;ret=writeOrBuffer(this,state,chunk,encoding,cb)}return ret};Writable.prototype.cork=function(){var state=this._writableState;state.corked++};Writable.prototype.uncork=function(){var state=this._writableState;if(state.corked){state.corked--;if(!state.writing&&!state.corked&&!state.finished&&!state.bufferProcessing&&state.bufferedRequest)clearBuffer(this,state)}};Writable.prototype.setDefaultEncoding=function setDefaultEncoding(encoding){if(typeof encoding==="string")encoding=encoding.toLowerCase();if(!(["hex","utf8","utf-8","ascii","binary","base64","ucs2","ucs-2","utf16le","utf-16le","raw"].indexOf((encoding+"").toLowerCase())>-1))throw new TypeError("Unknown encoding: "+encoding);this._writableState.defaultEncoding=encoding;return this};function decodeChunk(state,chunk,encoding){if(!state.objectMode&&state.decodeStrings!==false&&typeof chunk==="string"){chunk=bufferShim.from(chunk,encoding)}return chunk}function writeOrBuffer(stream,state,chunk,encoding,cb){chunk=decodeChunk(state,chunk,encoding);if(Buffer.isBuffer(chunk))encoding="buffer";var len=state.objectMode?1:chunk.length;state.length+=len;var ret=state.length0)this.tail.next=entry;else this.head=entry;this.tail=entry;++this.length};BufferList.prototype.unshift=function(v){var entry={data:v,next:this.head};if(this.length===0)this.tail=entry;this.head=entry;++this.length};BufferList.prototype.shift=function(){if(this.length===0)return;var ret=this.head.data;if(this.length===1)this.head=this.tail=null;else this.head=this.head.next;--this.length;return ret};BufferList.prototype.clear=function(){this.head=this.tail=null;this.length=0};BufferList.prototype.join=function(s){if(this.length===0)return"";var p=this.head;var ret=""+p.data;while(p=p.next){ret+=s+p.data}return ret};BufferList.prototype.concat=function(n){if(this.length===0)return bufferShim.alloc(0);if(this.length===1)return this.head.data;var ret=bufferShim.allocUnsafe(n>>>0);var p=this.head;var i=0;while(p){p.data.copy(ret,i);i+=p.data.length;p=p.next}return ret}},{buffer:5,"buffer-shims":4}],50:[function(require,module,exports){module.exports=require("./lib/_stream_passthrough.js")},{"./lib/_stream_passthrough.js":45}],51:[function(require,module,exports){(function(process){var Stream=function(){try{return require("st"+"ream")}catch(_){}}();exports=module.exports=require("./lib/_stream_readable.js");exports.Stream=Stream||exports;exports.Readable=exports;exports.Writable=require("./lib/_stream_writable.js");exports.Duplex=require("./lib/_stream_duplex.js");exports.Transform=require("./lib/_stream_transform.js");exports.PassThrough=require("./lib/_stream_passthrough.js");if(!process.browser&&process.env.READABLE_STREAM==="disable"&&Stream){module.exports=Stream}}).call(this,require("_process"))},{"./lib/_stream_duplex.js":44,"./lib/_stream_passthrough.js":45,"./lib/_stream_readable.js":46,"./lib/_stream_transform.js":47,"./lib/_stream_writable.js":48,_process:42}],52:[function(require,module,exports){module.exports=require("./lib/_stream_transform.js")},{"./lib/_stream_transform.js":47}],53:[function(require,module,exports){module.exports=require("./lib/_stream_writable.js")},{"./lib/_stream_writable.js":48}],54:[function(require,module,exports){module.exports=function(string){return string.replace(/[-\\^$*+?.()|[\]{}]/g,"\\$&")}},{}],55:[function(require,module,exports){module.exports=Stream;var EE=require("events").EventEmitter;var inherits=require("inherits");inherits(Stream,EE);Stream.Readable=require("readable-stream/readable.js");Stream.Writable=require("readable-stream/writable.js");Stream.Duplex=require("readable-stream/duplex.js");Stream.Transform=require("readable-stream/transform.js");Stream.PassThrough=require("readable-stream/passthrough.js");Stream.Stream=Stream;function Stream(){EE.call(this)}Stream.prototype.pipe=function(dest,options){var source=this;function ondata(chunk){if(dest.writable){if(false===dest.write(chunk)&&source.pause){source.pause()}}}source.on("data",ondata);function ondrain(){if(source.readable&&source.resume){source.resume()}}dest.on("drain",ondrain);if(!dest._isStdio&&(!options||options.end!==false)){source.on("end",onend);source.on("close",onclose)}var didOnEnd=false;function onend(){if(didOnEnd)return;didOnEnd=true;dest.end()}function onclose(){if(didOnEnd)return;didOnEnd=true;if(typeof dest.destroy==="function")dest.destroy()}function onerror(er){cleanup();if(EE.listenerCount(this,"error")===0){throw er}}source.on("error",onerror);dest.on("error",onerror);function cleanup(){source.removeListener("data",ondata);dest.removeListener("drain",ondrain);source.removeListener("end",onend);source.removeListener("close",onclose);source.removeListener("error",onerror);dest.removeListener("error",onerror);source.removeListener("end",cleanup);source.removeListener("close",cleanup);dest.removeListener("close",cleanup)}source.on("end",cleanup);source.on("close",cleanup);dest.on("close",cleanup);dest.emit("pipe",source);return dest}},{events:28,inherits:38,"readable-stream/duplex.js":43,"readable-stream/passthrough.js":50,"readable-stream/readable.js":51,"readable-stream/transform.js":52,"readable-stream/writable.js":53}],56:[function(require,module,exports){var Buffer=require("buffer").Buffer;var isBufferEncoding=Buffer.isEncoding||function(encoding){switch(encoding&&encoding.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return true;default:return false}};function assertEncoding(encoding){if(encoding&&!isBufferEncoding(encoding)){throw new Error("Unknown encoding: "+encoding)}}var StringDecoder=exports.StringDecoder=function(encoding){this.encoding=(encoding||"utf8").toLowerCase().replace(/[-_]/,"");assertEncoding(encoding);switch(this.encoding){case"utf8":this.surrogateSize=3;break;case"ucs2":case"utf16le":this.surrogateSize=2;this.detectIncompleteChar=utf16DetectIncompleteChar;break;case"base64":this.surrogateSize=3;this.detectIncompleteChar=base64DetectIncompleteChar;break;default:this.write=passThroughWrite;return}this.charBuffer=new Buffer(6);this.charReceived=0;this.charLength=0};StringDecoder.prototype.write=function(buffer){var charStr="";while(this.charLength){var available=buffer.length>=this.charLength-this.charReceived?this.charLength-this.charReceived:buffer.length;buffer.copy(this.charBuffer,this.charReceived,0,available);this.charReceived+=available;if(this.charReceived=55296&&charCode<=56319){this.charLength+=this.surrogateSize;charStr="";continue}this.charReceived=this.charLength=0;if(buffer.length===0){return charStr}break}this.detectIncompleteChar(buffer);var end=buffer.length;if(this.charLength){buffer.copy(this.charBuffer,0,buffer.length-this.charReceived,end);end-=this.charReceived}charStr+=buffer.toString(this.encoding,0,end);var end=charStr.length-1;var charCode=charStr.charCodeAt(end);if(charCode>=55296&&charCode<=56319){var size=this.surrogateSize;this.charLength+=size;this.charReceived+=size;this.charBuffer.copy(this.charBuffer,size,0,size);buffer.copy(this.charBuffer,0,0,size);return charStr.substring(0,end)}return charStr};StringDecoder.prototype.detectIncompleteChar=function(buffer){var i=buffer.length>=3?3:buffer.length;for(;i>0;i--){var c=buffer[buffer.length-i];if(i==1&&c>>5==6){this.charLength=2;break}if(i<=2&&c>>4==14){this.charLength=3;break}if(i<=3&&c>>3==30){this.charLength=4;break}}this.charReceived=i};StringDecoder.prototype.end=function(buffer){var res="";if(buffer&&buffer.length)res=this.write(buffer);if(this.charReceived){var cr=this.charReceived;var buf=this.charBuffer;var enc=this.encoding;res+=buf.slice(0,cr).toString(enc)}return res};function passThroughWrite(buffer){return buffer.toString(this.encoding)}function utf16DetectIncompleteChar(buffer){this.charReceived=buffer.length%2;this.charLength=this.charReceived?2:0}function base64DetectIncompleteChar(buffer){this.charReceived=buffer.length%3;this.charLength=this.charReceived?3:0}},{buffer:5}],57:[function(require,module,exports){(function(global){module.exports=deprecate;function deprecate(fn,msg){if(config("noDeprecation")){return fn}var warned=false;function deprecated(){if(!warned){if(config("throwDeprecation")){throw new Error(msg)}else if(config("traceDeprecation")){console.trace(msg)}else{console.warn(msg)}warned=true}return fn.apply(this,arguments)}return deprecated}function config(name){try{if(!global.localStorage)return false}catch(_){return false}var val=global.localStorage[name];if(null==val)return false;return String(val).toLowerCase()==="true"}}).call(this,typeof global!=="undefined"?global:typeof self!=="undefined"?self:typeof window!=="undefined"?window:{})},{}],58:[function(require,module,exports){module.exports=extend;var hasOwnProperty=Object.prototype.hasOwnProperty;function extend(){var target={};for(var i=0;i Date: Mon, 8 Jul 2019 19:46:26 +0530 Subject: [PATCH 019/268] [Jasoet|Bimozx] Refactor: introduce execution_context --- cmd/proctord/main.go | 4 +- internal/app/proctord/audit/auditor.go | 2 +- internal/app/proctord/http/client.go | 2 +- internal/app/proctord/http/client_test.go | 2 +- .../app/proctord/instrumentation/newrelic.go | 2 +- .../app/proctord/jobs/execution/handler.go | 2 +- internal/app/proctord/jobs/logs/handler.go | 4 +- .../app/proctord/jobs/metadata/handler.go | 2 +- .../app/proctord/jobs/schedule/handler.go | 2 +- internal/app/proctord/jobs/schedule/worker.go | 2 +- internal/app/proctord/jobs/secrets/handler.go | 2 +- internal/app/proctord/kubernetes/client.go | 4 +- .../app/proctord/kubernetes/client_test.go | 2 +- internal/app/proctord/kubernetes/utils.go | 4 +- internal/app/proctord/mail/mailer.go | 2 +- internal/app/proctord/mail/mailer_test.go | 2 +- .../middleware/validate_client_version.go | 4 +- internal/app/proctord/redis/pool.go | 2 +- internal/app/proctord/scheduler/scheduler.go | 6 +-- internal/app/proctord/server/api.go | 6 +-- internal/app/proctord/server/router.go | 32 +++++++-------- .../proctord/storage/postgres/migrations.go | 4 +- .../app/proctord/storage/postgres/schema.go | 2 +- internal/app/proctord/storage/store.go | 5 ++- internal/app/proctord/storage/store_test.go | 27 ++++++------ .../context/model/execution_context.go | 19 +++++++++ .../context/repository/execution_context.go | 23 +++++++++++ .../infra}/config/config.go | 0 .../infra}/config/config_test.go | 0 .../infra/db/postgresql}/client.go | 6 +-- .../infra/db/postgresql}/client_mock.go | 2 +- .../infra/db/postgresql}/client_test.go | 19 +++++---- .../app/service/infra/db/types/base64_map.go | 41 +++++++++++++++++++ .../service/infra/db/types/base64_map_test.go | 28 +++++++++++++ .../infra}/logger/logrus.go | 2 +- 35 files changed, 191 insertions(+), 77 deletions(-) create mode 100644 internal/app/service/context/model/execution_context.go create mode 100644 internal/app/service/context/repository/execution_context.go rename internal/app/{proctord => service/infra}/config/config.go (100%) rename internal/app/{proctord => service/infra}/config/config_test.go (100%) rename internal/app/{proctord/storage/postgres => service/infra/db/postgresql}/client.go (92%) rename internal/app/{proctord/storage/postgres => service/infra/db/postgresql}/client_mock.go (97%) rename internal/app/{proctord/storage/postgres => service/infra/db/postgresql}/client_test.go (87%) create mode 100644 internal/app/service/infra/db/types/base64_map.go create mode 100644 internal/app/service/infra/db/types/base64_map_test.go rename internal/app/{proctord => service/infra}/logger/logrus.go (95%) diff --git a/cmd/proctord/main.go b/cmd/proctord/main.go index 779691f0..bc064ccc 100644 --- a/cmd/proctord/main.go +++ b/cmd/proctord/main.go @@ -3,11 +3,11 @@ package main import ( "github.com/getsentry/raven-go" "os" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" "proctor/internal/app/proctord/scheduler" "proctor/internal/app/proctord/server" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "github.com/urfave/cli" ) diff --git a/internal/app/proctord/audit/auditor.go b/internal/app/proctord/audit/auditor.go index dd7a2311..9468df0a 100644 --- a/internal/app/proctord/audit/auditor.go +++ b/internal/app/proctord/audit/auditor.go @@ -3,9 +3,9 @@ package audit import ( "github.com/getsentry/raven-go" "proctor/internal/app/proctord/kubernetes" - "proctor/internal/app/proctord/logger" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" ) diff --git a/internal/app/proctord/http/client.go b/internal/app/proctord/http/client.go index 21a3384b..860a2f9b 100644 --- a/internal/app/proctord/http/client.go +++ b/internal/app/proctord/http/client.go @@ -5,7 +5,7 @@ import ( "crypto/x509" "encoding/base64" "net/http" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" ) func NewClient() (*http.Client, error) { diff --git a/internal/app/proctord/http/client_test.go b/internal/app/proctord/http/client_test.go index ce9f5d35..b5c295f2 100644 --- a/internal/app/proctord/http/client_test.go +++ b/internal/app/proctord/http/client_test.go @@ -5,7 +5,7 @@ import ( "crypto/x509" "encoding/base64" "net/http" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" "testing" "github.com/stretchr/testify/assert" diff --git a/internal/app/proctord/instrumentation/newrelic.go b/internal/app/proctord/instrumentation/newrelic.go index fa64aaa1..bd101649 100644 --- a/internal/app/proctord/instrumentation/newrelic.go +++ b/internal/app/proctord/instrumentation/newrelic.go @@ -3,7 +3,7 @@ package instrumentation import ( newrelic "github.com/newrelic/go-agent" "net/http" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" ) var NewRelicApp newrelic.Application diff --git a/internal/app/proctord/jobs/execution/handler.go b/internal/app/proctord/jobs/execution/handler.go index 60984617..1da2a949 100644 --- a/internal/app/proctord/jobs/execution/handler.go +++ b/internal/app/proctord/jobs/execution/handler.go @@ -8,9 +8,9 @@ import ( "github.com/gorilla/mux" "net/http" "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/logger" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" "time" ) diff --git a/internal/app/proctord/jobs/logs/handler.go b/internal/app/proctord/jobs/logs/handler.go index cece6734..3b5f6999 100644 --- a/internal/app/proctord/jobs/logs/handler.go +++ b/internal/app/proctord/jobs/logs/handler.go @@ -5,9 +5,9 @@ import ( "github.com/getsentry/raven-go" "io" "net/http" - "proctor/internal/app/proctord/config" "proctor/internal/app/proctord/kubernetes" - _logger "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + _logger "proctor/internal/app/service/infra/logger" "strings" "proctor/internal/pkg/constant" diff --git a/internal/app/proctord/jobs/metadata/handler.go b/internal/app/proctord/jobs/metadata/handler.go index da9f6654..703984d8 100644 --- a/internal/app/proctord/jobs/metadata/handler.go +++ b/internal/app/proctord/jobs/metadata/handler.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/getsentry/raven-go" "net/http" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" modelMetadata "proctor/internal/pkg/model/metadata" ) diff --git a/internal/app/proctord/jobs/schedule/handler.go b/internal/app/proctord/jobs/schedule/handler.go index eb1af60d..5b22b162 100644 --- a/internal/app/proctord/jobs/schedule/handler.go +++ b/internal/app/proctord/jobs/schedule/handler.go @@ -7,8 +7,8 @@ import ( "github.com/gorilla/mux" "net/http" jobMetadata "proctor/internal/app/proctord/jobs/metadata" - "proctor/internal/app/proctord/logger" "proctor/internal/app/proctord/storage" + "proctor/internal/app/service/infra/logger" "strings" "github.com/badoux/checkmail" diff --git a/internal/app/proctord/jobs/schedule/worker.go b/internal/app/proctord/jobs/schedule/worker.go index 7be829bd..00f9b80f 100644 --- a/internal/app/proctord/jobs/schedule/worker.go +++ b/internal/app/proctord/jobs/schedule/worker.go @@ -6,10 +6,10 @@ import ( "os" "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/logger" "proctor/internal/app/proctord/mail" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/logger" "strings" "time" diff --git a/internal/app/proctord/jobs/secrets/handler.go b/internal/app/proctord/jobs/secrets/handler.go index 9e07a938..d6f97b23 100644 --- a/internal/app/proctord/jobs/secrets/handler.go +++ b/internal/app/proctord/jobs/secrets/handler.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/getsentry/raven-go" "net/http" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" ) diff --git a/internal/app/proctord/kubernetes/client.go b/internal/app/proctord/kubernetes/client.go index 33721754..fbde73ce 100644 --- a/internal/app/proctord/kubernetes/client.go +++ b/internal/app/proctord/kubernetes/client.go @@ -4,8 +4,8 @@ import ( "fmt" "io" "net/http" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "time" uuid "github.com/satori/go.uuid" diff --git a/internal/app/proctord/kubernetes/client_test.go b/internal/app/proctord/kubernetes/client_test.go index 2b50367a..3b2b4681 100644 --- a/internal/app/proctord/kubernetes/client_test.go +++ b/internal/app/proctord/kubernetes/client_test.go @@ -4,7 +4,7 @@ import ( "bufio" "net/http" "os" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" "testing" "time" diff --git a/internal/app/proctord/kubernetes/utils.go b/internal/app/proctord/kubernetes/utils.go index f750a41a..a8815e79 100644 --- a/internal/app/proctord/kubernetes/utils.go +++ b/internal/app/proctord/kubernetes/utils.go @@ -4,8 +4,8 @@ import ( "flag" "os" "path/filepath" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" ) func KubeConfig() string { diff --git a/internal/app/proctord/mail/mailer.go b/internal/app/proctord/mail/mailer.go index e7075f56..a2953d74 100644 --- a/internal/app/proctord/mail/mailer.go +++ b/internal/app/proctord/mail/mailer.go @@ -2,7 +2,7 @@ package mail import ( "net/smtp" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" "proctor/internal/pkg/utility" ) diff --git a/internal/app/proctord/mail/mailer_test.go b/internal/app/proctord/mail/mailer_test.go index bbf0e9cd..9476b0a3 100644 --- a/internal/app/proctord/mail/mailer_test.go +++ b/internal/app/proctord/mail/mailer_test.go @@ -6,7 +6,7 @@ import ( "fmt" "net" "net/textproto" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" "strings" "testing" diff --git a/internal/app/proctord/middleware/validate_client_version.go b/internal/app/proctord/middleware/validate_client_version.go index 3947b771..f7e5977d 100644 --- a/internal/app/proctord/middleware/validate_client_version.go +++ b/internal/app/proctord/middleware/validate_client_version.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/hashicorp/go-version" "net/http" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" ) diff --git a/internal/app/proctord/redis/pool.go b/internal/app/proctord/redis/pool.go index d339035c..9b606cb5 100644 --- a/internal/app/proctord/redis/pool.go +++ b/internal/app/proctord/redis/pool.go @@ -1,7 +1,7 @@ package redis import ( - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" "time" "github.com/garyburd/redigo/redis" diff --git a/internal/app/proctord/scheduler/scheduler.go b/internal/app/proctord/scheduler/scheduler.go index 83f5f4b7..d0f96095 100644 --- a/internal/app/proctord/scheduler/scheduler.go +++ b/internal/app/proctord/scheduler/scheduler.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/config" "proctor/internal/app/proctord/http" "proctor/internal/app/proctord/jobs/execution" "proctor/internal/app/proctord/jobs/metadata" @@ -14,14 +13,15 @@ import ( "proctor/internal/app/proctord/mail" "proctor/internal/app/proctord/redis" "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/db/postgresql" "time" ) func Start() error { fmt.Println("started scheduler") - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() redisClient := redis.NewClient() store := storage.New(postgresClient) diff --git a/internal/app/proctord/server/api.go b/internal/app/proctord/server/api.go index 78405570..f89c29a2 100644 --- a/internal/app/proctord/server/api.go +++ b/internal/app/proctord/server/api.go @@ -1,9 +1,9 @@ package server import ( - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/instrumentation" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/proctord/instrumentation" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "time" "github.com/tylerb/graceful" diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index c43401c2..cf02c84e 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -4,32 +4,32 @@ import ( "fmt" "net/http" "path" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/docs" + "proctor/internal/app/proctord/audit" + "proctor/internal/app/proctord/docs" httpClient "proctor/internal/app/proctord/http" - "proctor/internal/app/proctord/instrumentation" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/jobs/logs" - "proctor/internal/app/proctord/jobs/metadata" - "proctor/internal/app/proctord/jobs/schedule" - "proctor/internal/app/proctord/jobs/secrets" - "proctor/internal/app/proctord/kubernetes" - "proctor/internal/app/proctord/middleware" - "proctor/internal/app/proctord/redis" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/proctord/instrumentation" + "proctor/internal/app/proctord/jobs/execution" + "proctor/internal/app/proctord/jobs/logs" + "proctor/internal/app/proctord/jobs/metadata" + "proctor/internal/app/proctord/jobs/schedule" + "proctor/internal/app/proctord/jobs/secrets" + "proctor/internal/app/proctord/kubernetes" + "proctor/internal/app/proctord/middleware" + "proctor/internal/app/proctord/redis" + "proctor/internal/app/proctord/storage" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/db/postgresql" "github.com/gorilla/mux" ) -var postgresClient postgres.Client +var postgresClient postgresql.Client func NewRouter() (*mux.Router, error) { router := mux.NewRouter() redisClient := redis.NewClient() - postgresClient = postgres.NewClient() + postgresClient = postgresql.NewClient() store := storage.New(postgresClient) metadataStore := metadata.NewStore(redisClient) diff --git a/internal/app/proctord/storage/postgres/migrations.go b/internal/app/proctord/storage/postgres/migrations.go index 27db0c0c..3f8c4797 100644 --- a/internal/app/proctord/storage/postgres/migrations.go +++ b/internal/app/proctord/storage/postgres/migrations.go @@ -2,8 +2,8 @@ package postgres import ( "fmt" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "github.com/mattes/migrate" //postgres driver diff --git a/internal/app/proctord/storage/postgres/schema.go b/internal/app/proctord/storage/postgres/schema.go index ae6b527e..30ef0854 100644 --- a/internal/app/proctord/storage/postgres/schema.go +++ b/internal/app/proctord/storage/postgres/schema.go @@ -4,7 +4,7 @@ import ( "database/sql" "encoding/base64" "encoding/json" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/logger" "time" ) diff --git a/internal/app/proctord/storage/store.go b/internal/app/proctord/storage/store.go index 81b0c0f2..6043e61e 100644 --- a/internal/app/proctord/storage/store.go +++ b/internal/app/proctord/storage/store.go @@ -5,6 +5,7 @@ import ( "encoding/json" uuid "github.com/satori/go.uuid" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/db/postgresql" "time" ) @@ -20,10 +21,10 @@ type Store interface { } type store struct { - postgresClient postgres.Client + postgresClient postgresql.Client } -func New(postgresClient postgres.Client) Store { +func New(postgresClient postgresql.Client) Store { return &store{ postgresClient: postgresClient, } diff --git a/internal/app/proctord/storage/store_test.go b/internal/app/proctord/storage/store_test.go index c1d81ec6..d8af6254 100644 --- a/internal/app/proctord/storage/store_test.go +++ b/internal/app/proctord/storage/store_test.go @@ -8,12 +8,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/pkg/constant" "testing" ) func TestJobsExecutionAuditLog(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobExecutionAuditLog := &postgres.JobsExecutionAuditLog{ @@ -38,7 +39,7 @@ func TestJobsExecutionAuditLog(t *testing.T) { } func TestJobsExecutionAuditLogPostgresClientFailure(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobExecutionAuditLog := &postgres.JobsExecutionAuditLog{ @@ -63,7 +64,7 @@ func TestJobsExecutionAuditLogPostgresClientFailure(t *testing.T) { } func TestUpdateJobsExecutionAuditLog(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) executionID := "any-submission" @@ -88,7 +89,7 @@ func TestUpdateJobsExecutionAuditLog(t *testing.T) { } func TestGetJobsStatusWhenJobIsPresent(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobName := "any-job" @@ -116,7 +117,7 @@ func TestGetJobsStatusWhenJobIsPresent(t *testing.T) { } func TestGetJobsStatusWhenJobIsNotPresent(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobName := "any-job" @@ -138,7 +139,7 @@ func TestGetJobsStatusWhenJobIsNotPresent(t *testing.T) { } func TestGetJobsStatusWhenError(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobName := "any-job" @@ -156,7 +157,7 @@ func TestGetJobsStatusWhenError(t *testing.T) { } func TestJobsScheduleInsertionSuccessfull(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) scheduledJobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) @@ -169,7 +170,7 @@ func TestJobsScheduleInsertionSuccessfull(t *testing.T) { } func TestJobsScheduleInsertionFailed(t *testing.T) { - mockPostgresClient := &postgres.ClientMock{} + mockPostgresClient := &postgresql.ClientMock{} testStore := New(mockPostgresClient) jobName := "job-name" @@ -194,7 +195,7 @@ func TestJobsScheduleInsertionFailed(t *testing.T) { } func TestGetScheduledJobByID(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) @@ -211,7 +212,7 @@ func TestGetScheduledJobByID(t *testing.T) { } func TestGetScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) resultJob, err := testStore.GetScheduledJob("86A7963B-3621-492D-8D6C-33076242256B") @@ -220,7 +221,7 @@ func TestGetScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { } func TestRemoveScheduledJobByID(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) @@ -235,7 +236,7 @@ func TestRemoveScheduledJobByID(t *testing.T) { } func TestRemoveScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) removedJobsCount, err := testStore.RemoveScheduledJob("86A7963B-3621-492D-8D6C-33076242256B") @@ -244,7 +245,7 @@ func TestRemoveScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { } func TestRemoveScheduledJobByIDReturnErrorIfIDIsInvalid(t *testing.T) { - postgresClient := postgres.NewClient() + postgresClient := postgresql.NewClient() testStore := New(postgresClient) removedJobsCount, err := testStore.RemoveScheduledJob("86A7963B") diff --git a/internal/app/service/context/model/execution_context.go b/internal/app/service/context/model/execution_context.go new file mode 100644 index 00000000..74aede84 --- /dev/null +++ b/internal/app/service/context/model/execution_context.go @@ -0,0 +1,19 @@ +package model + +import ( + sqlxTypes "github.com/jmoiron/sqlx/types" + dbTypes "proctor/internal/app/service/infra/db/types" + "time" +) + +type ExecutionContext struct { + ExecutionID string `db:"execution_id"` + JobName string `db:"job_name"` + UserEmail string `db:"user_email"` + ImageTag string `db:"image_tag"` + Args dbTypes.Base64Map `db:"args"` + Output sqlxTypes.GzippedText `db:"output"` + Status string `db:"status"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} diff --git a/internal/app/service/context/repository/execution_context.go b/internal/app/service/context/repository/execution_context.go new file mode 100644 index 00000000..1421a8b4 --- /dev/null +++ b/internal/app/service/context/repository/execution_context.go @@ -0,0 +1,23 @@ +package repository + +import ( + "github.com/jmoiron/sqlx/types" + "proctor/internal/app/service/context/model" + "time" +) + +type ExecutionContextRepository interface { + Insert(context model.ExecutionContext) error + UpdateJobOutput(executionId string, status types.GzippedText) error + UpdateStatus(executionId string, status string) error + Delete(executionId string) error + GetById(executionId string) (model.ExecutionContext, error) + GetByEmail(userEmail string) ([]model.ExecutionContext, error) + GetByJobName(jobName string) ([]model.ExecutionContext, error) + GetByStatus(status string) ([]model.ExecutionContext, error) + GetByCreationTime(start time.Time, end time.Time) ([]model.ExecutionContext, error) +} + +type executionContextRepository struct { + +} diff --git a/internal/app/proctord/config/config.go b/internal/app/service/infra/config/config.go similarity index 100% rename from internal/app/proctord/config/config.go rename to internal/app/service/infra/config/config.go diff --git a/internal/app/proctord/config/config_test.go b/internal/app/service/infra/config/config_test.go similarity index 100% rename from internal/app/proctord/config/config_test.go rename to internal/app/service/infra/config/config_test.go diff --git a/internal/app/proctord/storage/postgres/client.go b/internal/app/service/infra/db/postgresql/client.go similarity index 92% rename from internal/app/proctord/storage/postgres/client.go rename to internal/app/service/infra/db/postgresql/client.go index 252624b0..6d0e4424 100644 --- a/internal/app/proctord/storage/postgres/client.go +++ b/internal/app/service/infra/db/postgresql/client.go @@ -1,9 +1,9 @@ -package postgres +package postgresql import ( "fmt" - "proctor/internal/app/proctord/config" - "proctor/internal/app/proctord/logger" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" "time" "github.com/jmoiron/sqlx" diff --git a/internal/app/proctord/storage/postgres/client_mock.go b/internal/app/service/infra/db/postgresql/client_mock.go similarity index 97% rename from internal/app/proctord/storage/postgres/client_mock.go rename to internal/app/service/infra/db/postgresql/client_mock.go index 0babc2ef..6f4a2988 100644 --- a/internal/app/proctord/storage/postgres/client_mock.go +++ b/internal/app/service/infra/db/postgresql/client_mock.go @@ -1,4 +1,4 @@ -package postgres +package postgresql import ( "github.com/jmoiron/sqlx" diff --git a/internal/app/proctord/storage/postgres/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go similarity index 87% rename from internal/app/proctord/storage/postgres/client_test.go rename to internal/app/service/infra/db/postgresql/client_test.go index 65b372f3..f632319f 100644 --- a/internal/app/proctord/storage/postgres/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -1,8 +1,9 @@ -package postgres +package postgresql import ( "fmt" - config2 "proctor/internal/app/proctord/config" + "proctor/internal/app/proctord/storage/postgres" + config2 "proctor/internal/app/service/infra/config" "testing" "github.com/jmoiron/sqlx" @@ -18,10 +19,10 @@ func TestNamedExec(t *testing.T) { postgresClient := &client{db: db} defer postgresClient.db.Close() - jobsExecutionAuditLog := &JobsExecutionAuditLog{ + jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ JobName: "test-job-name", ImageName: "test-image-name", - ExecutionID: StringToSQLString("test-submission-name"), + ExecutionID: postgres.StringToSQLString("test-submission-name"), JobArgs: "test-job-args", JobSubmissionStatus: "test-job-status", JobExecutionStatus: "test-job-execution-status", @@ -30,7 +31,7 @@ func TestNamedExec(t *testing.T) { _, err = postgresClient.NamedExec("INSERT INTO jobs_execution_audit_log (job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", jobsExecutionAuditLog) assert.NoError(t, err) - var persistedJobsExecutionAuditLog JobsExecutionAuditLog + var persistedJobsExecutionAuditLog postgres.JobsExecutionAuditLog err = postgresClient.db.Get(&persistedJobsExecutionAuditLog, `SELECT job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status FROM jobs_execution_audit_log WHERE job_name='test-job-name'`) assert.NoError(t, err) @@ -55,10 +56,10 @@ func TestSelect(t *testing.T) { defer postgresClient.db.Close() jobName := "test-job-name" - jobsExecutionAuditLog := &JobsExecutionAuditLog{ + jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ JobName: jobName, ImageName: "test-image-name", - ExecutionID: StringToSQLString("test-submission-name"), + ExecutionID: postgres.StringToSQLString("test-submission-name"), JobArgs: "test-job-args", JobSubmissionStatus: "test-job-status", JobExecutionStatus: "test-job-execution-status", @@ -67,7 +68,7 @@ func TestSelect(t *testing.T) { _, err = postgresClient.NamedExec("INSERT INTO jobs_execution_audit_log (job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", jobsExecutionAuditLog) assert.NoError(t, err) - jobsExecutionAuditLogResult := []JobsExecutionAuditLog{} + jobsExecutionAuditLogResult := []postgres.JobsExecutionAuditLog{} err = postgresClient.db.Select(&jobsExecutionAuditLogResult, "SELECT job_execution_status from jobs_execution_audit_log where job_name = $1", jobName) assert.NoError(t, err) @@ -87,7 +88,7 @@ func TestSelectForNoRows(t *testing.T) { defer postgresClient.db.Close() jobName := "test-job-name" - jobsExecutionAuditLogResult := []JobsExecutionAuditLog{} + jobsExecutionAuditLogResult := []postgres.JobsExecutionAuditLog{} err = postgresClient.db.Select(&jobsExecutionAuditLogResult, "SELECT job_execution_status from jobs_execution_audit_log where job_name = $1", jobName) assert.NoError(t, err) diff --git a/internal/app/service/infra/db/types/base64_map.go b/internal/app/service/infra/db/types/base64_map.go new file mode 100644 index 00000000..fcc8a47a --- /dev/null +++ b/internal/app/service/infra/db/types/base64_map.go @@ -0,0 +1,41 @@ +package types + +import ( + "database/sql/driver" + "encoding/base64" + "encoding/json" + "errors" +) + +type Base64Map map[string]string + +// Value implements the driver.Valuer interface, convert map into json and encode it as Base64 +func (g Base64Map) Value() (driver.Value, error) { + jsonByte, err := json.Marshal(g) + if err != nil { + return nil, err + } + + return base64.StdEncoding.EncodeToString(jsonByte), nil +} + +func (g *Base64Map) Scan(src interface{}) error { + var source string + switch src.(type) { + case string: + source = src.(string) + default: + return errors.New("incompatible type for Base64Map") + } + + jsonByte, err := base64.StdEncoding.DecodeString(source) + if err != nil { + return err + } + + err = json.Unmarshal(jsonByte, g) + if err != nil { + return err + } + return nil +} diff --git a/internal/app/service/infra/db/types/base64_map_test.go b/internal/app/service/infra/db/types/base64_map_test.go new file mode 100644 index 00000000..d0102583 --- /dev/null +++ b/internal/app/service/infra/db/types/base64_map_test.go @@ -0,0 +1,28 @@ +package types + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestBase64Map(t *testing.T) { + maps := Base64Map{ + "name": "Bimo.zx", + "address": "Royal Orchid", + "age": "17", + } + v, err := maps.Value() + if err != nil { + t.Errorf("Was not expecting an error") + } + + expectedMaps := Base64Map{} + err = (&expectedMaps).Scan(v) + if err != nil { + t.Errorf("Was not expecting an error") + } + + assert.Equal(t, "Bimo.zx", expectedMaps["name"]) + assert.Equal(t, "Royal Orchid", expectedMaps["address"]) + assert.Equal(t, "17", expectedMaps["age"]) +} diff --git a/internal/app/proctord/logger/logrus.go b/internal/app/service/infra/logger/logrus.go similarity index 95% rename from internal/app/proctord/logger/logrus.go rename to internal/app/service/infra/logger/logrus.go index e9014f52..48c64f35 100644 --- a/internal/app/proctord/logger/logrus.go +++ b/internal/app/service/infra/logger/logrus.go @@ -2,7 +2,7 @@ package logger import ( "os" - "proctor/internal/app/proctord/config" + "proctor/internal/app/service/infra/config" log "github.com/sirupsen/logrus" ) From c9694622678942e28d0cd9439edfe0e40ee00370 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 9 Jul 2019 15:26:19 +0530 Subject: [PATCH 020/268] [Jasoet|Bimozx] Refactor: Implement new ExecutionContext entity and repository --- go.mod | 2 + go.sum | 5 + .../context/model/execution_context.go | 2 +- .../context/repository/execution_context.go | 119 ++++++++- .../repository/execution_context_test.go | 242 ++++++++++++++++++ .../infra/db/postgresql/client_test.go | 8 +- internal/app/service/infra/id/snowflake.go | 18 ++ .../10_CreateExecutionContextTable.down.sql | 1 + .../10_CreateExecutionContextTable.up.sql | 12 + 9 files changed, 398 insertions(+), 11 deletions(-) create mode 100644 internal/app/service/context/repository/execution_context_test.go create mode 100644 internal/app/service/infra/id/snowflake.go create mode 100644 migrations/10_CreateExecutionContextTable.down.sql create mode 100644 migrations/10_CreateExecutionContextTable.up.sql diff --git a/go.mod b/go.mod index b566eb87..10da86a4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Microsoft/go-winio v0.4.12 // indirect github.com/badoux/checkmail v0.0.0-20181210160741-9661bd69e9ad github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 + github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 // indirect @@ -37,6 +38,7 @@ require ( github.com/robfig/cron v1.1.0 github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.4.2 + github.com/sony/sonyflake v1.0.0 github.com/spf13/cobra v0.0.4 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 diff --git a/go.sum b/go.sum index b0fb1e56..399bb8d6 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 h1:GMmnK0dvr0Sf0gx3DvTbln0c8DE07B7sPVD9dgHOqo4= github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= +github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= +github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -36,6 +38,7 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= @@ -188,6 +191,8 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/sony/sonyflake v1.0.0 h1:MpU6Ro7tfXwgn2l5eluf9xQvQJDROTBImNCfRXn/YeM= +github.com/sony/sonyflake v1.0.0/go.mod h1:Jv3cfhf/UFtolOTTRd3q4Nl6ENqM+KfyZ5PseKfZGF4= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/internal/app/service/context/model/execution_context.go b/internal/app/service/context/model/execution_context.go index 74aede84..10f7ccc6 100644 --- a/internal/app/service/context/model/execution_context.go +++ b/internal/app/service/context/model/execution_context.go @@ -7,7 +7,7 @@ import ( ) type ExecutionContext struct { - ExecutionID string `db:"execution_id"` + ExecutionID uint64 `db:"id"` JobName string `db:"job_name"` UserEmail string `db:"user_email"` ImageTag string `db:"image_tag"` diff --git a/internal/app/service/context/repository/execution_context.go b/internal/app/service/context/repository/execution_context.go index 1421a8b4..5f73aa8e 100644 --- a/internal/app/service/context/repository/execution_context.go +++ b/internal/app/service/context/repository/execution_context.go @@ -2,22 +2,129 @@ package repository import ( "github.com/jmoiron/sqlx/types" + "github.com/pkg/errors" "proctor/internal/app/service/context/model" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/id" "time" ) type ExecutionContextRepository interface { - Insert(context model.ExecutionContext) error - UpdateJobOutput(executionId string, status types.GzippedText) error - UpdateStatus(executionId string, status string) error - Delete(executionId string) error - GetById(executionId string) (model.ExecutionContext, error) + Insert(context *model.ExecutionContext) (uint64, error) + UpdateJobOutput(executionId uint64, output types.GzippedText) error + UpdateStatus(executionId uint64, status string) error + Delete(executionId uint64) error + GetById(executionId uint64) (*model.ExecutionContext, error) GetByEmail(userEmail string) ([]model.ExecutionContext, error) GetByJobName(jobName string) ([]model.ExecutionContext, error) GetByStatus(status string) ([]model.ExecutionContext, error) - GetByCreationTime(start time.Time, end time.Time) ([]model.ExecutionContext, error) + deleteAll() error } type executionContextRepository struct { + postgresqlClient postgresql.Client +} + +func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRepository { + return &executionContextRepository{ + postgresqlClient: client, + } +} + +func (repository *executionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { + snowflakeId, _ := id.NextId() + context.ExecutionID = snowflakeId + sql := "INSERT INTO execution_context (id, job_name, user_email, image_tag, args, output, status) " + + " VALUES (:id, :job_name, :user_email, :image_tag, :args, :output, :status)" + _, err := repository.postgresqlClient.NamedExec(sql, &context) + if err != nil { + return 0, nil + } + return snowflakeId, nil +} + +func (repository *executionContextRepository) UpdateJobOutput(executionId uint64, output types.GzippedText) error { + sql := "UPDATE execution_context SET output = :output, updated_at = :updated_at WHERE id = :id" + context := model.ExecutionContext{ + ExecutionID: executionId, + UpdatedAt: time.Now(), + Output: output, + } + _, err := repository.postgresqlClient.NamedExec(sql, &context) + return err +} + +func (repository *executionContextRepository) UpdateStatus(executionId uint64, status string) error { + sql := "UPDATE execution_context SET status = :status, updated_at = :updated_at WHERE id = :id" + context := model.ExecutionContext{ + ExecutionID: executionId, + UpdatedAt: time.Now(), + Status: status, + } + _, err := repository.postgresqlClient.NamedExec(sql, &context) + return err +} + +func (repository *executionContextRepository) Delete(executionId uint64) error { + sql := "DELETE FROM execution_context WHERE id = :id" + context := model.ExecutionContext{ + ExecutionID: executionId, + } + _, err := repository.postgresqlClient.NamedExec(sql, &context) + return err +} + +func (repository *executionContextRepository) GetById(executionId uint64) (*model.ExecutionContext, error) { + sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE id=$1 " + var contexts []model.ExecutionContext + err := repository.postgresqlClient.Select(&contexts, sql, executionId) + if err != nil { + return nil, err + } + + if len(contexts) == 0 { + return nil, errors.Errorf("Execution context with id %v is not found!", executionId) + } + + return &contexts[0], nil +} + +func (repository *executionContextRepository) GetByEmail(userEmail string) ([]model.ExecutionContext, error) { + sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE user_email=$1 " + var contexts []model.ExecutionContext + err := repository.postgresqlClient.Select(&contexts, sql, userEmail) + if err != nil { + return nil, err + } + + return contexts, nil +} + +func (repository *executionContextRepository) GetByJobName(jobName string) ([]model.ExecutionContext, error) { + sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE job_name=$1 " + var contexts []model.ExecutionContext + err := repository.postgresqlClient.Select(&contexts, sql, jobName) + if err != nil { + return nil, err + } + + return contexts, nil +} + +func (repository *executionContextRepository) GetByStatus(status string) ([]model.ExecutionContext, error) { + sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE status=$1 " + var contexts []model.ExecutionContext + err := repository.postgresqlClient.Select(&contexts, sql, status) + if err != nil { + return nil, err + } + + return contexts, nil +} +func (repository *executionContextRepository) deleteAll() error { + sql := "DELETE FROM execution_context" + context := model.ExecutionContext{} + _, err := repository.postgresqlClient.NamedExec(sql, context) + return err } diff --git a/internal/app/service/context/repository/execution_context_test.go b/internal/app/service/context/repository/execution_context_test.go new file mode 100644 index 00000000..212e7725 --- /dev/null +++ b/internal/app/service/context/repository/execution_context_test.go @@ -0,0 +1,242 @@ +package repository + +import ( + fake "github.com/brianvoe/gofakeit" + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/assert" + "proctor/internal/app/service/context/model" + "proctor/internal/app/service/infra/db/postgresql" + "testing" +) + +func TestExecutionContextRepository_Insert(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + fake.Seed(0) + mapKey := fake.FirstName() + mapValue := fake.LastName() + + context := &model.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + mapKey: mapValue, + }, + Status: fake.State(), + } + + id, err := repository.Insert(context) + assert.Nil(t, err) + assert.NotZero(t, id) + + expectedContext, err := repository.GetById(id) + assert.Nil(t, err) + assert.NotNil(t, expectedContext) + assert.Equal(t, id, expectedContext.ExecutionID) + assert.NotNil(t, expectedContext.CreatedAt) + assert.NotNil(t, expectedContext.UpdatedAt) + assert.Equal(t, expectedContext.Args[mapKey], mapValue) +} + +func TestExecutionContextRepository_Delete(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + fake.Seed(0) + context := &model.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: fake.State(), + } + + id, err := repository.Insert(context) + assert.Nil(t, err) + assert.NotZero(t, id) + + err = repository.Delete(id) + assert.Nil(t, err) + + expectedContext, err := repository.GetById(id) + assert.NotNil(t, err) + assert.Nil(t, expectedContext) +} + +func TestExecutionContextRepository_UpdateStatus(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + fake.Seed(0) + context := &model.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: fake.State(), + } + + id, err := repository.Insert(context) + assert.Nil(t, err) + assert.NotZero(t, id) + + newStatus := fake.State() + err = repository.UpdateStatus(id, newStatus) + assert.Nil(t, err) + + expectedContext, err := repository.GetById(id) + assert.NotNil(t, expectedContext) + assert.Nil(t, err) + assert.Equal(t, newStatus, expectedContext.Status) +} + +func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + fake.Seed(0) + context := &model.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: fake.State(), + } + + id, err := repository.Insert(context) + assert.Nil(t, err) + assert.NotZero(t, id) + + newLog := ` + This ain't a log for the broken-hearted + No silent prayer for the faith-departed + I ain't gonna be just a face in the crowd + You're gonna hear my voice + When I shout it out loud + It's my log + It's now or never + I ain't gonna log forever + I just want to log while I'm alive + (It's my log) + My heart is like an open highway + Like Frankie said + I did it my way + I just want to log while I'm alive + It's my log + ` + + newOutput := types.GzippedText(newLog) + + err = repository.UpdateJobOutput(id, newOutput) + assert.Nil(t, err) + + expectedContext, err := repository.GetById(id) + assert.NotNil(t, expectedContext) + assert.Nil(t, err) + expectedLog := string(expectedContext.Output) + assert.Equal(t, newLog, expectedLog) +} + +func populateSeedDataForTest(repository ExecutionContextRepository, count int, seedField map[string]string) error { + for i := 0; i < count; i++ { + fake.Seed(0) + var jobName = fake.BuzzWord() + if val, ok := seedField["JobName"]; ok { + jobName = val + } + + var email = fake.Email() + if val, ok := seedField["UserEmail"]; ok { + email = val + } + + var status = fake.State() + if val, ok := seedField["Status"]; ok { + status = val + } + + context := &model.ExecutionContext{ + JobName: jobName, + UserEmail: email, + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: status, + } + + _, err := repository.Insert(context) + + if err != nil { + return err + } + } + return nil +} + +func TestExecutionContextRepository_GetByEmail(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + recordCount := 15 + userEmail := "bimo.horizon@go-pay.co.id" + err := populateSeedDataForTest(repository, recordCount, map[string]string{"UserEmail": userEmail}) + assert.Nil(t, err) + + contexts, err := repository.GetByEmail(userEmail) + assert.Nil(t, err) + assert.NotEmpty(t, contexts) + assert.Equal(t, recordCount, len(contexts)) +} + +func TestExecutionContextRepository_GetByJobName(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + recordCount := 15 + jobName := "some_job_that_only_exists_in_your_past" + err := populateSeedDataForTest(repository, recordCount, map[string]string{"JobName": jobName}) + assert.Nil(t, err) + + contexts, err := repository.GetByJobName(jobName) + assert.Nil(t, err) + assert.NotEmpty(t, contexts) + assert.Equal(t, recordCount, len(contexts)) +} + +func TestExecutionContextRepository_GetByStatus(t *testing.T) { + postgresqlClient := postgresql.NewClient() + defer postgresqlClient.Close() + repository := NewExecutionContextRepository(postgresqlClient) + defer repository.deleteAll() + + recordCount := 15 + status := "well_execution_status_here_must_be_cool" + err := populateSeedDataForTest(repository, recordCount, map[string]string{"Status": status}) + assert.Nil(t, err) + + contexts, err := repository.GetByStatus(status) + assert.Nil(t, err) + assert.NotEmpty(t, contexts) + assert.Equal(t, recordCount, len(contexts)) +} diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index f632319f..75d5cfc3 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -3,7 +3,7 @@ package postgresql import ( "fmt" "proctor/internal/app/proctord/storage/postgres" - config2 "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/config" "testing" "github.com/jmoiron/sqlx" @@ -11,7 +11,7 @@ import ( ) func TestNamedExec(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config2.PostgresDatabase(), config2.PostgresUser(), config2.PostgresPassword(), config2.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) @@ -47,7 +47,7 @@ func TestNamedExec(t *testing.T) { } func TestSelect(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config2.PostgresDatabase(), config2.PostgresUser(), config2.PostgresPassword(), config2.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) @@ -79,7 +79,7 @@ func TestSelect(t *testing.T) { } func TestSelectForNoRows(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config2.PostgresDatabase(), config2.PostgresUser(), config2.PostgresPassword(), config2.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) diff --git a/internal/app/service/infra/id/snowflake.go b/internal/app/service/infra/id/snowflake.go new file mode 100644 index 00000000..0621f972 --- /dev/null +++ b/internal/app/service/infra/id/snowflake.go @@ -0,0 +1,18 @@ +package id + +import "github.com/sony/sonyflake" + +var snowflake *sonyflake.Sonyflake + +func init() { + var snowflakeSetting sonyflake.Settings + snowflake = sonyflake.NewSonyflake(snowflakeSetting) +} + +func NextId() (uint64, error) { + return snowflake.NextID() +} + +func Extract(id uint64) map[string]uint64 { + return sonyflake.Decompose(id) +} diff --git a/migrations/10_CreateExecutionContextTable.down.sql b/migrations/10_CreateExecutionContextTable.down.sql new file mode 100644 index 00000000..3c967f65 --- /dev/null +++ b/migrations/10_CreateExecutionContextTable.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS execution_context; \ No newline at end of file diff --git a/migrations/10_CreateExecutionContextTable.up.sql b/migrations/10_CreateExecutionContextTable.up.sql new file mode 100644 index 00000000..8ec479a0 --- /dev/null +++ b/migrations/10_CreateExecutionContextTable.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE execution_context +( + id bigint, + job_name varchar(255), + user_email varchar(255), + image_tag text, + args text, + output bytea, + status varchar(255), + created_at timestamp default now(), + updated_at timestamp default now() +); \ No newline at end of file From 53d2c4b5b31855c62f63a6e60cd9b4a029dbfdb7 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 9 Jul 2019 16:15:07 +0530 Subject: [PATCH 021/268] [Jasoet|Bimozx] Refactor: Move Redis and Migration to Infra --- cmd/proctord/main.go | 6 ++-- internal/app/proctord/jobs/metadata/store.go | 2 +- .../app/proctord/jobs/metadata/store_test.go | 2 +- internal/app/proctord/jobs/schedule/worker.go | 2 +- .../app/proctord/jobs/schedule/worker_test.go | 10 +++--- internal/app/proctord/jobs/secrets/store.go | 2 +- .../app/proctord/jobs/secrets/store_test.go | 2 +- internal/app/proctord/redis/pool.go | 31 ------------------- internal/app/proctord/scheduler/scheduler.go | 4 +-- internal/app/proctord/server/router.go | 2 +- .../context/model/execution_context.go | 2 +- .../infra/db/migration}/migrations.go | 2 +- .../infra/db}/redis/client.go | 25 +++++++++++++++ .../infra/db}/redis/client_mock.go | 0 .../infra/db}/redis/client_test.go | 0 .../infra/db/{types => type}/base64_map.go | 2 +- .../db/{types => type}/base64_map_test.go | 2 +- .../infra}/mail/mailer.go | 0 .../infra}/mail/mailer_mock.go | 0 .../infra}/mail/mailer_test.go | 0 internal/pkg/model/metadata/metadata.go | 18 +++++------ 21 files changed, 54 insertions(+), 60 deletions(-) delete mode 100644 internal/app/proctord/redis/pool.go rename internal/app/{proctord/storage/postgres => service/infra/db/migration}/migrations.go (98%) rename internal/app/{proctord => service/infra/db}/redis/client.go (62%) rename internal/app/{proctord => service/infra/db}/redis/client_mock.go (100%) rename internal/app/{proctord => service/infra/db}/redis/client_test.go (100%) rename internal/app/service/infra/db/{types => type}/base64_map.go (98%) rename internal/app/service/infra/db/{types => type}/base64_map_test.go (97%) rename internal/app/{proctord => service/infra}/mail/mailer.go (100%) rename internal/app/{proctord => service/infra}/mail/mailer_mock.go (100%) rename internal/app/{proctord => service/infra}/mail/mailer_test.go (100%) diff --git a/cmd/proctord/main.go b/cmd/proctord/main.go index bc064ccc..7253a658 100644 --- a/cmd/proctord/main.go +++ b/cmd/proctord/main.go @@ -5,8 +5,8 @@ import ( "os" "proctor/internal/app/proctord/scheduler" "proctor/internal/app/proctord/server" - "proctor/internal/app/proctord/storage/postgres" "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/db/migration" "proctor/internal/app/service/infra/logger" "github.com/urfave/cli" @@ -25,7 +25,7 @@ func main() { Name: "migrate", Description: "Run database migrations for proctord", Action: func(c *cli.Context) { - err := postgres.Up() + err := migration.Up() if err != nil { panic(err.Error()) } @@ -36,7 +36,7 @@ func main() { Name: "rollback", Description: "Rollback database migrations by one step for proctord", Action: func(c *cli.Context) { - err := postgres.DownOneStep() + err := migration.DownOneStep() if err != nil { panic(err.Error()) } diff --git a/internal/app/proctord/jobs/metadata/store.go b/internal/app/proctord/jobs/metadata/store.go index 6f5c2121..06a64f5e 100644 --- a/internal/app/proctord/jobs/metadata/store.go +++ b/internal/app/proctord/jobs/metadata/store.go @@ -2,7 +2,7 @@ package metadata import ( "encoding/json" - "proctor/internal/app/proctord/redis" + "proctor/internal/app/service/infra/db/redis" modelMetadata "proctor/internal/pkg/model/metadata" ) diff --git a/internal/app/proctord/jobs/metadata/store_test.go b/internal/app/proctord/jobs/metadata/store_test.go index 0a493faa..16fe545d 100644 --- a/internal/app/proctord/jobs/metadata/store_test.go +++ b/internal/app/proctord/jobs/metadata/store_test.go @@ -3,7 +3,7 @@ package metadata import ( "encoding/json" "errors" - "proctor/internal/app/proctord/redis" + "proctor/internal/app/service/infra/db/redis" "testing" "github.com/stretchr/testify/assert" diff --git a/internal/app/proctord/jobs/schedule/worker.go b/internal/app/proctord/jobs/schedule/worker.go index 00f9b80f..597827a2 100644 --- a/internal/app/proctord/jobs/schedule/worker.go +++ b/internal/app/proctord/jobs/schedule/worker.go @@ -6,10 +6,10 @@ import ( "os" "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/mail" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/infra/mail" "strings" "time" diff --git a/internal/app/proctord/jobs/schedule/worker_test.go b/internal/app/proctord/jobs/schedule/worker_test.go index 1b756278..3e5d3cf3 100644 --- a/internal/app/proctord/jobs/schedule/worker_test.go +++ b/internal/app/proctord/jobs/schedule/worker_test.go @@ -4,11 +4,11 @@ import ( "encoding/base64" "encoding/json" "os" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/mail" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/proctord/audit" + "proctor/internal/app/proctord/jobs/execution" + "proctor/internal/app/proctord/storage" + "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/mail" "strings" "syscall" "testing" diff --git a/internal/app/proctord/jobs/secrets/store.go b/internal/app/proctord/jobs/secrets/store.go index 24ff4f4a..f989eb4f 100644 --- a/internal/app/proctord/jobs/secrets/store.go +++ b/internal/app/proctord/jobs/secrets/store.go @@ -2,7 +2,7 @@ package secrets import ( "encoding/json" - "proctor/internal/app/proctord/redis" + "proctor/internal/app/service/infra/db/redis" ) const JobsSecretsKeySuffix = "-secret" diff --git a/internal/app/proctord/jobs/secrets/store_test.go b/internal/app/proctord/jobs/secrets/store_test.go index 5e476932..3332dcca 100644 --- a/internal/app/proctord/jobs/secrets/store_test.go +++ b/internal/app/proctord/jobs/secrets/store_test.go @@ -2,7 +2,7 @@ package secrets import ( "encoding/json" - "proctor/internal/app/proctord/redis" + "proctor/internal/app/service/infra/db/redis" "testing" "github.com/pkg/errors" diff --git a/internal/app/proctord/redis/pool.go b/internal/app/proctord/redis/pool.go deleted file mode 100644 index 9b606cb5..00000000 --- a/internal/app/proctord/redis/pool.go +++ /dev/null @@ -1,31 +0,0 @@ -package redis - -import ( - "proctor/internal/app/service/infra/config" - "time" - - "github.com/garyburd/redigo/redis" -) - -func newPool() (*redis.Pool, error) { - pool := &redis.Pool{ - MaxIdle: config.RedisMaxActiveConnections() / 2, - MaxActive: config.RedisMaxActiveConnections(), - IdleTimeout: 5 * time.Second, - Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.RedisAddress()) }, - TestOnBorrow: func(c redis.Conn, t time.Time) error { - if time.Since(t) < time.Minute { - return nil - } - _, err := c.Do("PING") - return err - }, - Wait: true, - } - - conn := pool.Get() - defer conn.Close() - - _, err := conn.Do("PING") - return pool, err -} diff --git a/internal/app/proctord/scheduler/scheduler.go b/internal/app/proctord/scheduler/scheduler.go index d0f96095..2f66794c 100644 --- a/internal/app/proctord/scheduler/scheduler.go +++ b/internal/app/proctord/scheduler/scheduler.go @@ -10,11 +10,11 @@ import ( "proctor/internal/app/proctord/jobs/schedule" "proctor/internal/app/proctord/jobs/secrets" "proctor/internal/app/proctord/kubernetes" - "proctor/internal/app/proctord/mail" - "proctor/internal/app/proctord/redis" "proctor/internal/app/proctord/storage" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/db/redis" + "proctor/internal/app/service/infra/mail" "time" ) diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index cf02c84e..6ca86711 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -15,10 +15,10 @@ import ( "proctor/internal/app/proctord/jobs/secrets" "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/middleware" - "proctor/internal/app/proctord/redis" "proctor/internal/app/proctord/storage" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/db/redis" "github.com/gorilla/mux" ) diff --git a/internal/app/service/context/model/execution_context.go b/internal/app/service/context/model/execution_context.go index 10f7ccc6..5f57b73c 100644 --- a/internal/app/service/context/model/execution_context.go +++ b/internal/app/service/context/model/execution_context.go @@ -2,7 +2,7 @@ package model import ( sqlxTypes "github.com/jmoiron/sqlx/types" - dbTypes "proctor/internal/app/service/infra/db/types" + dbTypes "proctor/internal/app/service/infra/db/type" "time" ) diff --git a/internal/app/proctord/storage/postgres/migrations.go b/internal/app/service/infra/db/migration/migrations.go similarity index 98% rename from internal/app/proctord/storage/postgres/migrations.go rename to internal/app/service/infra/db/migration/migrations.go index 3f8c4797..fbedceda 100644 --- a/internal/app/proctord/storage/postgres/migrations.go +++ b/internal/app/service/infra/db/migration/migrations.go @@ -1,4 +1,4 @@ -package postgres +package migration import ( "fmt" diff --git a/internal/app/proctord/redis/client.go b/internal/app/service/infra/db/redis/client.go similarity index 62% rename from internal/app/proctord/redis/client.go rename to internal/app/service/infra/db/redis/client.go index dda4f083..41b6201f 100644 --- a/internal/app/proctord/redis/client.go +++ b/internal/app/service/infra/db/redis/client.go @@ -2,6 +2,8 @@ package redis import ( "github.com/garyburd/redigo/redis" + "proctor/internal/app/service/infra/config" + "time" ) type Client interface { @@ -24,6 +26,29 @@ func NewClient() Client { return &redisClient{connPool} } +func newPool() (*redis.Pool, error) { + pool := &redis.Pool{ + MaxIdle: config.RedisMaxActiveConnections() / 2, + MaxActive: config.RedisMaxActiveConnections(), + IdleTimeout: 5 * time.Second, + Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.RedisAddress()) }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + if time.Since(t) < time.Minute { + return nil + } + _, err := c.Do("PING") + return err + }, + Wait: true, + } + + conn := pool.Get() + defer conn.Close() + + _, err := conn.Do("PING") + return pool, err +} + func (c *redisClient) GET(key string) ([]byte, error) { conn := c.connPool.Get() defer conn.Close() diff --git a/internal/app/proctord/redis/client_mock.go b/internal/app/service/infra/db/redis/client_mock.go similarity index 100% rename from internal/app/proctord/redis/client_mock.go rename to internal/app/service/infra/db/redis/client_mock.go diff --git a/internal/app/proctord/redis/client_test.go b/internal/app/service/infra/db/redis/client_test.go similarity index 100% rename from internal/app/proctord/redis/client_test.go rename to internal/app/service/infra/db/redis/client_test.go diff --git a/internal/app/service/infra/db/types/base64_map.go b/internal/app/service/infra/db/type/base64_map.go similarity index 98% rename from internal/app/service/infra/db/types/base64_map.go rename to internal/app/service/infra/db/type/base64_map.go index fcc8a47a..a716d42e 100644 --- a/internal/app/service/infra/db/types/base64_map.go +++ b/internal/app/service/infra/db/type/base64_map.go @@ -1,4 +1,4 @@ -package types +package _type import ( "database/sql/driver" diff --git a/internal/app/service/infra/db/types/base64_map_test.go b/internal/app/service/infra/db/type/base64_map_test.go similarity index 97% rename from internal/app/service/infra/db/types/base64_map_test.go rename to internal/app/service/infra/db/type/base64_map_test.go index d0102583..18f3cad5 100644 --- a/internal/app/service/infra/db/types/base64_map_test.go +++ b/internal/app/service/infra/db/type/base64_map_test.go @@ -1,4 +1,4 @@ -package types +package _type import ( "github.com/stretchr/testify/assert" diff --git a/internal/app/proctord/mail/mailer.go b/internal/app/service/infra/mail/mailer.go similarity index 100% rename from internal/app/proctord/mail/mailer.go rename to internal/app/service/infra/mail/mailer.go diff --git a/internal/app/proctord/mail/mailer_mock.go b/internal/app/service/infra/mail/mailer_mock.go similarity index 100% rename from internal/app/proctord/mail/mailer_mock.go rename to internal/app/service/infra/mail/mailer_mock.go diff --git a/internal/app/proctord/mail/mailer_test.go b/internal/app/service/infra/mail/mailer_test.go similarity index 100% rename from internal/app/proctord/mail/mailer_test.go rename to internal/app/service/infra/mail/mailer_test.go diff --git a/internal/pkg/model/metadata/metadata.go b/internal/pkg/model/metadata/metadata.go index 5e43ab55..603653a4 100644 --- a/internal/pkg/model/metadata/metadata.go +++ b/internal/pkg/model/metadata/metadata.go @@ -1,16 +1,16 @@ package metadata import ( - env2 "proctor/internal/pkg/model/metadata/env" + "proctor/internal/pkg/model/metadata/env" ) type Metadata struct { - Name string `json:"name"` - Description string `json:"description"` - ImageName string `json:"image_name"` - EnvVars env2.Vars `json:"env_vars"` - AuthorizedGroups []string `json:"authorized_groups"` - Author string `json:"author"` - Contributors string `json:"contributors"` - Organization string `json:"organization"` + Name string `json:"name"` + Description string `json:"description"` + ImageName string `json:"image_name"` + EnvVars env.Vars `json:"env_vars"` + AuthorizedGroups []string `json:"authorized_groups"` + Author string `json:"author"` + Contributors string `json:"contributors"` + Organization string `json:"organization"` } From 2eaeeff67d2fc1db99c36e6245bf3b37b4f48ccc Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 10 Jul 2019 09:44:11 +0530 Subject: [PATCH 022/268] [Jasoet|Bimozx] Refactor: Move kubernetes client and http to infra --- go.mod | 2 +- go.sum | 2 ++ internal/app/proctord/audit/auditor.go | 6 ++--- internal/app/proctord/audit/auditor_test.go | 2 +- .../proctord/jobs/execution/executioner.go | 6 ++--- .../jobs/execution/executioner_test.go | 2 +- internal/app/proctord/jobs/logs/handler.go | 6 ++--- .../app/proctord/jobs/logs/handler_test.go | 2 +- internal/app/proctord/kubernetes/utils.go | 24 ------------------- internal/app/proctord/scheduler/scheduler.go | 7 +++--- internal/app/proctord/server/router.go | 7 +++--- .../infra}/http/client.go | 0 .../infra}/http/client_test.go | 0 .../infra}/kubernetes/client.go | 24 ++++++++++++++++--- .../infra}/kubernetes/client_mock.go | 0 .../infra}/kubernetes/client_test.go | 4 ++-- 16 files changed, 44 insertions(+), 50 deletions(-) delete mode 100644 internal/app/proctord/kubernetes/utils.go rename internal/app/{proctord => service/infra}/http/client.go (100%) rename internal/app/{proctord => service/infra}/http/client_test.go (100%) rename internal/app/{proctord => service/infra}/kubernetes/client.go (90%) rename internal/app/{proctord => service/infra}/kubernetes/client_mock.go (100%) rename internal/app/{proctord => service/infra}/kubernetes/client_test.go (98%) diff --git a/go.mod b/go.mod index 10da86a4..bfa15b5e 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/newrelic/go-agent v1.4.0 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/pkg/errors v0.8.1 - github.com/robfig/cron v1.1.0 + github.com/robfig/cron v1.2.0 github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.4.2 github.com/sony/sonyflake v1.0.0 diff --git a/go.sum b/go.sum index 399bb8d6..ca6210f3 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= diff --git a/internal/app/proctord/audit/auditor.go b/internal/app/proctord/audit/auditor.go index 9468df0a..8dea6b97 100644 --- a/internal/app/proctord/audit/auditor.go +++ b/internal/app/proctord/audit/auditor.go @@ -2,9 +2,9 @@ package audit import ( "github.com/getsentry/raven-go" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" ) @@ -17,10 +17,10 @@ type Auditor interface { type auditor struct { store storage.Store - kubeClient kubernetes.Client + kubeClient kubernetes.KubernetesClient } -func New(store storage.Store, kubeClient kubernetes.Client) Auditor { +func New(store storage.Store, kubeClient kubernetes.KubernetesClient) Auditor { return &auditor{ store: store, kubeClient: kubeClient, diff --git a/internal/app/proctord/audit/auditor_test.go b/internal/app/proctord/audit/auditor_test.go index 9e1848f5..7e7d241d 100644 --- a/internal/app/proctord/audit/auditor_test.go +++ b/internal/app/proctord/audit/auditor_test.go @@ -1,9 +1,9 @@ package audit import ( - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/kubernetes" "testing" "proctor/internal/pkg/constant" diff --git a/internal/app/proctord/jobs/execution/executioner.go b/internal/app/proctord/jobs/execution/executioner.go index 6860a5d0..cc97fa3e 100644 --- a/internal/app/proctord/jobs/execution/executioner.go +++ b/internal/app/proctord/jobs/execution/executioner.go @@ -5,15 +5,15 @@ import ( "fmt" jobMetadata "proctor/internal/app/proctord/jobs/metadata" jobSecrets "proctor/internal/app/proctord/jobs/secrets" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/kubernetes" "proctor/internal/pkg/constant" "proctor/internal/pkg/utility" ) type executioner struct { - kubeClient kubernetes.Client + kubeClient kubernetes.KubernetesClient metadataStore jobMetadata.Store secretsStore jobSecrets.Store } @@ -22,7 +22,7 @@ type Executioner interface { Execute(*postgres.JobsExecutionAuditLog, string, map[string]string) (string, error) } -func NewExecutioner(kubeClient kubernetes.Client, metadataStore jobMetadata.Store, secretsStore jobSecrets.Store) Executioner { +func NewExecutioner(kubeClient kubernetes.KubernetesClient, metadataStore jobMetadata.Store, secretsStore jobSecrets.Store) Executioner { return &executioner{ kubeClient: kubeClient, metadataStore: metadataStore, diff --git a/internal/app/proctord/jobs/execution/executioner_test.go b/internal/app/proctord/jobs/execution/executioner_test.go index 79679c44..eab3cc2e 100644 --- a/internal/app/proctord/jobs/execution/executioner_test.go +++ b/internal/app/proctord/jobs/execution/executioner_test.go @@ -4,8 +4,8 @@ import ( "errors" jobMetadata "proctor/internal/app/proctord/jobs/metadata" jobSecrets "proctor/internal/app/proctord/jobs/secrets" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/infra/kubernetes" "testing" "github.com/stretchr/testify/assert" diff --git a/internal/app/proctord/jobs/logs/handler.go b/internal/app/proctord/jobs/logs/handler.go index 3b5f6999..2ebd49af 100644 --- a/internal/app/proctord/jobs/logs/handler.go +++ b/internal/app/proctord/jobs/logs/handler.go @@ -5,8 +5,8 @@ import ( "github.com/getsentry/raven-go" "io" "net/http" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/kubernetes" _logger "proctor/internal/app/service/infra/logger" "strings" @@ -21,14 +21,14 @@ var upgrader = websocket.Upgrader{ } type logger struct { - kubeClient kubernetes.Client + kubeClient kubernetes.KubernetesClient } type Logger interface { Stream() http.HandlerFunc } -func NewLogger(kubeClient kubernetes.Client) Logger { +func NewLogger(kubeClient kubernetes.KubernetesClient) Logger { return &logger{ kubeClient: kubeClient, } diff --git a/internal/app/proctord/jobs/logs/handler_test.go b/internal/app/proctord/jobs/logs/handler_test.go index c7c455a8..a9be03ca 100644 --- a/internal/app/proctord/jobs/logs/handler_test.go +++ b/internal/app/proctord/jobs/logs/handler_test.go @@ -4,7 +4,7 @@ import ( "errors" "net/http" "net/http/httptest" - "proctor/internal/app/proctord/kubernetes" + "proctor/internal/app/service/infra/kubernetes" "strings" "testing" diff --git a/internal/app/proctord/kubernetes/utils.go b/internal/app/proctord/kubernetes/utils.go deleted file mode 100644 index a8815e79..00000000 --- a/internal/app/proctord/kubernetes/utils.go +++ /dev/null @@ -1,24 +0,0 @@ -package kubernetes - -import ( - "flag" - "os" - "path/filepath" - "proctor/internal/app/service/infra/config" - "proctor/internal/app/service/infra/logger" -) - -func KubeConfig() string { - if config.KubeConfig() == "out-of-cluster" { - logger.Info("service is running outside kube cluster") - home := os.Getenv("HOME") - - kubeConfig := flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") - flag.Parse() - - return *kubeConfig - } - logger.Info("Assuming service is running inside kube cluster") - logger.Info("Kube config provided is:", config.KubeConfig()) - return "" -} diff --git a/internal/app/proctord/scheduler/scheduler.go b/internal/app/proctord/scheduler/scheduler.go index 2f66794c..80216afe 100644 --- a/internal/app/proctord/scheduler/scheduler.go +++ b/internal/app/proctord/scheduler/scheduler.go @@ -4,16 +4,16 @@ import ( "fmt" "os" "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/http" "proctor/internal/app/proctord/jobs/execution" "proctor/internal/app/proctord/jobs/metadata" "proctor/internal/app/proctord/jobs/schedule" "proctor/internal/app/proctord/jobs/secrets" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/storage" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" + "proctor/internal/app/service/infra/http" + "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/mail" "time" ) @@ -32,8 +32,7 @@ func Start() error { if err != nil { return err } - kubeConfig := kubernetes.KubeConfig() - kubeClient := kubernetes.NewClient(kubeConfig, httpClient) + kubeClient := kubernetes.NewKubernetesClient(httpClient) jobExecutioner := execution.NewExecutioner(kubeClient, metadataStore, secretsStore) diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index 6ca86711..42bf8615 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -6,19 +6,19 @@ import ( "path" "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/docs" - httpClient "proctor/internal/app/proctord/http" "proctor/internal/app/proctord/instrumentation" "proctor/internal/app/proctord/jobs/execution" "proctor/internal/app/proctord/jobs/logs" "proctor/internal/app/proctord/jobs/metadata" "proctor/internal/app/proctord/jobs/schedule" "proctor/internal/app/proctord/jobs/secrets" - "proctor/internal/app/proctord/kubernetes" "proctor/internal/app/proctord/middleware" "proctor/internal/app/proctord/storage" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" + httpClient "proctor/internal/app/service/infra/http" + "proctor/internal/app/service/infra/kubernetes" "github.com/gorilla/mux" ) @@ -39,8 +39,7 @@ func NewRouter() (*mux.Router, error) { if err != nil { return router, err } - kubeConfig := kubernetes.KubeConfig() - kubeClient := kubernetes.NewClient(kubeConfig, httpClient) + kubeClient := kubernetes.NewKubernetesClient(httpClient) auditor := audit.New(store, kubeClient) jobExecutioner := execution.NewExecutioner(kubeClient, metadataStore, secretsStore) diff --git a/internal/app/proctord/http/client.go b/internal/app/service/infra/http/client.go similarity index 100% rename from internal/app/proctord/http/client.go rename to internal/app/service/infra/http/client.go diff --git a/internal/app/proctord/http/client_test.go b/internal/app/service/infra/http/client_test.go similarity index 100% rename from internal/app/proctord/http/client_test.go rename to internal/app/service/infra/http/client_test.go diff --git a/internal/app/proctord/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go similarity index 90% rename from internal/app/proctord/kubernetes/client.go rename to internal/app/service/infra/kubernetes/client.go index fbde73ce..f3bc5c0f 100644 --- a/internal/app/proctord/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -1,9 +1,12 @@ package kubernetes import ( + "flag" "fmt" "io" "net/http" + "os" + "path/filepath" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" "time" @@ -36,18 +39,18 @@ type client struct { httpClient *http.Client } -type Client interface { +type KubernetesClient interface { ExecuteJob(string, map[string]string) (string, error) StreamJobLogs(string) (io.ReadCloser, error) JobExecutionStatus(string) (string, error) } -func NewClient(kubeconfig string, httpClient *http.Client) Client { +func NewKubernetesClient(httpClient *http.Client) KubernetesClient { newClient := &client{ httpClient: httpClient, } - config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + config, err := clientcmd.BuildConfigFromFlags("", KubeConfig()) if err != nil { panic(err.Error()) } @@ -255,3 +258,18 @@ func (client *client) getLogsStreamReaderFor(podName string) (io.ReadCloser, err } return resp.Body, err } + +func KubeConfig() string { + if config.KubeConfig() == "out-of-cluster" { + logger.Info("service is running outside kube cluster") + home := os.Getenv("HOME") + + kubeConfig := flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") + flag.Parse() + + return *kubeConfig + } + logger.Info("Assuming service is running inside kube cluster") + logger.Info("Kube config provided is:", config.KubeConfig()) + return "" +} diff --git a/internal/app/proctord/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go similarity index 100% rename from internal/app/proctord/kubernetes/client_mock.go rename to internal/app/service/infra/kubernetes/client_mock.go diff --git a/internal/app/proctord/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go similarity index 98% rename from internal/app/proctord/kubernetes/client_test.go rename to internal/app/service/infra/kubernetes/client_test.go index 3b2b4681..95896b68 100644 --- a/internal/app/proctord/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -23,14 +23,14 @@ import ( type ClientTestSuite struct { suite.Suite - testClient Client + testClient KubernetesClient testKubernetesJobs batch_v1.JobInterface fakeClientSet *fakeclientset.Clientset jobName string podName string fakeClientSetStreaming *fakeclientset.Clientset fakeHttpClient *http.Client - testClientStreaming Client + testClientStreaming KubernetesClient } func (suite *ClientTestSuite) SetupTest() { From 79d538cceb3c5b30c9c765ce33ed8e2085018503 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 10 Jul 2019 16:42:19 +0530 Subject: [PATCH 023/268] [Jasoet|Bimozx] Refactor: Refactor kubernetes client and add integration test for it --- .env.v2.test | 35 ++++ internal/app/proctord/scheduler/scheduler.go | 2 +- internal/app/proctord/server/router.go | 2 +- internal/app/service/infra/config/config.go | 10 +- .../app/service/infra/kubernetes/client.go | 154 ++++++++++++++---- .../kubernetes/client_integration_test.go | 118 ++++++++++++++ .../service/infra/kubernetes/client_mock.go | 5 + .../service/infra/kubernetes/client_test.go | 8 +- .../infra/{ => kubernetes}/http/client.go | 0 .../{ => kubernetes}/http/client_test.go | 0 10 files changed, 291 insertions(+), 43 deletions(-) create mode 100644 .env.v2.test create mode 100644 internal/app/service/infra/kubernetes/client_integration_test.go rename internal/app/service/infra/{ => kubernetes}/http/client.go (100%) rename internal/app/service/infra/{ => kubernetes}/http/client_test.go (100%) diff --git a/.env.v2.test b/.env.v2.test new file mode 100644 index 00000000..1d9224a4 --- /dev/null +++ b/.env.v2.test @@ -0,0 +1,35 @@ +ENVIRONMENT=test +PROCTOR_KUBE_CONFIG=out-of-cluster +PROCTOR_KUBE_CONTEXT=minikube +PROCTOR_LOG_LEVEL=debug +PROCTOR_APP_PORT=5000 +PROCTOR_DEFAULT_NAMESPACE=default +PROCTOR_REDIS_ADDRESS=localhost:6379 +PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS=10 +PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 +PROCTOR_KUBE_JOB_RETRIES=0 +PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 +PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 +PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 +PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 +PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== +PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK +PROCTOR_POSTGRES_USER=postgres +PROCTOR_POSTGRES_PASSWORD= +PROCTOR_POSTGRES_HOST=localhost +PROCTOR_POSTGRES_PORT=5432 +PROCTOR_POSTGRES_DATABASE=proctord_test +PROCTOR_POSTGRES_MAX_CONNECTIONS=50 +PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 +PROCTOR_NEW_RELIC_APP_NAME="PROCTORD" +PROCTOR_NEW_RELIC_LICENCE_KEY="nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr" +PROCTOR_MIN_CLIENT_VERSION="v0.2.0" +PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS="5" +PROCTOR_MAIL_USERNAME="user@mail.com" +PROCTOR_MAIL_PASSWORD="password" +PROCTOR_MAIL_SERVER_HOST="smtp.mail.com" +PROCTOR_MAIL_SERVER_PORT="123" +PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" +PROCTOR_SENTRY_DSN="foo" +PROCTOR_DOCS_PATH="/path/to/docs/dir" +ENABLE_INTEGRATION_TEST=true diff --git a/internal/app/proctord/scheduler/scheduler.go b/internal/app/proctord/scheduler/scheduler.go index 80216afe..5de4c4c5 100644 --- a/internal/app/proctord/scheduler/scheduler.go +++ b/internal/app/proctord/scheduler/scheduler.go @@ -12,8 +12,8 @@ import ( "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" - "proctor/internal/app/service/infra/http" "proctor/internal/app/service/infra/kubernetes" + "proctor/internal/app/service/infra/kubernetes/http" "proctor/internal/app/service/infra/mail" "time" ) diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index 42bf8615..e5c32ba3 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -17,8 +17,8 @@ import ( "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" - httpClient "proctor/internal/app/service/infra/http" "proctor/internal/app/service/infra/kubernetes" + httpClient "proctor/internal/app/service/infra/kubernetes/http" "github.com/gorilla/mux" ) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 9097591b..8f34a8be 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/spf13/viper" + "time" ) func init() { @@ -15,6 +16,11 @@ func KubeConfig() string { return viper.GetString("KUBE_CONFIG") } +func KubeContext() string { + viper.SetDefault("KUBE_CONTEXT","default") + return viper.GetString("KUBE_CONTEXT") +} + func LogLevel() string { return viper.GetString("LOG_LEVEL") } @@ -55,8 +61,8 @@ func LogsStreamWriteBufferSize() int { return viper.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE") } -func KubePodsListWaitTime() int { - return viper.GetInt("KUBE_POD_LIST_WAIT_TIME") +func KubePodsListWaitTime() time.Duration { + return time.Duration(viper.GetInt("KUBE_POD_LIST_WAIT_TIME")) } func KubeJobActiveDeadlineSeconds() *int64 { diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index f3bc5c0f..529d573b 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -1,7 +1,6 @@ package kubernetes import ( - "flag" "fmt" "io" "net/http" @@ -20,6 +19,7 @@ import ( "proctor/internal/pkg/constant" //Package needed for kubernetes cluster in google cloud _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" + kubeRestClient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) @@ -40,22 +40,55 @@ type client struct { } type KubernetesClient interface { + ExecuteJobWithCommand(string, map[string]string, []string) (string, error) ExecuteJob(string, map[string]string) (string, error) StreamJobLogs(string) (io.ReadCloser, error) JobExecutionStatus(string) (string, error) } -func NewKubernetesClient(httpClient *http.Client) KubernetesClient { - newClient := &client{ - httpClient: httpClient, +func NewClientSet() (*kubernetes.Clientset, error) { + var kubeConfig *kubeRestClient.Config + if config.KubeConfig() == "out-of-cluster" { + logger.Info("service is running outside kube cluster") + home := os.Getenv("HOME") + + kubeConfigPath := filepath.Join(home, ".kube", "config") + + configOverrides := &clientcmd.ConfigOverrides{} + if config.KubeContext() != "default" { + configOverrides.CurrentContext = config.KubeContext() + } + + var err error + kubeConfig, err = clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeConfigPath}, + configOverrides).ClientConfig() + if err != nil { + return nil, err + } + + } else { + var err error + kubeConfig, err = kubeRestClient.InClusterConfig() + if err != nil { + return nil, err + } } - config, err := clientcmd.BuildConfigFromFlags("", KubeConfig()) + clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { - panic(err.Error()) + return nil, err + } + return clientSet, nil +} + +func NewKubernetesClient(httpClient *http.Client) KubernetesClient { + newClient := &client{ + httpClient: httpClient, } - newClient.clientSet, err = kubernetes.NewForConfig(config) + var err error + newClient.clientSet, err = NewClientSet() if err != nil { panic(err.Error()) } @@ -90,6 +123,10 @@ func jobLabelSelector(jobName string) string { } func (client *client) ExecuteJob(imageName string, envMap map[string]string) (string, error) { + return client.ExecuteJobWithCommand(imageName, envMap, []string{}) +} + +func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string]string, command []string) (string, error) { uniqueJobName := uniqueName() label := jobLabel(uniqueJobName) @@ -102,6 +139,10 @@ func (client *client) ExecuteJob(imageName string, envMap map[string]string) (st Env: getEnvVars(envMap), } + if len(command) != 0 { + container.Command = command + } + podSpec := v1.PodSpec{ Containers: []v1.Container{container}, RestartPolicy: v1.RestartPolicyNever, @@ -157,7 +198,7 @@ func (client *client) StreamJobLogs(jobName string) (io.ReadCloser, error) { if len(listOfPods.Items) > 0 { podJob := listOfPods.Items[0] if podJob.Status.Phase == v1.PodRunning || podJob.Status.Phase == v1.PodSucceeded || podJob.Status.Phase == v1.PodFailed { - return client.getLogsStreamReaderFor(podJob.ObjectMeta.Name) + return client.getPodLogs(podJob) } watchPod, err := kubernetesPods.Watch(listOptions) if err != nil { @@ -210,12 +251,73 @@ func (client *client) StreamJobLogs(jobName string) (io.ReadCloser, error) { } } -func (client *client) JobExecutionStatus(jobExecutionID string) (string, error) { +func (client *client) WaitForReadyJob(jobName string) error { + batchV1 := client.clientSet.BatchV1() + jobs := batchV1.Jobs(namespace) + listOptions := meta_v1.ListOptions{ + TypeMeta: typeMeta, + LabelSelector: jobLabelSelector(jobName), + } + + watchJob, err := jobs.Watch(listOptions) + if err != nil { + return err + } + + resultChan := watchJob.ResultChan() + defer watchJob.Stop() + + select { + case event := <-resultChan: + if event.Type == watch.Error { + return fmt.Errorf("error when waiting for job with list option %v", listOptions) + } + case <-time.After(config.KubePodsListWaitTime() * time.Second): + return fmt.Errorf("timeout when waiting for ready job") + } + + return nil +} + +func (client *client) WaitForReadyPod(jobName string) (*v1.Pod, error) { + coreV1 := client.clientSet.CoreV1() + kubernetesPods := coreV1.Pods(namespace) + listOptions := meta_v1.ListOptions{ + TypeMeta: typeMeta, + LabelSelector: jobLabelSelector(jobName), + } + + watchJob, err := kubernetesPods.Watch(listOptions) + if err != nil { + return nil, err + } + + resultChan := watchJob.ResultChan() + defer watchJob.Stop() + var pod *v1.Pod + + select { + case event := <-resultChan: + if event.Type == watch.Error { + return nil, fmt.Errorf("error when waiting for job with list option %v", listOptions) + } + pod = event.Object.(*v1.Pod) + if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + return pod, nil + } + case <-time.After(config.KubePodsListWaitTime() * time.Second): + return nil, fmt.Errorf("timeout when waiting for ready job") + } + + return pod, nil +} + +func (client *client) JobExecutionStatus(jobName string) (string, error) { batchV1 := client.clientSet.BatchV1() kubernetesJobs := batchV1.Jobs(namespace) listOptions := meta_v1.ListOptions{ TypeMeta: typeMeta, - LabelSelector: jobLabelSelector(jobExecutionID), + LabelSelector: jobLabelSelector(jobName), } watchJob, err := kubernetesJobs.Watch(listOptions) @@ -244,32 +346,16 @@ func (client *client) JobExecutionStatus(jobExecutionID string) (string, error) return constant.NoDefinitiveJobExecutionStatusFound, nil } -func (client *client) getLogsStreamReaderFor(podName string) (io.ReadCloser, error) { - logger.Debug("reading pod logs for: ", podName) - - req, err := http.NewRequest("GET", "https://"+config.KubeClusterHostName()+"/api/v1/namespaces/"+namespace+"/pods/"+podName+"/log?follow=true", nil) - if err != nil { - return nil, err +func (client *client) getPodLogs(pod v1.Pod) (io.ReadCloser, error) { + logger.Debug("reading pod logs for: ", pod.Name) + podLogOpts := v1.PodLogOptions{ + Follow: true, } - req.Header.Set("Authorization", "Basic "+config.KubeBasicAuthEncoded()) - resp, err := client.httpClient.Do(req) + request := client.clientSet.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + response, err := request.Stream() + if err != nil { return nil, err } - return resp.Body, err -} - -func KubeConfig() string { - if config.KubeConfig() == "out-of-cluster" { - logger.Info("service is running outside kube cluster") - home := os.Getenv("HOME") - - kubeConfig := flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file") - flag.Parse() - - return *kubeConfig - } - logger.Info("Assuming service is running inside kube cluster") - logger.Info("Kube config provided is:", config.KubeConfig()) - return "" + return response, nil } diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go new file mode 100644 index 00000000..772b0367 --- /dev/null +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -0,0 +1,118 @@ +package kubernetes + +import ( + "bufio" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "k8s.io/api/core/v1" + meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "os" + "proctor/internal/app/service/infra/config" + kubeHttpClient "proctor/internal/app/service/infra/kubernetes/http" + "proctor/internal/pkg/constant" + "testing" +) + +type IntegrationTestSuite struct { + suite.Suite + testClient KubernetesClient + clientSet kubernetes.Interface +} + +func (suite *IntegrationTestSuite) SetupTest() { + t := suite.T() + kubeHttpClient, err := kubeHttpClient.NewClient() + assert.NoError(t, err) + suite.testClient = NewKubernetesClient(kubeHttpClient) + suite.clientSet, err = NewClientSet() + assert.NoError(t, err) +} + +func (suite *IntegrationTestSuite) TestJobExecution() { + t := suite.T() + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + sampleImageName := "busybox" + + executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) + assert.NoError(t, err) + + typeMeta := meta_v1.TypeMeta{ + Kind: "Job", + APIVersion: "batch/v1", + } + + listOptions := meta_v1.ListOptions{ + TypeMeta: typeMeta, + LabelSelector: jobLabelSelector(executedJobname), + } + + namespace := config.DefaultNamespace() + listOfJobs, err := suite.clientSet.BatchV1().Jobs(namespace).List(listOptions) + assert.NoError(t, err) + executedJob := listOfJobs.Items[0] + + assert.Equal(t, executedJobname, executedJob.ObjectMeta.Name) + assert.Equal(t, executedJobname, executedJob.Spec.Template.ObjectMeta.Name) + + expectedLabel := jobLabel(executedJobname) + assert.Equal(t, expectedLabel, executedJob.ObjectMeta.Labels) + assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) + + assert.Equal(t, config.KubeJobActiveDeadlineSeconds(), executedJob.Spec.ActiveDeadlineSeconds) + assert.Equal(t, config.KubeJobRetries(), executedJob.Spec.BackoffLimit) + + assert.Equal(t, v1.RestartPolicyNever, executedJob.Spec.Template.Spec.RestartPolicy) + + container := executedJob.Spec.Template.Spec.Containers[0] + assert.Equal(t, executedJobname, container.Name) + + assert.Equal(t, sampleImageName, container.Image) + + expectedEnvVars := getEnvVars(envVarsForContainer) + assert.Equal(t, expectedEnvVars, container.Env) +} + +func (suite *IntegrationTestSuite) TestJobExecutionStatus() { + t := suite.T() + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + sampleImageName := "busybox" + + executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) + assert.NoError(t, err) + + status, err := suite.testClient.JobExecutionStatus(executedJobname) + assert.Equal(t, status, constant.JobSucceeded) +} + +func (suite *IntegrationTestSuite) TestStreamLogsSuccess() { + t := suite.T() + + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + sampleImageName := "busybox" + + executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) + assert.NoError(t, err) + + logStream, err := suite.testClient.StreamJobLogs(executedJobname) + assert.NoError(t, err) + + defer logStream.Close() + + bufioReader := bufio.NewReader(logStream) + + jobLogSingleLine, _, err := bufioReader.ReadLine() + assert.NoError(t, err) + + assert.Equal(t, "Bimo Horizon", string(jobLogSingleLine[:])) + +} +func TestIntegrationTestSuite(t *testing.T) { + value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") + if available == true || value == "true" { + suite.Run(t, new(IntegrationTestSuite)) + } +} diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index 03233f16..cd68b5b7 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -16,6 +16,11 @@ func (m *MockClient) ExecuteJob(jobName string, envMap map[string]string) (strin return args.String(0), args.Error(1) } +func (m *MockClient) ExecuteJobWithCommand(jobName string, envMap map[string]string, command []string) (string, error) { + args := m.Called(jobName, envMap) + return args.String(0), args.Error(1) +} + func (m *MockClient) StreamJobLogs(jobName string) (io.ReadCloser, error) { args := m.Called(jobName) return args.Get(0).(*utility.Buffer), args.Error(1) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 95896b68..b2a0110f 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -1,14 +1,12 @@ package kubernetes import ( - "bufio" "net/http" "os" - "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/config" "testing" "time" - "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" batchV1 "k8s.io/api/batch/v1" @@ -113,7 +111,7 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedEnvVars, container.Env) } -func (suite *ClientTestSuite) TestStreamLogsSuccess() { +/*func (suite *ClientTestSuite) TestStreamLogsSuccess() { t := suite.T() httpmock.ActivateNonDefault(suite.fakeHttpClient) @@ -135,7 +133,7 @@ func (suite *ClientTestSuite) TestStreamLogsSuccess() { assert.Equal(t, "logs are streaming", string(jobLogSingleLine[:])) -} +}*/ func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { t := suite.T() diff --git a/internal/app/service/infra/http/client.go b/internal/app/service/infra/kubernetes/http/client.go similarity index 100% rename from internal/app/service/infra/http/client.go rename to internal/app/service/infra/kubernetes/http/client.go diff --git a/internal/app/service/infra/http/client_test.go b/internal/app/service/infra/kubernetes/http/client_test.go similarity index 100% rename from internal/app/service/infra/http/client_test.go rename to internal/app/service/infra/kubernetes/http/client_test.go From e56d22c9a57ce7144bb9743989d77788a99a6303 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 10 Jul 2019 17:40:55 +0530 Subject: [PATCH 024/268] [Jasoet|Bimozx] Refactor: Enable Timeout when waiting to job and pod to be ready --- .../app/service/infra/kubernetes/client.go | 144 +++++++----------- 1 file changed, 52 insertions(+), 92 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 529d573b..4a76b189 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -11,9 +11,9 @@ import ( "time" uuid "github.com/satori/go.uuid" - batch_v1 "k8s.io/api/batch/v1" + batch "k8s.io/api/batch/v1" "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" "proctor/internal/pkg/constant" @@ -23,11 +23,11 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -var typeMeta meta_v1.TypeMeta +var typeMeta meta.TypeMeta var namespace string func init() { - typeMeta = meta_v1.TypeMeta{ + typeMeta = meta.TypeMeta{ Kind: "Job", APIVersion: "batch/v1", } @@ -148,7 +148,7 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] RestartPolicy: v1.RestartPolicyNever, } - objectMeta := meta_v1.ObjectMeta{ + objectMeta := meta.ObjectMeta{ Name: uniqueJobName, Labels: label, Annotations: config.JobPodAnnotations(), @@ -159,13 +159,13 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] Spec: podSpec, } - jobSpec := batch_v1.JobSpec{ + jobSpec := batch.JobSpec{ Template: template, ActiveDeadlineSeconds: config.KubeJobActiveDeadlineSeconds(), BackoffLimit: config.KubeJobRetries(), } - jobToRun := batch_v1.Job{ + jobToRun := batch.Job{ TypeMeta: typeMeta, ObjectMeta: objectMeta, Spec: jobSpec, @@ -179,82 +179,29 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] } func (client *client) StreamJobLogs(jobName string) (io.ReadCloser, error) { - listOptions := meta_v1.ListOptions{ - TypeMeta: typeMeta, - LabelSelector: jobLabelSelector(jobName), + err := client.waitForReadyJob(jobName) + if err != nil { + return nil, err } - coreV1 := client.clientSet.CoreV1() - kubernetesPods := coreV1.Pods(namespace) - - logger.Debug("list of pods") + pod, err := client.waitForReadyPod(jobName) + if err != nil { + return nil, err + } - for { - listOfPods, err := kubernetesPods.List(listOptions) - if err != nil { - return nil, fmt.Errorf("Error fetching kubernetes Pods list %v", err) - } + result, err := client.getPodLogs(pod) - if len(listOfPods.Items) > 0 { - podJob := listOfPods.Items[0] - if podJob.Status.Phase == v1.PodRunning || podJob.Status.Phase == v1.PodSucceeded || podJob.Status.Phase == v1.PodFailed { - return client.getPodLogs(podJob) - } - watchPod, err := kubernetesPods.Watch(listOptions) - if err != nil { - return nil, fmt.Errorf("Error watching kubernetes Pods %v", err) - } - - resultChan := watchPod.ResultChan() - defer watchPod.Stop() - - waitingForKubePods := make(chan bool) - go func() { - defer close(waitingForKubePods) - time.Sleep(time.Duration(config.KubePodsListWaitTime()) * time.Second) - waitingForKubePods <- true - }() - - select { - case <-resultChan: - continue - case <-waitingForKubePods: - return nil, fmt.Errorf("Pod didn't reach active state after waiting for %d minutes", config.KubePodsListWaitTime()) - } - - } else { - batchV1 := client.clientSet.BatchV1() - kubernetesJobs := batchV1.Jobs(namespace) - - watchJob, err := kubernetesJobs.Watch(listOptions) - if err != nil { - return nil, fmt.Errorf("Error watching kubernetes Jobs %v", err) - } - - resultChan := watchJob.ResultChan() - defer watchJob.Stop() - - waitingForKubeJobs := make(chan bool) - go func() { - defer close(waitingForKubeJobs) - time.Sleep(time.Duration(config.KubePodsListWaitTime()) * time.Second) - waitingForKubeJobs <- true - }() - - select { - case <-resultChan: - continue - case <-waitingForKubeJobs: - return nil, fmt.Errorf("Couldn't find a pod for job's given list options %v after waiting for %d minutes", listOptions, config.KubePodsListWaitTime()) - } - } + if err != nil { + return nil, err } + + return result, nil } -func (client *client) WaitForReadyJob(jobName string) error { +func (client *client) waitForReadyJob(jobName string) error { batchV1 := client.clientSet.BatchV1() jobs := batchV1.Jobs(namespace) - listOptions := meta_v1.ListOptions{ + listOptions := meta.ListOptions{ TypeMeta: typeMeta, LabelSelector: jobLabelSelector(jobName), } @@ -264,25 +211,34 @@ func (client *client) WaitForReadyJob(jobName string) error { return err } + timeoutChan := time.After(config.KubePodsListWaitTime() * time.Second) resultChan := watchJob.ResultChan() defer watchJob.Stop() - select { - case event := <-resultChan: + for event := range resultChan { if event.Type == watch.Error { - return fmt.Errorf("error when waiting for job with list option %v", listOptions) + return fmt.Errorf("watch error when waiting for job with list option %v", listOptions) + } + job := event.Object.(*batch.Job) + if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { + return nil + } + + select { + case <-resultChan: + continue + case <-timeoutChan: + return fmt.Errorf("timeout when waiting pod to be ready") } - case <-time.After(config.KubePodsListWaitTime() * time.Second): - return fmt.Errorf("timeout when waiting for ready job") } - return nil + return fmt.Errorf("job never reach the active status") } -func (client *client) WaitForReadyPod(jobName string) (*v1.Pod, error) { +func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { coreV1 := client.clientSet.CoreV1() kubernetesPods := coreV1.Pods(namespace) - listOptions := meta_v1.ListOptions{ + listOptions := meta.ListOptions{ TypeMeta: typeMeta, LabelSelector: jobLabelSelector(jobName), } @@ -292,30 +248,34 @@ func (client *client) WaitForReadyPod(jobName string) (*v1.Pod, error) { return nil, err } + timeoutChan := time.After(config.KubePodsListWaitTime() * time.Second) resultChan := watchJob.ResultChan() defer watchJob.Stop() var pod *v1.Pod - select { - case event := <-resultChan: + for event := range resultChan { if event.Type == watch.Error { - return nil, fmt.Errorf("error when waiting for job with list option %v", listOptions) + return nil, fmt.Errorf("watch error when waiting for pod with list option %v", listOptions) } pod = event.Object.(*v1.Pod) if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { return pod, nil } - case <-time.After(config.KubePodsListWaitTime() * time.Second): - return nil, fmt.Errorf("timeout when waiting for ready job") + select { + case <-resultChan: + continue + case <-timeoutChan: + return nil, fmt.Errorf("timeout when waiting pod to be ready") + } } - return pod, nil + return nil, fmt.Errorf("pod never get the intended state") } func (client *client) JobExecutionStatus(jobName string) (string, error) { batchV1 := client.clientSet.BatchV1() kubernetesJobs := batchV1.Jobs(namespace) - listOptions := meta_v1.ListOptions{ + listOptions := meta.ListOptions{ TypeMeta: typeMeta, LabelSelector: jobLabelSelector(jobName), } @@ -328,14 +288,14 @@ func (client *client) JobExecutionStatus(jobName string) (string, error) { resultChan := watchJob.ResultChan() defer watchJob.Stop() var event watch.Event - var jobEvent *batch_v1.Job + var jobEvent *batch.Job for event = range resultChan { if event.Type == watch.Error { return constant.JobExecutionStatusFetchError, nil } - jobEvent = event.Object.(*batch_v1.Job) + jobEvent = event.Object.(*batch.Job) if jobEvent.Status.Succeeded >= int32(1) { return constant.JobSucceeded, nil } else if jobEvent.Status.Failed >= int32(1) { @@ -346,7 +306,7 @@ func (client *client) JobExecutionStatus(jobName string) (string, error) { return constant.NoDefinitiveJobExecutionStatusFound, nil } -func (client *client) getPodLogs(pod v1.Pod) (io.ReadCloser, error) { +func (client *client) getPodLogs(pod *v1.Pod) (io.ReadCloser, error) { logger.Debug("reading pod logs for: ", pod.Name) podLogOpts := v1.PodLogOptions{ Follow: true, From d715d3f099ff9a1d19639849baaceacdabcf6d41 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 10 Jul 2019 17:42:57 +0530 Subject: [PATCH 025/268] [Jasoet|Bimozx] V2: Reorder timeout select --- internal/app/service/infra/kubernetes/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 4a76b189..81ca8dd1 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -225,10 +225,10 @@ func (client *client) waitForReadyJob(jobName string) error { } select { - case <-resultChan: - continue case <-timeoutChan: return fmt.Errorf("timeout when waiting pod to be ready") + case <-resultChan: + continue } } @@ -262,10 +262,10 @@ func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { return pod, nil } select { - case <-resultChan: - continue case <-timeoutChan: return nil, fmt.Errorf("timeout when waiting pod to be ready") + case <-resultChan: + continue } } From d1f5608eb773b6177f9e60a36bb469d44c69ab92 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 10 Jul 2019 17:44:28 +0530 Subject: [PATCH 026/268] [Jasoet|Bimozx] V2: Remove Invalid test case --- .../service/infra/kubernetes/client_test.go | 46 +++++-------------- 1 file changed, 11 insertions(+), 35 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index b2a0110f..266ae657 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -11,10 +11,10 @@ import ( "github.com/stretchr/testify/suite" batchV1 "k8s.io/api/batch/v1" "k8s.io/api/core/v1" - meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" fakeclientset "k8s.io/client-go/kubernetes/fake" - batch_v1 "k8s.io/client-go/kubernetes/typed/batch/v1" + batch "k8s.io/client-go/kubernetes/typed/batch/v1" testing_kubernetes "k8s.io/client-go/testing" utility "proctor/internal/pkg/constant" ) @@ -22,7 +22,7 @@ import ( type ClientTestSuite struct { suite.Suite testClient KubernetesClient - testKubernetesJobs batch_v1.JobInterface + testKubernetesJobs batch.JobInterface fakeClientSet *fakeclientset.Clientset jobName string podName string @@ -40,11 +40,11 @@ func (suite *ClientTestSuite) SetupTest() { suite.podName = "pod1" namespace := config.DefaultNamespace() suite.fakeClientSetStreaming = fakeclientset.NewSimpleClientset(&v1.Pod{ - TypeMeta: meta_v1.TypeMeta{ + TypeMeta: meta.TypeMeta{ Kind: "Pod", APIVersion: "v1", }, - ObjectMeta: meta_v1.ObjectMeta{ + ObjectMeta: meta.ObjectMeta{ Name: suite.podName, Namespace: namespace, Labels: map[string]string{ @@ -73,12 +73,12 @@ func (suite *ClientTestSuite) TestJobExecution() { executedJobname, err := suite.testClient.ExecuteJob(sampleImageName, envVarsForContainer) assert.NoError(t, err) - typeMeta := meta_v1.TypeMeta{ + typeMeta := meta.TypeMeta{ Kind: "Job", APIVersion: "batch/v1", } - listOptions := meta_v1.ListOptions{ + listOptions := meta.ListOptions{ TypeMeta: typeMeta, LabelSelector: jobLabelSelector(executedJobname), } @@ -111,30 +111,6 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedEnvVars, container.Env) } -/*func (suite *ClientTestSuite) TestStreamLogsSuccess() { - t := suite.T() - - httpmock.ActivateNonDefault(suite.fakeHttpClient) - defer httpmock.DeactivateAndReset() - - namespace := config.DefaultNamespace() - httpmock.RegisterResponder("GET", "https://"+config.KubeClusterHostName()+"/api/v1/namespaces/"+namespace+"/pods/"+suite.podName+"/log?follow=true", - httpmock.NewStringResponder(200, "logs are streaming")) - - logStream, err := suite.testClientStreaming.StreamJobLogs(suite.jobName) - assert.NoError(t, err) - - defer logStream.Close() - - bufioReader := bufio.NewReader(logStream) - - jobLogSingleLine, _, err := bufioReader.ReadLine() - assert.NoError(t, err) - - assert.Equal(t, "logs are streaming", string(jobLogSingleLine[:])) - -}*/ - func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { t := suite.T() @@ -152,7 +128,7 @@ func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { var succeededJob batchV1.Job uniqueJobName := "proctor-job-2" label := jobLabel(uniqueJobName) - objectMeta := meta_v1.ObjectMeta{ + objectMeta := meta.ObjectMeta{ Name: uniqueJobName, Labels: label, } @@ -187,7 +163,7 @@ func (suite *ClientTestSuite) TestShouldReturnFailedJobExecutionStatus() { var failedJob batchV1.Job uniqueJobName := "proctor-job-1" label := jobLabel(uniqueJobName) - objectMeta := meta_v1.ObjectMeta{ + objectMeta := meta.ObjectMeta{ Name: uniqueJobName, Labels: label, } @@ -220,7 +196,7 @@ func (suite *ClientTestSuite) TestJobExecutionStatusForNonDefinitiveStatus() { var testJob batchV1.Job uniqueJobName := "proctor-job-1" label := jobLabel(uniqueJobName) - objectMeta := meta_v1.ObjectMeta{ + objectMeta := meta.ObjectMeta{ Name: uniqueJobName, Labels: label, } @@ -249,7 +225,7 @@ func (suite *ClientTestSuite) TestShouldReturnJobExecutionStatusFetchError() { var testJob batchV1.Job uniqueJobName := "proctor-job-3" label := jobLabel(uniqueJobName) - objectMeta := meta_v1.ObjectMeta{ + objectMeta := meta.ObjectMeta{ Name: uniqueJobName, Labels: label, } From e0944d351e7c7efd125d6397f2842377ad44933f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 11 Jul 2019 12:43:56 +0530 Subject: [PATCH 027/268] [Jasoet|Bimozx] V2: Add Schedule Table --- .env.v2.test | 2 +- .../context/repository/execution_context.go | 3 +- .../app/service/infra/kubernetes/client.go | 64 ++++++---- .../kubernetes/client_integration_test.go | 3 +- .../service/infra/kubernetes/client_test.go | 1 + .../app/service/schedule/model/schedule.go | 20 +++ .../service/schedule/repository/schedule.go | 115 ++++++++++++++++++ .../schedule/repository/schedule_test.go | 97 +++++++++++++++ migrations/11_CreateScheduleTable.down.sql | 1 + migrations/11_CreateScheduleTable.up.sql | 13 ++ 10 files changed, 288 insertions(+), 31 deletions(-) create mode 100644 internal/app/service/schedule/model/schedule.go create mode 100644 internal/app/service/schedule/repository/schedule.go create mode 100644 internal/app/service/schedule/repository/schedule_test.go create mode 100644 migrations/11_CreateScheduleTable.down.sql create mode 100644 migrations/11_CreateScheduleTable.up.sql diff --git a/.env.v2.test b/.env.v2.test index 1d9224a4..dd44cad2 100644 --- a/.env.v2.test +++ b/.env.v2.test @@ -32,4 +32,4 @@ PROCTOR_MAIL_SERVER_PORT="123" PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" PROCTOR_SENTRY_DSN="foo" PROCTOR_DOCS_PATH="/path/to/docs/dir" -ENABLE_INTEGRATION_TEST=true +ENABLE_INTEGRATION_TEST=false diff --git a/internal/app/service/context/repository/execution_context.go b/internal/app/service/context/repository/execution_context.go index 5f73aa8e..0912855a 100644 --- a/internal/app/service/context/repository/execution_context.go +++ b/internal/app/service/context/repository/execution_context.go @@ -34,8 +34,7 @@ func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRep func (repository *executionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { snowflakeId, _ := id.NextId() context.ExecutionID = snowflakeId - sql := "INSERT INTO execution_context (id, job_name, user_email, image_tag, args, output, status) " + - " VALUES (:id, :job_name, :user_email, :image_tag, :args, :output, :status)" + sql := "INSERT INTO execution_context (id, job_name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :user_email, :image_tag, :args, :output, :status)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { return 0, nil diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 81ca8dd1..20512872 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -215,20 +215,25 @@ func (client *client) waitForReadyJob(jobName string) error { resultChan := watchJob.ResultChan() defer watchJob.Stop() - for event := range resultChan { - if event.Type == watch.Error { - return fmt.Errorf("watch error when waiting for job with list option %v", listOptions) - } - job := event.Object.(*batch.Job) - if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { - return nil - } - - select { - case <-timeoutChan: - return fmt.Errorf("timeout when waiting pod to be ready") - case <-resultChan: - continue + select { + case <-timeoutChan: + return fmt.Errorf("timeout when waiting job to be available") + case <-resultChan: + for event := range resultChan { + if event.Type == watch.Error { + return fmt.Errorf("watch error when waiting for job with list option %v", listOptions) + } + job := event.Object.(*batch.Job) + if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { + return nil + } + + select { + case <-timeoutChan: + return fmt.Errorf("timeout when waiting job to be ready") + case <-resultChan: + continue + } } } @@ -253,19 +258,24 @@ func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { defer watchJob.Stop() var pod *v1.Pod - for event := range resultChan { - if event.Type == watch.Error { - return nil, fmt.Errorf("watch error when waiting for pod with list option %v", listOptions) - } - pod = event.Object.(*v1.Pod) - if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { - return pod, nil - } - select { - case <-timeoutChan: - return nil, fmt.Errorf("timeout when waiting pod to be ready") - case <-resultChan: - continue + select { + case <-timeoutChan: + return nil,fmt.Errorf("timeout when waiting pod to be available") + case <-resultChan: + for event := range resultChan { + if event.Type == watch.Error { + return nil, fmt.Errorf("watch error when waiting for pod with list option %v", listOptions) + } + pod = event.Object.(*v1.Pod) + if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + return pod, nil + } + select { + case <-timeoutChan: + return nil, fmt.Errorf("timeout when waiting pod to be ready") + case <-resultChan: + continue + } } } diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 772b0367..8531ec34 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -110,9 +110,10 @@ func (suite *IntegrationTestSuite) TestStreamLogsSuccess() { assert.Equal(t, "Bimo Horizon", string(jobLogSingleLine[:])) } + func TestIntegrationTestSuite(t *testing.T) { value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") - if available == true || value == "true" { + if available == true && value == "true" { suite.Run(t, new(IntegrationTestSuite)) } } diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 266ae657..99f8ecb7 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -118,6 +118,7 @@ func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { assert.Error(t, err) } + func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { t := suite.T() diff --git a/internal/app/service/schedule/model/schedule.go b/internal/app/service/schedule/model/schedule.go new file mode 100644 index 00000000..9678d983 --- /dev/null +++ b/internal/app/service/schedule/model/schedule.go @@ -0,0 +1,20 @@ +package model + +import ( + dbTypes "proctor/internal/app/service/infra/db/type" + "time" +) + +type Schedule struct { + ID uint64 `db:"id"` + JobName string `db:"job_name"` + Args dbTypes.Base64Map `db:"args"` + Tags string `db:"tags"` + Cron string `db:"cron"` + NotificationEmails string `db:"notification_emails"` + UserEmail string `db:"user_email"` + Group string `db:"group"` + Enabled bool `db:"enabled"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` +} diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go new file mode 100644 index 00000000..8dca223a --- /dev/null +++ b/internal/app/service/schedule/repository/schedule.go @@ -0,0 +1,115 @@ +package repository + +import ( + "github.com/pkg/errors" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/id" + "proctor/internal/app/service/schedule/model" +) + +type ScheduleRepository interface { + Insert(context *model.Schedule) (uint64, error) + Delete(id uint64) error + GetById(id uint64) (*model.Schedule, error) + GetByUserEmail(userEmail string) ([]model.Schedule, error) + GetAllEnabled() ([]model.Schedule, error) + GetAll() ([]model.Schedule, error) + GetEnabledById(id uint64) (*model.Schedule, error) + deleteAll() error +} + +type scheduleRepository struct { + postgresqlClient postgresql.Client +} + +func NewScheduleRepository(client postgresql.Client) ScheduleRepository { + return &scheduleRepository{ + postgresqlClient: client, + } +} + +func (repository *scheduleRepository) Insert(context *model.Schedule) (uint64, error) { + snowflakeId, _ := id.NextId() + context.ID = snowflakeId + sql := "INSERT INTO schedule (id, job_name, args,cron,notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :cron, :notification_emails, :user_email, :group, :enabled)" + _, err := repository.postgresqlClient.NamedExec(sql, &context) + if err != nil { + return 0, nil + } + return snowflakeId, nil +} + +func (repository *scheduleRepository) Delete(id uint64) error { + sql := "DELETE FROM schedule WHERE id = :id" + schedule := model.Schedule{ + ID: id, + } + _, err := repository.postgresqlClient.NamedExec(sql, &schedule) + return err +} + +func (repository *scheduleRepository) GetById(id uint64) (*model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email,\"group\", enabled, created_at, updated_at FROM schedule WHERE id=$1 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, id) + if err != nil { + return nil, err + } + + if len(schedules) == 0 { + return nil, errors.Errorf("Execution context with id %v is not found!", id) + } + + return &schedules[0], nil +} + +func (repository *scheduleRepository) GetByUserEmail(userEmail string) ([]model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE user_email=$1 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, userEmail) + if err != nil { + return nil, err + } + + return schedules, nil +} + +func (repository *scheduleRepository) GetAllEnabled() ([]model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, true) + if err != nil { + return nil, err + } + + return schedules, nil +} + +func (repository *scheduleRepository) GetAll() ([]model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql) + if err != nil { + return nil, err + } + + return schedules, nil +} + +func (repository *scheduleRepository) GetEnabledById(id uint64) (*model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 AND id=$2 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, true, id) + if err != nil { + return nil, err + } + + return &schedules[0], nil +} + +func (repository *scheduleRepository) deleteAll() error { + sql := "DELETE FROM schedule" + schedules := model.Schedule{} + _, err := repository.postgresqlClient.NamedExec(sql, schedules) + return err +} diff --git a/internal/app/service/schedule/repository/schedule_test.go b/internal/app/service/schedule/repository/schedule_test.go new file mode 100644 index 00000000..1c96b897 --- /dev/null +++ b/internal/app/service/schedule/repository/schedule_test.go @@ -0,0 +1,97 @@ +package repository + +import ( + fake "github.com/brianvoe/gofakeit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/schedule/model" + "testing" +) + +type ScheduleTestSuite struct { + suite.Suite + repository ScheduleRepository +} + +var postgresqlClient postgresql.Client + +func (suite *ScheduleTestSuite) SetupSuite() { + postgresqlClient = postgresql.NewClient() +} + +func (suite *ScheduleTestSuite) SetupTest() { + t := suite.T() + suite.repository = NewScheduleRepository(postgresqlClient) + err := suite.repository.deleteAll() + assert.NoError(t, err) + fake.Seed(0) +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_Insert() { + t := suite.T() + mapKey := fake.FirstName() + mapValue := fake.LastName() + schedule := &model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + mapKey: mapValue, + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: fake.Bool(), + } + + id, err := suite.repository.Insert(schedule) + assert.NotNil(t, id) + assert.NoError(t, err) + + expectedSchedule, err := suite.repository.GetById(id) + assert.NoError(t, err) + assert.NotNil(t, expectedSchedule) + + assert.Equal(t, id, expectedSchedule.ID) + assert.NotNil(t, expectedSchedule.CreatedAt) + assert.NotNil(t, expectedSchedule.UpdatedAt) + assert.Equal(t, expectedSchedule.Args[mapKey], mapValue) +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_Delete() { + t := suite.T() + mapKey := fake.FirstName() + mapValue := fake.LastName() + schedule := &model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + mapKey: mapValue, + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: fake.Bool(), + } + + id, err := suite.repository.Insert(schedule) + assert.NotNil(t, id) + assert.NoError(t, err) + + err = suite.repository.Delete(id) + assert.NoError(t, err) + + expectedSchedule, err := suite.repository.GetById(id) + assert.Error(t, err) + assert.Nil(t, expectedSchedule) +} + +func (suite *ScheduleTestSuite) TearDownSuite() { + postgresqlClient.Close() +} + +func TestScheduleTestSuite(t *testing.T) { + suite.Run(t, new(ScheduleTestSuite)) +} diff --git a/migrations/11_CreateScheduleTable.down.sql b/migrations/11_CreateScheduleTable.down.sql new file mode 100644 index 00000000..8a5bf074 --- /dev/null +++ b/migrations/11_CreateScheduleTable.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS schedule; diff --git a/migrations/11_CreateScheduleTable.up.sql b/migrations/11_CreateScheduleTable.up.sql new file mode 100644 index 00000000..a37d1a3f --- /dev/null +++ b/migrations/11_CreateScheduleTable.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE schedule +( + id bigint, + job_name varchar(255), + args text, + cron varchar(255), + notification_emails varchar(255), + user_email varchar(255), + "group" varchar(255), + enabled bool, + created_at timestamp default now(), + updated_at timestamp default now() +); From 8dc4c3b178e84af7129770b52343d566cb86947e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 11 Jul 2019 13:57:18 +0530 Subject: [PATCH 028/268] [Jasoet|Bimozx] V2: Complete Schedule Repository Test --- .../service/schedule/repository/schedule.go | 41 ++++ .../schedule/repository/schedule_test.go | 197 ++++++++++++++++++ 2 files changed, 238 insertions(+) diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go index 8dca223a..196d9471 100644 --- a/internal/app/service/schedule/repository/schedule.go +++ b/internal/app/service/schedule/repository/schedule.go @@ -5,13 +5,17 @@ import ( "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/id" "proctor/internal/app/service/schedule/model" + "time" ) type ScheduleRepository interface { Insert(context *model.Schedule) (uint64, error) Delete(id uint64) error GetById(id uint64) (*model.Schedule, error) + Disable(id uint64) error + Enable(id uint64) error GetByUserEmail(userEmail string) ([]model.Schedule, error) + GetByJobName(jobName string) ([]model.Schedule, error) GetAllEnabled() ([]model.Schedule, error) GetAll() ([]model.Schedule, error) GetEnabledById(id uint64) (*model.Schedule, error) @@ -28,6 +32,28 @@ func NewScheduleRepository(client postgresql.Client) ScheduleRepository { } } +func (repository *scheduleRepository) Enable(id uint64) error { + sql := "UPDATE schedule SET enabled = :enabled, updated_at = :updated_at WHERE id = :id" + context := model.Schedule{ + ID: id, + UpdatedAt: time.Now(), + Enabled: true, + } + _, err := repository.postgresqlClient.NamedExec(sql, &context) + return err +} + +func (repository *scheduleRepository) Disable(id uint64) error { + sql := "UPDATE schedule SET enabled = :enabled, updated_at = :updated_at WHERE id = :id" + context := model.Schedule{ + ID: id, + UpdatedAt: time.Now(), + Enabled: false, + } + _, err := repository.postgresqlClient.NamedExec(sql, &context) + return err +} + func (repository *scheduleRepository) Insert(context *model.Schedule) (uint64, error) { snowflakeId, _ := id.NextId() context.ID = snowflakeId @@ -74,6 +100,17 @@ func (repository *scheduleRepository) GetByUserEmail(userEmail string) ([]model. return schedules, nil } +func (repository *scheduleRepository) GetByJobName(jobName string) ([]model.Schedule, error) { + sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE job_name=$1 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, jobName) + if err != nil { + return nil, err + } + + return schedules, nil +} + func (repository *scheduleRepository) GetAllEnabled() ([]model.Schedule, error) { sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 " var schedules []model.Schedule @@ -104,6 +141,10 @@ func (repository *scheduleRepository) GetEnabledById(id uint64) (*model.Schedule return nil, err } + if len(schedules) == 0 { + return nil, errors.Errorf("Execution context with id %v is not found!", id) + } + return &schedules[0], nil } diff --git a/internal/app/service/schedule/repository/schedule_test.go b/internal/app/service/schedule/repository/schedule_test.go index 1c96b897..27229d7b 100644 --- a/internal/app/service/schedule/repository/schedule_test.go +++ b/internal/app/service/schedule/repository/schedule_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/suite" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/schedule/model" + "strconv" "testing" ) @@ -88,6 +89,202 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Delete() { assert.Nil(t, expectedSchedule) } +func (suite *ScheduleTestSuite) TestScheduleRepository_GetAll() { + t := suite.T() + recordCount := 15 + err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{}) + assert.NoError(t, err) + + schedules, err := suite.repository.GetAll() + assert.NoError(t, err) + assert.NotNil(t, schedules) + size := len(schedules) + assert.Equal(t, recordCount, size) +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_GetByUserEmail() { + t := suite.T() + recordCount := 15 + err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{}) + assert.NoError(t, err) + + suppliedEmail := "bimo.horizon@gojek.co.id" + withEmailCount := 14 + err = populateSeedDataForTest(suite.repository, withEmailCount, map[string]string{"UserEmail": suppliedEmail}) + + schedules, err := suite.repository.GetByUserEmail(suppliedEmail) + assert.NoError(t, err) + assert.NotNil(t, schedules) + size := len(schedules) + assert.Equal(t, withEmailCount, size) + + for _, schedule := range schedules { + assert.Equal(t, suppliedEmail, schedule.UserEmail) + } +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_GetByJobName() { + t := suite.T() + recordCount := 15 + err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{}) + assert.NoError(t, err) + + suppliedJobName := "bimo-awesome-job" + withJobName := 14 + err = populateSeedDataForTest(suite.repository, withJobName, map[string]string{"JobName": suppliedJobName}) + + schedules, err := suite.repository.GetByJobName(suppliedJobName) + assert.NoError(t, err) + assert.NotNil(t, schedules) + size := len(schedules) + assert.Equal(t, withJobName, size) + + for _, schedule := range schedules { + assert.Equal(t, suppliedJobName, schedule.JobName) + } +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_GetAllEnabled() { + t := suite.T() + recordCount := 15 + err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{"Enabled": "false"}) + assert.NoError(t, err) + + withJobName := 14 + err = populateSeedDataForTest(suite.repository, withJobName, map[string]string{"Enabled": "true"}) + + schedules, err := suite.repository.GetAllEnabled() + assert.NoError(t, err) + assert.NotNil(t, schedules) + size := len(schedules) + assert.Equal(t, withJobName, size) + + for _, schedule := range schedules { + assert.True(t, schedule.Enabled) + } +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledById() { + t := suite.T() + recordCount := 15 + err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{}) + assert.NoError(t, err) + + mapKey := fake.FirstName() + mapValue := fake.LastName() + schedule := &model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + mapKey: mapValue, + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: true, + } + + id, err := suite.repository.Insert(schedule) + assert.NotNil(t, id) + assert.NoError(t, err) + + expectedSchedule, err := suite.repository.GetEnabledById(id) + assert.NoError(t, err) + assert.NotNil(t, expectedSchedule) + assert.True(t, expectedSchedule.Enabled) + + willNotExistsId := uint64(17777717) + unexpectedSchedule, err := suite.repository.GetEnabledById(willNotExistsId) + assert.Error(t, err) + assert.Nil(t, unexpectedSchedule) + +} + +func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { + t := suite.T() + mapKey := fake.FirstName() + mapValue := fake.LastName() + schedule := &model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + mapKey: mapValue, + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: true, + } + + id, err := suite.repository.Insert(schedule) + assert.NotNil(t, id) + assert.NoError(t, err) + + expectedSchedule, err := suite.repository.GetById(id) + assert.NoError(t, err) + assert.NotNil(t, expectedSchedule) + assert.True(t, expectedSchedule.Enabled) + + err = suite.repository.Disable(id) + assert.NoError(t, err) + + expectedSchedule, err = suite.repository.GetById(id) + assert.NoError(t, err) + assert.NotNil(t, expectedSchedule) + assert.False(t, expectedSchedule.Enabled) + + err = suite.repository.Enable(id) + assert.NoError(t, err) + + expectedSchedule, err = suite.repository.GetById(id) + assert.NoError(t, err) + assert.NotNil(t, expectedSchedule) + assert.True(t, expectedSchedule.Enabled) + +} + +func populateSeedDataForTest(repository ScheduleRepository, count int, seedField map[string]string) error { + for i := 0; i < count; i++ { + fake.Seed(0) + var jobName = fake.BuzzWord() + if val, ok := seedField["JobName"]; ok { + jobName = val + } + + var email = fake.Email() + if val, ok := seedField["UserEmail"]; ok { + email = val + } + + var enabled = fake.Bool() + if val, ok := seedField["Enabled"]; ok { + enabled, _ = strconv.ParseBool(val) + } + + schedule := &model.Schedule{ + JobName: jobName, + UserEmail: email, + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: enabled, + } + + _, err := repository.Insert(schedule) + + if err != nil { + return err + } + } + return nil +} + func (suite *ScheduleTestSuite) TearDownSuite() { postgresqlClient.Close() } From c32c0cd9cbcb73b0cbb93be5a10c94b39a7b004a Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 11 Jul 2019 14:28:23 +0530 Subject: [PATCH 029/268] [Jasoet|Bimozx] V2: Add metadata repository --- .../service/metadata/repository/metadata.go | 84 +++++++++ .../metadata/repository/metadata_test.go | 159 ++++++++++++++++++ 2 files changed, 243 insertions(+) create mode 100644 internal/app/service/metadata/repository/metadata.go create mode 100644 internal/app/service/metadata/repository/metadata_test.go diff --git a/internal/app/service/metadata/repository/metadata.go b/internal/app/service/metadata/repository/metadata.go new file mode 100644 index 00000000..b7d4e4fc --- /dev/null +++ b/internal/app/service/metadata/repository/metadata.go @@ -0,0 +1,84 @@ +package repository + +import ( + "encoding/json" + "proctor/internal/app/service/infra/db/redis" + "proctor/internal/pkg/model/metadata" +) + +const KeySuffix = "-metadata" + +type MetadataRepository interface { + Save(metadata *metadata.Metadata) error + GetAll() ([]metadata.Metadata, error) + GetByName(name string) (*metadata.Metadata, error) +} + +type metadataRepository struct { + redisClient redis.Client +} + +func applySuffix(name string) string { + return name + KeySuffix +} + +func NewMetadataRepository(client redis.Client) MetadataRepository { + return &metadataRepository{ + redisClient: client, + } +} + +func (repository *metadataRepository) Save(metadata *metadata.Metadata) error { + key := applySuffix(metadata.Name) + + jsonMetadata, err := json.Marshal(metadata) + if err != nil { + return err + } + + return repository.redisClient.SET(key, jsonMetadata) +} + +func (repository *metadataRepository) GetAll() ([]metadata.Metadata, error) { + searchKey := "*" + KeySuffix + + keys, err := repository.redisClient.KEYS(searchKey) + if err != nil { + return nil, err + } + + availableKeys := make([]interface{}, len(keys)) + for i := range keys { + availableKeys[i] = keys[i] + } + + values, err := repository.redisClient.MGET(availableKeys...) + if err != nil { + return nil, err + } + + metadataSlice := make([]metadata.Metadata, len(values)) + for i := range values { + err = json.Unmarshal(values[i], &metadataSlice[i]) + if err != nil { + return nil, err + } + } + + return metadataSlice, nil +} + +func (repository *metadataRepository) GetByName(name string) (*metadata.Metadata, error) { + binaryMetadata, err := repository.redisClient.GET(applySuffix(name)) + if err != nil { + return nil, err + } + + var jobMetadata metadata.Metadata + err = json.Unmarshal(binaryMetadata, &jobMetadata) + if err != nil { + return nil, err + } + + return &jobMetadata, nil +} diff --git a/internal/app/service/metadata/repository/metadata_test.go b/internal/app/service/metadata/repository/metadata_test.go new file mode 100644 index 00000000..90da6794 --- /dev/null +++ b/internal/app/service/metadata/repository/metadata_test.go @@ -0,0 +1,159 @@ +package repository + +import ( + "encoding/json" + "errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "proctor/internal/app/service/infra/db/redis" + modelMetadata "proctor/internal/pkg/model/metadata" + "testing" +) + +type MetadataRepositoryTestSuite struct { + suite.Suite + mockRedisClient *redis.MockClient + testMetadataStore MetadataRepository +} + +func (s *MetadataRepositoryTestSuite) SetupTest() { + s.mockRedisClient = &redis.MockClient{} + + s.testMetadataStore = NewMetadataRepository(s.mockRedisClient) +} + +func (s *MetadataRepositoryTestSuite) TestSave() { + t := s.T() + + metadata := modelMetadata.Metadata{ + Name: "any-name", + ImageName: "any-image-name", + Description: "any-description", + Author: "Test User", + Contributors: "Test User, Test Admin", + Organization: "Test Org", + } + + jsonData, err := json.Marshal(metadata) + assert.NoError(t, err) + + s.mockRedisClient.On("SET", "any-name-metadata", jsonData).Return(nil).Once() + + err = s.testMetadataStore.Save(&metadata) + assert.NoError(t, err) + s.mockRedisClient.AssertExpectations(t) +} + +func (s *MetadataRepositoryTestSuite) TestSaveFailure() { + t := s.T() + + metadata := modelMetadata.Metadata{} + + expectedError := errors.New("any-error") + s.mockRedisClient.On("SET", mock.Anything, mock.Anything).Return(expectedError).Once() + + err := s.testMetadataStore.Save(&metadata) + assert.EqualError(t, err, "any-error") + s.mockRedisClient.AssertExpectations(t) +} + +func (s *MetadataRepositoryTestSuite) TestGetAll() { + t := s.T() + + metadata1 := modelMetadata.Metadata{ + Name: "job1", + ImageName: "job1-image-name", + Description: "desc1", + Author: "Test User Date: Thu, 11 Jul 2019 16:10:41 +0530 Subject: [PATCH 030/268] [Jasoet|Bimozx] V2: Fix metadata and secret handler --- .../proctord/jobs/execution/executioner.go | 14 +- .../jobs/execution/executioner_test.go | 26 +-- .../app/proctord/jobs/metadata/handler.go | 81 --------- internal/app/proctord/jobs/metadata/store.go | 83 --------- .../app/proctord/jobs/metadata/store_mock.go | 25 --- .../app/proctord/jobs/metadata/store_test.go | 159 ------------------ .../app/proctord/jobs/schedule/handler.go | 8 +- .../proctord/jobs/schedule/handler_test.go | 16 +- internal/app/proctord/jobs/secrets/store.go | 48 ------ .../app/proctord/jobs/secrets/store_mock.go | 19 --- internal/app/proctord/scheduler/scheduler.go | 8 +- internal/app/proctord/server/router.go | 24 +-- .../model/execution_context.go | 0 .../repository/execution_context.go | 2 +- .../repository/execution_context_test.go | 2 +- .../service/metadata/handler/http_handler.go | 82 +++++++++ .../metadata/handler/http_handler_test.go} | 39 ++--- .../service/metadata/repository/metadata.go | 4 +- .../metadata/repository/metadata_mock.go | 25 +++ .../metadata/repository/metadata_test.go | 4 +- .../secret/handler/http.go} | 26 +-- .../secret/handler/http_test.go} | 38 +++-- .../secret/model}/secret.go | 2 +- .../app/service/secret/repository/secret.go | 49 ++++++ .../service/secret/repository/secret_mock.go | 20 +++ .../secret/repository/secret_test.go} | 21 +-- 26 files changed, 297 insertions(+), 528 deletions(-) delete mode 100644 internal/app/proctord/jobs/metadata/handler.go delete mode 100644 internal/app/proctord/jobs/metadata/store.go delete mode 100644 internal/app/proctord/jobs/metadata/store_mock.go delete mode 100644 internal/app/proctord/jobs/metadata/store_test.go delete mode 100644 internal/app/proctord/jobs/secrets/store.go delete mode 100644 internal/app/proctord/jobs/secrets/store_mock.go rename internal/app/service/{context => execution}/model/execution_context.go (100%) rename internal/app/service/{context => execution}/repository/execution_context.go (98%) rename internal/app/service/{context => execution}/repository/execution_context_test.go (99%) create mode 100644 internal/app/service/metadata/handler/http_handler.go rename internal/app/{proctord/jobs/metadata/handler_test.go => service/metadata/handler/http_handler_test.go} (76%) create mode 100644 internal/app/service/metadata/repository/metadata_mock.go rename internal/app/{proctord/jobs/secrets/handler.go => service/secret/handler/http.go} (50%) rename internal/app/{proctord/jobs/secrets/handler_test.go => service/secret/handler/http_test.go} (59%) rename internal/app/{proctord/jobs/secrets => service/secret/model}/secret.go (87%) create mode 100644 internal/app/service/secret/repository/secret.go create mode 100644 internal/app/service/secret/repository/secret_mock.go rename internal/app/{proctord/jobs/secrets/store_test.go => service/secret/repository/secret_test.go} (80%) diff --git a/internal/app/proctord/jobs/execution/executioner.go b/internal/app/proctord/jobs/execution/executioner.go index cc97fa3e..2d275fe2 100644 --- a/internal/app/proctord/jobs/execution/executioner.go +++ b/internal/app/proctord/jobs/execution/executioner.go @@ -3,10 +3,10 @@ package execution import ( "errors" "fmt" - jobMetadata "proctor/internal/app/proctord/jobs/metadata" - jobSecrets "proctor/internal/app/proctord/jobs/secrets" "proctor/internal/app/proctord/storage/postgres" "proctor/internal/app/service/infra/kubernetes" + repository2 "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/secret/repository" "proctor/internal/pkg/constant" "proctor/internal/pkg/utility" @@ -14,15 +14,15 @@ import ( type executioner struct { kubeClient kubernetes.KubernetesClient - metadataStore jobMetadata.Store - secretsStore jobSecrets.Store + metadataStore repository2.MetadataRepository + secretsStore repository.SecretRepository } type Executioner interface { Execute(*postgres.JobsExecutionAuditLog, string, map[string]string) (string, error) } -func NewExecutioner(kubeClient kubernetes.KubernetesClient, metadataStore jobMetadata.Store, secretsStore jobSecrets.Store) Executioner { +func NewExecutioner(kubeClient kubernetes.KubernetesClient, metadataStore repository2.MetadataRepository, secretsStore repository.SecretRepository) Executioner { return &executioner{ kubeClient: kubeClient, metadataStore: metadataStore, @@ -33,14 +33,14 @@ func NewExecutioner(kubeClient kubernetes.KubernetesClient, metadataStore jobMet func (executioner *executioner) Execute(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog, jobName string, jobArgs map[string]string) (string, error) { jobsExecutionAuditLog.JobName = jobName - jobMetadata, err := executioner.metadataStore.GetJobMetadata(jobName) + jobMetadata, err := executioner.metadataStore.GetByName(jobName) if err != nil { return "", errors.New(fmt.Sprintf("Error finding image for job: %s. Error: %s", jobName, err.Error())) } imageName := jobMetadata.ImageName jobsExecutionAuditLog.ImageName = imageName - jobSecrets, err := executioner.secretsStore.GetJobSecrets(jobName) + jobSecrets, err := executioner.secretsStore.GetByJobName(jobName) if err != nil && err.Error() != "redigo: nil returned" { return "", errors.New(fmt.Sprintf("Error retrieving secrets for job: %s. Error: %s", jobName, err.Error())) } diff --git a/internal/app/proctord/jobs/execution/executioner_test.go b/internal/app/proctord/jobs/execution/executioner_test.go index eab3cc2e..cd79516f 100644 --- a/internal/app/proctord/jobs/execution/executioner_test.go +++ b/internal/app/proctord/jobs/execution/executioner_test.go @@ -2,10 +2,10 @@ package execution import ( "errors" - jobMetadata "proctor/internal/app/proctord/jobs/metadata" - jobSecrets "proctor/internal/app/proctord/jobs/secrets" "proctor/internal/app/proctord/storage/postgres" "proctor/internal/app/service/infra/kubernetes" + metadataRepository "proctor/internal/app/service/metadata/repository" + secretRepository "proctor/internal/app/service/secret/repository" "testing" "github.com/stretchr/testify/assert" @@ -18,15 +18,15 @@ import ( type ExecutionerTestSuite struct { suite.Suite mockKubeClient kubernetes.MockClient - mockMetadataStore *jobMetadata.MockStore - mockSecretsStore *jobSecrets.MockStore + mockMetadataStore *metadataRepository.MockMetadataRepository + mockSecretsStore *secretRepository.MockSecretRepository testExecutioner Executioner } func (suite *ExecutionerTestSuite) SetupTest() { suite.mockKubeClient = kubernetes.MockClient{} - suite.mockMetadataStore = &jobMetadata.MockStore{} - suite.mockSecretsStore = &jobSecrets.MockStore{} + suite.mockMetadataStore = &metadataRepository.MockMetadataRepository{} + suite.mockSecretsStore = &secretRepository.MockSecretRepository{} suite.testExecutioner = NewExecutioner(&suite.mockKubeClient, suite.mockMetadataStore, suite.mockSecretsStore) } @@ -42,12 +42,12 @@ func (suite *ExecutionerTestSuite) TestSuccessfulJobExecution() { jobMetadata := modelMetadata.Metadata{ ImageName: "img", } - suite.mockMetadataStore.On("GetJobMetadata", jobName).Return(&jobMetadata, nil).Once() + suite.mockMetadataStore.On("GetByName", jobName).Return(&jobMetadata, nil).Once() jobSecrets := map[string]string{ "secretOne": "sample-secrets", } - suite.mockSecretsStore.On("GetJobSecrets", jobName).Return(jobSecrets, nil).Once() + suite.mockSecretsStore.On("GetByJobName", jobName).Return(jobSecrets, nil).Once() jobExecutionID := "proctor-ipsum-lorem" envVarsForJob := utility.MergeMaps(jobArgs, jobSecrets) @@ -67,7 +67,7 @@ func (suite *ExecutionerTestSuite) TestSuccessfulJobExecution() { func (suite *ExecutionerTestSuite) TestJobExecutionOnImageLookupFailure() { t := suite.T() - suite.mockMetadataStore.On("GetJobMetadata", mock.Anything).Return(&modelMetadata.Metadata{}, errors.New("image-fetch-error")).Once() + suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&modelMetadata.Metadata{}, errors.New("image-fetch-error")).Once() _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) assert.EqualError(t, err, "Error finding image for job: any-job. Error: image-fetch-error") @@ -77,9 +77,9 @@ func (suite *ExecutionerTestSuite) TestJobExecutionOnSecretsFetchFailure() { t := suite.T() jobMetadata := modelMetadata.Metadata{ImageName: "img"} - suite.mockMetadataStore.On("GetJobMetadata", mock.Anything).Return(&jobMetadata, nil).Once() + suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&jobMetadata, nil).Once() - suite.mockSecretsStore.On("GetJobSecrets", mock.Anything).Return(map[string]string{}, errors.New("secret-store-error")).Once() + suite.mockSecretsStore.On("GetByJobName", mock.Anything).Return(map[string]string{}, errors.New("secret-store-error")).Once() _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) assert.EqualError(t, err, "Error retrieving secrets for job: any-job. Error: secret-store-error") @@ -89,9 +89,9 @@ func (suite *ExecutionerTestSuite) TestJobExecutionOnKubernetesJobExecutionFailu t := suite.T() jobMetadata := modelMetadata.Metadata{ImageName: "img"} - suite.mockMetadataStore.On("GetJobMetadata", mock.Anything).Return(&jobMetadata, nil).Once() + suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&jobMetadata, nil).Once() - suite.mockSecretsStore.On("GetJobSecrets", mock.Anything).Return(map[string]string{}, nil).Once() + suite.mockSecretsStore.On("GetByJobName", mock.Anything).Return(map[string]string{}, nil).Once() suite.mockKubeClient.On("ExecuteJob", mock.Anything, mock.Anything).Return("", errors.New("kube-client-error")).Once() _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) diff --git a/internal/app/proctord/jobs/metadata/handler.go b/internal/app/proctord/jobs/metadata/handler.go deleted file mode 100644 index 703984d8..00000000 --- a/internal/app/proctord/jobs/metadata/handler.go +++ /dev/null @@ -1,81 +0,0 @@ -package metadata - -import ( - "encoding/json" - "github.com/getsentry/raven-go" - "net/http" - "proctor/internal/app/service/infra/logger" - "proctor/internal/pkg/constant" - modelMetadata "proctor/internal/pkg/model/metadata" -) - -type handler struct { - store Store -} - -type Handler interface { - HandleSubmission() http.HandlerFunc - HandleBulkDisplay() http.HandlerFunc -} - -func NewHandler(store Store) Handler { - return &handler{ - store: store, - } -} - -func (handler *handler) HandleSubmission() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - var jobMetadata []modelMetadata.Metadata - err := json.NewDecoder(req.Body).Decode(&jobMetadata) - defer req.Body.Close() - if err != nil { - logger.Error("Error parsing request body", err.Error()) - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.ClientError)) - return - } - - for _, metadata := range jobMetadata { - err = handler.store.CreateOrUpdateJobMetadata(metadata) - if err != nil { - logger.Error("Error updating metadata", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(constant.ServerError)) - return - } - } - - w.WriteHeader(http.StatusCreated) - } -} - -func (handler *handler) HandleBulkDisplay() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - - jobMetadata, err := handler.store.GetAllJobsMetadata() - if err != nil { - logger.Error("Error fetching metadata", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(constant.ServerError)) - return - } - - jobsMetadataInJSON, err := json.Marshal(jobMetadata) - if err != nil { - logger.Error("Error marshalling jobs metadata in json", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - w.Write([]byte(constant.ServerError)) - return - } - - w.Write(jobsMetadataInJSON) - } -} diff --git a/internal/app/proctord/jobs/metadata/store.go b/internal/app/proctord/jobs/metadata/store.go deleted file mode 100644 index 06a64f5e..00000000 --- a/internal/app/proctord/jobs/metadata/store.go +++ /dev/null @@ -1,83 +0,0 @@ -package metadata - -import ( - "encoding/json" - "proctor/internal/app/service/infra/db/redis" - modelMetadata "proctor/internal/pkg/model/metadata" -) - -const JobNameKeySuffix = "-metadata" - -type Store interface { - CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error - GetAllJobsMetadata() ([]modelMetadata.Metadata, error) - GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) -} - -type store struct { - redisClient redis.Client -} - -func NewStore(redisClient redis.Client) Store { - return &store{ - redisClient: redisClient, - } -} - -func jobMetadataKey(jobName string) string { - return jobName + JobNameKeySuffix -} - -func (store *store) CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error { - jobNameKey := jobMetadataKey(metadata.Name) - - binaryJobMetadata, err := json.Marshal(metadata) - if err != nil { - return err - } - - return store.redisClient.SET(jobNameKey, binaryJobMetadata) -} - -func (store *store) GetAllJobsMetadata() ([]modelMetadata.Metadata, error) { - jobNameKeyRegex := "*" + JobNameKeySuffix - - keys, err := store.redisClient.KEYS(jobNameKeyRegex) - if err != nil { - return nil, err - } - - jobKeys := make([]interface{}, len(keys)) - for i := range keys { - jobKeys[i] = keys[i] - } - values, err := store.redisClient.MGET(jobKeys...) - if err != nil { - return nil, err - } - - jobsMetadata := make([]modelMetadata.Metadata, len(values)) - for i := range values { - err = json.Unmarshal(values[i], &jobsMetadata[i]) - if err != nil { - return nil, err - } - } - - return jobsMetadata, nil -} - -func (store *store) GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) { - binaryJobMetadata, err := store.redisClient.GET(jobMetadataKey(jobName)) - if err != nil { - return nil, err - } - - var jobMetadata modelMetadata.Metadata - err = json.Unmarshal(binaryJobMetadata, &jobMetadata) - if err != nil { - return nil, err - } - - return &jobMetadata, nil -} diff --git a/internal/app/proctord/jobs/metadata/store_mock.go b/internal/app/proctord/jobs/metadata/store_mock.go deleted file mode 100644 index 2fefd7b5..00000000 --- a/internal/app/proctord/jobs/metadata/store_mock.go +++ /dev/null @@ -1,25 +0,0 @@ -package metadata - -import ( - "github.com/stretchr/testify/mock" - modelMetadata "proctor/internal/pkg/model/metadata" -) - -type MockStore struct { - mock.Mock -} - -func (m *MockStore) CreateOrUpdateJobMetadata(metadata modelMetadata.Metadata) error { - args := m.Called(metadata) - return args.Error(0) -} - -func (m *MockStore) GetAllJobsMetadata() ([]modelMetadata.Metadata, error) { - args := m.Called() - return args.Get(0).([]modelMetadata.Metadata), args.Error(1) -} - -func (m *MockStore) GetJobMetadata(jobName string) (*modelMetadata.Metadata, error) { - args := m.Called(jobName) - return args.Get(0).(*modelMetadata.Metadata), args.Error(1) -} diff --git a/internal/app/proctord/jobs/metadata/store_test.go b/internal/app/proctord/jobs/metadata/store_test.go deleted file mode 100644 index 16fe545d..00000000 --- a/internal/app/proctord/jobs/metadata/store_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package metadata - -import ( - "encoding/json" - "errors" - "proctor/internal/app/service/infra/db/redis" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - modelMetadata "proctor/internal/pkg/model/metadata" -) - -type MetadataStoreTestSuite struct { - suite.Suite - mockRedisClient *redis.MockClient - testMetadataStore Store -} - -func (s *MetadataStoreTestSuite) SetupTest() { - s.mockRedisClient = &redis.MockClient{} - - s.testMetadataStore = NewStore(s.mockRedisClient) -} - -func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadata() { - t := s.T() - - metadata := modelMetadata.Metadata{ - Name: "any-name", - ImageName: "any-image-name", - Description: "any-description", - Author: "Test User", - Contributors: "Test User, Test Admin", - Organization: "Test Org", - } - - binaryJobMetadata, err := json.Marshal(metadata) - assert.NoError(t, err) - - s.mockRedisClient.On("SET", "any-name-metadata", binaryJobMetadata).Return(nil).Once() - - err = s.testMetadataStore.CreateOrUpdateJobMetadata(metadata) - assert.NoError(t, err) - s.mockRedisClient.AssertExpectations(t) -} - -func (s *MetadataStoreTestSuite) TestCreateOrUpdateJobMetadataForRedisClientFailure() { - t := s.T() - - metadata := modelMetadata.Metadata{} - - expectedError := errors.New("any-error") - s.mockRedisClient.On("SET", mock.Anything, mock.Anything).Return(expectedError).Once() - - err := s.testMetadataStore.CreateOrUpdateJobMetadata(metadata) - assert.EqualError(t, err, "any-error") - s.mockRedisClient.AssertExpectations(t) -} - -func (s *MetadataStoreTestSuite) TestGetAllJobsMetadata() { - t := s.T() - - metadata1 := modelMetadata.Metadata{ - Name: "job1", - ImageName: "job1-image-name", - Description: "desc1", - Author: "Test User Date: Thu, 11 Jul 2019 18:30:20 +0530 Subject: [PATCH 031/268] [Jasoet|Bimozx] V2: Create Execution Service --- .../execution/model/execution_context.go | 21 ++-- .../execution/repository/execution_context.go | 5 +- .../repository/execution_context_test.go | 17 +-- .../service/execution/service/execution.go | 106 ++++++++++++++++++ .../app/service/execution/status/execution.go | 10 ++ .../infra/db/{type => types}/base64_map.go | 2 +- .../db/{type => types}/base64_map_test.go | 2 +- .../app/service/infra/kubernetes/client.go | 46 ++++---- .../service/infra/kubernetes/client_mock.go | 8 +- internal/app/service/infra/logger/logrus.go | 8 ++ .../app/service/schedule/model/schedule.go | 2 +- 11 files changed, 177 insertions(+), 50 deletions(-) create mode 100644 internal/app/service/execution/service/execution.go create mode 100644 internal/app/service/execution/status/execution.go rename internal/app/service/infra/db/{type => types}/base64_map.go (98%) rename internal/app/service/infra/db/{type => types}/base64_map_test.go (97%) diff --git a/internal/app/service/execution/model/execution_context.go b/internal/app/service/execution/model/execution_context.go index 5f57b73c..6f414081 100644 --- a/internal/app/service/execution/model/execution_context.go +++ b/internal/app/service/execution/model/execution_context.go @@ -2,18 +2,19 @@ package model import ( sqlxTypes "github.com/jmoiron/sqlx/types" - dbTypes "proctor/internal/app/service/infra/db/type" + "proctor/internal/app/service/execution/status" + dbTypes "proctor/internal/app/service/infra/db/types" "time" ) type ExecutionContext struct { - ExecutionID uint64 `db:"id"` - JobName string `db:"job_name"` - UserEmail string `db:"user_email"` - ImageTag string `db:"image_tag"` - Args dbTypes.Base64Map `db:"args"` - Output sqlxTypes.GzippedText `db:"output"` - Status string `db:"status"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ExecutionID uint64 `db:"id"` + JobName string `db:"job_name"` + UserEmail string `db:"user_email"` + ImageTag string `db:"image_tag"` + Args dbTypes.Base64Map `db:"args"` + Output sqlxTypes.GzippedText `db:"output"` + Status status.ExecutionStatus `db:"status"` + CreatedAt time.Time `db:"created_at"` + UpdatedAt time.Time `db:"updated_at"` } diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 2b837e38..2d9b1948 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -4,6 +4,7 @@ import ( "github.com/jmoiron/sqlx/types" "github.com/pkg/errors" "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/id" "time" @@ -12,7 +13,7 @@ import ( type ExecutionContextRepository interface { Insert(context *model.ExecutionContext) (uint64, error) UpdateJobOutput(executionId uint64, output types.GzippedText) error - UpdateStatus(executionId uint64, status string) error + UpdateStatus(executionId uint64, status status.ExecutionStatus) error Delete(executionId uint64) error GetById(executionId uint64) (*model.ExecutionContext, error) GetByEmail(userEmail string) ([]model.ExecutionContext, error) @@ -53,7 +54,7 @@ func (repository *executionContextRepository) UpdateJobOutput(executionId uint64 return err } -func (repository *executionContextRepository) UpdateStatus(executionId uint64, status string) error { +func (repository *executionContextRepository) UpdateStatus(executionId uint64, status status.ExecutionStatus) error { sql := "UPDATE execution_context SET status = :status, updated_at = :updated_at WHERE id = :id" context := model.ExecutionContext{ ExecutionID: executionId, diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index c0306816..2ee81f35 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -5,6 +5,7 @@ import ( "github.com/jmoiron/sqlx/types" "github.com/stretchr/testify/assert" "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/db/postgresql" "testing" ) @@ -26,7 +27,7 @@ func TestExecutionContextRepository_Insert(t *testing.T) { Args: map[string]string{ mapKey: mapValue, }, - Status: fake.State(), + Status: status.Received, } id, err := repository.Insert(context) @@ -56,7 +57,7 @@ func TestExecutionContextRepository_Delete(t *testing.T) { Args: map[string]string{ fake.FirstName(): fake.LastName(), }, - Status: fake.State(), + Status: status.Received, } id, err := repository.Insert(context) @@ -85,14 +86,14 @@ func TestExecutionContextRepository_UpdateStatus(t *testing.T) { Args: map[string]string{ fake.FirstName(): fake.LastName(), }, - Status: fake.State(), + Status: status.Received, } id, err := repository.Insert(context) assert.Nil(t, err) assert.NotZero(t, id) - newStatus := fake.State() + newStatus := status.Created err = repository.UpdateStatus(id, newStatus) assert.Nil(t, err) @@ -116,7 +117,7 @@ func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { Args: map[string]string{ fake.FirstName(): fake.LastName(), }, - Status: fake.State(), + Status: status.Received, } id, err := repository.Insert(context) @@ -166,9 +167,9 @@ func populateSeedDataForTest(repository ExecutionContextRepository, count int, s email = val } - var status = fake.State() + var defaultStatus = status.RequirementNotMet if val, ok := seedField["Status"]; ok { - status = val + defaultStatus = status.ExecutionStatus(val) } context := &model.ExecutionContext{ @@ -178,7 +179,7 @@ func populateSeedDataForTest(repository ExecutionContextRepository, count int, s Args: map[string]string{ fake.FirstName(): fake.LastName(), }, - Status: status, + Status: defaultStatus, } _, err := repository.Insert(context) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go new file mode 100644 index 00000000..5ea8739d --- /dev/null +++ b/internal/app/service/execution/service/execution.go @@ -0,0 +1,106 @@ +package service + +import ( + "errors" + "fmt" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/kubernetes" + "proctor/internal/app/service/infra/logger" + svcMetadataRepository "proctor/internal/app/service/metadata/repository" + svcSecretRepository "proctor/internal/app/service/secret/repository" +) + +type ExecutionService interface { + Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) + Save(executionContext *model.ExecutionContext) +} + +type executionService struct { + kubernetesClient kubernetes.KubernetesClient + repository repository.ExecutionContextRepository + metadataRepository svcMetadataRepository.MetadataRepository + secretRepository svcSecretRepository.SecretRepository +} + +func NewExecutionService( + kubernetesClient kubernetes.KubernetesClient, + repository repository.ExecutionContextRepository, + metadataRepository svcMetadataRepository.MetadataRepository, + secretRepository svcSecretRepository.SecretRepository, +) ExecutionService { + return &executionService{ + kubernetesClient: kubernetesClient, + repository: repository, + metadataRepository: metadataRepository, + secretRepository: secretRepository, + } +} + +func (service *executionService) Save(executionContext *model.ExecutionContext) { + if executionContext.ExecutionID == 0 { + _, err := service.repository.Insert(executionContext) + logger.LogErrors(err, "save execution context to db ", executionContext) + } else { + context, err := service.repository.GetById(executionContext.ExecutionID) + if err != nil || context == nil { + service.repository.Insert(executionContext) + logger.LogErrors(err, "save execution context to db ", executionContext) + } else { + err = service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) + logger.LogErrors(err, "update execution context status", executionContext) + if len(executionContext.Output) > 0 { + err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) + logger.LogErrors(err, "update execution context output", executionContext) + } + } + } +} + +func (service *executionService) Execute(name string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { + context := &model.ExecutionContext{ + UserEmail: userEmail, + JobName: name, + Args: args, + Status: status.Created, + } + + defer service.Save(context) + + metadata, err := service.metadataRepository.GetByName(name) + if err != nil { + context.Status = status.RequirementNotMet + return context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", name, err.Error())) + } + + secret, err := service.secretRepository.GetByJobName(name) + if err != nil { + context.Status = status.RequirementNotMet + return context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", name, err.Error())) + } + + executionArgs := mergeArgs(args, secret) + + context.Status = status.Created + executionName, err := service.kubernetesClient.ExecuteJob(metadata.ImageName, executionArgs) + logger.Info("Executed Job on Kubernetes got ", executionName, " execution name and ", err, "errors") + if err != nil { + context.Status = status.CreationFailed + return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", name, args, err.Error())) + } + + return context, executionName, nil +} + +func mergeArgs(argsOne, argsTwo map[string]string) map[string]string { + result := make(map[string]string) + + for k, v := range argsOne { + result[k] = v + } + for k, v := range argsTwo { + result[k] = v + } + return result +} diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go new file mode 100644 index 00000000..265d4074 --- /dev/null +++ b/internal/app/service/execution/status/execution.go @@ -0,0 +1,10 @@ +package status + +type ExecutionStatus string + +const ( + Received ExecutionStatus = "RECEIVED" + RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" + Created ExecutionStatus = "CREATED" + CreationFailed ExecutionStatus = "CREATION_FAILED" +) diff --git a/internal/app/service/infra/db/type/base64_map.go b/internal/app/service/infra/db/types/base64_map.go similarity index 98% rename from internal/app/service/infra/db/type/base64_map.go rename to internal/app/service/infra/db/types/base64_map.go index a716d42e..fcc8a47a 100644 --- a/internal/app/service/infra/db/type/base64_map.go +++ b/internal/app/service/infra/db/types/base64_map.go @@ -1,4 +1,4 @@ -package _type +package types import ( "database/sql/driver" diff --git a/internal/app/service/infra/db/type/base64_map_test.go b/internal/app/service/infra/db/types/base64_map_test.go similarity index 97% rename from internal/app/service/infra/db/type/base64_map_test.go rename to internal/app/service/infra/db/types/base64_map_test.go index 18f3cad5..d0102583 100644 --- a/internal/app/service/infra/db/type/base64_map_test.go +++ b/internal/app/service/infra/db/types/base64_map_test.go @@ -1,4 +1,4 @@ -package _type +package types import ( "github.com/stretchr/testify/assert" diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 20512872..7a71e2a7 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -40,10 +40,10 @@ type client struct { } type KubernetesClient interface { - ExecuteJobWithCommand(string, map[string]string, []string) (string, error) - ExecuteJob(string, map[string]string) (string, error) - StreamJobLogs(string) (io.ReadCloser, error) - JobExecutionStatus(string) (string, error) + ExecuteJobWithCommand(imageName string, args map[string]string, commands []string) (string, error) + ExecuteJob(executionName string, args map[string]string) (string, error) + StreamJobLogs(executionName string) (io.ReadCloser, error) + JobExecutionStatus(executionName string) (string, error) } func NewClientSet() (*kubernetes.Clientset, error) { @@ -112,14 +112,14 @@ func uniqueName() string { return "proctor" + "-" + uuid.NewV4().String() } -func jobLabel(jobName string) map[string]string { +func jobLabel(executionName string) map[string]string { return map[string]string{ - "job": jobName, + "job": executionName, } } -func jobLabelSelector(jobName string) string { - return fmt.Sprintf("job=%s", jobName) +func jobLabelSelector(executionName string) string { + return fmt.Sprintf("job=%s", executionName) } func (client *client) ExecuteJob(imageName string, envMap map[string]string) (string, error) { @@ -127,14 +127,14 @@ func (client *client) ExecuteJob(imageName string, envMap map[string]string) (st } func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string]string, command []string) (string, error) { - uniqueJobName := uniqueName() - label := jobLabel(uniqueJobName) + executionName := uniqueName() + label := jobLabel(executionName) batchV1 := client.clientSet.BatchV1() kubernetesJobs := batchV1.Jobs(namespace) container := v1.Container{ - Name: uniqueJobName, + Name: executionName, Image: imageName, Env: getEnvVars(envMap), } @@ -149,7 +149,7 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] } objectMeta := meta.ObjectMeta{ - Name: uniqueJobName, + Name: executionName, Labels: label, Annotations: config.JobPodAnnotations(), } @@ -175,16 +175,16 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] if err != nil { return "", err } - return uniqueJobName, nil + return executionName, nil } -func (client *client) StreamJobLogs(jobName string) (io.ReadCloser, error) { - err := client.waitForReadyJob(jobName) +func (client *client) StreamJobLogs(executionName string) (io.ReadCloser, error) { + err := client.waitForReadyJob(executionName) if err != nil { return nil, err } - pod, err := client.waitForReadyPod(jobName) + pod, err := client.waitForReadyPod(executionName) if err != nil { return nil, err } @@ -198,12 +198,12 @@ func (client *client) StreamJobLogs(jobName string) (io.ReadCloser, error) { return result, nil } -func (client *client) waitForReadyJob(jobName string) error { +func (client *client) waitForReadyJob(executionName string) error { batchV1 := client.clientSet.BatchV1() jobs := batchV1.Jobs(namespace) listOptions := meta.ListOptions{ TypeMeta: typeMeta, - LabelSelector: jobLabelSelector(jobName), + LabelSelector: jobLabelSelector(executionName), } watchJob, err := jobs.Watch(listOptions) @@ -240,12 +240,12 @@ func (client *client) waitForReadyJob(jobName string) error { return fmt.Errorf("job never reach the active status") } -func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { +func (client *client) waitForReadyPod(executionName string) (*v1.Pod, error) { coreV1 := client.clientSet.CoreV1() kubernetesPods := coreV1.Pods(namespace) listOptions := meta.ListOptions{ TypeMeta: typeMeta, - LabelSelector: jobLabelSelector(jobName), + LabelSelector: jobLabelSelector(executionName), } watchJob, err := kubernetesPods.Watch(listOptions) @@ -260,7 +260,7 @@ func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { select { case <-timeoutChan: - return nil,fmt.Errorf("timeout when waiting pod to be available") + return nil, fmt.Errorf("timeout when waiting pod to be available") case <-resultChan: for event := range resultChan { if event.Type == watch.Error { @@ -282,12 +282,12 @@ func (client *client) waitForReadyPod(jobName string) (*v1.Pod, error) { return nil, fmt.Errorf("pod never get the intended state") } -func (client *client) JobExecutionStatus(jobName string) (string, error) { +func (client *client) JobExecutionStatus(executionName string) (string, error) { batchV1 := client.clientSet.BatchV1() kubernetesJobs := batchV1.Jobs(namespace) listOptions := meta.ListOptions{ TypeMeta: typeMeta, - LabelSelector: jobLabelSelector(jobName), + LabelSelector: jobLabelSelector(executionName), } watchJob, err := kubernetesJobs.Watch(listOptions) diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index cd68b5b7..006c4038 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -21,12 +21,12 @@ func (m *MockClient) ExecuteJobWithCommand(jobName string, envMap map[string]str return args.String(0), args.Error(1) } -func (m *MockClient) StreamJobLogs(jobName string) (io.ReadCloser, error) { - args := m.Called(jobName) +func (m *MockClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { + args := m.Called(executionName) return args.Get(0).(*utility.Buffer), args.Error(1) } -func (m *MockClient) JobExecutionStatus(jobExecutionID string) (string, error) { - args := m.Called(jobExecutionID) +func (m *MockClient) JobExecutionStatus(executionName string) (string, error) { + args := m.Called(executionName) return args.String(0), args.Error(1) } diff --git a/internal/app/service/infra/logger/logrus.go b/internal/app/service/infra/logger/logrus.go index 48c64f35..a105c813 100644 --- a/internal/app/service/infra/logger/logrus.go +++ b/internal/app/service/infra/logger/logrus.go @@ -60,3 +60,11 @@ func Fatal(args ...interface{}) { func Panic(args ...interface{}) { log.Panic(args...) } + +func LogErrors(err error, action string, args ...interface{}) { + if err != nil { + logger.Error("Failed to", action, "with errors", err, "and data", args) + } else { + logger.Debug("Success to", action, "with data", args) + } +} diff --git a/internal/app/service/schedule/model/schedule.go b/internal/app/service/schedule/model/schedule.go index 9678d983..e8cea32c 100644 --- a/internal/app/service/schedule/model/schedule.go +++ b/internal/app/service/schedule/model/schedule.go @@ -1,7 +1,7 @@ package model import ( - dbTypes "proctor/internal/app/service/infra/db/type" + dbTypes "proctor/internal/app/service/infra/db/types" "time" ) From ab2424981258a3c889e09381208c8746c5d15f1e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 12 Jul 2019 13:24:35 +0530 Subject: [PATCH 032/268] [Jasoet|Bimozx] V2: Add Mock and test for Executor Service --- go.mod | 2 +- internal/app/proctord/audit/auditor_test.go | 6 +- .../jobs/execution/executioner_test.go | 4 +- .../app/proctord/jobs/logs/handler_test.go | 4 +- .../repository/execution_context_mock.go | 57 ++++++ .../service/execution/service/execution.go | 41 ++-- .../execution/service/execution_mock.go | 19 ++ .../execution/service/execution_test.go | 180 ++++++++++++++++++ .../app/service/infra/kubernetes/client.go | 26 +-- .../service/infra/kubernetes/client_mock.go | 10 +- .../service/infra/kubernetes/client_test.go | 4 +- internal/app/service/infra/logger/logrus.go | 4 +- .../schedule/repository/schedule_mock.go | 65 +++++++ 13 files changed, 373 insertions(+), 49 deletions(-) create mode 100644 internal/app/service/execution/repository/execution_context_mock.go create mode 100644 internal/app/service/execution/service/execution_mock.go create mode 100644 internal/app/service/execution/service/execution_test.go create mode 100644 internal/app/service/schedule/repository/schedule_mock.go diff --git a/go.mod b/go.mod index bfa15b5e..26f60693 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/brianvoe/gofakeit v3.18.0+incompatible github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/docker/docker v1.13.1 // indirect + github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect github.com/evanphx/json-patch v4.4.0+incompatible // indirect diff --git a/internal/app/proctord/audit/auditor_test.go b/internal/app/proctord/audit/auditor_test.go index 7e7d241d..db58a5de 100644 --- a/internal/app/proctord/audit/auditor_test.go +++ b/internal/app/proctord/audit/auditor_test.go @@ -11,7 +11,7 @@ import ( func TestJobsExecutionAuditing(t *testing.T) { mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockClient{} + mockKubeClient := &kubernetes.MockKubernetesClient{} testAuditor := New(mockStore, mockKubeClient) jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ JobName: "any-job-name", @@ -27,7 +27,7 @@ func TestJobsExecutionAuditing(t *testing.T) { func TestAuditJobsExecutionStatusAuditing(t *testing.T) { mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockClient{} + mockKubeClient := &kubernetes.MockKubernetesClient{} testAuditor := New(mockStore, mockKubeClient) jobExecutionID := "job-execution-id" @@ -44,7 +44,7 @@ func TestAuditJobsExecutionStatusAuditing(t *testing.T) { func TestAuditJobsExecutionAndStatusAuditing(t *testing.T) { mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockClient{} + mockKubeClient := &kubernetes.MockKubernetesClient{} testAuditor := New(mockStore, mockKubeClient) jobExecutionID := "job-execution-id" diff --git a/internal/app/proctord/jobs/execution/executioner_test.go b/internal/app/proctord/jobs/execution/executioner_test.go index cd79516f..ee204206 100644 --- a/internal/app/proctord/jobs/execution/executioner_test.go +++ b/internal/app/proctord/jobs/execution/executioner_test.go @@ -17,14 +17,14 @@ import ( type ExecutionerTestSuite struct { suite.Suite - mockKubeClient kubernetes.MockClient + mockKubeClient kubernetes.MockKubernetesClient mockMetadataStore *metadataRepository.MockMetadataRepository mockSecretsStore *secretRepository.MockSecretRepository testExecutioner Executioner } func (suite *ExecutionerTestSuite) SetupTest() { - suite.mockKubeClient = kubernetes.MockClient{} + suite.mockKubeClient = kubernetes.MockKubernetesClient{} suite.mockMetadataStore = &metadataRepository.MockMetadataRepository{} suite.mockSecretsStore = &secretRepository.MockSecretRepository{} suite.testExecutioner = NewExecutioner(&suite.mockKubeClient, suite.mockMetadataStore, suite.mockSecretsStore) diff --git a/internal/app/proctord/jobs/logs/handler_test.go b/internal/app/proctord/jobs/logs/handler_test.go index a9be03ca..ae3dd499 100644 --- a/internal/app/proctord/jobs/logs/handler_test.go +++ b/internal/app/proctord/jobs/logs/handler_test.go @@ -19,11 +19,11 @@ import ( type LoggerTestSuite struct { suite.Suite testLogger Logger - mockKubeClient *kubernetes.MockClient + mockKubeClient *kubernetes.MockKubernetesClient } func (suite *LoggerTestSuite) SetupTest() { - suite.mockKubeClient = &kubernetes.MockClient{} + suite.mockKubeClient = &kubernetes.MockKubernetesClient{} suite.testLogger = NewLogger(suite.mockKubeClient) } diff --git a/internal/app/service/execution/repository/execution_context_mock.go b/internal/app/service/execution/repository/execution_context_mock.go new file mode 100644 index 00000000..9195af14 --- /dev/null +++ b/internal/app/service/execution/repository/execution_context_mock.go @@ -0,0 +1,57 @@ +package repository + +import ( + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/mock" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/status" +) + +type MockExecutionContextRepository struct { + mock.Mock +} + +func (mockRepository *MockExecutionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { + args := mockRepository.Called(context) + return uint64(args.Int(0)), args.Error(1) +} + +func (mockRepository *MockExecutionContextRepository) UpdateJobOutput(executionId uint64, output types.GzippedText) error { + args := mockRepository.Called(executionId, output) + return args.Error(0) +} + +func (mockRepository *MockExecutionContextRepository) UpdateStatus(executionId uint64, status status.ExecutionStatus) error { + args := mockRepository.Called(executionId, status) + return args.Error(0) +} + +func (mockRepository *MockExecutionContextRepository) Delete(executionId uint64) error { + args := mockRepository.Called(executionId) + return args.Error(0) +} + +func (mockRepository *MockExecutionContextRepository) GetById(executionId uint64) (*model.ExecutionContext, error) { + args := mockRepository.Called(executionId) + return args.Get(0).(*model.ExecutionContext), args.Error(1) +} + +func (mockRepository *MockExecutionContextRepository) GetByEmail(userEmail string) ([]model.ExecutionContext, error) { + args := mockRepository.Called(userEmail) + return args.Get(0).([]model.ExecutionContext), args.Error(1) +} + +func (mockRepository *MockExecutionContextRepository) GetByJobName(jobName string) ([]model.ExecutionContext, error) { + args := mockRepository.Called(jobName) + return args.Get(0).([]model.ExecutionContext), args.Error(1) +} + +func (mockRepository *MockExecutionContextRepository) GetByStatus(status string) ([]model.ExecutionContext, error) { + args := mockRepository.Called(status) + return args.Get(0).([]model.ExecutionContext), args.Error(1) +} + +func (mockRepository *MockExecutionContextRepository) deleteAll() error { + args := mockRepository.Called() + return args.Error(0) +} diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 5ea8739d..185d35f5 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -14,7 +14,7 @@ import ( type ExecutionService interface { Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) - Save(executionContext *model.ExecutionContext) + save(executionContext *model.ExecutionContext) error } type executionService struct { @@ -38,56 +38,59 @@ func NewExecutionService( } } -func (service *executionService) Save(executionContext *model.ExecutionContext) { +func (service *executionService) save(executionContext *model.ExecutionContext) error { + var err error if executionContext.ExecutionID == 0 { - _, err := service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db ", executionContext) + _, err = service.repository.Insert(executionContext) + logger.LogErrors(err, "save execution context to db", *executionContext) } else { - context, err := service.repository.GetById(executionContext.ExecutionID) - if err != nil || context == nil { - service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db ", executionContext) + context, _err := service.repository.GetById(executionContext.ExecutionID) + logger.LogErrors(_err, "get context from db by execution id", *executionContext) + if _err != nil || context == nil { + _, err = service.repository.Insert(executionContext) + logger.LogErrors(err, "save execution context to db", *executionContext) } else { err = service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) - logger.LogErrors(err, "update execution context status", executionContext) + logger.LogErrors(err, "update execution context status", *executionContext) if len(executionContext.Output) > 0 { err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) - logger.LogErrors(err, "update execution context output", executionContext) + logger.LogErrors(err, "update execution context output", *executionContext) } } } + return err } -func (service *executionService) Execute(name string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { +func (service *executionService) Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { context := &model.ExecutionContext{ UserEmail: userEmail, - JobName: name, + JobName: jobName, Args: args, Status: status.Created, } - defer service.Save(context) + defer service.save(context) - metadata, err := service.metadataRepository.GetByName(name) + metadata, err := service.metadataRepository.GetByName(jobName) if err != nil { context.Status = status.RequirementNotMet - return context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", name, err.Error())) + return context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", jobName, err.Error())) } - secret, err := service.secretRepository.GetByJobName(name) + secret, err := service.secretRepository.GetByJobName(jobName) if err != nil { context.Status = status.RequirementNotMet - return context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", name, err.Error())) + return context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", jobName, err.Error())) } executionArgs := mergeArgs(args, secret) context.Status = status.Created executionName, err := service.kubernetesClient.ExecuteJob(metadata.ImageName, executionArgs) - logger.Info("Executed Job on Kubernetes got ", executionName, " execution name and ", err, "errors") + logger.Info("Executed Job on Kubernetes got ", executionName, " execution jobName and ", err, "errors") if err != nil { context.Status = status.CreationFailed - return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", name, args, err.Error())) + return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } return context, executionName, nil diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go new file mode 100644 index 00000000..cbf13cb1 --- /dev/null +++ b/internal/app/service/execution/service/execution_mock.go @@ -0,0 +1,19 @@ +package service + +import ( + "github.com/stretchr/testify/mock" + "proctor/internal/app/service/execution/model" +) + +type MockExecutionService struct { + mock.Mock +} + +func (mockService *MockExecutionService) Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { + arguments := mockService.Called(jobName, userEmail, args) + return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) +} + +func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) { + mockService.Called(executionContext) +} diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go new file mode 100644 index 00000000..3af13ddd --- /dev/null +++ b/internal/app/service/execution/service/execution_test.go @@ -0,0 +1,180 @@ +package service + +import ( + fake "github.com/brianvoe/gofakeit" + "github.com/docker/docker/pkg/testutil/assert" + "github.com/jmoiron/sqlx/types" + "github.com/pkg/errors" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/id" + "proctor/internal/app/service/infra/kubernetes" + svcMetadataRepository "proctor/internal/app/service/metadata/repository" + svcSecretRepository "proctor/internal/app/service/secret/repository" + "proctor/internal/pkg/model/metadata" + "testing" +) + +type TestExecutionServiceSuite struct { + suite.Suite + service ExecutionService + mockKubernetesClient *kubernetes.MockKubernetesClient + mockRepository *repository.MockExecutionContextRepository + mockMetadataRepository *svcMetadataRepository.MockMetadataRepository + mockSecretRepository *svcSecretRepository.MockSecretRepository +} + +func (suite *TestExecutionServiceSuite) SetupTest() { + suite.mockKubernetesClient = &kubernetes.MockKubernetesClient{} + suite.mockRepository = &repository.MockExecutionContextRepository{} + suite.mockMetadataRepository = &svcMetadataRepository.MockMetadataRepository{} + suite.mockSecretRepository = &svcSecretRepository.MockSecretRepository{} + suite.service = NewExecutionService( + suite.mockKubernetesClient, + suite.mockRepository, + suite.mockMetadataRepository, + suite.mockSecretRepository, + ) +} + +func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { + t := suite.T() + context := &model.ExecutionContext{} + + suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() + err := suite.service.save(context) + assert.Error(t, err, "Insert Failed") + + suite.mockRepository.On("Insert", context).Return(0, nil).Once() + err = suite.service.save(context) + assert.NilError(t, err) +} + +func (suite *TestExecutionServiceSuite) TestSaveWithExecutionId() { + t := suite.T() + id, _ := id.NextId() + context := &model.ExecutionContext{ + ExecutionID: id, + Status: status.Created, + } + + suite.mockRepository.On("GetById", id).Return(context, errors.New("Get By Id Error")).Once() + suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() + err := suite.service.save(context) + assert.Error(t, err, "Insert Failed") + + suite.mockRepository.On("GetById", id).Return(context, nil).Once() + suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(errors.New("Update Status Failed")).Once() + err = suite.service.save(context) + assert.Error(t, err, "Update Status Failed") + + context.Output = types.GzippedText("This is some output") + suite.mockRepository.On("GetById", id).Return(context, nil).Once() + suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(nil).Once() + suite.mockRepository.On("UpdateJobOutput", context.ExecutionID, context.Output).Return(errors.New("Update Output Failed")).Once() + err = suite.service.save(context) + assert.Error(t, err, "Update Output Failed") +} + +func (suite *TestExecutionServiceSuite) TestExecuteMetadataNotFound() { + t := suite.T() + jobName := fake.Username() + userEmail := fake.Email() + mapKey := fake.FirstName() + mapValue := fake.LastName() + + jobArgs := map[string]string{ + mapKey: mapValue, + } + + suite.mockMetadataRepository.On("GetByName", jobName).Return(&metadata.Metadata{}, errors.New("metadataNotFound")).Once() + suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() + + context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) + + assert.Error(t, err, "metadata not found") + assert.NotNil(t, context) + assert.Equal(t, context.Status, status.RequirementNotMet) +} + +func (suite *TestExecutionServiceSuite) TestExecuteSecretNotFound() { + t := suite.T() + jobName := fake.Username() + userEmail := fake.Email() + mapKey := fake.FirstName() + mapValue := fake.LastName() + + jobArgs := map[string]string{ + mapKey: mapValue, + } + + suite.mockMetadataRepository.On("GetByName", jobName).Return(&metadata.Metadata{}, nil).Once() + suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, errors.New("secret not found")).Once() + suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() + + context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) + assert.Error(t, err, "secret not found") + assert.NotNil(t, context) + assert.Equal(t, context.Status, status.RequirementNotMet) +} + +func (suite *TestExecutionServiceSuite) TestExecuteJobFailed() { + t := suite.T() + jobName := fake.Username() + userEmail := fake.Email() + mapKey := fake.FirstName() + mapValue := fake.LastName() + + jobArgs := map[string]string{ + mapKey: mapValue, + } + + imageName := fake.BeerYeast() + fakeMetadata := &metadata.Metadata{ + ImageName: imageName, + } + + suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() + suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() + suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() + suite.mockKubernetesClient.On("ExecuteJob", imageName, jobArgs).Return("", errors.New("Execution Failed")) + + context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) + assert.Error(t, err, "error when executing image") + assert.NotNil(t, context) + assert.Equal(t, context.Status, status.CreationFailed) +} + +func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { + t := suite.T() + jobName := fake.Username() + userEmail := fake.Email() + mapKey := fake.FirstName() + mapValue := fake.LastName() + + jobArgs := map[string]string{ + mapKey: mapValue, + } + + imageName := fake.BeerYeast() + fakeMetadata := &metadata.Metadata{ + ImageName: imageName, + } + + suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() + suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() + suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() + suite.mockKubernetesClient.On("ExecuteJob", imageName, jobArgs).Return("", nil) + + context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) + assert.NilError(t, err) + assert.NotNil(t, context) + assert.Equal(t, context.Status, status.Created) +} + +func TestExecutionServiceSuiteTest(t *testing.T) { + suite.Run(t, new(TestExecutionServiceSuite)) +} diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 7a71e2a7..c14ec40b 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -34,11 +34,6 @@ func init() { namespace = config.DefaultNamespace() } -type client struct { - clientSet kubernetes.Interface - httpClient *http.Client -} - type KubernetesClient interface { ExecuteJobWithCommand(imageName string, args map[string]string, commands []string) (string, error) ExecuteJob(executionName string, args map[string]string) (string, error) @@ -46,6 +41,11 @@ type KubernetesClient interface { JobExecutionStatus(executionName string) (string, error) } +type kubernetesClient struct { + clientSet kubernetes.Interface + httpClient *http.Client +} + func NewClientSet() (*kubernetes.Clientset, error) { var kubeConfig *kubeRestClient.Config if config.KubeConfig() == "out-of-cluster" { @@ -83,7 +83,7 @@ func NewClientSet() (*kubernetes.Clientset, error) { } func NewKubernetesClient(httpClient *http.Client) KubernetesClient { - newClient := &client{ + newClient := &kubernetesClient{ httpClient: httpClient, } @@ -122,11 +122,11 @@ func jobLabelSelector(executionName string) string { return fmt.Sprintf("job=%s", executionName) } -func (client *client) ExecuteJob(imageName string, envMap map[string]string) (string, error) { +func (client *kubernetesClient) ExecuteJob(imageName string, envMap map[string]string) (string, error) { return client.ExecuteJobWithCommand(imageName, envMap, []string{}) } -func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string]string, command []string) (string, error) { +func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap map[string]string, command []string) (string, error) { executionName := uniqueName() label := jobLabel(executionName) @@ -178,7 +178,7 @@ func (client *client) ExecuteJobWithCommand(imageName string, envMap map[string] return executionName, nil } -func (client *client) StreamJobLogs(executionName string) (io.ReadCloser, error) { +func (client *kubernetesClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { err := client.waitForReadyJob(executionName) if err != nil { return nil, err @@ -198,7 +198,7 @@ func (client *client) StreamJobLogs(executionName string) (io.ReadCloser, error) return result, nil } -func (client *client) waitForReadyJob(executionName string) error { +func (client *kubernetesClient) waitForReadyJob(executionName string) error { batchV1 := client.clientSet.BatchV1() jobs := batchV1.Jobs(namespace) listOptions := meta.ListOptions{ @@ -240,7 +240,7 @@ func (client *client) waitForReadyJob(executionName string) error { return fmt.Errorf("job never reach the active status") } -func (client *client) waitForReadyPod(executionName string) (*v1.Pod, error) { +func (client *kubernetesClient) waitForReadyPod(executionName string) (*v1.Pod, error) { coreV1 := client.clientSet.CoreV1() kubernetesPods := coreV1.Pods(namespace) listOptions := meta.ListOptions{ @@ -282,7 +282,7 @@ func (client *client) waitForReadyPod(executionName string) (*v1.Pod, error) { return nil, fmt.Errorf("pod never get the intended state") } -func (client *client) JobExecutionStatus(executionName string) (string, error) { +func (client *kubernetesClient) JobExecutionStatus(executionName string) (string, error) { batchV1 := client.clientSet.BatchV1() kubernetesJobs := batchV1.Jobs(namespace) listOptions := meta.ListOptions{ @@ -316,7 +316,7 @@ func (client *client) JobExecutionStatus(executionName string) (string, error) { return constant.NoDefinitiveJobExecutionStatusFound, nil } -func (client *client) getPodLogs(pod *v1.Pod) (io.ReadCloser, error) { +func (client *kubernetesClient) getPodLogs(pod *v1.Pod) (io.ReadCloser, error) { logger.Debug("reading pod logs for: ", pod.Name) podLogOpts := v1.PodLogOptions{ Follow: true, diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index 006c4038..f6abf58b 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -7,26 +7,26 @@ import ( "proctor/internal/pkg/utility" ) -type MockClient struct { +type MockKubernetesClient struct { mock.Mock } -func (m *MockClient) ExecuteJob(jobName string, envMap map[string]string) (string, error) { +func (m *MockKubernetesClient) ExecuteJob(jobName string, envMap map[string]string) (string, error) { args := m.Called(jobName, envMap) return args.String(0), args.Error(1) } -func (m *MockClient) ExecuteJobWithCommand(jobName string, envMap map[string]string, command []string) (string, error) { +func (m *MockKubernetesClient) ExecuteJobWithCommand(jobName string, envMap map[string]string, command []string) (string, error) { args := m.Called(jobName, envMap) return args.String(0), args.Error(1) } -func (m *MockClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { +func (m *MockKubernetesClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { args := m.Called(executionName) return args.Get(0).(*utility.Buffer), args.Error(1) } -func (m *MockClient) JobExecutionStatus(executionName string) (string, error) { +func (m *MockKubernetesClient) JobExecutionStatus(executionName string) (string, error) { args := m.Called(executionName) return args.String(0), args.Error(1) } diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 99f8ecb7..f8c32d1f 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -33,7 +33,7 @@ type ClientTestSuite struct { func (suite *ClientTestSuite) SetupTest() { suite.fakeClientSet = fakeclientset.NewSimpleClientset() - suite.testClient = &client{ + suite.testClient = &kubernetesClient{ clientSet: suite.fakeClientSet, } suite.jobName = "job1" @@ -58,7 +58,7 @@ func (suite *ClientTestSuite) SetupTest() { }) suite.fakeHttpClient = &http.Client{} - suite.testClientStreaming = &client{ + suite.testClientStreaming = &kubernetesClient{ clientSet: suite.fakeClientSetStreaming, httpClient: suite.fakeHttpClient, } diff --git a/internal/app/service/infra/logger/logrus.go b/internal/app/service/infra/logger/logrus.go index a105c813..6adb0b39 100644 --- a/internal/app/service/infra/logger/logrus.go +++ b/internal/app/service/infra/logger/logrus.go @@ -63,8 +63,8 @@ func Panic(args ...interface{}) { func LogErrors(err error, action string, args ...interface{}) { if err != nil { - logger.Error("Failed to", action, "with errors", err, "and data", args) + Error("Failed to ", action, " with errors ", err, " and data ", args) } else { - logger.Debug("Success to", action, "with data", args) + Debug("Success to ", action, " with data ", args) } } diff --git a/internal/app/service/schedule/repository/schedule_mock.go b/internal/app/service/schedule/repository/schedule_mock.go new file mode 100644 index 00000000..b1fd5848 --- /dev/null +++ b/internal/app/service/schedule/repository/schedule_mock.go @@ -0,0 +1,65 @@ +package repository + +import ( + "github.com/stretchr/testify/mock" + "proctor/internal/app/service/schedule/model" +) + +type MockScheduleRepository struct { + mock.Mock +} + +func (repository *MockScheduleRepository) Insert(context *model.Schedule) (uint64, error) { + args := repository.Called(context) + return uint64(args.Int(0)), args.Error(1) +} + +func (repository *MockScheduleRepository) Delete(id uint64) error { + args := repository.Called(id) + return args.Error(0) +} + +func (repository *MockScheduleRepository) GetById(id uint64) (*model.Schedule, error) { + args := repository.Called(id) + return args.Get(0).(*model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) Disable(id uint64) error { + args := repository.Called(id) + return args.Error(0) +} + +func (repository *MockScheduleRepository) Enable(id uint64) error { + args := repository.Called(id) + return args.Error(0) +} + +func (repository *MockScheduleRepository) GetByUserEmail(userEmail string) ([]model.Schedule, error) { + args := repository.Called(userEmail) + return args.Get(0).([]model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) GetByJobName(jobName string) ([]model.Schedule, error) { + args := repository.Called(jobName) + return args.Get(0).([]model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) GetAllEnabled() ([]model.Schedule, error) { + args := repository.Called() + return args.Get(0).([]model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) GetAll() ([]model.Schedule, error) { + args := repository.Called() + return args.Get(0).([]model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) GetEnabledById(id uint64) (*model.Schedule, error) { + args := repository.Called(id) + return args.Get(0).(*model.Schedule), args.Error(1) +} + +func (repository *MockScheduleRepository) deleteAll() error { + args := repository.Called() + return args.Error(0) +} From 4b5aacf1d21a9a733154d5d6b802ee2b82c3de7a Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 15 Jul 2019 11:50:29 +0530 Subject: [PATCH 033/268] [Jasoet|Bimozx] V2: Finish Watch pod Status --- internal/app/proctord/jobs/logs/handler.go | 4 +- .../app/proctord/jobs/logs/handler_test.go | 8 +-- .../service/execution/service/execution.go | 51 +++++++++++++++++++ .../app/service/execution/status/execution.go | 6 +++ .../app/service/infra/kubernetes/client.go | 27 ++++++---- .../kubernetes/client_integration_test.go | 4 +- .../service/infra/kubernetes/client_mock.go | 19 ++++++- .../service/infra/kubernetes/client_test.go | 4 +- 8 files changed, 102 insertions(+), 21 deletions(-) diff --git a/internal/app/proctord/jobs/logs/handler.go b/internal/app/proctord/jobs/logs/handler.go index 2ebd49af..33fd54a0 100644 --- a/internal/app/proctord/jobs/logs/handler.go +++ b/internal/app/proctord/jobs/logs/handler.go @@ -9,6 +9,7 @@ import ( "proctor/internal/app/service/infra/kubernetes" _logger "proctor/internal/app/service/infra/logger" "strings" + "time" "proctor/internal/pkg/constant" @@ -63,7 +64,8 @@ func (l *logger) Stream() http.HandlerFunc { return } - logStream, err := l.kubeClient.StreamJobLogs(jobName) + waitTime := config.KubePodsListWaitTime() * time.Second + logStream, err := l.kubeClient.StreamJobLogs(jobName, waitTime) if err != nil { _logger.Error("Error streaming logs from kube client: ", err) raven.CaptureError(err, map[string]string{"job_name": jobName}) diff --git a/internal/app/proctord/jobs/logs/handler_test.go b/internal/app/proctord/jobs/logs/handler_test.go index ae3dd499..222f5ff3 100644 --- a/internal/app/proctord/jobs/logs/handler_test.go +++ b/internal/app/proctord/jobs/logs/handler_test.go @@ -62,7 +62,7 @@ func (suite *LoggerTestSuite) TestLoggerStream() { buffer := utility.NewBuffer() _, _ = buffer.Write([]byte("first line\nsecond line\n")) - suite.mockKubeClient.On("StreamJobLogs", "sample").Return(buffer, nil).Once() + suite.mockKubeClient.On("StreamJobLogs", "sample", mock.Anything).Return(buffer, nil).Once() c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) assert.NoError(t, err) @@ -93,7 +93,7 @@ func (suite *LoggerTestSuite) TestLoggerStreamConnectionUpgradeFailure() { suite.testLogger.Stream()(responseRecorder, req) - suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything) + suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything, mock.Anything) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) assert.Equal(t, "Bad Request\n"+constant.ClientError, responseRecorder.Body.String()) @@ -109,7 +109,7 @@ func (suite *LoggerTestSuite) TestLoggerStreamForNoJobName() { assert.NoError(t, err) defer c.Close() - suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything) + suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything, mock.Anything) _, finalMessage, err := c.ReadMessage() assert.Error(t, err) @@ -123,7 +123,7 @@ func (suite *LoggerTestSuite) TestLoggerStreamKubeClientFailure() { s := suite.newServer() defer s.Close() - suite.mockKubeClient.On("StreamJobLogs", "sample").Return(&utility.Buffer{}, errors.New("error")).Once() + suite.mockKubeClient.On("StreamJobLogs", "sample", mock.Anything).Return(&utility.Buffer{}, errors.New("error")).Once() c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) assert.NoError(t, err) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 185d35f5..5df0b968 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -1,8 +1,11 @@ package service import ( + "bufio" + "bytes" "errors" "fmt" + "github.com/jmoiron/sqlx/types" "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" @@ -10,6 +13,7 @@ import ( "proctor/internal/app/service/infra/logger" svcMetadataRepository "proctor/internal/app/service/metadata/repository" svcSecretRepository "proctor/internal/app/service/secret/repository" + "time" ) type ExecutionService interface { @@ -93,9 +97,56 @@ func (service *executionService) Execute(jobName string, userEmail string, args return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } + service.watchProcess(executionName, context) + return context, executionName, nil } +func (service *executionService) watchProcess(executionName string, context *model.ExecutionContext) { + waitTime := 15 * time.Minute + err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) + + if err != nil { + context.Status = status.JobCreationFailed + return + } + + context.Status = status.JobReady + logger.Info("Job Ready for ", context.ExecutionID) + + pod, err := service.kubernetesClient.WaitForReadyPod(executionName, waitTime) + if err != nil { + context.Status = status.PodCreationFailed + return + } + + context.Status = status.PodReady + logger.Info("Job Ready for ", context.ExecutionID) + + podLog, err := service.kubernetesClient.GetPodLogs(pod) + if err != nil { + context.Status = status.FetchPodLogFailed + return + } + + scanner := bufio.NewScanner(podLog) + scanner.Split(bufio.ScanLines) + + var buffer bytes.Buffer + for scanner.Scan() { + buffer.WriteString(scanner.Text()) + } + + output := types.GzippedText(buffer.Bytes()) + + context.Output = output + logger.Info("Execution Output Produced ", context.ExecutionID, " with length ", len(output)) + + context.Status = status.Finished + + return +} + func mergeArgs(argsOne, argsTwo map[string]string) map[string]string { result := make(map[string]string) diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go index 265d4074..2f596f8d 100644 --- a/internal/app/service/execution/status/execution.go +++ b/internal/app/service/execution/status/execution.go @@ -7,4 +7,10 @@ const ( RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" Created ExecutionStatus = "CREATED" CreationFailed ExecutionStatus = "CREATION_FAILED" + JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" + JobReady ExecutionStatus = "JOB_READY" + PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" + PodReady ExecutionStatus = "POD_READY" + FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" + Finished ExecutionStatus = "FINISHED" ) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index c14ec40b..adee55dc 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -23,6 +23,8 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +const JobFailed = int32(0) + var typeMeta meta.TypeMeta var namespace string @@ -36,9 +38,12 @@ func init() { type KubernetesClient interface { ExecuteJobWithCommand(imageName string, args map[string]string, commands []string) (string, error) - ExecuteJob(executionName string, args map[string]string) (string, error) - StreamJobLogs(executionName string) (io.ReadCloser, error) + ExecuteJob(imageName string, args map[string]string) (string, error) + StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) JobExecutionStatus(executionName string) (string, error) + WaitForReadyJob(executionName string, waitTime time.Duration) error + WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) + GetPodLogs(pod *v1.Pod) (io.ReadCloser, error) } type kubernetesClient struct { @@ -178,18 +183,18 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m return executionName, nil } -func (client *kubernetesClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { - err := client.waitForReadyJob(executionName) +func (client *kubernetesClient) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { + err := client.WaitForReadyJob(executionName, waitTime) if err != nil { return nil, err } - pod, err := client.waitForReadyPod(executionName) + pod, err := client.WaitForReadyPod(executionName, waitTime) if err != nil { return nil, err } - result, err := client.getPodLogs(pod) + result, err := client.GetPodLogs(pod) if err != nil { return nil, err @@ -198,7 +203,7 @@ func (client *kubernetesClient) StreamJobLogs(executionName string) (io.ReadClos return result, nil } -func (client *kubernetesClient) waitForReadyJob(executionName string) error { +func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime time.Duration) error { batchV1 := client.clientSet.BatchV1() jobs := batchV1.Jobs(namespace) listOptions := meta.ListOptions{ @@ -211,7 +216,7 @@ func (client *kubernetesClient) waitForReadyJob(executionName string) error { return err } - timeoutChan := time.After(config.KubePodsListWaitTime() * time.Second) + timeoutChan := time.After(waitTime) resultChan := watchJob.ResultChan() defer watchJob.Stop() @@ -240,7 +245,7 @@ func (client *kubernetesClient) waitForReadyJob(executionName string) error { return fmt.Errorf("job never reach the active status") } -func (client *kubernetesClient) waitForReadyPod(executionName string) (*v1.Pod, error) { +func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) { coreV1 := client.clientSet.CoreV1() kubernetesPods := coreV1.Pods(namespace) listOptions := meta.ListOptions{ @@ -253,7 +258,7 @@ func (client *kubernetesClient) waitForReadyPod(executionName string) (*v1.Pod, return nil, err } - timeoutChan := time.After(config.KubePodsListWaitTime() * time.Second) + timeoutChan := time.After(waitTime) resultChan := watchJob.ResultChan() defer watchJob.Stop() var pod *v1.Pod @@ -316,7 +321,7 @@ func (client *kubernetesClient) JobExecutionStatus(executionName string) (string return constant.NoDefinitiveJobExecutionStatusFound, nil } -func (client *kubernetesClient) getPodLogs(pod *v1.Pod) (io.ReadCloser, error) { +func (client *kubernetesClient) GetPodLogs(pod *v1.Pod) (io.ReadCloser, error) { logger.Debug("reading pod logs for: ", pod.Name) podLogOpts := v1.PodLogOptions{ Follow: true, diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 8531ec34..d3dd0858 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -12,6 +12,7 @@ import ( kubeHttpClient "proctor/internal/app/service/infra/kubernetes/http" "proctor/internal/pkg/constant" "testing" + "time" ) type IntegrationTestSuite struct { @@ -97,7 +98,8 @@ func (suite *IntegrationTestSuite) TestStreamLogsSuccess() { executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) assert.NoError(t, err) - logStream, err := suite.testClient.StreamJobLogs(executedJobname) + waitTime := config.KubePodsListWaitTime() * time.Second + logStream, err := suite.testClient.StreamJobLogs(executedJobname, waitTime) assert.NoError(t, err) defer logStream.Close() diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index f6abf58b..44eca538 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -2,6 +2,8 @@ package kubernetes import ( "io" + v1 "k8s.io/api/core/v1" + "time" "github.com/stretchr/testify/mock" "proctor/internal/pkg/utility" @@ -21,8 +23,8 @@ func (m *MockKubernetesClient) ExecuteJobWithCommand(jobName string, envMap map[ return args.String(0), args.Error(1) } -func (m *MockKubernetesClient) StreamJobLogs(executionName string) (io.ReadCloser, error) { - args := m.Called(executionName) +func (m *MockKubernetesClient) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { + args := m.Called(executionName, waitTime) return args.Get(0).(*utility.Buffer), args.Error(1) } @@ -30,3 +32,16 @@ func (m *MockKubernetesClient) JobExecutionStatus(executionName string) (string, args := m.Called(executionName) return args.String(0), args.Error(1) } + +func (m *MockKubernetesClient) WaitForReadyJob(executionName string, waitTime time.Duration) error { + args := m.Called(executionName, waitTime) + return args.Error(0) +} +func (m *MockKubernetesClient) WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) { + args := m.Called(executionName) + return args.Get(0).(*v1.Pod), args.Error(1) +} +func (m *MockKubernetesClient) GetPodLogs(pod *v1.Pod) (io.ReadCloser, error) { + args := m.Called(pod) + return args.Get(0).(io.ReadCloser), args.Error(1) +} diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index f8c32d1f..35521fea 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -114,11 +114,11 @@ func (suite *ClientTestSuite) TestJobExecution() { func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { t := suite.T() - _, err := suite.testClientStreaming.StreamJobLogs("unknown-job") + waitTime := config.KubePodsListWaitTime() * time.Second + _, err := suite.testClientStreaming.StreamJobLogs("unknown-job", waitTime) assert.Error(t, err) } - func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { t := suite.T() From d3891a0c3408f0dd3a1489d57f7e637abaf557e6 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 15 Jul 2019 12:29:25 +0530 Subject: [PATCH 034/268] [Jasoet|Bimozx] V2: Fix test after adding watch pod --- .../service/execution/service/execution.go | 7 +++- .../service/execution_integration_test.go | 42 +++++++++++++++++++ .../execution/service/execution_mock.go | 5 +++ .../execution/service/execution_test.go | 26 ++++++++---- .../service/infra/kubernetes/client_mock.go | 2 +- 5 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 internal/app/service/execution/service/execution_integration_test.go diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 5df0b968..35608e2d 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -18,6 +18,7 @@ import ( type ExecutionService interface { Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) + ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) save(executionContext *model.ExecutionContext) error } @@ -66,6 +67,10 @@ func (service *executionService) save(executionContext *model.ExecutionContext) } func (service *executionService) Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { + return service.ExecuteWithCommand(jobName, userEmail, args, []string{}) +} + +func (service *executionService) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) { context := &model.ExecutionContext{ UserEmail: userEmail, JobName: jobName, @@ -90,7 +95,7 @@ func (service *executionService) Execute(jobName string, userEmail string, args executionArgs := mergeArgs(args, secret) context.Status = status.Created - executionName, err := service.kubernetesClient.ExecuteJob(metadata.ImageName, executionArgs) + executionName, err := service.kubernetesClient.ExecuteJobWithCommand(metadata.ImageName, executionArgs, commands) logger.Info("Executed Job on Kubernetes got ", executionName, " execution jobName and ", err, "errors") if err != nil { context.Status = status.CreationFailed diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go new file mode 100644 index 00000000..91528c88 --- /dev/null +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -0,0 +1,42 @@ +package service + +import ( + "github.com/stretchr/testify/suite" + "os" + "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/infra/kubernetes" + "proctor/internal/app/service/infra/kubernetes/http" + svcMetadataRepository "proctor/internal/app/service/metadata/repository" + svcSecretRepository "proctor/internal/app/service/secret/repository" + "testing" +) + +type TestExecutionIntegrationSuite struct { + suite.Suite + service ExecutionService + mockKubernetesClient kubernetes.KubernetesClient + mockRepository *repository.MockExecutionContextRepository + mockMetadataRepository *svcMetadataRepository.MockMetadataRepository + mockSecretRepository *svcSecretRepository.MockSecretRepository +} + +func (suite *TestExecutionIntegrationSuite) SetupTest() { + httpClient, _ := http.NewClient() + suite.mockKubernetesClient = kubernetes.NewKubernetesClient(httpClient) + suite.mockRepository = &repository.MockExecutionContextRepository{} + suite.mockMetadataRepository = &svcMetadataRepository.MockMetadataRepository{} + suite.mockSecretRepository = &svcSecretRepository.MockSecretRepository{} + suite.service = NewExecutionService( + suite.mockKubernetesClient, + suite.mockRepository, + suite.mockMetadataRepository, + suite.mockSecretRepository, + ) +} + +func TestExecutionIntegrationSuiteTest(t *testing.T) { + value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") + if available == true && value == "true" { + suite.Run(t, new(TestExecutionIntegrationSuite)) + } +} diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index cbf13cb1..d1e0a2a8 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -14,6 +14,11 @@ func (mockService *MockExecutionService) Execute(jobName string, userEmail strin return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) } +func (mockService *MockExecutionService) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) { + arguments := mockService.Called(jobName, userEmail, args, commands) + return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) +} + func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) { mockService.Called(executionContext) } diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 3af13ddd..6a52a5b1 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -7,6 +7,8 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + "io/ioutil" + v1 "k8s.io/api/core/v1" "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" @@ -15,6 +17,7 @@ import ( svcMetadataRepository "proctor/internal/app/service/metadata/repository" svcSecretRepository "proctor/internal/app/service/secret/repository" "proctor/internal/pkg/model/metadata" + "strings" "testing" ) @@ -55,24 +58,24 @@ func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { func (suite *TestExecutionServiceSuite) TestSaveWithExecutionId() { t := suite.T() - id, _ := id.NextId() + _id, _ := id.NextId() context := &model.ExecutionContext{ - ExecutionID: id, + ExecutionID: _id, Status: status.Created, } - suite.mockRepository.On("GetById", id).Return(context, errors.New("Get By Id Error")).Once() + suite.mockRepository.On("GetById", _id).Return(context, errors.New("Get By Id Error")).Once() suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() err := suite.service.save(context) assert.Error(t, err, "Insert Failed") - suite.mockRepository.On("GetById", id).Return(context, nil).Once() + suite.mockRepository.On("GetById", _id).Return(context, nil).Once() suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(errors.New("Update Status Failed")).Once() err = suite.service.save(context) assert.Error(t, err, "Update Status Failed") context.Output = types.GzippedText("This is some output") - suite.mockRepository.On("GetById", id).Return(context, nil).Once() + suite.mockRepository.On("GetById", _id).Return(context, nil).Once() suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(nil).Once() suite.mockRepository.On("UpdateJobOutput", context.ExecutionID, context.Output).Return(errors.New("Update Output Failed")).Once() err = suite.service.save(context) @@ -140,7 +143,7 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobFailed() { suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() - suite.mockKubernetesClient.On("ExecuteJob", imageName, jobArgs).Return("", errors.New("Execution Failed")) + suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, jobArgs, []string{}).Return("", errors.New("Execution Failed")) context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) assert.Error(t, err, "error when executing image") @@ -167,12 +170,19 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() - suite.mockKubernetesClient.On("ExecuteJob", imageName, jobArgs).Return("", nil) + + executionName := "execution-name" + suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, jobArgs, []string{}).Return(executionName, nil) + suite.mockKubernetesClient.On("WaitForReadyJob", executionName, mock.Anything).Return(nil) + podDetail := &v1.Pod{} + suite.mockKubernetesClient.On("WaitForReadyPod", executionName, mock.Anything).Return(podDetail, nil) + mockLog := ioutil.NopCloser(strings.NewReader("hello world")) + suite.mockKubernetesClient.On("GetPodLogs", podDetail).Return(mockLog, nil) context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) assert.NilError(t, err) assert.NotNil(t, context) - assert.Equal(t, context.Status, status.Created) + assert.Equal(t, context.Status, status.Finished) } func TestExecutionServiceSuiteTest(t *testing.T) { diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index 44eca538..da8d2e0c 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -19,7 +19,7 @@ func (m *MockKubernetesClient) ExecuteJob(jobName string, envMap map[string]stri } func (m *MockKubernetesClient) ExecuteJobWithCommand(jobName string, envMap map[string]string, command []string) (string, error) { - args := m.Called(jobName, envMap) + args := m.Called(jobName, envMap, command) return args.String(0), args.Error(1) } From b00f5ea201aa1ce78aa3862717207a3746903b88 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 15 Jul 2019 15:59:39 +0530 Subject: [PATCH 035/268] [Jasoet|Bimozx] V2: Add integration test for execution service --- .env.test | 9 +-- .env.v2.test | 35 ----------- .../service/execution/service/execution.go | 23 ++++++-- .../service/execution_integration_test.go | 58 ++++++++++++++++++- .../app/service/execution/status/execution.go | 1 + internal/app/service/infra/config/config.go | 6 +- .../app/service/infra/kubernetes/client.go | 44 +++++--------- 7 files changed, 96 insertions(+), 80 deletions(-) delete mode 100644 .env.v2.test diff --git a/.env.test b/.env.test index 16837d39..8bc21b9e 100644 --- a/.env.test +++ b/.env.test @@ -1,5 +1,6 @@ export ENVIRONMENT=test export PROCTOR_KUBE_CONFIG=out-of-cluster +export PROCTOR_KUBE_CONTEXT=minikube export PROCTOR_LOG_LEVEL=debug export PROCTOR_APP_PORT=5000 export PROCTOR_DEFAULT_NAMESPACE=default @@ -9,9 +10,10 @@ export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 export PROCTOR_KUBE_JOB_RETRIES=0 export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 -export PROCTOR_KUBE_CLUSTER_HOST_NAME=localhost:8001 -export PROCTOR_KUBE_POD_LIST_WAIT_TIME=5 -export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K +export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 +export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 +export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60 +export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== export PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK export PROCTOR_POSTGRES_USER=postgres export PROCTOR_POSTGRES_PASSWORD= @@ -31,4 +33,3 @@ export PROCTOR_MAIL_SERVER_PORT="123" export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" export PROCTOR_SENTRY_DSN="foo" export PROCTOR_DOCS_PATH="/path/to/docs/dir" - diff --git a/.env.v2.test b/.env.v2.test deleted file mode 100644 index dd44cad2..00000000 --- a/.env.v2.test +++ /dev/null @@ -1,35 +0,0 @@ -ENVIRONMENT=test -PROCTOR_KUBE_CONFIG=out-of-cluster -PROCTOR_KUBE_CONTEXT=minikube -PROCTOR_LOG_LEVEL=debug -PROCTOR_APP_PORT=5000 -PROCTOR_DEFAULT_NAMESPACE=default -PROCTOR_REDIS_ADDRESS=localhost:6379 -PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS=10 -PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 -PROCTOR_KUBE_JOB_RETRIES=0 -PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 -PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 -PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 -PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 -PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== -PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK -PROCTOR_POSTGRES_USER=postgres -PROCTOR_POSTGRES_PASSWORD= -PROCTOR_POSTGRES_HOST=localhost -PROCTOR_POSTGRES_PORT=5432 -PROCTOR_POSTGRES_DATABASE=proctord_test -PROCTOR_POSTGRES_MAX_CONNECTIONS=50 -PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 -PROCTOR_NEW_RELIC_APP_NAME="PROCTORD" -PROCTOR_NEW_RELIC_LICENCE_KEY="nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr" -PROCTOR_MIN_CLIENT_VERSION="v0.2.0" -PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS="5" -PROCTOR_MAIL_USERNAME="user@mail.com" -PROCTOR_MAIL_PASSWORD="password" -PROCTOR_MAIL_SERVER_HOST="smtp.mail.com" -PROCTOR_MAIL_SERVER_PORT="123" -PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" -PROCTOR_SENTRY_DSN="foo" -PROCTOR_DOCS_PATH="/path/to/docs/dir" -ENABLE_INTEGRATION_TEST=false diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 35608e2d..db63528d 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "github.com/jmoiron/sqlx/types" + v1 "k8s.io/api/core/v1" "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/logger" svcMetadataRepository "proctor/internal/app/service/metadata/repository" @@ -102,13 +104,15 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } - service.watchProcess(executionName, context) + go service.watchProcess(executionName, context) return context, executionName, nil } func (service *executionService) watchProcess(executionName string, context *model.ExecutionContext) { - waitTime := 15 * time.Minute + defer service.save(context) + + waitTime := config.KubeLogProcessWaitTime() * time.Second err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) if err != nil { @@ -125,8 +129,13 @@ func (service *executionService) watchProcess(executionName string, context *mod return } - context.Status = status.PodReady - logger.Info("Job Ready for ", context.ExecutionID) + if pod.Status.Phase == v1.PodFailed { + context.Status = status.PodFailed + logger.Info("Pod Failed for ", context.ExecutionID, " reason: ", pod.Status.Reason, " message: ", pod.Status.Message) + } else { + context.Status = status.PodReady + logger.Info("Pod Ready for ", context.ExecutionID) + } podLog, err := service.kubernetesClient.GetPodLogs(pod) if err != nil { @@ -139,7 +148,7 @@ func (service *executionService) watchProcess(executionName string, context *mod var buffer bytes.Buffer for scanner.Scan() { - buffer.WriteString(scanner.Text()) + buffer.WriteString(scanner.Text() + "\n") } output := types.GzippedText(buffer.Bytes()) @@ -147,7 +156,9 @@ func (service *executionService) watchProcess(executionName string, context *mod context.Output = output logger.Info("Execution Output Produced ", context.ExecutionID, " with length ", len(output)) - context.Status = status.Finished + if context.Status == status.PodReady { + context.Status = status.Finished + } return } diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 91528c88..1638cfe8 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -1,21 +1,28 @@ package service import ( + fake "github.com/brianvoe/gofakeit" + "github.com/docker/docker/pkg/testutil/assert" "github.com/stretchr/testify/suite" "os" "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/kubernetes/http" svcMetadataRepository "proctor/internal/app/service/metadata/repository" svcSecretRepository "proctor/internal/app/service/secret/repository" + "proctor/internal/pkg/model/metadata" + "proctor/internal/pkg/model/metadata/env" "testing" + "time" ) type TestExecutionIntegrationSuite struct { suite.Suite service ExecutionService mockKubernetesClient kubernetes.KubernetesClient - mockRepository *repository.MockExecutionContextRepository + repository repository.ExecutionContextRepository mockMetadataRepository *svcMetadataRepository.MockMetadataRepository mockSecretRepository *svcSecretRepository.MockSecretRepository } @@ -23,17 +30,62 @@ type TestExecutionIntegrationSuite struct { func (suite *TestExecutionIntegrationSuite) SetupTest() { httpClient, _ := http.NewClient() suite.mockKubernetesClient = kubernetes.NewKubernetesClient(httpClient) - suite.mockRepository = &repository.MockExecutionContextRepository{} + pgClient := postgresql.NewClient() + suite.repository = repository.NewExecutionContextRepository(pgClient) suite.mockMetadataRepository = &svcMetadataRepository.MockMetadataRepository{} suite.mockSecretRepository = &svcSecretRepository.MockSecretRepository{} suite.service = NewExecutionService( suite.mockKubernetesClient, - suite.mockRepository, + suite.repository, suite.mockMetadataRepository, suite.mockSecretRepository, ) } +func (suite *TestExecutionIntegrationSuite) TestExecuteJobSuccess() { + t := suite.T() + jobName := fake.Username() + userEmail := fake.Email() + mapKey := fake.FirstName() + mapValue := fake.LastName() + + jobArgs := map[string]string{ + mapKey: mapValue, + } + + imageName := "ubuntu" + fakeMetadata := &metadata.Metadata{ + ImageName: imageName, + Author: "bimo.horizon", + Description: fake.HackerIngverb(), + Organization: fake.BuzzWord(), + AuthorizedGroups: []string{}, + EnvVars: env.Vars{ + Args: []env.VarMetadata{ + { + Name: fake.BeerYeast(), + Description: fake.JobDescriptor(), + }, + }, + Secrets: []env.VarMetadata{}, + }, + } + + suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() + suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() + + context, _, err := suite.service.ExecuteWithCommand(jobName, userEmail, jobArgs, []string{"bash", "-c", "for run in {1..10}; do sleep 1 && echo bimo; done"}) + assert.NilError(t, err) + assert.NotNil(t, context) + + time.Sleep(30 * time.Second) + expectedContext, err := suite.repository.GetById(context.ExecutionID) + assert.NilError(t, err) + assert.NotNil(t, expectedContext) + assert.Equal(t, expectedContext.Status, status.Finished) + assert.NotNil(t, expectedContext.Output) +} + func TestExecutionIntegrationSuiteTest(t *testing.T) { value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") if available == true && value == "true" { diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go index 2f596f8d..422556ed 100644 --- a/internal/app/service/execution/status/execution.go +++ b/internal/app/service/execution/status/execution.go @@ -11,6 +11,7 @@ const ( JobReady ExecutionStatus = "JOB_READY" PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" PodReady ExecutionStatus = "POD_READY" + PodFailed ExecutionStatus = "POD_FAILED" FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" Finished ExecutionStatus = "FINISHED" ) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 8f34a8be..71c00f9d 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -17,7 +17,7 @@ func KubeConfig() string { } func KubeContext() string { - viper.SetDefault("KUBE_CONTEXT","default") + viper.SetDefault("KUBE_CONTEXT", "default") return viper.GetString("KUBE_CONTEXT") } @@ -65,6 +65,10 @@ func KubePodsListWaitTime() time.Duration { return time.Duration(viper.GetInt("KUBE_POD_LIST_WAIT_TIME")) } +func KubeLogProcessWaitTime() time.Duration { + return time.Duration(viper.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")) +} + func KubeJobActiveDeadlineSeconds() *int64 { kubeJobActiveDeadlineSeconds := viper.GetInt64("KUBE_JOB_ACTIVE_DEADLINE_SECONDS") return &kubeJobActiveDeadlineSeconds diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index adee55dc..a47c8476 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -23,8 +23,6 @@ import ( "k8s.io/client-go/tools/clientcmd" ) -const JobFailed = int32(0) - var typeMeta meta.TypeMeta var namespace string @@ -220,36 +218,27 @@ func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime t resultChan := watchJob.ResultChan() defer watchJob.Stop() - select { - case <-timeoutChan: - return fmt.Errorf("timeout when waiting job to be available") - case <-resultChan: - for event := range resultChan { - if event.Type == watch.Error { + for { + select { + case watchEvent := <-resultChan: + if watchEvent.Type == watch.Error { return fmt.Errorf("watch error when waiting for job with list option %v", listOptions) } - job := event.Object.(*batch.Job) + + job := watchEvent.Object.(*batch.Job) if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { return nil } - - select { - case <-timeoutChan: - return fmt.Errorf("timeout when waiting job to be ready") - case <-resultChan: - continue - } + case <-timeoutChan: + return fmt.Errorf("timeout when waiting job to be available") } } - - return fmt.Errorf("job never reach the active status") } func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) { coreV1 := client.clientSet.CoreV1() kubernetesPods := coreV1.Pods(namespace) listOptions := meta.ListOptions{ - TypeMeta: typeMeta, LabelSelector: jobLabelSelector(executionName), } @@ -263,11 +252,9 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t defer watchJob.Stop() var pod *v1.Pod - select { - case <-timeoutChan: - return nil, fmt.Errorf("timeout when waiting pod to be available") - case <-resultChan: - for event := range resultChan { + for { + select { + case event := <-resultChan: if event.Type == watch.Error { return nil, fmt.Errorf("watch error when waiting for pod with list option %v", listOptions) } @@ -275,16 +262,11 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { return pod, nil } - select { - case <-timeoutChan: - return nil, fmt.Errorf("timeout when waiting pod to be ready") - case <-resultChan: - continue - } + case <-timeoutChan: + return nil, fmt.Errorf("timeout when waiting pod to be available") } } - return nil, fmt.Errorf("pod never get the intended state") } func (client *kubernetesClient) JobExecutionStatus(executionName string) (string, error) { From adcf8f6b11bd1b4ef5084eba75ef620421703ea0 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 15 Jul 2019 16:10:48 +0530 Subject: [PATCH 036/268] [Jasoet|Bimozx] V2: Change makefile adds integration test --- .env.test | 4 ++-- Makefile | 6 ++++++ internal/app/service/execution/service/execution_test.go | 5 +++-- internal/app/service/infra/kubernetes/client_test.go | 2 +- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.env.test b/.env.test index 8bc21b9e..94079980 100644 --- a/.env.test +++ b/.env.test @@ -11,8 +11,8 @@ export PROCTOR_KUBE_JOB_RETRIES=0 export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 -export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 -export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60 +export PROCTOR_KUBE_POD_LIST_WAIT_TIME=30 +export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=30 export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== export PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK export PROCTOR_POSTGRES_USER=postgres diff --git a/Makefile b/Makefile index b9131564..c159ce26 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,12 @@ test-with-race: test .PHONY: test test: + ENABLE_INTEGRATION_TEST=false \ + go test $(RACE_FLAG) -coverprofile=$(OUT_DIR)/coverage.out ./... + +.PHONY: itest +itest: + ENABLE_INTEGRATION_TEST=true \ go test $(RACE_FLAG) -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: server diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 6a52a5b1..9b191910 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -169,7 +169,8 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() - suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() + suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Times(3) + suite.mockRepository.On("GetById", mock.Anything).Return(0, nil).Times(3) executionName := "execution-name" suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, jobArgs, []string{}).Return(executionName, nil) @@ -182,7 +183,7 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) assert.NilError(t, err) assert.NotNil(t, context) - assert.Equal(t, context.Status, status.Finished) + assert.Equal(t, context.Status, status.Created) } func TestExecutionServiceSuiteTest(t *testing.T) { diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 35521fea..d16104da 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -114,7 +114,7 @@ func (suite *ClientTestSuite) TestJobExecution() { func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { t := suite.T() - waitTime := config.KubePodsListWaitTime() * time.Second + waitTime := 3 * time.Second _, err := suite.testClientStreaming.StreamJobLogs("unknown-job", waitTime) assert.Error(t, err) } From bbfa18cdd634f3652cd85f4d2434b5f749e82f91 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 15 Jul 2019 18:22:46 +0530 Subject: [PATCH 037/268] [Jasoet|Bimozx] V2: Implement Execution Http Handler --- .../app/proctord/jobs/execution/handler.go | 3 +- .../proctord/jobs/execution/handler_test.go | 7 +- internal/app/proctord/jobs/logs/handler.go | 46 +--- .../app/proctord/jobs/logs/handler_test.go | 142 ------------- .../service/execution/handler/http_handler.go | 198 ++++++++++++++++++ .../execution/handler/parameter}/job.go | 2 +- .../execution/handler/parameter/parameter.go | 8 + .../execution/handler/status/status.go | 11 + .../execution/model/execution_context.go | 1 + .../execution/repository/execution_context.go | 10 +- .../repository/execution_context_test.go | 2 + .../service/execution/service/execution.go | 24 +++ .../service/execution_integration_test.go | 39 +++- .../execution/service/execution_mock.go | 8 + .../app/service/infra/kubernetes/client.go | 21 -- .../kubernetes/client_integration_test.go | 26 --- .../service/infra/kubernetes/client_mock.go | 6 - .../service/infra/kubernetes/client_test.go | 8 - .../service/metadata/handler/http_handler.go | 8 +- ...ColumnNameToExecutionContextTable.down.sql | 1 + ...ddColumnNameToExecutionContextTable.up.sql | 1 + 21 files changed, 306 insertions(+), 266 deletions(-) delete mode 100644 internal/app/proctord/jobs/logs/handler_test.go create mode 100644 internal/app/service/execution/handler/http_handler.go rename internal/app/{proctord/jobs/execution => service/execution/handler/parameter}/job.go (90%) create mode 100644 internal/app/service/execution/handler/parameter/parameter.go create mode 100644 internal/app/service/execution/handler/status/status.go create mode 100644 migrations/12_AddColumnNameToExecutionContextTable.down.sql create mode 100644 migrations/12_AddColumnNameToExecutionContextTable.up.sql diff --git a/internal/app/proctord/jobs/execution/handler.go b/internal/app/proctord/jobs/execution/handler.go index 1da2a949..7f175711 100644 --- a/internal/app/proctord/jobs/execution/handler.go +++ b/internal/app/proctord/jobs/execution/handler.go @@ -10,6 +10,7 @@ import ( "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/storage" "proctor/internal/app/proctord/storage/postgres" + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/infra/logger" "proctor/internal/pkg/constant" "time" @@ -66,7 +67,7 @@ func (handler *executionHandler) Handle() http.HandlerFunc { userEmail := req.Header.Get(constant.UserEmailHeaderKey) jobsExecutionAuditLog.UserEmail = userEmail - var job Job + var job parameter.Job err := json.NewDecoder(req.Body).Decode(&job) defer req.Body.Close() if err != nil { diff --git a/internal/app/proctord/jobs/execution/handler_test.go b/internal/app/proctord/jobs/execution/handler_test.go index e2bf0388..21375d84 100644 --- a/internal/app/proctord/jobs/execution/handler_test.go +++ b/internal/app/proctord/jobs/execution/handler_test.go @@ -14,6 +14,7 @@ import ( "net/http/httptest" "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/storage" + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/pkg/constant" "testing" ) @@ -68,7 +69,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandler() { remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) userEmail := "mrproctor@example.com" - job := Job{ + job := parameter.Job{ Name: "sample-job-name", Args: map[string]string{"argOne": "sample-arg"}, CallbackURL: remoteCallerURL, @@ -112,7 +113,7 @@ func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandlerWithout defer ts.Close() userEmail := "mrproctor@example.com" - job := Job{ + job := parameter.Job{ Name: "sample-job-name", Args: map[string]string{"argOne": "sample-arg"}, } @@ -167,7 +168,7 @@ func (suite *ExecutionHandlerTestSuite) TestJobExecutionOnMalformedRequest() { func (suite *ExecutionHandlerTestSuite) TestJobExecutionServerFailure() { t := suite.T() - job := Job{ + job := parameter.Job{ Name: "sample-job-name", Args: map[string]string{"argOne": "sample-arg"}, } diff --git a/internal/app/proctord/jobs/logs/handler.go b/internal/app/proctord/jobs/logs/handler.go index 33fd54a0..9301d0c9 100644 --- a/internal/app/proctord/jobs/logs/handler.go +++ b/internal/app/proctord/jobs/logs/handler.go @@ -1,17 +1,13 @@ package logs import ( - "bufio" "github.com/getsentry/raven-go" - "io" "net/http" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/kubernetes" _logger "proctor/internal/app/service/infra/logger" - "strings" - "time" - "proctor/internal/pkg/constant" + "strings" "github.com/gorilla/websocket" ) @@ -64,44 +60,8 @@ func (l *logger) Stream() http.HandlerFunc { return } - waitTime := config.KubePodsListWaitTime() * time.Second - logStream, err := l.kubeClient.StreamJobLogs(jobName, waitTime) - if err != nil { - _logger.Error("Error streaming logs from kube client: ", err) - raven.CaptureError(err, map[string]string{"job_name": jobName}) - - CloseWebSocket("Something went wrong", conn) - return - } - defer logStream.Close() + CloseWebSocket("No job name provided while requesting for logs", conn) + return - bufioReader := bufio.NewReader(logStream) - - for { - jobLogSingleLine, _, err := bufioReader.ReadLine() - if err != nil { - if err == io.EOF { - _logger.Debug("Finished streaming logs for job: ", jobName) - CloseWebSocket("All logs are read", conn) - return - } - - _logger.Error("Error reading from reader: ", err.Error()) - raven.CaptureError(err, nil) - - CloseWebSocket("Something went wrong", conn) - return - } - - _logger.Debug("writing to web socket ", string(jobLogSingleLine[:])) - err = conn.WriteMessage(websocket.TextMessage, jobLogSingleLine[:]) - if err != nil { - _logger.Error("Error writing logs to client: ", err) - raven.CaptureError(err, nil) - - CloseWebSocket("Something went wrong", conn) - return - } - } } } diff --git a/internal/app/proctord/jobs/logs/handler_test.go b/internal/app/proctord/jobs/logs/handler_test.go deleted file mode 100644 index 222f5ff3..00000000 --- a/internal/app/proctord/jobs/logs/handler_test.go +++ /dev/null @@ -1,142 +0,0 @@ -package logs - -import ( - "errors" - "net/http" - "net/http/httptest" - "proctor/internal/app/service/infra/kubernetes" - "strings" - "testing" - - "github.com/gorilla/websocket" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "proctor/internal/pkg/constant" - "proctor/internal/pkg/utility" -) - -type LoggerTestSuite struct { - suite.Suite - testLogger Logger - mockKubeClient *kubernetes.MockKubernetesClient -} - -func (suite *LoggerTestSuite) SetupTest() { - suite.mockKubeClient = &kubernetes.MockKubernetesClient{} - suite.testLogger = NewLogger(suite.mockKubeClient) -} - -type logsHandlerServer struct { - *httptest.Server -} - -var logsHandlerDialer = websocket.Dialer{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, -} - -const ( - logsHandlerPath = "/jobs/logs" - logsHandlerRawQuery = "job_name=sample" - logsHandlerRequestURI = logsHandlerPath -) - -func (suite *LoggerTestSuite) newServer() *logsHandlerServer { - var s logsHandlerServer - s.Server = httptest.NewServer(suite.testLogger.Stream()) - s.Server.URL += logsHandlerRequestURI - s.URL = makeWsProto(s.Server.URL) - return &s -} - -func makeWsProto(s string) string { - return "ws" + strings.TrimPrefix(s, "http") -} - -func (suite *LoggerTestSuite) TestLoggerStream() { - t := suite.T() - - s := suite.newServer() - defer s.Close() - - buffer := utility.NewBuffer() - _, _ = buffer.Write([]byte("first line\nsecond line\n")) - suite.mockKubeClient.On("StreamJobLogs", "sample", mock.Anything).Return(buffer, nil).Once() - - c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) - assert.NoError(t, err) - defer c.Close() - - _, firstMessage, err := c.ReadMessage() - assert.NoError(t, err) - assert.Equal(t, "first line", string(firstMessage)) - - _, secondMessage, err := c.ReadMessage() - assert.NoError(t, err) - assert.Equal(t, "second line", string(secondMessage)) - - _, finalMessage, err := c.ReadMessage() - assert.Error(t, err) - assert.Equal(t, "", string(finalMessage)) - assert.Equal(t, "websocket: close 1000 (normal): All logs are read", err.Error()) - - suite.mockKubeClient.AssertExpectations(t) - assert.True(t, buffer.WasClosed()) -} - -func (suite *LoggerTestSuite) TestLoggerStreamConnectionUpgradeFailure() { - t := suite.T() - - req := httptest.NewRequest("GET", "/jobs/logs", &utility.Buffer{}) - responseRecorder := httptest.NewRecorder() - - suite.testLogger.Stream()(responseRecorder, req) - - suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything, mock.Anything) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, "Bad Request\n"+constant.ClientError, responseRecorder.Body.String()) -} - -func (suite *LoggerTestSuite) TestLoggerStreamForNoJobName() { - t := suite.T() - - s := suite.newServer() - defer s.Close() - - c, _, err := websocket.DefaultDialer.Dial(s.URL, nil) - assert.NoError(t, err) - defer c.Close() - - suite.mockKubeClient.AssertNotCalled(t, "StreamJobLogs", mock.Anything, mock.Anything) - - _, finalMessage, err := c.ReadMessage() - assert.Error(t, err) - assert.Equal(t, "", string(finalMessage)) - assert.Equal(t, "websocket: close 1000 (normal): No job name provided while requesting for logs", err.Error()) -} - -func (suite *LoggerTestSuite) TestLoggerStreamKubeClientFailure() { - t := suite.T() - - s := suite.newServer() - defer s.Close() - - suite.mockKubeClient.On("StreamJobLogs", "sample", mock.Anything).Return(&utility.Buffer{}, errors.New("error")).Once() - - c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) - assert.NoError(t, err) - defer c.Close() - - _, finalMessage, err := c.ReadMessage() - assert.Error(t, err) - assert.Equal(t, "", string(finalMessage)) - assert.Equal(t, "websocket: close 1000 (normal): Something went wrong", err.Error()) - - suite.mockKubeClient.AssertExpectations(t) -} - -func TestLoggerTestSuite(t *testing.T) { - suite.Run(t, new(LoggerTestSuite)) -} diff --git a/internal/app/service/execution/handler/http_handler.go b/internal/app/service/execution/handler/http_handler.go new file mode 100644 index 00000000..e2be1bf4 --- /dev/null +++ b/internal/app/service/execution/handler/http_handler.go @@ -0,0 +1,198 @@ +package handler + +import ( + "bufio" + "encoding/json" + "fmt" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + "net/http" + "proctor/internal/app/service/execution/handler/parameter" + "proctor/internal/app/service/execution/handler/status" + "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/service" + executionStatus "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" + "strconv" + "strings" + "time" +) + +type ExecutionHttpHandler interface { + Post() http.HandlerFunc + Status() http.HandlerFunc + Logs() http.HandlerFunc +} + +type executionHttpHandler struct { + service service.ExecutionService + repository repository.ExecutionContextRepository +} + +var upgrader = websocket.Upgrader{ + ReadBufferSize: config.LogsStreamReadBufferSize(), + WriteBufferSize: config.LogsStreamWriteBufferSize(), +} + +func NewExecutionHttpHandler( + executionService service.ExecutionService, + repository repository.ExecutionContextRepository, +) ExecutionHttpHandler { + return &executionHttpHandler{ + service: executionService, + repository: repository, + } +} + +func (httpHandler *executionHttpHandler) closeWebSocket(message string, conn *websocket.Conn) { + err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, message)) + logger.LogErrors(err, "close WebSocket") + return +} + +func (httpHandler *executionHttpHandler) Logs() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + conn, err := upgrader.Upgrade(response, request, nil) + logger.LogErrors(err, "upgrade http server connection to WebSocket") + if err != nil { + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(status.WebSocketInitError)) + return + } + defer conn.Close() + + contextIdParam := strings.TrimLeft(request.URL.RawQuery, "job_name=") + executionContextId, err := strconv.ParseUint(contextIdParam, 10, 64) + if contextIdParam == "" || err != nil { + logger.Error("No valid execution context id provided as part of URL ", request.URL.RawQuery) + httpHandler.closeWebSocket("No valid execution context id provided when fetching logs", conn) + return + } + + context, err := httpHandler.repository.GetById(executionContextId) + + if err != nil { + logger.Error("No execution context id found ", executionContextId) + httpHandler.closeWebSocket(fmt.Sprintf("No execution context found with id %v", executionContextId), conn) + return + } + + if context.Status == executionStatus.Finished { + err = conn.WriteMessage(websocket.TextMessage, context.Output) + logger.LogErrors(err, "write output to socket:", string(context.Output)) + httpHandler.closeWebSocket("Finished Streaming log", conn) + return + } + + if context.Status == executionStatus.PodReady { + waitTime := config.KubeLogProcessWaitTime() * time.Second + podLog, _err := httpHandler.service.StreamJobLogs(context.Name, waitTime) + + logger.LogErrors(_err, "stream job log by execution name", context.Name) + if _err != nil { + logger.Error("failed to stream job log by execution name ", context.Name) + httpHandler.closeWebSocket(fmt.Sprintf("failed to stream job log by execution name %s:", context.Name), conn) + return + } + + defer podLog.Close() + + scanner := bufio.NewScanner(podLog) + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + _ = conn.WriteMessage(websocket.TextMessage, scanner.Bytes()) + } + httpHandler.closeWebSocket("Finished Streaming log", conn) + return + } + + httpHandler.closeWebSocket("No Logs Found", conn) + return + } +} + +func (httpHandler *executionHttpHandler) Status() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + contextId := mux.Vars(request)["name"] + executionContextId, err := strconv.ParseUint(contextId, 10, 64) + logger.LogErrors(err, "parse execution context id from path parameter:", contextId) + if err != nil { + response.WriteHeader(http.StatusNotFound) + _, _ = response.Write([]byte(status.PathParameterError)) + return + } + + context, err := httpHandler.repository.GetById(executionContextId) + logger.LogErrors(err, "fetch execution context by id:", contextId) + + if err != nil { + response.WriteHeader(http.StatusNotFound) + _, _ = response.Write([]byte(status.ExecutionContextNotFound)) + return + } + + responseMap := map[string]string{ + "ExecutionId": string(context.ExecutionID), + "JobName": context.JobName, + "ImageTag": context.ImageTag, + "CreatedAt": context.CreatedAt.String(), + "Status": string(context.Status), + } + + response.WriteHeader(http.StatusOK) + + responseJson, err := json.Marshal(responseMap) + logger.LogErrors(err, "marshal json from: ", responseMap) + + _, _ = response.Write(responseJson) + + } +} + +func (httpHandler *executionHttpHandler) Post() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + userEmail := request.Header.Get(parameter.UserEmailHeader) + + var job parameter.Job + + err := json.NewDecoder(request.Body).Decode(&job) + defer request.Body.Close() + + logger.LogErrors(err, "parsing request body to Job object") + + if err != nil { + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(status.MalformedRequest)) + return + } + + context, executionName, err := httpHandler.service.Execute(job.Name, userEmail, job.Args) + + logger.LogErrors(err, "execute job: ", job) + + if err != nil { + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(fmt.Sprintf("%s , Errors Detail %s", status.JobExecutionError, err.Error()))) + return + } + + responseMap := map[string]string{ + "ExecutionId": string(context.ExecutionID), + "JobName": context.JobName, + "ExecutionName": executionName, + "ImageTag": context.ImageTag, + "CreatedAt": context.CreatedAt.String(), + "Status": string(context.Status), + } + + response.WriteHeader(http.StatusCreated) + + responseJson, err := json.Marshal(responseMap) + logger.LogErrors(err, "marshal json from: ", responseMap) + + _, _ = response.Write(responseJson) + return + } +} diff --git a/internal/app/proctord/jobs/execution/job.go b/internal/app/service/execution/handler/parameter/job.go similarity index 90% rename from internal/app/proctord/jobs/execution/job.go rename to internal/app/service/execution/handler/parameter/job.go index d31ef4d3..932defe9 100644 --- a/internal/app/proctord/jobs/execution/job.go +++ b/internal/app/service/execution/handler/parameter/job.go @@ -1,4 +1,4 @@ -package execution +package parameter type Job struct { Name string `json:"name"` diff --git a/internal/app/service/execution/handler/parameter/parameter.go b/internal/app/service/execution/handler/parameter/parameter.go new file mode 100644 index 00000000..0851f8dd --- /dev/null +++ b/internal/app/service/execution/handler/parameter/parameter.go @@ -0,0 +1,8 @@ +package parameter + + +const ( + UserEmailHeader string = "Email-Id" + AccessTokenHeader string = "Access-Token" + ClientVersionHeader string = "Client-Version" +) diff --git a/internal/app/service/execution/handler/status/status.go b/internal/app/service/execution/handler/status/status.go new file mode 100644 index 00000000..d1a1e7f8 --- /dev/null +++ b/internal/app/service/execution/handler/status/status.go @@ -0,0 +1,11 @@ +package status + +type ExecutionHandlerStatus string + +const ( + MalformedRequest ExecutionHandlerStatus = "Failed to parse request body from CLI" + JobExecutionError ExecutionHandlerStatus = "Failed to execute Job into Executor" + PathParameterError ExecutionHandlerStatus = "Failed to translate path parameter to uint64" + WebSocketInitError ExecutionHandlerStatus = "WebSocket Initializer Error" + ExecutionContextNotFound ExecutionHandlerStatus = "Execution context not Found" +) diff --git a/internal/app/service/execution/model/execution_context.go b/internal/app/service/execution/model/execution_context.go index 6f414081..28d44ccc 100644 --- a/internal/app/service/execution/model/execution_context.go +++ b/internal/app/service/execution/model/execution_context.go @@ -10,6 +10,7 @@ import ( type ExecutionContext struct { ExecutionID uint64 `db:"id"` JobName string `db:"job_name"` + Name string `db:"name"` UserEmail string `db:"user_email"` ImageTag string `db:"image_tag"` Args dbTypes.Base64Map `db:"args"` diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 2d9b1948..4c93d944 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -35,7 +35,7 @@ func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRep func (repository *executionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { snowflakeId, _ := id.NextId() context.ExecutionID = snowflakeId - sql := "INSERT INTO execution_context (id, job_name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :user_email, :image_tag, :args, :output, :status)" + sql := "INSERT INTO execution_context (id, job_name,name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :name, :user_email, :image_tag, :args, :output, :status)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { return 0, nil @@ -75,7 +75,7 @@ func (repository *executionContextRepository) Delete(executionId uint64) error { } func (repository *executionContextRepository) GetById(executionId uint64) (*model.ExecutionContext, error) { - sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE id=$1 " + sql := "SELECT id, job_name, name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE id=$1 " var contexts []model.ExecutionContext err := repository.postgresqlClient.Select(&contexts, sql, executionId) if err != nil { @@ -90,7 +90,7 @@ func (repository *executionContextRepository) GetById(executionId uint64) (*mode } func (repository *executionContextRepository) GetByEmail(userEmail string) ([]model.ExecutionContext, error) { - sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE user_email=$1 " + sql := "SELECT id, job_name, name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE user_email=$1 " var contexts []model.ExecutionContext err := repository.postgresqlClient.Select(&contexts, sql, userEmail) if err != nil { @@ -101,7 +101,7 @@ func (repository *executionContextRepository) GetByEmail(userEmail string) ([]mo } func (repository *executionContextRepository) GetByJobName(jobName string) ([]model.ExecutionContext, error) { - sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE job_name=$1 " + sql := "SELECT id, job_name, name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE job_name=$1 " var contexts []model.ExecutionContext err := repository.postgresqlClient.Select(&contexts, sql, jobName) if err != nil { @@ -112,7 +112,7 @@ func (repository *executionContextRepository) GetByJobName(jobName string) ([]mo } func (repository *executionContextRepository) GetByStatus(status string) ([]model.ExecutionContext, error) { - sql := "SELECT id, job_name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE status=$1 " + sql := "SELECT id, job_name, name, user_email, image_tag, args, output, status, created_at, updated_at FROM execution_context WHERE status=$1 " var contexts []model.ExecutionContext err := repository.postgresqlClient.Select(&contexts, sql, status) if err != nil { diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index 2ee81f35..439a6ce2 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -112,6 +112,7 @@ func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { fake.Seed(0) context := &model.ExecutionContext{ JobName: fake.BuzzWord(), + Name: fake.HackerAdjective(), UserEmail: fake.Email(), ImageTag: fake.BeerStyle(), Args: map[string]string{ @@ -174,6 +175,7 @@ func populateSeedDataForTest(repository ExecutionContextRepository, count int, s context := &model.ExecutionContext{ JobName: jobName, + Name: fake.HackerAdjective(), UserEmail: email, ImageTag: fake.BeerStyle(), Args: map[string]string{ diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index db63528d..e40f9256 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "github.com/jmoiron/sqlx/types" + "io" v1 "k8s.io/api/core/v1" "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/repository" @@ -21,6 +22,7 @@ import ( type ExecutionService interface { Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) + StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) save(executionContext *model.ExecutionContext) error } @@ -68,6 +70,26 @@ func (service *executionService) save(executionContext *model.ExecutionContext) return err } +func (service *executionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { + err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) + if err != nil { + return nil, err + } + + pod, err := service.kubernetesClient.WaitForReadyPod(executionName, waitTime) + if err != nil { + return nil, err + } + + result, err := service.kubernetesClient.GetPodLogs(pod) + + if err != nil { + return nil, err + } + + return result, nil +} + func (service *executionService) Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) { return service.ExecuteWithCommand(jobName, userEmail, args, []string{}) } @@ -104,6 +126,8 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } + context.Name = executionName + go service.watchProcess(executionName, context) return context, executionName, nil diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 1638cfe8..34f75e8b 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -1,12 +1,14 @@ package service import ( + "bufio" fake "github.com/brianvoe/gofakeit" - "github.com/docker/docker/pkg/testutil/assert" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "os" "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/kubernetes/http" @@ -21,7 +23,7 @@ import ( type TestExecutionIntegrationSuite struct { suite.Suite service ExecutionService - mockKubernetesClient kubernetes.KubernetesClient + kubernetesClient kubernetes.KubernetesClient repository repository.ExecutionContextRepository mockMetadataRepository *svcMetadataRepository.MockMetadataRepository mockSecretRepository *svcSecretRepository.MockSecretRepository @@ -29,13 +31,13 @@ type TestExecutionIntegrationSuite struct { func (suite *TestExecutionIntegrationSuite) SetupTest() { httpClient, _ := http.NewClient() - suite.mockKubernetesClient = kubernetes.NewKubernetesClient(httpClient) + suite.kubernetesClient = kubernetes.NewKubernetesClient(httpClient) pgClient := postgresql.NewClient() suite.repository = repository.NewExecutionContextRepository(pgClient) suite.mockMetadataRepository = &svcMetadataRepository.MockMetadataRepository{} suite.mockSecretRepository = &svcSecretRepository.MockSecretRepository{} suite.service = NewExecutionService( - suite.mockKubernetesClient, + suite.kubernetesClient, suite.repository, suite.mockMetadataRepository, suite.mockSecretRepository, @@ -75,17 +77,42 @@ func (suite *TestExecutionIntegrationSuite) TestExecuteJobSuccess() { suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() context, _, err := suite.service.ExecuteWithCommand(jobName, userEmail, jobArgs, []string{"bash", "-c", "for run in {1..10}; do sleep 1 && echo bimo; done"}) - assert.NilError(t, err) + assert.NoError(t, err) assert.NotNil(t, context) time.Sleep(30 * time.Second) expectedContext, err := suite.repository.GetById(context.ExecutionID) - assert.NilError(t, err) + assert.NoError(t, err) assert.NotNil(t, expectedContext) assert.Equal(t, expectedContext.Status, status.Finished) assert.NotNil(t, expectedContext.Output) } +func (suite *TestExecutionIntegrationSuite) TestStreamLogsSuccess() { + t := suite.T() + + _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + sampleImageName := "busybox" + + executedJobname, err := suite.kubernetesClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) + assert.NoError(t, err) + + waitTime := config.KubePodsListWaitTime() * time.Second + logStream, err := suite.service.StreamJobLogs(executedJobname, waitTime) + assert.NoError(t, err) + + defer logStream.Close() + + bufioReader := bufio.NewReader(logStream) + + jobLogSingleLine, _, err := bufioReader.ReadLine() + assert.NoError(t, err) + + assert.Equal(t, "Bimo Horizon", string(jobLogSingleLine[:])) + +} + func TestExecutionIntegrationSuiteTest(t *testing.T) { value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") if available == true && value == "true" { diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index d1e0a2a8..b48277e2 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -2,7 +2,10 @@ package service import ( "github.com/stretchr/testify/mock" + "io" "proctor/internal/app/service/execution/model" + "proctor/internal/pkg/utility" + "time" ) type MockExecutionService struct { @@ -22,3 +25,8 @@ func (mockService *MockExecutionService) ExecuteWithCommand(jobName string, user func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) { mockService.Called(executionContext) } + +func (mockService *MockExecutionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { + args := mockService.Called(executionName, waitTime) + return args.Get(0).(*utility.Buffer), args.Error(1) +} diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index a47c8476..f44403c3 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -37,7 +37,6 @@ func init() { type KubernetesClient interface { ExecuteJobWithCommand(imageName string, args map[string]string, commands []string) (string, error) ExecuteJob(imageName string, args map[string]string) (string, error) - StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) JobExecutionStatus(executionName string) (string, error) WaitForReadyJob(executionName string, waitTime time.Duration) error WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) @@ -181,26 +180,6 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m return executionName, nil } -func (client *kubernetesClient) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { - err := client.WaitForReadyJob(executionName, waitTime) - if err != nil { - return nil, err - } - - pod, err := client.WaitForReadyPod(executionName, waitTime) - if err != nil { - return nil, err - } - - result, err := client.GetPodLogs(pod) - - if err != nil { - return nil, err - } - - return result, nil -} - func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime time.Duration) error { batchV1 := client.clientSet.BatchV1() jobs := batchV1.Jobs(namespace) diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index d3dd0858..114f80b5 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -1,7 +1,6 @@ package kubernetes import ( - "bufio" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "k8s.io/api/core/v1" @@ -12,7 +11,6 @@ import ( kubeHttpClient "proctor/internal/app/service/infra/kubernetes/http" "proctor/internal/pkg/constant" "testing" - "time" ) type IntegrationTestSuite struct { @@ -88,30 +86,6 @@ func (suite *IntegrationTestSuite) TestJobExecutionStatus() { assert.Equal(t, status, constant.JobSucceeded) } -func (suite *IntegrationTestSuite) TestStreamLogsSuccess() { - t := suite.T() - - _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} - sampleImageName := "busybox" - - executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) - assert.NoError(t, err) - - waitTime := config.KubePodsListWaitTime() * time.Second - logStream, err := suite.testClient.StreamJobLogs(executedJobname, waitTime) - assert.NoError(t, err) - - defer logStream.Close() - - bufioReader := bufio.NewReader(logStream) - - jobLogSingleLine, _, err := bufioReader.ReadLine() - assert.NoError(t, err) - - assert.Equal(t, "Bimo Horizon", string(jobLogSingleLine[:])) - -} func TestIntegrationTestSuite(t *testing.T) { value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index da8d2e0c..555ba50d 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -6,7 +6,6 @@ import ( "time" "github.com/stretchr/testify/mock" - "proctor/internal/pkg/utility" ) type MockKubernetesClient struct { @@ -23,11 +22,6 @@ func (m *MockKubernetesClient) ExecuteJobWithCommand(jobName string, envMap map[ return args.String(0), args.Error(1) } -func (m *MockKubernetesClient) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { - args := m.Called(executionName, waitTime) - return args.Get(0).(*utility.Buffer), args.Error(1) -} - func (m *MockKubernetesClient) JobExecutionStatus(executionName string) (string, error) { args := m.Called(executionName) return args.String(0), args.Error(1) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index d16104da..cf6dfa29 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -111,14 +111,6 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedEnvVars, container.Env) } -func (suite *ClientTestSuite) TestStreamLogsPodNotFoundFailure() { - t := suite.T() - - waitTime := 3 * time.Second - _, err := suite.testClientStreaming.StreamJobLogs("unknown-job", waitTime) - assert.Error(t, err) -} - func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { t := suite.T() diff --git a/internal/app/service/metadata/handler/http_handler.go b/internal/app/service/metadata/handler/http_handler.go index 4fb1eee6..cd8245c2 100644 --- a/internal/app/service/metadata/handler/http_handler.go +++ b/internal/app/service/metadata/handler/http_handler.go @@ -10,7 +10,7 @@ import ( modelMetadata "proctor/internal/pkg/model/metadata" ) -type handler struct { +type metadataHttpHandler struct { repository repository.MetadataRepository } @@ -20,12 +20,12 @@ type MetadataHttpHandler interface { } func NewMetadataHttpHandler(repository repository.MetadataRepository) MetadataHttpHandler { - return &handler{ + return &metadataHttpHandler{ repository: repository, } } -func (handler *handler) Post() http.HandlerFunc { +func (handler *metadataHttpHandler) Post() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { var metadata []modelMetadata.Metadata err := json.NewDecoder(request.Body).Decode(&metadata) @@ -54,7 +54,7 @@ func (handler *handler) Post() http.HandlerFunc { } } -func (handler *handler) GetAll() http.HandlerFunc { +func (handler *metadataHttpHandler) GetAll() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { metadataSlice, err := handler.repository.GetAll() diff --git a/migrations/12_AddColumnNameToExecutionContextTable.down.sql b/migrations/12_AddColumnNameToExecutionContextTable.down.sql new file mode 100644 index 00000000..07879aef --- /dev/null +++ b/migrations/12_AddColumnNameToExecutionContextTable.down.sql @@ -0,0 +1 @@ +alter table execution_context drop column if exists name; diff --git a/migrations/12_AddColumnNameToExecutionContextTable.up.sql b/migrations/12_AddColumnNameToExecutionContextTable.up.sql new file mode 100644 index 00000000..1ccec823 --- /dev/null +++ b/migrations/12_AddColumnNameToExecutionContextTable.up.sql @@ -0,0 +1 @@ +alter table execution_context add column name varchar(255) default NULL; \ No newline at end of file From bd0f99797dfd6b0ab06c3c5e1e2db785180b3f76 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 14:38:30 +0530 Subject: [PATCH 038/268] [bimo.horizon|jasoet] Renamed http handler to just http --- .../service/execution/handler/{http_handler.go => http.go} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename internal/app/service/execution/handler/{http_handler.go => http.go} (97%) diff --git a/internal/app/service/execution/handler/http_handler.go b/internal/app/service/execution/handler/http.go similarity index 97% rename from internal/app/service/execution/handler/http_handler.go rename to internal/app/service/execution/handler/http.go index e2be1bf4..48bf51ed 100644 --- a/internal/app/service/execution/handler/http_handler.go +++ b/internal/app/service/execution/handler/http.go @@ -119,7 +119,7 @@ func (httpHandler *executionHttpHandler) Status() http.HandlerFunc { executionContextId, err := strconv.ParseUint(contextId, 10, 64) logger.LogErrors(err, "parse execution context id from path parameter:", contextId) if err != nil { - response.WriteHeader(http.StatusNotFound) + response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.PathParameterError)) return } @@ -134,7 +134,7 @@ func (httpHandler *executionHttpHandler) Status() http.HandlerFunc { } responseMap := map[string]string{ - "ExecutionId": string(context.ExecutionID), + "ExecutionId": fmt.Sprint(context.ExecutionID), "JobName": context.JobName, "ImageTag": context.ImageTag, "CreatedAt": context.CreatedAt.String(), @@ -179,7 +179,7 @@ func (httpHandler *executionHttpHandler) Post() http.HandlerFunc { } responseMap := map[string]string{ - "ExecutionId": string(context.ExecutionID), + "ExecutionId": fmt.Sprint(context.ExecutionID), "JobName": context.JobName, "ExecutionName": executionName, "ImageTag": context.ImageTag, From dc6fbf8baa2c9a6108104a4e7d82ebaf89b5e43e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 14:38:52 +0530 Subject: [PATCH 039/268] [bimo.horizon|jasoet] Fix insufficient mock --- internal/app/service/execution/service/execution_mock.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index b48277e2..f881d7a6 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -22,8 +22,9 @@ func (mockService *MockExecutionService) ExecuteWithCommand(jobName string, user return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) } -func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) { - mockService.Called(executionContext) +func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) error { + args := mockService.Called(executionContext) + return args.Error(0) } func (mockService *MockExecutionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { From 72ff223f44145ed27ff469900c08b0ce6ee61c7b Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 14:39:14 +0530 Subject: [PATCH 040/268] [bimo.horizon|jasoet] Add test for execution handler --- .../service/execution/handler/http_test.go | 318 ++++++++++++++++++ 1 file changed, 318 insertions(+) create mode 100644 internal/app/service/execution/handler/http_test.go diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go new file mode 100644 index 00000000..d68aff87 --- /dev/null +++ b/internal/app/service/execution/handler/http_test.go @@ -0,0 +1,318 @@ +package handler + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "proctor/internal/app/service/execution/handler/parameter" + handlerStatus "proctor/internal/app/service/execution/handler/status" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/service" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/kubernetes" + "proctor/internal/pkg/constant" + "proctor/internal/pkg/utility" + + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/urfave/negroni" +) + +type ExecutionHttpHandlerTestSuite struct { + suite.Suite + mockExecutionerService *service.MockExecutionService + mockExecutionerContextRepository *repository.MockExecutionContextRepository + mockKubernetesClient kubernetes.MockKubernetesClient + testExecutionHttpHandler ExecutionHttpHandler + + Client *http.Client + TestServer *httptest.Server +} + +func (suite *ExecutionHttpHandlerTestSuite) SetupTest() { + suite.mockExecutionerService = &service.MockExecutionService{} + suite.mockExecutionerContextRepository = &repository.MockExecutionContextRepository{} + suite.mockKubernetesClient = kubernetes.MockKubernetesClient{} + suite.testExecutionHttpHandler = NewExecutionHttpHandler(suite.mockExecutionerService, suite.mockExecutionerContextRepository) + + suite.Client = &http.Client{} + router := mux.NewRouter() + router.HandleFunc("/jobs/execute/{name}/status", suite.testExecutionHttpHandler.Status()).Methods("GET") + n := negroni.Classic() + n.UseHandler(router) + suite.TestServer = httptest.NewServer(n) +} + +type logsHandlerServer struct { + *httptest.Server +} + +var logsHandlerDialer = websocket.Dialer{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, +} + +const ( + logsHandlerPath = "/jobs/logs" + logsHandlerRawQuery = "job_name=1" + logsHandlerRequestURI = logsHandlerPath +) + +func (suite *ExecutionHttpHandlerTestSuite) newServer() *logsHandlerServer { + var s logsHandlerServer + s.Server = httptest.NewServer(suite.testExecutionHttpHandler.Logs()) + s.Server.URL += logsHandlerRequestURI + s.URL = makeWsProto(s.Server.URL) + return &s +} + +func makeWsProto(s string) string { + return "ws" + strings.TrimPrefix(s, "http") +} + +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFinishedHttpHandler() { + t := suite.T() + + s := suite.newServer() + defer s.Close() + + executionContextId := uint64(1) + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.Finished, + Output: types.GzippedText("test"), + } + + buffer := utility.NewBuffer() + buffer.Write([]byte("test\n")) + + suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(10)).Return(buffer, nil).Once() + suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + + c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) + assert.NoError(t, err) + defer c.Close() + + _, firstMessage, err := c.ReadMessage() + assert.NoError(t, err) + assert.Equal(t, "test", string(firstMessage)) + + suite.mockKubernetesClient.AssertExpectations(t) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenReadyHttpHandler() { + t := suite.T() + + s := suite.newServer() + defer s.Close() + + executionContextId := uint64(1) + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + Name: "1", + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.PodReady, + Output: types.GzippedText("test"), + } + + buffer := utility.NewBuffer() + buffer.Write([]byte("test1\ntest2\ntest3\n")) + + suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(30)*time.Second).Return(buffer, nil).Once() + suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + + c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) + assert.NoError(t, err) + defer c.Close() + + _, firstMessage, err := c.ReadMessage() + assert.NoError(t, err) + assert.Equal(t, "test1", string(firstMessage)) + + _, secondMessage, err := c.ReadMessage() + assert.NoError(t, err) + assert.Equal(t, "test2", string(secondMessage)) + + _, thirdMessage, err := c.ReadMessage() + assert.NoError(t, err) + assert.Equal(t, "test3", string(thirdMessage)) + + suite.mockKubernetesClient.AssertExpectations(t) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttpHandler() { + t := suite.T() + + executionContextId := uint64(1) + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.Finished, + } + responseMap := map[string]string{ + "ExecutionId": fmt.Sprint(executionContextId), + "JobName": context.JobName, + "ImageTag": context.ImageTag, + "CreatedAt": context.CreatedAt.String(), + "Status": string(context.Status), + } + + responseBody, err := json.Marshal(responseMap) + assert.NoError(t, err) + + req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"name": fmt.Sprint(executionContextId)}) + responseRecorder := httptest.NewRecorder() + + suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + + suite.testExecutionHttpHandler.Status()(responseRecorder, req) + suite.mockExecutionerService.AssertExpectations(t) + + assert.Equal(t, http.StatusOK, responseRecorder.Code) + assert.Equal(t, string(responseBody), responseRecorder.Body.String()) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionStatusHttpHandler() { + t := suite.T() + + executionContextId := uint64(1) + + req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte("test"))) + req = mux.SetURLVars(req, map[string]string{"name": "notfound"}) + responseRecorder := httptest.NewRecorder() + + suite.testExecutionHttpHandler.Status()(responseRecorder, req) + + assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) + assert.Equal(t, string(handlerStatus.PathParameterError), responseRecorder.Body.String()) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionPostHttpHandler() { + t := suite.T() + + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + UserEmail: userEmail, + JobName: job.Name, + Args: job.Args, + Status: status.Finished, + } + responseMap := map[string]string{ + "CreatedAt": context.CreatedAt.String(), + "ExecutionId": "0", + "ExecutionName": "test", + "ImageTag": "", + "JobName": context.JobName, + "Status": string(context.Status), + } + + requestBody, err := json.Marshal(job) + assert.NoError(t, err) + + responseBody, err := json.Marshal(responseMap) + assert.NoError(t, err) + + req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) + req.Header.Set(constant.UserEmailHeaderKey, userEmail) + responseRecorder := httptest.NewRecorder() + + suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", nil).Once() + + suite.testExecutionHttpHandler.Post()(responseRecorder, req) + suite.mockExecutionerService.AssertExpectations(t) + + assert.Equal(t, http.StatusCreated, responseRecorder.Code) + assert.Equal(t, string(responseBody), responseRecorder.Body.String()) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestJobExecutionPostHttpHandler() { + t := suite.T() + + req := httptest.NewRequest("POST", "/execute", bytes.NewReader([]byte("test"))) + responseRecorder := httptest.NewRecorder() + + suite.testExecutionHttpHandler.Post()(responseRecorder, req) + + assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) + assert.Equal(t, string(handlerStatus.MalformedRequest), responseRecorder.Body.String()) +} + +func (suite *ExecutionHttpHandlerTestSuite) TestGenericErrorJobExecutionPostHttpHandler() { + t := suite.T() + + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + UserEmail: userEmail, + JobName: job.Name, + Args: job.Args, + Status: status.Finished, + } + genericError := errors.New("Something went wrong") + + requestBody, err := json.Marshal(job) + assert.NoError(t, err) + + req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) + req.Header.Set(constant.UserEmailHeaderKey, userEmail) + responseRecorder := httptest.NewRecorder() + + suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", genericError).Once() + + suite.testExecutionHttpHandler.Post()(responseRecorder, req) + suite.mockExecutionerService.AssertExpectations(t) + + assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) + assert.Equal(t, fmt.Sprintf("%s , Errors Detail %s", handlerStatus.JobExecutionError, genericError), responseRecorder.Body.String()) +} + +func TestExecutionHttpHandlerTestSuite(t *testing.T) { + suite.Run(t, new(ExecutionHttpHandlerTestSuite)) +} From d70c0558f359cef5320b471ec06c655cf48484b5 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 15:44:04 +0530 Subject: [PATCH 041/268] [bimo.horizon|jasoet] Change assert expectations location to be nearer to each mocked method --- .../app/service/execution/handler/http_test.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index d68aff87..b2bbc292 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -107,8 +107,8 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFi buffer := utility.NewBuffer() buffer.Write([]byte("test\n")) - suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(10)).Return(buffer, nil).Once() suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + defer suite.mockExecutionerContextRepository.AssertExpectations(t) c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) assert.NoError(t, err) @@ -117,8 +117,6 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFi _, firstMessage, err := c.ReadMessage() assert.NoError(t, err) assert.Equal(t, "test", string(firstMessage)) - - suite.mockKubernetesClient.AssertExpectations(t) } func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenReadyHttpHandler() { @@ -149,7 +147,9 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenRe buffer.Write([]byte("test1\ntest2\ntest3\n")) suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(30)*time.Second).Return(buffer, nil).Once() + defer suite.mockExecutionerService.AssertExpectations(t) suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + defer suite.mockExecutionerContextRepository.AssertExpectations(t) c, _, err := websocket.DefaultDialer.Dial(s.URL+"?"+logsHandlerRawQuery, nil) assert.NoError(t, err) @@ -166,8 +166,6 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenRe _, thirdMessage, err := c.ReadMessage() assert.NoError(t, err) assert.Equal(t, "test3", string(thirdMessage)) - - suite.mockKubernetesClient.AssertExpectations(t) } func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttpHandler() { @@ -204,9 +202,9 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttp responseRecorder := httptest.NewRecorder() suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() + defer suite.mockExecutionerContextRepository.AssertExpectations(t) suite.testExecutionHttpHandler.Status()(responseRecorder, req) - suite.mockExecutionerService.AssertExpectations(t) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) @@ -261,9 +259,9 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionPostHttpHa responseRecorder := httptest.NewRecorder() suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", nil).Once() + defer suite.mockExecutionerService.AssertExpectations(t) suite.testExecutionHttpHandler.Post()(responseRecorder, req) - suite.mockExecutionerService.AssertExpectations(t) assert.Equal(t, http.StatusCreated, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) @@ -305,9 +303,9 @@ func (suite *ExecutionHttpHandlerTestSuite) TestGenericErrorJobExecutionPostHttp responseRecorder := httptest.NewRecorder() suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", genericError).Once() + defer suite.mockExecutionerService.AssertExpectations(t) suite.testExecutionHttpHandler.Post()(responseRecorder, req) - suite.mockExecutionerService.AssertExpectations(t) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) assert.Equal(t, fmt.Sprintf("%s , Errors Detail %s", handlerStatus.JobExecutionError, genericError), responseRecorder.Body.String()) From 49c7c84b41435fc90cae5b9f3cbdb712fb104703 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 15:44:24 +0530 Subject: [PATCH 042/268] [bimo.horizon|jasoet] Add not found test of job execution status handler --- .../service/execution/handler/http_test.go | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index b2bbc292..f2bfcdcf 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -225,6 +225,39 @@ func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionS assert.Equal(t, string(handlerStatus.PathParameterError), responseRecorder.Body.String()) } +func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionStatusHttpHandler() { + t := suite.T() + + executionContextId := uint64(1) + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + context := &model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.Finished, + } + notFoundErr := errors.New("Execution context not found") + + req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"name": fmt.Sprint(executionContextId)}) + responseRecorder := httptest.NewRecorder() + + suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, notFoundErr).Once() + defer suite.mockExecutionerContextRepository.AssertExpectations(t) + + suite.testExecutionHttpHandler.Status()(responseRecorder, req) + + assert.Equal(t, http.StatusNotFound, responseRecorder.Code) + assert.Equal(t, string(handlerStatus.ExecutionContextNotFound), responseRecorder.Body.String()) +} + func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionPostHttpHandler() { t := suite.T() From 54e15674cba32762449f8c899d29d1e13496adde Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 16:34:30 +0530 Subject: [PATCH 043/268] [bimo.horizon|jasoet] Swap job execution related routes to use the new handler instead --- internal/app/proctord/server/router.go | 27 +++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index ab7f7e3c..706b752f 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -4,14 +4,14 @@ import ( "fmt" "net/http" "path" - "proctor/internal/app/proctord/audit" "proctor/internal/app/proctord/docs" "proctor/internal/app/proctord/instrumentation" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/jobs/logs" "proctor/internal/app/proctord/jobs/schedule" "proctor/internal/app/proctord/middleware" "proctor/internal/app/proctord/storage" + executionHttpHandler "proctor/internal/app/service/execution/handler" + executionContextRepository "proctor/internal/app/service/execution/repository" + executionService "proctor/internal/app/service/execution/service" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" @@ -32,21 +32,20 @@ func NewRouter() (*mux.Router, error) { redisClient := redis.NewClient() postgresClient = postgresql.NewClient() - - store := storage.New(postgresClient) - metadataStore := metadataRepository.NewMetadataRepository(redisClient) - secretsStore := secretRepository.NewSecretRepository(redisClient) - httpClient, err := kubernetesHttpClient.NewClient() if err != nil { return router, err } kubeClient := kubernetes.NewKubernetesClient(httpClient) - auditor := audit.New(store, kubeClient) - jobExecutioner := execution.NewExecutioner(kubeClient, metadataStore, secretsStore) - jobExecutionHandler := execution.NewExecutionHandler(auditor, store, jobExecutioner) - jobLogger := logs.NewLogger(kubeClient) + store := storage.New(postgresClient) + executionStore := executionContextRepository.NewExecutionContextRepository(postgresClient) + metadataStore := metadataRepository.NewMetadataRepository(redisClient) + secretsStore := secretRepository.NewSecretRepository(redisClient) + + executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) + + jobExecutionHandler := executionHttpHandler.NewExecutionHttpHandler(executionService, executionStore) jobMetadataHandler := metadataHandler.NewMetadataHttpHandler(metadataStore) jobSecretsHandler := secretHttpHandler.NewSecretHttpHandler(secretsStore) @@ -62,9 +61,9 @@ func NewRouter() (*mux.Router, error) { http.ServeFile(w, r, path.Join(config.DocsPath(), "swagger.yml")) }) - router.HandleFunc(instrumentation.Wrap("/jobs/execute", middleware.ValidateClientVersion(jobExecutionHandler.Handle()))).Methods("POST") + router.HandleFunc(instrumentation.Wrap("/jobs/execute", middleware.ValidateClientVersion(jobExecutionHandler.Post()))).Methods("POST") router.HandleFunc(instrumentation.Wrap("/jobs/execute/{name}/status", middleware.ValidateClientVersion(jobExecutionHandler.Status()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/logs", middleware.ValidateClientVersion(jobLogger.Stream()))).Methods("GET") + router.HandleFunc(instrumentation.Wrap("/jobs/logs", middleware.ValidateClientVersion(jobExecutionHandler.Logs()))).Methods("GET") router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.Post()))).Methods("POST") router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.GetAll()))).Methods("GET") router.HandleFunc(instrumentation.Wrap("/jobs/secrets", middleware.ValidateClientVersion(jobSecretsHandler.Post()))).Methods("POST") From 9feebe9816e22e230a3e98997d8fe1c690f68b7e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 16 Jul 2019 16:37:18 +0530 Subject: [PATCH 044/268] [bimo.horizon|jasoet] Change imports placement on router according to style guide According to https://github.com/golang/go/wiki/CodeReviewComments#imports --- internal/app/service/execution/handler/http.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 48bf51ed..4036ba27 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -4,9 +4,11 @@ import ( "bufio" "encoding/json" "fmt" - "github.com/gorilla/mux" - "github.com/gorilla/websocket" "net/http" + "strconv" + "strings" + "time" + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/execution/handler/status" "proctor/internal/app/service/execution/repository" @@ -14,9 +16,9 @@ import ( executionStatus "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" - "strconv" - "strings" - "time" + + "github.com/gorilla/mux" + "github.com/gorilla/websocket" ) type ExecutionHttpHandler interface { From 99f2654c89c4091b037fdc0f6308a5dd63f4a60f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 17 Jul 2019 10:39:05 +0530 Subject: [PATCH 045/268] [Jasoet|Bimozx] V2: Change Router route --- internal/app/proctord/server/router.go | 10 ++-- .../app/service/execution/handler/http.go | 4 +- .../service/execution/handler/http_test.go | 46 ++++++++----------- .../execution/service/execution_mock.go | 3 +- 4 files changed, 29 insertions(+), 34 deletions(-) diff --git a/internal/app/proctord/server/router.go b/internal/app/proctord/server/router.go index 706b752f..e7424a58 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/proctord/server/router.go @@ -45,7 +45,7 @@ func NewRouter() (*mux.Router, error) { executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) - jobExecutionHandler := executionHttpHandler.NewExecutionHttpHandler(executionService, executionStore) + executionHandler := executionHttpHandler.NewExecutionHttpHandler(executionService, executionStore) jobMetadataHandler := metadataHandler.NewMetadataHttpHandler(metadataStore) jobSecretsHandler := secretHttpHandler.NewSecretHttpHandler(secretsStore) @@ -61,12 +61,14 @@ func NewRouter() (*mux.Router, error) { http.ServeFile(w, r, path.Join(config.DocsPath(), "swagger.yml")) }) - router.HandleFunc(instrumentation.Wrap("/jobs/execute", middleware.ValidateClientVersion(jobExecutionHandler.Post()))).Methods("POST") - router.HandleFunc(instrumentation.Wrap("/jobs/execute/{name}/status", middleware.ValidateClientVersion(jobExecutionHandler.Status()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/logs", middleware.ValidateClientVersion(jobExecutionHandler.Logs()))).Methods("GET") + router.HandleFunc(instrumentation.Wrap("/execute", middleware.ValidateClientVersion(executionHandler.Post()))).Methods("POST") + router.HandleFunc(instrumentation.Wrap("/execution/{contextId}/status", middleware.ValidateClientVersion(executionHandler.Status()))).Methods("GET") + router.HandleFunc(instrumentation.Wrap("/execution/logs", middleware.ValidateClientVersion(executionHandler.Logs()))).Methods("GET") + router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.Post()))).Methods("POST") router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.GetAll()))).Methods("GET") router.HandleFunc(instrumentation.Wrap("/jobs/secrets", middleware.ValidateClientVersion(jobSecretsHandler.Post()))).Methods("POST") + router.HandleFunc(instrumentation.Wrap("/jobs/schedule", middleware.ValidateClientVersion(scheduledJobsHandler.Schedule()))).Methods("POST") router.HandleFunc(instrumentation.Wrap("/jobs/schedule", middleware.ValidateClientVersion(scheduledJobsHandler.GetScheduledJobs()))).Methods("GET") router.HandleFunc(instrumentation.Wrap("/jobs/schedule/{id}", middleware.ValidateClientVersion(scheduledJobsHandler.GetScheduledJob()))).Methods("GET") diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 4036ba27..0e816409 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -64,7 +64,7 @@ func (httpHandler *executionHttpHandler) Logs() http.HandlerFunc { } defer conn.Close() - contextIdParam := strings.TrimLeft(request.URL.RawQuery, "job_name=") + contextIdParam := strings.TrimLeft(request.URL.RawQuery, "context_id=") executionContextId, err := strconv.ParseUint(contextIdParam, 10, 64) if contextIdParam == "" || err != nil { logger.Error("No valid execution context id provided as part of URL ", request.URL.RawQuery) @@ -117,7 +117,7 @@ func (httpHandler *executionHttpHandler) Logs() http.HandlerFunc { func (httpHandler *executionHttpHandler) Status() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - contextId := mux.Vars(request)["name"] + contextId := mux.Vars(request)["contextId"] executionContextId, err := strconv.ParseUint(contextId, 10, 64) logger.LogErrors(err, "parse execution context id from path parameter:", contextId) if err != nil { diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index f2bfcdcf..e24bd675 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -5,12 +5,19 @@ import ( "encoding/json" "errors" "fmt" + "io/ioutil" "net/http" "net/http/httptest" "strings" "testing" "time" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "github.com/urfave/negroni" "proctor/internal/app/service/execution/handler/parameter" handlerStatus "proctor/internal/app/service/execution/handler/status" "proctor/internal/app/service/execution/model" @@ -19,14 +26,6 @@ import ( "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/kubernetes" "proctor/internal/pkg/constant" - "proctor/internal/pkg/utility" - - "github.com/gorilla/mux" - "github.com/gorilla/websocket" - "github.com/jmoiron/sqlx/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "github.com/urfave/negroni" ) type ExecutionHttpHandlerTestSuite struct { @@ -48,7 +47,7 @@ func (suite *ExecutionHttpHandlerTestSuite) SetupTest() { suite.Client = &http.Client{} router := mux.NewRouter() - router.HandleFunc("/jobs/execute/{name}/status", suite.testExecutionHttpHandler.Status()).Methods("GET") + router.HandleFunc("/execute/{contextId}/status", suite.testExecutionHttpHandler.Status()).Methods("GET") n := negroni.Classic() n.UseHandler(router) suite.TestServer = httptest.NewServer(n) @@ -64,9 +63,8 @@ var logsHandlerDialer = websocket.Dialer{ } const ( - logsHandlerPath = "/jobs/logs" - logsHandlerRawQuery = "job_name=1" - logsHandlerRequestURI = logsHandlerPath + logsHandlerRawQuery = "context_id=1" + logsHandlerRequestURI = "/execution/logs" ) func (suite *ExecutionHttpHandlerTestSuite) newServer() *logsHandlerServer { @@ -104,9 +102,6 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFi Output: types.GzippedText("test"), } - buffer := utility.NewBuffer() - buffer.Write([]byte("test\n")) - suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) @@ -143,10 +138,9 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenRe Output: types.GzippedText("test"), } - buffer := utility.NewBuffer() - buffer.Write([]byte("test1\ntest2\ntest3\n")) - - suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(30)*time.Second).Return(buffer, nil).Once() + readCloser := ioutil.NopCloser(bytes.NewReader([]byte("test1\ntest2\ntest3\n"))) + defer readCloser.Close() + suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(30)*time.Second).Return(readCloser, nil).Once() defer suite.mockExecutionerService.AssertExpectations(t) suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) @@ -197,8 +191,8 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttp responseBody, err := json.Marshal(responseMap) assert.NoError(t, err) - req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"name": fmt.Sprint(executionContextId)}) + req := httptest.NewRequest("GET", fmt.Sprintf("/execution/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"contextId": fmt.Sprint(executionContextId)}) responseRecorder := httptest.NewRecorder() suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() @@ -215,8 +209,8 @@ func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionS executionContextId := uint64(1) - req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte("test"))) - req = mux.SetURLVars(req, map[string]string{"name": "notfound"}) + req := httptest.NewRequest("GET", fmt.Sprintf("/execution/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte("test"))) + req = mux.SetURLVars(req, map[string]string{"contextId": "notfound"}) responseRecorder := httptest.NewRecorder() suite.testExecutionHttpHandler.Status()(responseRecorder, req) @@ -243,10 +237,10 @@ func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionStatusHttpHa CreatedAt: time.Now(), Status: status.Finished, } - notFoundErr := errors.New("Execution context not found") + notFoundErr := errors.New("execution context not found") - req := httptest.NewRequest("GET", fmt.Sprintf("/execute/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"name": fmt.Sprint(executionContextId)}) + req := httptest.NewRequest("GET", fmt.Sprintf("/execution/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"contextId": fmt.Sprint(executionContextId)}) responseRecorder := httptest.NewRecorder() suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, notFoundErr).Once() diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index f881d7a6..23620655 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -4,7 +4,6 @@ import ( "github.com/stretchr/testify/mock" "io" "proctor/internal/app/service/execution/model" - "proctor/internal/pkg/utility" "time" ) @@ -29,5 +28,5 @@ func (mockService *MockExecutionService) save(executionContext *model.ExecutionC func (mockService *MockExecutionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { args := mockService.Called(executionName, waitTime) - return args.Get(0).(*utility.Buffer), args.Error(1) + return args.Get(0).(io.ReadCloser), args.Error(1) } From 2bf96aaa2616a4f36551d4d5bdf72ee0715819f6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 11:30:39 +0530 Subject: [PATCH 046/268] [bimo.horizon|jasoet] Change store to repo in schedule handler --- internal/app/service/schedule/handler/http.go | 254 ++++++++++++++++++ .../service/schedule/handler/status/status.go | 7 + 2 files changed, 261 insertions(+) create mode 100644 internal/app/service/schedule/handler/http.go create mode 100644 internal/app/service/schedule/handler/status/status.go diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go new file mode 100644 index 00000000..f8aec476 --- /dev/null +++ b/internal/app/service/schedule/handler/http.go @@ -0,0 +1,254 @@ +package handler + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/badoux/checkmail" + "github.com/getsentry/raven-go" + "github.com/gorilla/mux" + "github.com/robfig/cron" + + "proctor/internal/app/service/infra/logger" + metadataRepository "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/schedule/handler/status" + modelSchedule "proctor/internal/app/service/schedule/model" + scheduleRepository "proctor/internal/app/service/schedule/repository" + "proctor/internal/pkg/constant" +) + +type scheduler struct { + repository scheduleRepository.ScheduleRepository + metadataRepository metadataRepository.MetadataRepository +} + +type Scheduler interface { + Schedule() http.HandlerFunc + GetScheduledJobs() http.HandlerFunc + GetScheduledJob() http.HandlerFunc + RemoveScheduledJob() http.HandlerFunc +} + +func NewScheduler(repository scheduleRepository.ScheduleRepository, metadataRepository metadataRepository.MetadataRepository) Scheduler { + return &scheduler{ + metadataRepository: metadataRepository, + repository: repository, + } +} + +func (scheduler *scheduler) Schedule() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + var schedule modelSchedule.Schedule + err := json.NewDecoder(request.Body).Decode(&schedule) + defer request.Body.Close() + if err != nil { + logger.Error("Error parsing request body for scheduling jobs: ", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(constant.ClientError)) + + return + } + + if schedule.Tags == "" { + logger.Error("Tag(s) are missing") + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(constant.InvalidTagError)) + return + } + + _, err = cron.Parse(schedule.Cron) + if err != nil { + logger.Error(fmt.Sprintf("Client provided invalid cron expression: %s ", schedule.Tags), schedule.JobName, schedule.Cron) + + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(constant.InvalidCronExpressionClientError)) + return + } + + notificationEmails := strings.Split(schedule.NotificationEmails, ",") + + for _, notificationEmail := range notificationEmails { + err = checkmail.ValidateFormat(notificationEmail) + if err != nil { + logger.Error(fmt.Sprintf("Client provided invalid email address: %s: ", schedule.Tags), schedule.JobName, notificationEmail) + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(constant.InvalidEmailIdClientError)) + return + } + } + + if schedule.Group == "" { + logger.Error(fmt.Sprintf("Group Name is missing %s: ", schedule.Tags), schedule.JobName) + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(constant.GroupNameMissingError)) + return + } + + _, err = scheduler.metadataRepository.GetByName(schedule.JobName) + if err != nil { + if err.Error() == "redigo: nil returned" { + logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", schedule.Tags), schedule.JobName) + + response.WriteHeader(http.StatusNotFound) + _, _ = response.Write([]byte(constant.NonExistentProcClientError)) + } else { + logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", schedule.Tags), schedule.JobName, err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + } + + return + } + + schedule.Cron = fmt.Sprintf("0 %s", schedule.Cron) + schedule.ID, err = scheduler.repository.Insert(&schedule) + if err != nil { + if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { + logger.Error(fmt.Sprintf("Client provided duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) + + response.WriteHeader(http.StatusConflict) + _, _ = response.Write([]byte(constant.DuplicateJobNameArgsClientError)) + + return + } else { + logger.Error(fmt.Sprintf("Error persisting scheduled job %s ", schedule.Tags), schedule.JobName, err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + + return + } + } + + responseBody, err := json.Marshal(schedule) + if err != nil { + logger.Error(fmt.Sprintf("Error marshaling response body %s ", schedule.Tags), schedule.JobName, err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + + return + } + + response.WriteHeader(http.StatusCreated) + _, _ = response.Write(responseBody) + return + } +} + +func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + schedules, err := scheduler.repository.GetAllEnabled() + if err != nil { + logger.Error("Error fetching scheduled jobs", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + return + } + + if len(schedules) == 0 { + logger.Error(constant.NoScheduledJobsError, nil) + + response.WriteHeader(http.StatusNoContent) + return + } + + schedulesJson, err := json.Marshal(schedules) + if err != nil { + logger.Error("Error marshalling scheduled jobs", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + return + } + + _, _ = response.Write(schedulesJson) + } +} + +func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + jobId := mux.Vars(request)["id"] + scheduleId, err := strconv.ParseUint(jobId, 10, 64) + logger.LogErrors(err, "parse execution context id from path parameter:", jobId) + if err != nil { + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(status.PathParameterError)) + return + } + + schedule, err := scheduler.repository.GetById(scheduleId) + if err != nil { + if strings.Contains(err.Error(), "invalid input syntax") { + logger.Error(err.Error()) + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte("Invalid Job ID")) + return + } + logger.Error("Error fetching scheduled job", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + return + } + + scheduleJson, err := json.Marshal(schedule) + if err != nil { + logger.Error("Error marshalling scheduled job", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + return + } + + _, _ = response.Write(scheduleJson) + } +} + +func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc { + return func(response http.ResponseWriter, request *http.Request) { + jobId := mux.Vars(request)["id"] + scheduleId, err := strconv.ParseUint(jobId, 10, 64) + logger.LogErrors(err, "parse execution context id from path parameter:", jobId) + if err != nil { + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte(status.PathParameterError)) + return + } + + err = scheduler.repository.Delete(scheduleId) + if err != nil { + if strings.Contains(err.Error(), "invalid input syntax") { + logger.Error(err.Error()) + + response.WriteHeader(http.StatusBadRequest) + _, _ = response.Write([]byte("Invalid Job ID")) + return + } + logger.Error("Error fetching scheduled job", err.Error()) + raven.CaptureError(err, nil) + + response.WriteHeader(http.StatusInternalServerError) + _, _ = response.Write([]byte(constant.ServerError)) + return + } + + response.WriteHeader(http.StatusOK) + _, _ = response.Write([]byte(fmt.Sprintf("Successfully unscheduled Job ID: %d", scheduleId))) + } +} diff --git a/internal/app/service/schedule/handler/status/status.go b/internal/app/service/schedule/handler/status/status.go new file mode 100644 index 00000000..d3c29814 --- /dev/null +++ b/internal/app/service/schedule/handler/status/status.go @@ -0,0 +1,7 @@ +package status + +type ScheduleHandlerStatus string + +const ( + PathParameterError ScheduleHandlerStatus = "Failed to translate path parameter to uint64" +) From 76796e0bc1d756feffec621f099568f7cca971c7 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 17 Jul 2019 11:38:34 +0530 Subject: [PATCH 047/268] [Jasoet|Bimozx] V2: Migrate middleware to server --- cmd/proctord/main.go | 2 +- go.mod | 2 +- go.sum | 4 +- .../proctord/instrumentation/newrelic_stub.go | 62 ------------------- .../execution/handler/parameter/parameter.go | 2 - .../metadata/handler/http_handler_test.go | 10 +-- .../app/service/secret/handler/http_test.go | 6 +- .../app/{proctord => service}/server/api.go | 4 +- .../server/middleware}/newrelic.go | 13 ++-- .../server/middleware/parameter/parameter.go | 7 +++ .../server/middleware/status/status.go | 6 ++ .../middleware/validate_client_version.go | 13 ++-- .../validate_client_version_test.go | 12 ++-- .../{proctord => service}/server/router.go | 30 ++++----- 14 files changed, 63 insertions(+), 110 deletions(-) delete mode 100644 internal/app/proctord/instrumentation/newrelic_stub.go rename internal/app/{proctord => service}/server/api.go (87%) rename internal/app/{proctord/instrumentation => service/server/middleware}/newrelic.go (59%) create mode 100644 internal/app/service/server/middleware/parameter/parameter.go create mode 100644 internal/app/service/server/middleware/status/status.go rename internal/app/{proctord => service/server}/middleware/validate_client_version.go (63%) rename internal/app/{proctord => service/server}/middleware/validate_client_version_test.go (80%) rename internal/app/{proctord => service}/server/router.go (58%) diff --git a/cmd/proctord/main.go b/cmd/proctord/main.go index 7253a658..3b871991 100644 --- a/cmd/proctord/main.go +++ b/cmd/proctord/main.go @@ -4,10 +4,10 @@ import ( "github.com/getsentry/raven-go" "os" "proctor/internal/app/proctord/scheduler" - "proctor/internal/app/proctord/server" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/migration" "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/server" "github.com/urfave/cli" ) diff --git a/go.mod b/go.mod index 26f60693..b5a7f0eb 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/mattes/migrate v3.0.1+incompatible github.com/mattn/go-colorable v0.1.2 // indirect github.com/modern-go/reflect2 v1.0.1 // indirect - github.com/newrelic/go-agent v1.4.0 + github.com/newrelic/go-agent v2.9.0+incompatible github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/pkg/errors v0.8.1 github.com/robfig/cron v1.2.0 diff --git a/go.sum b/go.sum index ca6210f3..56afd109 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/newrelic/go-agent v1.4.0 h1:wY5acGl5vqk/4VJ+plJRTA87Utrr3k0sWNvlN857yH8= -github.com/newrelic/go-agent v1.4.0/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ= +github.com/newrelic/go-agent v2.9.0+incompatible h1:f47VHKmZ/3KXzMSg77CGroizhcyP4JaE6zMYOeXxkDg= +github.com/newrelic/go-agent v2.9.0+incompatible/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c h1:Hww8mOyEKTeON4bZn7FrlLismspbPc1teNRUVH7wLQ8= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= diff --git a/internal/app/proctord/instrumentation/newrelic_stub.go b/internal/app/proctord/instrumentation/newrelic_stub.go deleted file mode 100644 index 233bca31..00000000 --- a/internal/app/proctord/instrumentation/newrelic_stub.go +++ /dev/null @@ -1,62 +0,0 @@ -package instrumentation - -import ( - "net/http" - "time" - - newrelic "github.com/newrelic/go-agent" -) - -type StubNewRelicTransaction struct{} - -func (snrt *StubNewRelicTransaction) End() error { - return nil -} - -func (snrt *StubNewRelicTransaction) Ignore() error { - return nil -} - -func (snrt *StubNewRelicTransaction) SetName(name string) error { - return nil -} - -func (snrt *StubNewRelicTransaction) NoticeError(err error) error { - return nil -} - -func (snrt *StubNewRelicTransaction) AddAttribute(key string, value interface{}) error { - return nil -} - -func (snrt *StubNewRelicTransaction) StartSegmentNow() newrelic.SegmentStartTime { - return newrelic.SegmentStartTime{} -} - -func (snrt *StubNewRelicTransaction) Header() http.Header { - return http.Header{} -} - -func (snrt *StubNewRelicTransaction) Write([]byte) (int, error) { - return 0, nil -} - -func (snrt *StubNewRelicTransaction) WriteHeader(int) { - return -} - -type StubNewrelicApp struct{} - -func (sna *StubNewrelicApp) StartTransaction(name string, w http.ResponseWriter, r *http.Request) newrelic.Transaction { - return &StubNewRelicTransaction{} -} - -func (sna *StubNewrelicApp) RecordCustomEvent(eventType string, params map[string]interface{}) error { - return nil -} -func (sna *StubNewrelicApp) WaitForConnection(timeout time.Duration) error { - return nil -} -func (sna *StubNewrelicApp) Shutdown(timeout time.Duration) { - return -} diff --git a/internal/app/service/execution/handler/parameter/parameter.go b/internal/app/service/execution/handler/parameter/parameter.go index 0851f8dd..b0c9ff75 100644 --- a/internal/app/service/execution/handler/parameter/parameter.go +++ b/internal/app/service/execution/handler/parameter/parameter.go @@ -3,6 +3,4 @@ package parameter const ( UserEmailHeader string = "Email-Id" - AccessTokenHeader string = "Access-Token" - ClientVersionHeader string = "Client-Version" ) diff --git a/internal/app/service/metadata/handler/http_handler_test.go b/internal/app/service/metadata/handler/http_handler_test.go index 8a01bdc6..0baa6b5b 100644 --- a/internal/app/service/metadata/handler/http_handler_test.go +++ b/internal/app/service/metadata/handler/http_handler_test.go @@ -68,7 +68,7 @@ func (s *MetadataHandlerTestSuite) TestSuccessfulMetadataSubmission() { metadataSubmissionRequestBody, err := json.Marshal(jobsMetadata) assert.NoError(t, err) - req := httptest.NewRequest("PUT", "/jobs/metadata", bytes.NewReader(metadataSubmissionRequestBody)) + req := httptest.NewRequest("PUT", "/metadata", bytes.NewReader(metadataSubmissionRequestBody)) responseRecorder := httptest.NewRecorder() s.mockRepository.On("Save", metadata).Return(nil).Once() @@ -84,7 +84,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionMalformedRequest() { t := s.T() jobMetadataSubmissionRequest := fmt.Sprintf("{ some-malformed-reque") - req := httptest.NewRequest("PUT", "/jobs/metadata", bytes.NewReader([]byte(jobMetadataSubmissionRequest))) + req := httptest.NewRequest("PUT", "/metadata", bytes.NewReader([]byte(jobMetadataSubmissionRequest))) responseRecorder := httptest.NewRecorder() s.metadataHttpHandler.Post()(responseRecorder, req) @@ -104,7 +104,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionForStoreFailure() { metadataSubmissionRequestBody, err := json.Marshal(jobMetadata) assert.NoError(t, err) - req := httptest.NewRequest("PUT", "/jobs/metadata", bytes.NewReader(metadataSubmissionRequestBody)) + req := httptest.NewRequest("PUT", "/metadata", bytes.NewReader(metadataSubmissionRequestBody)) responseRecorder := httptest.NewRecorder() s.mockRepository.On("Save", metadata).Return(errors.New("error")).Once() @@ -120,7 +120,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionForStoreFailure() { func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { t := s.T() - req := httptest.NewRequest("GET", "/jobs/metadata", bytes.NewReader([]byte{})) + req := httptest.NewRequest("GET", "/metadata", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() jobsMetadata := []modelMetadata.Metadata{} @@ -140,7 +140,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { t := s.T() - req := httptest.NewRequest("GET", "/jobs/metadata", bytes.NewReader([]byte{})) + req := httptest.NewRequest("GET", "/metadata", bytes.NewReader([]byte{})) responseRecorder := httptest.NewRecorder() jobsMetadata := []modelMetadata.Metadata{} diff --git a/internal/app/service/secret/handler/http_test.go b/internal/app/service/secret/handler/http_test.go index dee7e235..ecde3fb0 100644 --- a/internal/app/service/secret/handler/http_test.go +++ b/internal/app/service/secret/handler/http_test.go @@ -40,7 +40,7 @@ func (suite *SecretsHandlerTestSuite) TestPostSecretSuccess() { requestBody, err := json.Marshal(secret) assert.NoError(t, err) - req := httptest.NewRequest("POST", "/job-secrets", bytes.NewReader(requestBody)) + req := httptest.NewRequest("POST", "/secrets", bytes.NewReader(requestBody)) responseRecorder := httptest.NewRecorder() suite.mockSecretRepository.On("Save", secret).Return(nil).Once() @@ -58,7 +58,7 @@ func (suite *SecretsHandlerTestSuite) TestPostSecretsMalformedData() { requestBody, err := json.Marshal("any-malformed-requ") assert.NoError(t, err) - req := httptest.NewRequest("POST", "/job-secrets", bytes.NewReader(requestBody)) + req := httptest.NewRequest("POST", "/secrets", bytes.NewReader(requestBody)) responseRecorder := httptest.NewRecorder() suite.secretHandler.Post()(responseRecorder, req) @@ -79,7 +79,7 @@ func (suite *SecretsHandlerTestSuite) TestPostSecretsStoreFailure() { requestBody, err := json.Marshal(secret) assert.NoError(t, err) - req := httptest.NewRequest("POST", "/job-secrets", bytes.NewReader(requestBody)) + req := httptest.NewRequest("POST", "/secrets", bytes.NewReader(requestBody)) responseRecorder := httptest.NewRecorder() suite.mockSecretRepository.On("Save", secret).Return(errors.New("error")).Once() diff --git a/internal/app/proctord/server/api.go b/internal/app/service/server/api.go similarity index 87% rename from internal/app/proctord/server/api.go rename to internal/app/service/server/api.go index f89c29a2..e71efb28 100644 --- a/internal/app/proctord/server/api.go +++ b/internal/app/service/server/api.go @@ -1,9 +1,9 @@ package server import ( - "proctor/internal/app/proctord/instrumentation" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/server/middleware" "time" "github.com/tylerb/graceful" @@ -11,7 +11,7 @@ import ( ) func Start() error { - err := instrumentation.InitNewRelic() + err := middleware.InitNewRelic() if err != nil { logger.Fatal(err) } diff --git a/internal/app/proctord/instrumentation/newrelic.go b/internal/app/service/server/middleware/newrelic.go similarity index 59% rename from internal/app/proctord/instrumentation/newrelic.go rename to internal/app/service/server/middleware/newrelic.go index bd101649..fe11b423 100644 --- a/internal/app/proctord/instrumentation/newrelic.go +++ b/internal/app/service/server/middleware/newrelic.go @@ -1,12 +1,13 @@ -package instrumentation +package middleware import ( + "github.com/gorilla/mux" newrelic "github.com/newrelic/go-agent" - "net/http" + "github.com/newrelic/go-agent/_integrations/nrgorilla/v1" "proctor/internal/app/service/infra/config" ) -var NewRelicApp newrelic.Application +var newRelicApp newrelic.Application func InitNewRelic() error { appName := config.NewRelicAppName() @@ -17,10 +18,10 @@ func InitNewRelic() error { if err != nil { return err } - NewRelicApp = app + newRelicApp = app return nil } -func Wrap(pattern string, handlerFunc http.HandlerFunc) (string, func(http.ResponseWriter, *http.Request)) { - return newrelic.WrapHandleFunc(NewRelicApp, pattern, handlerFunc) +func InstrumentNewRelic(r *mux.Router) *mux.Router { + return nrgorilla.InstrumentRoutes(r, newRelicApp) } diff --git a/internal/app/service/server/middleware/parameter/parameter.go b/internal/app/service/server/middleware/parameter/parameter.go new file mode 100644 index 00000000..b4f54009 --- /dev/null +++ b/internal/app/service/server/middleware/parameter/parameter.go @@ -0,0 +1,7 @@ +package parameter + + +const ( + AccessTokenHeader string = "Access-Token" + ClientVersionHeader string = "Client-Version" +) diff --git a/internal/app/service/server/middleware/status/status.go b/internal/app/service/server/middleware/status/status.go new file mode 100644 index 00000000..41c9c962 --- /dev/null +++ b/internal/app/service/server/middleware/status/status.go @@ -0,0 +1,6 @@ +package status + + +const ( + ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s . To continue using proctor, please upgrade to latest version." +) diff --git a/internal/app/proctord/middleware/validate_client_version.go b/internal/app/service/server/middleware/validate_client_version.go similarity index 63% rename from internal/app/proctord/middleware/validate_client_version.go rename to internal/app/service/server/middleware/validate_client_version.go index f7e5977d..08d98ba3 100644 --- a/internal/app/proctord/middleware/validate_client_version.go +++ b/internal/app/service/server/middleware/validate_client_version.go @@ -6,13 +6,14 @@ import ( "net/http" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" - "proctor/internal/pkg/constant" + "proctor/internal/app/service/server/middleware/parameter" + "proctor/internal/app/service/server/middleware/status" ) -func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func ValidateClientVersion(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requestHeaderClientVersion := r.Header.Get(constant.ClientVersionHeaderKey) + requestHeaderClientVersion := r.Header.Get(parameter.ClientVersionHeader) if requestHeaderClientVersion != "" { clientVersion, err := version.NewVersion(requestHeaderClientVersion) @@ -27,12 +28,12 @@ func ValidateClientVersion(next http.HandlerFunc) http.HandlerFunc { if clientVersion.LessThan(minClientVersion) { w.WriteHeader(400) - _, _ = w.Write([]byte(fmt.Sprintf(constant.ClientOutdatedErrorMessage, clientVersion))) + _, _ = w.Write([]byte(fmt.Sprintf(status.ClientOutdatedErrorMessage, clientVersion))) return } next.ServeHTTP(w, r) } else { next.ServeHTTP(w, r) } - } + }) } diff --git a/internal/app/proctord/middleware/validate_client_version_test.go b/internal/app/service/server/middleware/validate_client_version_test.go similarity index 80% rename from internal/app/proctord/middleware/validate_client_version_test.go rename to internal/app/service/server/middleware/validate_client_version_test.go index fe5f9b2a..dfdef9a6 100644 --- a/internal/app/proctord/middleware/validate_client_version_test.go +++ b/internal/app/service/server/middleware/validate_client_version_test.go @@ -6,7 +6,7 @@ import ( "net/http" "net/http/httptest" "os" - "proctor/internal/pkg/constant" + "proctor/internal/app/service/server/middleware/parameter" "testing" ) @@ -25,8 +25,8 @@ func TestValidClientVersionHttpHeader(t *testing.T) { client := &http.Client{} - req, _ := http.NewRequest("GET", ts.URL+"/jobs/metadata", nil) - req.Header.Add(constant.ClientVersionHeaderKey, "0.2.0") + req, _ := http.NewRequest("GET", ts.URL+"/metadata", nil) + req.Header.Add(parameter.ClientVersionHeader, "0.2.0") resp, _ := client.Do(req) defer resp.Body.Close() @@ -43,7 +43,7 @@ func TestEmptyClientVersionHttpHeader(t *testing.T) { client := &http.Client{} - req, _ := http.NewRequest("GET", ts.URL+"/jobs/metadata", nil) + req, _ := http.NewRequest("GET", ts.URL+"/metadata", nil) resp, _ := client.Do(req) defer resp.Body.Close() @@ -60,8 +60,8 @@ func TestInvalidClientVersionHttpHeader(t *testing.T) { client := &http.Client{} - req, _ := http.NewRequest("GET", ts.URL+"/jobs/metadata", nil) - req.Header.Add(constant.ClientVersionHeaderKey, "0.1.0") + req, _ := http.NewRequest("GET", ts.URL+"/metadata", nil) + req.Header.Add(parameter.ClientVersionHeader, "0.1.0") resp, _ := client.Do(req) defer resp.Body.Close() diff --git a/internal/app/proctord/server/router.go b/internal/app/service/server/router.go similarity index 58% rename from internal/app/proctord/server/router.go rename to internal/app/service/server/router.go index e7424a58..81c18f2e 100644 --- a/internal/app/proctord/server/router.go +++ b/internal/app/service/server/router.go @@ -5,9 +5,7 @@ import ( "net/http" "path" "proctor/internal/app/proctord/docs" - "proctor/internal/app/proctord/instrumentation" "proctor/internal/app/proctord/jobs/schedule" - "proctor/internal/app/proctord/middleware" "proctor/internal/app/proctord/storage" executionHttpHandler "proctor/internal/app/service/execution/handler" executionContextRepository "proctor/internal/app/service/execution/repository" @@ -21,6 +19,7 @@ import ( metadataRepository "proctor/internal/app/service/metadata/repository" secretHttpHandler "proctor/internal/app/service/secret/handler" secretRepository "proctor/internal/app/service/secret/repository" + "proctor/internal/app/service/server/middleware" "github.com/gorilla/mux" ) @@ -43,9 +42,9 @@ func NewRouter() (*mux.Router, error) { metadataStore := metadataRepository.NewMetadataRepository(redisClient) secretsStore := secretRepository.NewSecretRepository(redisClient) - executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) + _executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) - executionHandler := executionHttpHandler.NewExecutionHttpHandler(executionService, executionStore) + executionHandler := executionHttpHandler.NewExecutionHttpHandler(_executionService, executionStore) jobMetadataHandler := metadataHandler.NewMetadataHttpHandler(metadataStore) jobSecretsHandler := secretHttpHandler.NewSecretHttpHandler(secretsStore) @@ -61,18 +60,21 @@ func NewRouter() (*mux.Router, error) { http.ServeFile(w, r, path.Join(config.DocsPath(), "swagger.yml")) }) - router.HandleFunc(instrumentation.Wrap("/execute", middleware.ValidateClientVersion(executionHandler.Post()))).Methods("POST") - router.HandleFunc(instrumentation.Wrap("/execution/{contextId}/status", middleware.ValidateClientVersion(executionHandler.Status()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/execution/logs", middleware.ValidateClientVersion(executionHandler.Logs()))).Methods("GET") + router = middleware.InstrumentNewRelic(router) + router.Use(middleware.ValidateClientVersion) - router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.Post()))).Methods("POST") - router.HandleFunc(instrumentation.Wrap("/jobs/metadata", middleware.ValidateClientVersion(jobMetadataHandler.GetAll()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/secrets", middleware.ValidateClientVersion(jobSecretsHandler.Post()))).Methods("POST") + router.HandleFunc("/execute", executionHandler.Post()).Methods("POST") + router.HandleFunc("/execution/{contextId}/status", executionHandler.Status()).Methods("GET") + router.HandleFunc("/execution/logs", executionHandler.Logs()).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/schedule", middleware.ValidateClientVersion(scheduledJobsHandler.Schedule()))).Methods("POST") - router.HandleFunc(instrumentation.Wrap("/jobs/schedule", middleware.ValidateClientVersion(scheduledJobsHandler.GetScheduledJobs()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/schedule/{id}", middleware.ValidateClientVersion(scheduledJobsHandler.GetScheduledJob()))).Methods("GET") - router.HandleFunc(instrumentation.Wrap("/jobs/schedule/{id}", middleware.ValidateClientVersion(scheduledJobsHandler.RemoveScheduledJob()))).Methods("DELETE") + router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") + router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") + router.HandleFunc("/secrets", jobSecretsHandler.Post()).Methods("POST") + + router.HandleFunc("/jobs/schedule", scheduledJobsHandler.Schedule()).Methods("POST") + router.HandleFunc("/jobs/schedule", scheduledJobsHandler.GetScheduledJobs()).Methods("GET") + router.HandleFunc("/jobs/schedule/{id}", scheduledJobsHandler.GetScheduledJob()).Methods("GET") + router.HandleFunc("/jobs/schedule/{id}", scheduledJobsHandler.RemoveScheduledJob()).Methods("DELETE") return router, nil } From 2c26b754f0c5b4ca8c24aaed7861e82b165eadae Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:29:22 +0530 Subject: [PATCH 048/268] [bimo.horizon|jasoet] Add handler status constants --- internal/pkg/status/status.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 internal/pkg/status/status.go diff --git a/internal/pkg/status/status.go b/internal/pkg/status/status.go new file mode 100644 index 00000000..09fc70f5 --- /dev/null +++ b/internal/pkg/status/status.go @@ -0,0 +1,21 @@ +package status + +type HandlerStatus string + +const ( + EmailInvalidError HandlerStatus = "Email is invalid" + GenericServerError HandlerStatus = "Something went wrong" + MalformedRequestError HandlerStatus = "Malformed request" + PathParameterError HandlerStatus = "Failed to translate path parameter to uint64" + + MetadataNotFoundError HandlerStatus = "Metadata not found" + + ScheduleDeleteSuccess HandlerStatus = "Schedule delete is successful" + + ScheduleCronFormatInvalidError HandlerStatus = "Schedule cron format is invalid" + ScheduleDuplicateJobNameArgsError HandlerStatus = "Schedule job name and args duplicate is found" + ScheduleIdInvalidError HandlerStatus = "Schedule ID is invalid" + ScheduleGroupMissingError HandlerStatus = "Schedule group is missing" + ScheduleListNotFoundError HandlerStatus = "Schedule list is not found" + ScheduleTagMissingError HandlerStatus = "Schedule tag(s) are missing" +) From 06944a6b68bcc80641bcb93e30b49d4564851e5f Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:29:42 +0530 Subject: [PATCH 049/268] [bimo.horizon|jasoet] Update dependency --- go.mod | 2 ++ go.sum | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/go.mod b/go.mod index b5a7f0eb..85b0fe21 100644 --- a/go.mod +++ b/go.mod @@ -28,9 +28,11 @@ require ( github.com/jarcoal/httpmock v1.0.4 github.com/jmoiron/sqlx v1.2.0 github.com/json-iterator/go v1.1.6 // indirect + github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/lib/pq v1.1.1 github.com/mattes/migrate v3.0.1+incompatible github.com/mattn/go-colorable v0.1.2 // indirect + github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/newrelic/go-agent v2.9.0+incompatible github.com/opencontainers/go-digest v1.0.0-rc1 // indirect diff --git a/go.sum b/go.sum index 56afd109..b450fe31 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBv github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -145,6 +147,8 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f h1:ee+twVCignaZjt7jpbMSLxAeTN/Nfq9W/nm91E7QO1A= +github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f/go.mod h1:hltEC42XzfMNgg0S1v6JTywwra2Mu6F6cLR03debVQ8= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -275,6 +279,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= From bca22a3f2bc9a5e13ba4a8652057409cd3e59cd3 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:33:56 +0530 Subject: [PATCH 050/268] [bimo.horizon|jasoet] Rename metadata handler to maintain consistency --- .../app/service/metadata/handler/{http_handler.go => http.go} | 0 .../metadata/handler/{http_handler_test.go => http_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename internal/app/service/metadata/handler/{http_handler.go => http.go} (100%) rename internal/app/service/metadata/handler/{http_handler_test.go => http_test.go} (100%) diff --git a/internal/app/service/metadata/handler/http_handler.go b/internal/app/service/metadata/handler/http.go similarity index 100% rename from internal/app/service/metadata/handler/http_handler.go rename to internal/app/service/metadata/handler/http.go diff --git a/internal/app/service/metadata/handler/http_handler_test.go b/internal/app/service/metadata/handler/http_test.go similarity index 100% rename from internal/app/service/metadata/handler/http_handler_test.go rename to internal/app/service/metadata/handler/http_test.go From aa15d88dec450546e8fa1e199e7f9d93483df9eb Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:39:55 +0530 Subject: [PATCH 051/268] [bimo.horizon|jasoet] Change handler method name --- .../app/service/execution/handler/http.go | 8 ++--- .../service/execution/handler/http_test.go | 30 +++++++------------ internal/app/service/server/router.go | 4 +-- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 0e816409..3864db5a 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -23,8 +23,8 @@ import ( type ExecutionHttpHandler interface { Post() http.HandlerFunc - Status() http.HandlerFunc - Logs() http.HandlerFunc + GetStatus() http.HandlerFunc + GetLogs() http.HandlerFunc } type executionHttpHandler struct { @@ -53,7 +53,7 @@ func (httpHandler *executionHttpHandler) closeWebSocket(message string, conn *we return } -func (httpHandler *executionHttpHandler) Logs() http.HandlerFunc { +func (httpHandler *executionHttpHandler) GetLogs() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { conn, err := upgrader.Upgrade(response, request, nil) logger.LogErrors(err, "upgrade http server connection to WebSocket") @@ -115,7 +115,7 @@ func (httpHandler *executionHttpHandler) Logs() http.HandlerFunc { } } -func (httpHandler *executionHttpHandler) Status() http.HandlerFunc { +func (httpHandler *executionHttpHandler) GetStatus() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { contextId := mux.Vars(request)["contextId"] executionContextId, err := strconv.ParseUint(contextId, 10, 64) diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index e24bd675..c99e5d03 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -17,7 +17,7 @@ import ( "github.com/jmoiron/sqlx/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "github.com/urfave/negroni" + "proctor/internal/app/service/execution/handler/parameter" handlerStatus "proctor/internal/app/service/execution/handler/status" "proctor/internal/app/service/execution/model" @@ -34,9 +34,6 @@ type ExecutionHttpHandlerTestSuite struct { mockExecutionerContextRepository *repository.MockExecutionContextRepository mockKubernetesClient kubernetes.MockKubernetesClient testExecutionHttpHandler ExecutionHttpHandler - - Client *http.Client - TestServer *httptest.Server } func (suite *ExecutionHttpHandlerTestSuite) SetupTest() { @@ -44,13 +41,6 @@ func (suite *ExecutionHttpHandlerTestSuite) SetupTest() { suite.mockExecutionerContextRepository = &repository.MockExecutionContextRepository{} suite.mockKubernetesClient = kubernetes.MockKubernetesClient{} suite.testExecutionHttpHandler = NewExecutionHttpHandler(suite.mockExecutionerService, suite.mockExecutionerContextRepository) - - suite.Client = &http.Client{} - router := mux.NewRouter() - router.HandleFunc("/execute/{contextId}/status", suite.testExecutionHttpHandler.Status()).Methods("GET") - n := negroni.Classic() - n.UseHandler(router) - suite.TestServer = httptest.NewServer(n) } type logsHandlerServer struct { @@ -69,7 +59,7 @@ const ( func (suite *ExecutionHttpHandlerTestSuite) newServer() *logsHandlerServer { var s logsHandlerServer - s.Server = httptest.NewServer(suite.testExecutionHttpHandler.Logs()) + s.Server = httptest.NewServer(suite.testExecutionHttpHandler.GetLogs()) s.Server.URL += logsHandlerRequestURI s.URL = makeWsProto(s.Server.URL) return &s @@ -79,7 +69,7 @@ func makeWsProto(s string) string { return "ws" + strings.TrimPrefix(s, "http") } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFinishedHttpHandler() { +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenFinishedHttpHandler() { t := suite.T() s := suite.newServer() @@ -114,7 +104,7 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenFi assert.Equal(t, "test", string(firstMessage)) } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenReadyHttpHandler() { +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenReadyHttpHandler() { t := suite.T() s := suite.newServer() @@ -162,7 +152,7 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionLogsWhenRe assert.Equal(t, "test3", string(thirdMessage)) } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttpHandler() { +func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetStatusHttpHandler() { t := suite.T() executionContextId := uint64(1) @@ -198,13 +188,13 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionStatusHttp suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) - suite.testExecutionHttpHandler.Status()(responseRecorder, req) + suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionStatusHttpHandler() { +func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionGetStatusHttpHandler() { t := suite.T() executionContextId := uint64(1) @@ -213,13 +203,13 @@ func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionS req = mux.SetURLVars(req, map[string]string{"contextId": "notfound"}) responseRecorder := httptest.NewRecorder() - suite.testExecutionHttpHandler.Status()(responseRecorder, req) + suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) assert.Equal(t, string(handlerStatus.PathParameterError), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionStatusHttpHandler() { +func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionGetStatusHttpHandler() { t := suite.T() executionContextId := uint64(1) @@ -246,7 +236,7 @@ func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionStatusHttpHa suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, notFoundErr).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) - suite.testExecutionHttpHandler.Status()(responseRecorder, req) + suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusNotFound, responseRecorder.Code) assert.Equal(t, string(handlerStatus.ExecutionContextNotFound), responseRecorder.Body.String()) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 81c18f2e..76e5ac75 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -64,8 +64,8 @@ func NewRouter() (*mux.Router, error) { router.Use(middleware.ValidateClientVersion) router.HandleFunc("/execute", executionHandler.Post()).Methods("POST") - router.HandleFunc("/execution/{contextId}/status", executionHandler.Status()).Methods("GET") - router.HandleFunc("/execution/logs", executionHandler.Logs()).Methods("GET") + router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") + router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") From b6a7aedec56abbb0e122f43d1479a9a8af7b3036 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 17 Jul 2019 18:41:04 +0530 Subject: [PATCH 052/268] [Jasoet|Bimozx] V2: Fix race condition --- .travis.yml | 6 +-- Makefile | 4 +- .../execution/repository/execution_context.go | 4 +- .../repository/execution_context_mock.go | 2 +- .../repository/execution_context_test.go | 10 ++--- .../service/execution/service/execution.go | 38 ++++++++++++------- .../service/execution_integration_test.go | 2 +- .../execution/service/execution_mock.go | 2 +- .../execution/service/execution_test.go | 10 ++--- .../app/service/execution/status/execution.go | 23 +++++------ 10 files changed, 56 insertions(+), 45 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44daf6b7..b6b21e34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ jobs: include: - stage: test script: - - make db.setup test-with-race + - make db.setup test -after_success: - - scripts/release.sh +#after_success: +# - scripts/release.sh diff --git a/Makefile b/Makefile index c159ce26..e0a38364 100644 --- a/Makefile +++ b/Makefile @@ -24,12 +24,12 @@ test-with-race: test .PHONY: test test: ENABLE_INTEGRATION_TEST=false \ - go test $(RACE_FLAG) -coverprofile=$(OUT_DIR)/coverage.out ./... + go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: itest itest: ENABLE_INTEGRATION_TEST=true \ - go test $(RACE_FLAG) -coverprofile=$(OUT_DIR)/coverage.out ./... + go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: server server: diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 4c93d944..36ceb3eb 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -11,7 +11,7 @@ import ( ) type ExecutionContextRepository interface { - Insert(context *model.ExecutionContext) (uint64, error) + Insert(context model.ExecutionContext) (uint64, error) UpdateJobOutput(executionId uint64, output types.GzippedText) error UpdateStatus(executionId uint64, status status.ExecutionStatus) error Delete(executionId uint64) error @@ -32,7 +32,7 @@ func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRep } } -func (repository *executionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { +func (repository *executionContextRepository) Insert(context model.ExecutionContext) (uint64, error) { snowflakeId, _ := id.NextId() context.ExecutionID = snowflakeId sql := "INSERT INTO execution_context (id, job_name,name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :name, :user_email, :image_tag, :args, :output, :status)" diff --git a/internal/app/service/execution/repository/execution_context_mock.go b/internal/app/service/execution/repository/execution_context_mock.go index 9195af14..1908dfd0 100644 --- a/internal/app/service/execution/repository/execution_context_mock.go +++ b/internal/app/service/execution/repository/execution_context_mock.go @@ -11,7 +11,7 @@ type MockExecutionContextRepository struct { mock.Mock } -func (mockRepository *MockExecutionContextRepository) Insert(context *model.ExecutionContext) (uint64, error) { +func (mockRepository *MockExecutionContextRepository) Insert(context model.ExecutionContext) (uint64, error) { args := mockRepository.Called(context) return uint64(args.Int(0)), args.Error(1) } diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index 439a6ce2..7289b023 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -20,7 +20,7 @@ func TestExecutionContextRepository_Insert(t *testing.T) { mapKey := fake.FirstName() mapValue := fake.LastName() - context := &model.ExecutionContext{ + context := model.ExecutionContext{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), ImageTag: fake.BeerStyle(), @@ -50,7 +50,7 @@ func TestExecutionContextRepository_Delete(t *testing.T) { defer repository.deleteAll() fake.Seed(0) - context := &model.ExecutionContext{ + context := model.ExecutionContext{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), ImageTag: fake.BeerStyle(), @@ -79,7 +79,7 @@ func TestExecutionContextRepository_UpdateStatus(t *testing.T) { defer repository.deleteAll() fake.Seed(0) - context := &model.ExecutionContext{ + context := model.ExecutionContext{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), ImageTag: fake.BeerStyle(), @@ -110,7 +110,7 @@ func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { defer repository.deleteAll() fake.Seed(0) - context := &model.ExecutionContext{ + context := model.ExecutionContext{ JobName: fake.BuzzWord(), Name: fake.HackerAdjective(), UserEmail: fake.Email(), @@ -173,7 +173,7 @@ func populateSeedDataForTest(repository ExecutionContextRepository, count int, s defaultStatus = status.ExecutionStatus(val) } - context := &model.ExecutionContext{ + context := model.ExecutionContext{ JobName: jobName, Name: fake.HackerAdjective(), UserEmail: email, diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index e40f9256..06e5d75d 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -23,7 +23,7 @@ type ExecutionService interface { Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) - save(executionContext *model.ExecutionContext) error + save(executionContext model.ExecutionContext) error } type executionService struct { @@ -47,23 +47,23 @@ func NewExecutionService( } } -func (service *executionService) save(executionContext *model.ExecutionContext) error { +func (service *executionService) save(executionContext model.ExecutionContext) error { var err error if executionContext.ExecutionID == 0 { _, err = service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db", *executionContext) + logger.LogErrors(err, "save execution context to db", executionContext) } else { context, _err := service.repository.GetById(executionContext.ExecutionID) - logger.LogErrors(_err, "get context from db by execution id", *executionContext) + logger.LogErrors(_err, "get context from db by execution id", executionContext) if _err != nil || context == nil { _, err = service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db", *executionContext) + logger.LogErrors(err, "save execution context to db", executionContext) } else { err = service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) - logger.LogErrors(err, "update execution context status", *executionContext) + logger.LogErrors(err, "update execution context status", executionContext) if len(executionContext.Output) > 0 { err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) - logger.LogErrors(err, "update execution context output", *executionContext) + logger.LogErrors(err, "update execution context output", executionContext) } } } @@ -95,7 +95,7 @@ func (service *executionService) Execute(jobName string, userEmail string, args } func (service *executionService) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) { - context := &model.ExecutionContext{ + context := model.ExecutionContext{ UserEmail: userEmail, JobName: jobName, Args: args, @@ -107,13 +107,13 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st metadata, err := service.metadataRepository.GetByName(jobName) if err != nil { context.Status = status.RequirementNotMet - return context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", jobName, err.Error())) + return &context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", jobName, err.Error())) } secret, err := service.secretRepository.GetByJobName(jobName) if err != nil { context.Status = status.RequirementNotMet - return context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", jobName, err.Error())) + return &context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", jobName, err.Error())) } executionArgs := mergeArgs(args, secret) @@ -123,18 +123,26 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st logger.Info("Executed Job on Kubernetes got ", executionName, " execution jobName and ", err, "errors") if err != nil { context.Status = status.CreationFailed - return context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) + return &context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } context.Name = executionName + contextId, err := service.repository.Insert(context) + logger.LogErrors(err, "save execution context to db", context) + if err != nil { + context.Status = status.ContextSavingFailed + return &context, "", errors.New(fmt.Sprintf("error when saving execution context %v with errors %v", context, err.Error())) + } + + context.ExecutionID = contextId + go service.watchProcess(executionName, context) - return context, executionName, nil + return &context, executionName, nil } -func (service *executionService) watchProcess(executionName string, context *model.ExecutionContext) { - defer service.save(context) +func (service *executionService) watchProcess(executionName string, context model.ExecutionContext) { waitTime := config.KubeLogProcessWaitTime() * time.Second err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) @@ -184,6 +192,8 @@ func (service *executionService) watchProcess(executionName string, context *mod context.Status = status.Finished } + defer service.save(context) + return } diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 34f75e8b..159ae7f1 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -84,7 +84,7 @@ func (suite *TestExecutionIntegrationSuite) TestExecuteJobSuccess() { expectedContext, err := suite.repository.GetById(context.ExecutionID) assert.NoError(t, err) assert.NotNil(t, expectedContext) - assert.Equal(t, expectedContext.Status, status.Finished) + assert.Equal(t, status.Finished, expectedContext.Status) assert.NotNil(t, expectedContext.Output) } diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index 23620655..c7afaebb 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -21,7 +21,7 @@ func (mockService *MockExecutionService) ExecuteWithCommand(jobName string, user return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) } -func (mockService *MockExecutionService) save(executionContext *model.ExecutionContext) error { +func (mockService *MockExecutionService) save(executionContext model.ExecutionContext) error { args := mockService.Called(executionContext) return args.Error(0) } diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 9b191910..3b05e92e 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -45,7 +45,7 @@ func (suite *TestExecutionServiceSuite) SetupTest() { func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { t := suite.T() - context := &model.ExecutionContext{} + context := model.ExecutionContext{} suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() err := suite.service.save(context) @@ -59,23 +59,23 @@ func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { func (suite *TestExecutionServiceSuite) TestSaveWithExecutionId() { t := suite.T() _id, _ := id.NextId() - context := &model.ExecutionContext{ + context := model.ExecutionContext{ ExecutionID: _id, Status: status.Created, } - suite.mockRepository.On("GetById", _id).Return(context, errors.New("Get By Id Error")).Once() + suite.mockRepository.On("GetById", _id).Return(&context, errors.New("Get By Id Error")).Once() suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() err := suite.service.save(context) assert.Error(t, err, "Insert Failed") - suite.mockRepository.On("GetById", _id).Return(context, nil).Once() + suite.mockRepository.On("GetById", _id).Return(&context, nil).Once() suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(errors.New("Update Status Failed")).Once() err = suite.service.save(context) assert.Error(t, err, "Update Status Failed") context.Output = types.GzippedText("This is some output") - suite.mockRepository.On("GetById", _id).Return(context, nil).Once() + suite.mockRepository.On("GetById", _id).Return(&context, nil).Once() suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(nil).Once() suite.mockRepository.On("UpdateJobOutput", context.ExecutionID, context.Output).Return(errors.New("Update Output Failed")).Once() err = suite.service.save(context) diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go index 422556ed..28d6a25c 100644 --- a/internal/app/service/execution/status/execution.go +++ b/internal/app/service/execution/status/execution.go @@ -3,15 +3,16 @@ package status type ExecutionStatus string const ( - Received ExecutionStatus = "RECEIVED" - RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" - Created ExecutionStatus = "CREATED" - CreationFailed ExecutionStatus = "CREATION_FAILED" - JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" - JobReady ExecutionStatus = "JOB_READY" - PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" - PodReady ExecutionStatus = "POD_READY" - PodFailed ExecutionStatus = "POD_FAILED" - FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" - Finished ExecutionStatus = "FINISHED" + Received ExecutionStatus = "RECEIVED" + RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" + Created ExecutionStatus = "CREATED" + CreationFailed ExecutionStatus = "CREATION_FAILED" + ContextSavingFailed ExecutionStatus = "CONTEXT_SAVING_FAILED" + JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" + JobReady ExecutionStatus = "JOB_READY" + PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" + PodReady ExecutionStatus = "POD_READY" + PodFailed ExecutionStatus = "POD_FAILED" + FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" + Finished ExecutionStatus = "FINISHED" ) From ca2a1b9cc8619b2fad318cfe273b041b29b267c6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:41:41 +0530 Subject: [PATCH 053/268] [bimo.horizon|jasoet] Add json notation to schedule model --- .../app/service/schedule/model/schedule.go | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/app/service/schedule/model/schedule.go b/internal/app/service/schedule/model/schedule.go index e8cea32c..d97fab50 100644 --- a/internal/app/service/schedule/model/schedule.go +++ b/internal/app/service/schedule/model/schedule.go @@ -6,15 +6,15 @@ import ( ) type Schedule struct { - ID uint64 `db:"id"` - JobName string `db:"job_name"` - Args dbTypes.Base64Map `db:"args"` - Tags string `db:"tags"` - Cron string `db:"cron"` - NotificationEmails string `db:"notification_emails"` - UserEmail string `db:"user_email"` - Group string `db:"group"` - Enabled bool `db:"enabled"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` + ID uint64 `json:"id" db:"id"` + JobName string `json:"jobName" db:"job_name"` + Args dbTypes.Base64Map `json:"args" db:"args"` + Tags string `json:"tags" db:"tags"` + Cron string `json:"cron" db:"cron"` + NotificationEmails string `json:"notificationEmails" db:"notification_emails"` + UserEmail string `json:"userEmail" db:"user_email"` + Group string `json:"group" db:"group"` + Enabled bool `json:"enabled" db:"enabled"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` } From 2ae733164995d6975c022fe331e25acf3769b794 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:48:55 +0530 Subject: [PATCH 054/268] [bimo.horizon|jasoet] Swap constant usage with a more specific handler status --- internal/app/service/schedule/handler/http.go | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index f8aec476..264b3c8b 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -14,10 +14,9 @@ import ( "proctor/internal/app/service/infra/logger" metadataRepository "proctor/internal/app/service/metadata/repository" - "proctor/internal/app/service/schedule/handler/status" modelSchedule "proctor/internal/app/service/schedule/model" scheduleRepository "proctor/internal/app/service/schedule/repository" - "proctor/internal/pkg/constant" + "proctor/internal/pkg/status" ) type scheduler struct { @@ -45,11 +44,11 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { err := json.NewDecoder(request.Body).Decode(&schedule) defer request.Body.Close() if err != nil { - logger.Error("Error parsing request body for scheduling jobs: ", err.Error()) + logger.Error("Error parsing request body for schedule: ", err.Error()) raven.CaptureError(err, nil) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(constant.ClientError)) + _, _ = response.Write([]byte(status.MalformedRequestError)) return } @@ -57,16 +56,16 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { if schedule.Tags == "" { logger.Error("Tag(s) are missing") response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(constant.InvalidTagError)) + _, _ = response.Write([]byte(status.ScheduleTagMissingError)) return } _, err = cron.Parse(schedule.Cron) if err != nil { - logger.Error(fmt.Sprintf("Client provided invalid cron expression: %s ", schedule.Tags), schedule.JobName, schedule.Cron) + logger.Error(fmt.Sprintf("Cron format is invalid: %s ", schedule.Tags), schedule.JobName, schedule.Cron) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(constant.InvalidCronExpressionClientError)) + _, _ = response.Write([]byte(status.ScheduleCronFormatInvalidError)) return } @@ -75,33 +74,33 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { for _, notificationEmail := range notificationEmails { err = checkmail.ValidateFormat(notificationEmail) if err != nil { - logger.Error(fmt.Sprintf("Client provided invalid email address: %s: ", schedule.Tags), schedule.JobName, notificationEmail) + logger.Error(fmt.Sprintf("Email address provided is invalid: %s: ", schedule.Tags), schedule.JobName, notificationEmail) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(constant.InvalidEmailIdClientError)) + _, _ = response.Write([]byte(status.EmailInvalidError)) return } } if schedule.Group == "" { - logger.Error(fmt.Sprintf("Group Name is missing %s: ", schedule.Tags), schedule.JobName) + logger.Error(fmt.Sprintf("Group is missing %s: ", schedule.Tags), schedule.JobName) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(constant.GroupNameMissingError)) + _, _ = response.Write([]byte(status.ScheduleGroupMissingError)) return } _, err = scheduler.metadataRepository.GetByName(schedule.JobName) if err != nil { if err.Error() == "redigo: nil returned" { - logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", schedule.Tags), schedule.JobName) + logger.Error(fmt.Sprintf("Metadata not found: %s ", schedule.Tags), schedule.JobName) response.WriteHeader(http.StatusNotFound) - _, _ = response.Write([]byte(constant.NonExistentProcClientError)) + _, _ = response.Write([]byte(status.MetadataNotFoundError)) } else { logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", schedule.Tags), schedule.JobName, err.Error()) raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) } return @@ -111,11 +110,11 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { schedule.ID, err = scheduler.repository.Insert(&schedule) if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { - logger.Error(fmt.Sprintf("Client provided duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) + logger.Error(fmt.Sprintf("Duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusConflict) - _, _ = response.Write([]byte(constant.DuplicateJobNameArgsClientError)) + _, _ = response.Write([]byte(status.ScheduleDuplicateJobNameArgsError)) return } else { @@ -123,7 +122,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } @@ -135,7 +134,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } @@ -154,12 +153,12 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } - if len(schedules) == 0 { - logger.Error(constant.NoScheduledJobsError, nil) + if len(scheduleList) == 0 { + logger.Error(status.ScheduleListNotFoundError, nil) response.WriteHeader(http.StatusNoContent) return @@ -167,11 +166,11 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { schedulesJson, err := json.Marshal(schedules) if err != nil { - logger.Error("Error marshalling scheduled jobs", err.Error()) + logger.Error("Error marshalling schedule list", err.Error()) raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } @@ -195,14 +194,14 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte("Invalid Job ID")) + _, _ = response.Write([]byte(status.ScheduleIdInvalidError)) return } logger.Error("Error fetching scheduled job", err.Error()) raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } @@ -212,7 +211,7 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } @@ -237,18 +236,18 @@ func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc { logger.Error(err.Error()) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte("Invalid Job ID")) + _, _ = response.Write([]byte(status.ScheduleIdInvalidError)) return } - logger.Error("Error fetching scheduled job", err.Error()) + logger.Error("Error fetching schedule", err.Error()) raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(constant.ServerError)) + _, _ = response.Write([]byte(status.GenericServerError)) return } response.WriteHeader(http.StatusOK) - _, _ = response.Write([]byte(fmt.Sprintf("Successfully unscheduled Job ID: %d", scheduleId))) + _, _ = response.Write([]byte(status.ScheduleDeleteSuccess)) } } From 321ca304808c516e4b1f94c12179e7fa298c86e8 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:51:47 +0530 Subject: [PATCH 055/268] [bimo.horizon|jasoet] Change handler name to be more descriptive --- internal/app/service/schedule/handler/http.go | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 264b3c8b..4b45d878 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -19,26 +19,26 @@ import ( "proctor/internal/pkg/status" ) -type scheduler struct { +type scheduleHttpHandler struct { repository scheduleRepository.ScheduleRepository metadataRepository metadataRepository.MetadataRepository } -type Scheduler interface { - Schedule() http.HandlerFunc - GetScheduledJobs() http.HandlerFunc - GetScheduledJob() http.HandlerFunc - RemoveScheduledJob() http.HandlerFunc +type ScheduleHttpHandler interface { + Post() http.HandlerFunc + GetAll() http.HandlerFunc + Get() http.HandlerFunc + Delete() http.HandlerFunc } -func NewScheduler(repository scheduleRepository.ScheduleRepository, metadataRepository metadataRepository.MetadataRepository) Scheduler { - return &scheduler{ - metadataRepository: metadataRepository, +func NewScheduleHttpHandler(repository scheduleRepository.ScheduleRepository, metadataRepository metadataRepository.MetadataRepository) ScheduleHttpHandler { + return &scheduleHttpHandler{ repository: repository, + metadataRepository: metadataRepository, } } -func (scheduler *scheduler) Schedule() http.HandlerFunc { +func (httpHandler *scheduleHttpHandler) Post() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { var schedule modelSchedule.Schedule err := json.NewDecoder(request.Body).Decode(&schedule) @@ -88,7 +88,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { return } - _, err = scheduler.metadataRepository.GetByName(schedule.JobName) + _, err = httpHandler.metadataRepository.GetByName(schedule.JobName) if err != nil { if err.Error() == "redigo: nil returned" { logger.Error(fmt.Sprintf("Metadata not found: %s ", schedule.Tags), schedule.JobName) @@ -107,7 +107,7 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { } schedule.Cron = fmt.Sprintf("0 %s", schedule.Cron) - schedule.ID, err = scheduler.repository.Insert(&schedule) + schedule.ID, err = httpHandler.repository.Insert(&schedule) if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { logger.Error(fmt.Sprintf("Duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) @@ -145,9 +145,9 @@ func (scheduler *scheduler) Schedule() http.HandlerFunc { } } -func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { +func (httpHandler *scheduleHttpHandler) GetAll() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - schedules, err := scheduler.repository.GetAllEnabled() + scheduleList, err := httpHandler.repository.GetAllEnabled() if err != nil { logger.Error("Error fetching scheduled jobs", err.Error()) raven.CaptureError(err, nil) @@ -164,7 +164,7 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { return } - schedulesJson, err := json.Marshal(schedules) + scheduleListJson, err := json.Marshal(scheduleList) if err != nil { logger.Error("Error marshalling schedule list", err.Error()) raven.CaptureError(err, nil) @@ -174,22 +174,22 @@ func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { return } - _, _ = response.Write(schedulesJson) + _, _ = response.Write(scheduleListJson) } } -func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { +func (httpHandler *scheduleHttpHandler) Get() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - jobId := mux.Vars(request)["id"] - scheduleId, err := strconv.ParseUint(jobId, 10, 64) - logger.LogErrors(err, "parse execution context id from path parameter:", jobId) + scheduleIdParam := mux.Vars(request)["scheduleId"] + scheduleId, err := strconv.ParseUint(scheduleIdParam, 10, 64) + logger.LogErrors(err, "parse execution context id from path parameter:", scheduleIdParam) if err != nil { response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.PathParameterError)) return } - schedule, err := scheduler.repository.GetById(scheduleId) + schedule, err := httpHandler.repository.GetById(scheduleId) if err != nil { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) @@ -219,18 +219,18 @@ func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { } } -func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc { +func (httpHandler *scheduleHttpHandler) Delete() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - jobId := mux.Vars(request)["id"] - scheduleId, err := strconv.ParseUint(jobId, 10, 64) - logger.LogErrors(err, "parse execution context id from path parameter:", jobId) + scheduleIdParam := mux.Vars(request)["scheduleId"] + scheduleId, err := strconv.ParseUint(scheduleIdParam, 10, 64) + logger.LogErrors(err, "parse execution context id from path parameter:", scheduleIdParam) if err != nil { response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.PathParameterError)) return } - err = scheduler.repository.Delete(scheduleId) + err = httpHandler.repository.Delete(scheduleId) if err != nil { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) From 9789a0c51517627d49bb250558dd785d06127873 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:54:26 +0530 Subject: [PATCH 056/268] [bimo.horizon|jasoet] Update schedule handler --- internal/app/service/server/router.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 76e5ac75..a78e04a7 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -4,9 +4,10 @@ import ( "fmt" "net/http" "path" + + "github.com/gorilla/mux" + "proctor/internal/app/proctord/docs" - "proctor/internal/app/proctord/jobs/schedule" - "proctor/internal/app/proctord/storage" executionHttpHandler "proctor/internal/app/service/execution/handler" executionContextRepository "proctor/internal/app/service/execution/repository" executionService "proctor/internal/app/service/execution/service" @@ -17,11 +18,11 @@ import ( kubernetesHttpClient "proctor/internal/app/service/infra/kubernetes/http" metadataHandler "proctor/internal/app/service/metadata/handler" metadataRepository "proctor/internal/app/service/metadata/repository" + scheduleHttpHandler "proctor/internal/app/service/schedule/handler" + scheduleRepository "proctor/internal/app/service/schedule/repository" secretHttpHandler "proctor/internal/app/service/secret/handler" secretRepository "proctor/internal/app/service/secret/repository" "proctor/internal/app/service/server/middleware" - - "github.com/gorilla/mux" ) var postgresClient postgresql.Client @@ -37,8 +38,8 @@ func NewRouter() (*mux.Router, error) { } kubeClient := kubernetes.NewKubernetesClient(httpClient) - store := storage.New(postgresClient) executionStore := executionContextRepository.NewExecutionContextRepository(postgresClient) + scheduleStore := scheduleRepository.NewScheduleRepository(postgresClient) metadataStore := metadataRepository.NewMetadataRepository(redisClient) secretsStore := secretRepository.NewSecretRepository(redisClient) @@ -47,8 +48,7 @@ func NewRouter() (*mux.Router, error) { executionHandler := executionHttpHandler.NewExecutionHttpHandler(_executionService, executionStore) jobMetadataHandler := metadataHandler.NewMetadataHttpHandler(metadataStore) jobSecretsHandler := secretHttpHandler.NewSecretHttpHandler(secretsStore) - - scheduledJobsHandler := schedule.NewScheduler(store, metadataStore) + scheduleHandler := scheduleHttpHandler.NewScheduleHttpHandler(scheduleStore, metadataStore) router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { _, _ = fmt.Fprintf(w, "pong") @@ -71,10 +71,10 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") router.HandleFunc("/secrets", jobSecretsHandler.Post()).Methods("POST") - router.HandleFunc("/jobs/schedule", scheduledJobsHandler.Schedule()).Methods("POST") - router.HandleFunc("/jobs/schedule", scheduledJobsHandler.GetScheduledJobs()).Methods("GET") - router.HandleFunc("/jobs/schedule/{id}", scheduledJobsHandler.GetScheduledJob()).Methods("GET") - router.HandleFunc("/jobs/schedule/{id}", scheduledJobsHandler.RemoveScheduledJob()).Methods("DELETE") + router.HandleFunc("/schedule", scheduleHandler.Post()).Methods("POST") + router.HandleFunc("/schedule", scheduleHandler.GetAll()).Methods("GET") + router.HandleFunc("/schedule/{scheduleId}", scheduleHandler.Get()).Methods("GET") + router.HandleFunc("/schedule/{scheduleId}", scheduleHandler.Delete()).Methods("DELETE") return router, nil } From cb9e74c1ed05e7230f49f2c831be6fc8b600efa3 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 17 Jul 2019 18:54:52 +0530 Subject: [PATCH 057/268] [bimo.horizon|jasoet] Add schedule handler test --- .../app/service/schedule/handler/http_test.go | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 internal/app/service/schedule/handler/http_test.go diff --git a/internal/app/service/schedule/handler/http_test.go b/internal/app/service/schedule/handler/http_test.go new file mode 100644 index 00000000..ca99d1dc --- /dev/null +++ b/internal/app/service/schedule/handler/http_test.go @@ -0,0 +1,197 @@ +package handler + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "proctor/internal/app/service/infra/db/types" + metadataRepository "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/schedule/model" + scheduleRepository "proctor/internal/app/service/schedule/repository" + metadataModel "proctor/internal/pkg/model/metadata" + "proctor/internal/pkg/status" +) + +type ScheduleHttpHandlerTestSuite struct { + suite.Suite + mockScheduleRepository *scheduleRepository.MockScheduleRepository + mockMetadataRepository *metadataRepository.MockMetadataRepository + testScheduleHttpHandler ScheduleHttpHandler +} + +func (suite *ScheduleHttpHandlerTestSuite) SetupTest() { + suite.mockScheduleRepository = &scheduleRepository.MockScheduleRepository{} + suite.mockMetadataRepository = &metadataRepository.MockMetadataRepository{} + suite.testScheduleHttpHandler = NewScheduleHttpHandler(suite.mockScheduleRepository, suite.mockMetadataRepository) +} + +func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulSchedulePostHttpHandler() { + t := suite.T() + + scheduleId := uint64(0) + argsMap := types.Base64Map{ + "COMMAND": "test", + } + requestSchedule := model.Schedule{ + JobName: "test", + Args: argsMap, + Tags: "test", + Cron: "* * * * *", + UserEmail: "mrproctor@example.com", + NotificationEmails: "mrproctor@example.com", + Group: "mrproctor", + } + responseSchedule := requestSchedule + responseSchedule.ID = scheduleId + responseSchedule.Cron = "0 * * * * *" + + requestBody, err := json.Marshal(requestSchedule) + assert.NoError(t, err) + + responseBody, err := json.Marshal(responseSchedule) + assert.NoError(t, err) + + req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) + responseRecorder := httptest.NewRecorder() + + suite.mockMetadataRepository.On("GetByName", requestSchedule.JobName).Return(&metadataModel.Metadata{}, nil).Once() + defer suite.mockMetadataRepository.AssertExpectations(t) + + requestSchedule.Cron = "0 * * * * *" + suite.mockScheduleRepository.On("Insert", &requestSchedule).Return(0, nil).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + suite.testScheduleHttpHandler.Post()(responseRecorder, req) + + assert.Equal(t, http.StatusCreated, responseRecorder.Code) + assert.Equal(t, string(responseBody), responseRecorder.Body.String()) +} + +func (suite *ScheduleHttpHandlerTestSuite) TestErrorSchedulePostHttpHandler() { + t := suite.T() + + argsMap := types.Base64Map{ + "COMMAND": "test", + } + requestSchedule := model.Schedule{ + JobName: "test", + Args: argsMap, + Tags: "test", + Cron: "* * * * *", + UserEmail: "mrproctor@example.com", + NotificationEmails: "mrproctor@example.com", + Group: "mrproctor", + } + tagMissingSchedule := requestSchedule + tagMissingSchedule.Tags = "" + cronFormatInvalidSchedule := requestSchedule + cronFormatInvalidSchedule.Cron = "test" + emailInvalidSchedule := requestSchedule + emailInvalidSchedule.NotificationEmails = "test" + groupMissingSchedule := requestSchedule + groupMissingSchedule.Group = "" + + requestBody, err := json.Marshal(requestSchedule) + assert.NoError(t, err) + tagMissingRequestBody, err := json.Marshal(tagMissingSchedule) + assert.NoError(t, err) + cronFormatInvalidRequestBody, err := json.Marshal(cronFormatInvalidSchedule) + assert.NoError(t, err) + emailInvalidRequestBody, err := json.Marshal(emailInvalidSchedule) + assert.NoError(t, err) + groupMissingRequestBody, err := json.Marshal(groupMissingSchedule) + assert.NoError(t, err) + + requestSchedule.Cron = fmt.Sprintf("0 %s", requestSchedule.Cron) + + schedulePostErrorTests := []struct { + requestBody []byte + httpErrorResponse int + errorResponse status.HandlerStatus + }{ + {tagMissingRequestBody, http.StatusBadRequest, status.ScheduleTagMissingError}, + {cronFormatInvalidRequestBody, http.StatusBadRequest, status.ScheduleCronFormatInvalidError}, + {emailInvalidRequestBody, http.StatusBadRequest, status.EmailInvalidError}, + {groupMissingRequestBody, http.StatusBadRequest, status.ScheduleGroupMissingError}, + + // Metadata not found error + {requestBody, http.StatusNotFound, status.MetadataNotFoundError}, + + // Duplicate schedule job name and args error + {requestBody, http.StatusConflict, status.ScheduleDuplicateJobNameArgsError}, + } + + // Metadata not found error + suite.mockMetadataRepository.On("GetByName", requestSchedule.JobName).Return(&metadataModel.Metadata{}, errors.New("redigo: nil returned")).Once() + // Metadata success + suite.mockMetadataRepository.On("GetByName", requestSchedule.JobName).Return(&metadataModel.Metadata{}, nil).Once() + defer suite.mockMetadataRepository.AssertExpectations(t) + // Schedule duplicate job name and args error + suite.mockScheduleRepository.On("Insert", &requestSchedule).Return(0, errors.New("duplicate key value violates unique constraint")).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + for _, errorTest := range schedulePostErrorTests { + req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(errorTest.requestBody)) + responseRecorder := httptest.NewRecorder() + + suite.testScheduleHttpHandler.Post()(responseRecorder, req) + + assert.Equal(t, errorTest.httpErrorResponse, responseRecorder.Code) + assert.Equal(t, string(errorTest.errorResponse), responseRecorder.Body.String()) + } +} + +func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetAllHttpHandler() { + t := suite.T() + + argsMap := types.Base64Map{ + "COMMAND": "test", + } + responseScheduleList := []model.Schedule{ + { + ID: uint64(1), + JobName: "test1", + Args: argsMap, + Tags: "test", + Cron: "0 * * * * *", + UserEmail: "mrproctor@example.com", + NotificationEmails: "mrproctor@example.com", + Group: "mrproctor", + }, + { + ID: uint64(2), + JobName: "test2", + Args: argsMap, + Tags: "test", + Cron: "0 * * * * *", + UserEmail: "mrproctor@example.com", + NotificationEmails: "mrproctor@example.com", + Group: "mrproctor", + }, + } + responseBody, err := json.Marshal(responseScheduleList) + assert.NoError(t, err) + + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte("test"))) + responseRecorder := httptest.NewRecorder() + + suite.mockScheduleRepository.On("GetAllEnabled").Return(responseScheduleList, nil).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + suite.testScheduleHttpHandler.GetAll()(responseRecorder, req) + + assert.Equal(t, http.StatusOK, responseRecorder.Code) + assert.Equal(t, string(responseBody), responseRecorder.Body.String()) +} + +func TestScheduleHttpHandlerTestSuite(t *testing.T) { + suite.Run(t, new(ScheduleHttpHandlerTestSuite)) +} From e7839dd951f1525301c21adbfcd58d3180f2d759 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 18 Jul 2019 10:54:18 +0530 Subject: [PATCH 058/268] [bimo.horizon|jasoet] Add missing schedule handler test --- .../app/service/schedule/handler/http_test.go | 157 +++++++++++++++++- 1 file changed, 151 insertions(+), 6 deletions(-) diff --git a/internal/app/service/schedule/handler/http_test.go b/internal/app/service/schedule/handler/http_test.go index ca99d1dc..87c765e4 100644 --- a/internal/app/service/schedule/handler/http_test.go +++ b/internal/app/service/schedule/handler/http_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "testing" + "github.com/gorilla/mux" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" @@ -113,9 +114,9 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorSchedulePostHttpHandler() { requestSchedule.Cron = fmt.Sprintf("0 %s", requestSchedule.Cron) schedulePostErrorTests := []struct { - requestBody []byte - httpErrorResponse int - errorResponse status.HandlerStatus + requestBody []byte + responseStatus int + responseBody status.HandlerStatus }{ {tagMissingRequestBody, http.StatusBadRequest, status.ScheduleTagMissingError}, {cronFormatInvalidRequestBody, http.StatusBadRequest, status.ScheduleCronFormatInvalidError}, @@ -144,8 +145,8 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorSchedulePostHttpHandler() { suite.testScheduleHttpHandler.Post()(responseRecorder, req) - assert.Equal(t, errorTest.httpErrorResponse, responseRecorder.Code) - assert.Equal(t, string(errorTest.errorResponse), responseRecorder.Body.String()) + assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) + assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) } } @@ -180,7 +181,7 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetAllHttpHandl responseBody, err := json.Marshal(responseScheduleList) assert.NoError(t, err) - req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte("test"))) + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) responseRecorder := httptest.NewRecorder() suite.mockScheduleRepository.On("GetAllEnabled").Return(responseScheduleList, nil).Once() @@ -192,6 +193,150 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetAllHttpHandl assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } +func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetAllHttpHandler() { + t := suite.T() + + scheduleGetAllErrorTests := []struct { + responseStatus int + responseBody status.HandlerStatus + }{ + // Error fetching + {http.StatusInternalServerError, status.GenericServerError}, + // Empty schedule list + {http.StatusNoContent, ""}, + } + + suite.mockScheduleRepository.On("GetAllEnabled").Return([]model.Schedule{}, errors.New("test")).Once() + suite.mockScheduleRepository.On("GetAllEnabled").Return([]model.Schedule{}, nil).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + for _, errorTest := range scheduleGetAllErrorTests { + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) + responseRecorder := httptest.NewRecorder() + + suite.testScheduleHttpHandler.GetAll()(responseRecorder, req) + + assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) + assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) + } +} + +func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler() { + t := suite.T() + + scheduleId := uint64(1) + argsMap := types.Base64Map{ + "COMMAND": "test", + } + responseSchedule := model.Schedule{ + ID: scheduleId, + JobName: "test1", + Args: argsMap, + Tags: "test", + Cron: "0 * * * * *", + UserEmail: "mrproctor@example.com", + NotificationEmails: "mrproctor@example.com", + Group: "mrproctor", + } + + responseBody, err := json.Marshal(responseSchedule) + assert.NoError(t, err) + + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"scheduleId": fmt.Sprint(scheduleId)}) + responseRecorder := httptest.NewRecorder() + + suite.mockScheduleRepository.On("GetById", scheduleId).Return(&responseSchedule, nil).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + suite.testScheduleHttpHandler.Get()(responseRecorder, req) + + assert.Equal(t, http.StatusOK, responseRecorder.Code) + assert.Equal(t, string(responseBody), responseRecorder.Body.String()) +} + +func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { + t := suite.T() + + scheduleId := uint64(1) + + scheduleGetErrorTests := []struct { + requestParams string + responseStatus int + responseBody status.HandlerStatus + }{ + {"test", http.StatusBadRequest, status.PathParameterError}, + {"1", http.StatusBadRequest, status.ScheduleIdInvalidError}, + // Schedule fetch error + {"1", http.StatusInternalServerError, status.GenericServerError}, + } + + suite.mockScheduleRepository.On("GetById", scheduleId).Return(&model.Schedule{}, errors.New("invalid input syntax")).Once() + suite.mockScheduleRepository.On("GetById", scheduleId).Return(&model.Schedule{}, errors.New("")).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + for _, errorTest := range scheduleGetErrorTests { + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"scheduleId": errorTest.requestParams}) + responseRecorder := httptest.NewRecorder() + + suite.testScheduleHttpHandler.Get()(responseRecorder, req) + + assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) + assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) + } +} + +func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleDeleteHttpHandler() { + t := suite.T() + + scheduleId := uint64(1) + + req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"scheduleId": fmt.Sprint(scheduleId)}) + responseRecorder := httptest.NewRecorder() + + suite.mockScheduleRepository.On("Delete", scheduleId).Return(nil).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + suite.testScheduleHttpHandler.Delete()(responseRecorder, req) + + assert.Equal(t, http.StatusOK, responseRecorder.Code) + assert.Equal(t, string(status.ScheduleDeleteSuccess), responseRecorder.Body.String()) +} + +func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleDeleteHttpHandler() { + t := suite.T() + + scheduleId := uint64(1) + + scheduleDeleteErrorTests := []struct { + requestParams string + responseStatus int + responseBody status.HandlerStatus + }{ + {"test", http.StatusBadRequest, status.PathParameterError}, + {"1", http.StatusBadRequest, status.ScheduleIdInvalidError}, + // Schedule delete error + {"1", http.StatusInternalServerError, status.GenericServerError}, + } + + suite.mockScheduleRepository.On("Delete", scheduleId).Return(errors.New("invalid input syntax")).Once() + suite.mockScheduleRepository.On("Delete", scheduleId).Return(errors.New("")).Once() + defer suite.mockScheduleRepository.AssertExpectations(t) + + for _, errorTest := range scheduleDeleteErrorTests { + req := httptest.NewRequest("DELETE", "/schedule", bytes.NewReader([]byte(""))) + req = mux.SetURLVars(req, map[string]string{"scheduleId": errorTest.requestParams}) + responseRecorder := httptest.NewRecorder() + + suite.testScheduleHttpHandler.Delete()(responseRecorder, req) + + assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) + assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) + } +} + func TestScheduleHttpHandlerTestSuite(t *testing.T) { suite.Run(t, new(ScheduleHttpHandlerTestSuite)) } From 89a5473262e54112e4c060bbc3ae3b66954db8b4 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 24 Jul 2019 17:00:55 +0700 Subject: [PATCH 059/268] [bimo.horizon] Change inconsistent 'id' naming to 'ID' instead --- .../execution/repository/execution_context.go | 2 +- .../execution/service/execution_test.go | 2 +- internal/app/service/infra/id/snowflake.go | 2 +- internal/app/service/schedule/handler/http.go | 20 +++++----- .../app/service/schedule/handler/http_test.go | 38 +++++++++---------- .../service/schedule/repository/schedule.go | 14 +++---- .../schedule/repository/schedule_mock.go | 4 +- .../schedule/repository/schedule_test.go | 18 ++++----- internal/app/service/server/router.go | 6 +-- internal/pkg/status/status.go | 2 +- 10 files changed, 54 insertions(+), 54 deletions(-) diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 36ceb3eb..0d927631 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -33,7 +33,7 @@ func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRep } func (repository *executionContextRepository) Insert(context model.ExecutionContext) (uint64, error) { - snowflakeId, _ := id.NextId() + snowflakeId, _ := id.NextID() context.ExecutionID = snowflakeId sql := "INSERT INTO execution_context (id, job_name,name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :name, :user_email, :image_tag, :args, :output, :status)" _, err := repository.postgresqlClient.NamedExec(sql, &context) diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 3b05e92e..54c54791 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -58,7 +58,7 @@ func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { func (suite *TestExecutionServiceSuite) TestSaveWithExecutionId() { t := suite.T() - _id, _ := id.NextId() + _id, _ := id.NextID() context := model.ExecutionContext{ ExecutionID: _id, Status: status.Created, diff --git a/internal/app/service/infra/id/snowflake.go b/internal/app/service/infra/id/snowflake.go index 0621f972..eb844ee0 100644 --- a/internal/app/service/infra/id/snowflake.go +++ b/internal/app/service/infra/id/snowflake.go @@ -9,7 +9,7 @@ func init() { snowflake = sonyflake.NewSonyflake(snowflakeSetting) } -func NextId() (uint64, error) { +func NextID() (uint64, error) { return snowflake.NextID() } diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 4b45d878..7a699ecd 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -180,21 +180,21 @@ func (httpHandler *scheduleHttpHandler) GetAll() http.HandlerFunc { func (httpHandler *scheduleHttpHandler) Get() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - scheduleIdParam := mux.Vars(request)["scheduleId"] - scheduleId, err := strconv.ParseUint(scheduleIdParam, 10, 64) - logger.LogErrors(err, "parse execution context id from path parameter:", scheduleIdParam) + scheduleIDParam := mux.Vars(request)["scheduleID"] + scheduleID, err := strconv.ParseUint(scheduleIDParam, 10, 64) + logger.LogErrors(err, "parse schedule id from path parameter:", scheduleIDParam) if err != nil { response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.PathParameterError)) return } - schedule, err := httpHandler.repository.GetById(scheduleId) + schedule, err := httpHandler.repository.GetByID(scheduleID) if err != nil { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(status.ScheduleIdInvalidError)) + _, _ = response.Write([]byte(status.ScheduleIDInvalidError)) return } logger.Error("Error fetching scheduled job", err.Error()) @@ -221,22 +221,22 @@ func (httpHandler *scheduleHttpHandler) Get() http.HandlerFunc { func (httpHandler *scheduleHttpHandler) Delete() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { - scheduleIdParam := mux.Vars(request)["scheduleId"] - scheduleId, err := strconv.ParseUint(scheduleIdParam, 10, 64) - logger.LogErrors(err, "parse execution context id from path parameter:", scheduleIdParam) + scheduleIDParam := mux.Vars(request)["scheduleID"] + scheduleID, err := strconv.ParseUint(scheduleIDParam, 10, 64) + logger.LogErrors(err, "parse schedule id from path parameter:", scheduleIDParam) if err != nil { response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.PathParameterError)) return } - err = httpHandler.repository.Delete(scheduleId) + err = httpHandler.repository.Delete(scheduleID) if err != nil { if strings.Contains(err.Error(), "invalid input syntax") { logger.Error(err.Error()) response.WriteHeader(http.StatusBadRequest) - _, _ = response.Write([]byte(status.ScheduleIdInvalidError)) + _, _ = response.Write([]byte(status.ScheduleIDInvalidError)) return } logger.Error("Error fetching schedule", err.Error()) diff --git a/internal/app/service/schedule/handler/http_test.go b/internal/app/service/schedule/handler/http_test.go index 87c765e4..203292ed 100644 --- a/internal/app/service/schedule/handler/http_test.go +++ b/internal/app/service/schedule/handler/http_test.go @@ -37,7 +37,7 @@ func (suite *ScheduleHttpHandlerTestSuite) SetupTest() { func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulSchedulePostHttpHandler() { t := suite.T() - scheduleId := uint64(0) + scheduleID := uint64(0) argsMap := types.Base64Map{ "COMMAND": "test", } @@ -51,7 +51,7 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulSchedulePostHttpHandler Group: "mrproctor", } responseSchedule := requestSchedule - responseSchedule.ID = scheduleId + responseSchedule.ID = scheduleID responseSchedule.Cron = "0 * * * * *" requestBody, err := json.Marshal(requestSchedule) @@ -224,12 +224,12 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetAllHttpHandler() func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler() { t := suite.T() - scheduleId := uint64(1) + scheduleID := uint64(1) argsMap := types.Base64Map{ "COMMAND": "test", } responseSchedule := model.Schedule{ - ID: scheduleId, + ID: scheduleID, JobName: "test1", Args: argsMap, Tags: "test", @@ -243,10 +243,10 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler( assert.NoError(t, err) req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"scheduleId": fmt.Sprint(scheduleId)}) + req = mux.SetURLVars(req, map[string]string{"scheduleID": fmt.Sprint(scheduleID)}) responseRecorder := httptest.NewRecorder() - suite.mockScheduleRepository.On("GetById", scheduleId).Return(&responseSchedule, nil).Once() + suite.mockScheduleRepository.On("GetByID", scheduleID).Return(&responseSchedule, nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) suite.testScheduleHttpHandler.Get()(responseRecorder, req) @@ -258,7 +258,7 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler( func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { t := suite.T() - scheduleId := uint64(1) + scheduleID := uint64(1) scheduleGetErrorTests := []struct { requestParams string @@ -266,18 +266,18 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { responseBody status.HandlerStatus }{ {"test", http.StatusBadRequest, status.PathParameterError}, - {"1", http.StatusBadRequest, status.ScheduleIdInvalidError}, + {"1", http.StatusBadRequest, status.ScheduleIDInvalidError}, // Schedule fetch error {"1", http.StatusInternalServerError, status.GenericServerError}, } - suite.mockScheduleRepository.On("GetById", scheduleId).Return(&model.Schedule{}, errors.New("invalid input syntax")).Once() - suite.mockScheduleRepository.On("GetById", scheduleId).Return(&model.Schedule{}, errors.New("")).Once() + suite.mockScheduleRepository.On("GetByID", scheduleID).Return(&model.Schedule{}, errors.New("invalid input syntax")).Once() + suite.mockScheduleRepository.On("GetByID", scheduleID).Return(&model.Schedule{}, errors.New("")).Once() defer suite.mockScheduleRepository.AssertExpectations(t) for _, errorTest := range scheduleGetErrorTests { req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"scheduleId": errorTest.requestParams}) + req = mux.SetURLVars(req, map[string]string{"scheduleID": errorTest.requestParams}) responseRecorder := httptest.NewRecorder() suite.testScheduleHttpHandler.Get()(responseRecorder, req) @@ -290,13 +290,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleDeleteHttpHandler() { t := suite.T() - scheduleId := uint64(1) + scheduleID := uint64(1) req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"scheduleId": fmt.Sprint(scheduleId)}) + req = mux.SetURLVars(req, map[string]string{"scheduleID": fmt.Sprint(scheduleID)}) responseRecorder := httptest.NewRecorder() - suite.mockScheduleRepository.On("Delete", scheduleId).Return(nil).Once() + suite.mockScheduleRepository.On("Delete", scheduleID).Return(nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) suite.testScheduleHttpHandler.Delete()(responseRecorder, req) @@ -308,7 +308,7 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleDeleteHttpHandl func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleDeleteHttpHandler() { t := suite.T() - scheduleId := uint64(1) + scheduleID := uint64(1) scheduleDeleteErrorTests := []struct { requestParams string @@ -316,18 +316,18 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleDeleteHttpHandler() responseBody status.HandlerStatus }{ {"test", http.StatusBadRequest, status.PathParameterError}, - {"1", http.StatusBadRequest, status.ScheduleIdInvalidError}, + {"1", http.StatusBadRequest, status.ScheduleIDInvalidError}, // Schedule delete error {"1", http.StatusInternalServerError, status.GenericServerError}, } - suite.mockScheduleRepository.On("Delete", scheduleId).Return(errors.New("invalid input syntax")).Once() - suite.mockScheduleRepository.On("Delete", scheduleId).Return(errors.New("")).Once() + suite.mockScheduleRepository.On("Delete", scheduleID).Return(errors.New("invalid input syntax")).Once() + suite.mockScheduleRepository.On("Delete", scheduleID).Return(errors.New("")).Once() defer suite.mockScheduleRepository.AssertExpectations(t) for _, errorTest := range scheduleDeleteErrorTests { req := httptest.NewRequest("DELETE", "/schedule", bytes.NewReader([]byte(""))) - req = mux.SetURLVars(req, map[string]string{"scheduleId": errorTest.requestParams}) + req = mux.SetURLVars(req, map[string]string{"scheduleID": errorTest.requestParams}) responseRecorder := httptest.NewRecorder() suite.testScheduleHttpHandler.Delete()(responseRecorder, req) diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go index 196d9471..7275b7fb 100644 --- a/internal/app/service/schedule/repository/schedule.go +++ b/internal/app/service/schedule/repository/schedule.go @@ -11,14 +11,14 @@ import ( type ScheduleRepository interface { Insert(context *model.Schedule) (uint64, error) Delete(id uint64) error - GetById(id uint64) (*model.Schedule, error) + GetByID(id uint64) (*model.Schedule, error) Disable(id uint64) error Enable(id uint64) error GetByUserEmail(userEmail string) ([]model.Schedule, error) GetByJobName(jobName string) ([]model.Schedule, error) GetAllEnabled() ([]model.Schedule, error) GetAll() ([]model.Schedule, error) - GetEnabledById(id uint64) (*model.Schedule, error) + GetEnabledByID(id uint64) (*model.Schedule, error) deleteAll() error } @@ -55,14 +55,14 @@ func (repository *scheduleRepository) Disable(id uint64) error { } func (repository *scheduleRepository) Insert(context *model.Schedule) (uint64, error) { - snowflakeId, _ := id.NextId() - context.ID = snowflakeId + snowflakeID, _ := id.NextID() + context.ID = snowflakeID sql := "INSERT INTO schedule (id, job_name, args,cron,notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :cron, :notification_emails, :user_email, :group, :enabled)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { return 0, nil } - return snowflakeId, nil + return snowflakeID, nil } func (repository *scheduleRepository) Delete(id uint64) error { @@ -74,7 +74,7 @@ func (repository *scheduleRepository) Delete(id uint64) error { return err } -func (repository *scheduleRepository) GetById(id uint64) (*model.Schedule, error) { +func (repository *scheduleRepository) GetByID(id uint64) (*model.Schedule, error) { sql := "SELECT id, job_name, args, cron, notification_emails, user_email,\"group\", enabled, created_at, updated_at FROM schedule WHERE id=$1 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, id) @@ -133,7 +133,7 @@ func (repository *scheduleRepository) GetAll() ([]model.Schedule, error) { return schedules, nil } -func (repository *scheduleRepository) GetEnabledById(id uint64) (*model.Schedule, error) { +func (repository *scheduleRepository) GetEnabledByID(id uint64) (*model.Schedule, error) { sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 AND id=$2 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, true, id) diff --git a/internal/app/service/schedule/repository/schedule_mock.go b/internal/app/service/schedule/repository/schedule_mock.go index b1fd5848..7d33c1bc 100644 --- a/internal/app/service/schedule/repository/schedule_mock.go +++ b/internal/app/service/schedule/repository/schedule_mock.go @@ -19,7 +19,7 @@ func (repository *MockScheduleRepository) Delete(id uint64) error { return args.Error(0) } -func (repository *MockScheduleRepository) GetById(id uint64) (*model.Schedule, error) { +func (repository *MockScheduleRepository) GetByID(id uint64) (*model.Schedule, error) { args := repository.Called(id) return args.Get(0).(*model.Schedule), args.Error(1) } @@ -54,7 +54,7 @@ func (repository *MockScheduleRepository) GetAll() ([]model.Schedule, error) { return args.Get(0).([]model.Schedule), args.Error(1) } -func (repository *MockScheduleRepository) GetEnabledById(id uint64) (*model.Schedule, error) { +func (repository *MockScheduleRepository) GetEnabledByID(id uint64) (*model.Schedule, error) { args := repository.Called(id) return args.Get(0).(*model.Schedule), args.Error(1) } diff --git a/internal/app/service/schedule/repository/schedule_test.go b/internal/app/service/schedule/repository/schedule_test.go index 27229d7b..143797a6 100644 --- a/internal/app/service/schedule/repository/schedule_test.go +++ b/internal/app/service/schedule/repository/schedule_test.go @@ -50,7 +50,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Insert() { assert.NotNil(t, id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetById(id) + expectedSchedule, err := suite.repository.GetByID(id) assert.NoError(t, err) assert.NotNil(t, expectedSchedule) @@ -84,7 +84,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Delete() { err = suite.repository.Delete(id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetById(id) + expectedSchedule, err := suite.repository.GetByID(id) assert.Error(t, err) assert.Nil(t, expectedSchedule) } @@ -164,7 +164,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetAllEnabled() { } } -func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledById() { +func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledByID() { t := suite.T() recordCount := 15 err := populateSeedDataForTest(suite.repository, recordCount, map[string]string{}) @@ -189,13 +189,13 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledById() { assert.NotNil(t, id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetEnabledById(id) + expectedSchedule, err := suite.repository.GetEnabledByID(id) assert.NoError(t, err) assert.NotNil(t, expectedSchedule) assert.True(t, expectedSchedule.Enabled) - willNotExistsId := uint64(17777717) - unexpectedSchedule, err := suite.repository.GetEnabledById(willNotExistsId) + willNotExistsID := uint64(17777717) + unexpectedSchedule, err := suite.repository.GetEnabledByID(willNotExistsID) assert.Error(t, err) assert.Nil(t, unexpectedSchedule) @@ -222,7 +222,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { assert.NotNil(t, id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetById(id) + expectedSchedule, err := suite.repository.GetByID(id) assert.NoError(t, err) assert.NotNil(t, expectedSchedule) assert.True(t, expectedSchedule.Enabled) @@ -230,7 +230,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { err = suite.repository.Disable(id) assert.NoError(t, err) - expectedSchedule, err = suite.repository.GetById(id) + expectedSchedule, err = suite.repository.GetByID(id) assert.NoError(t, err) assert.NotNil(t, expectedSchedule) assert.False(t, expectedSchedule.Enabled) @@ -238,7 +238,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { err = suite.repository.Enable(id) assert.NoError(t, err) - expectedSchedule, err = suite.repository.GetById(id) + expectedSchedule, err = suite.repository.GetByID(id) assert.NoError(t, err) assert.NotNil(t, expectedSchedule) assert.True(t, expectedSchedule.Enabled) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index a78e04a7..5b0f62d9 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -64,7 +64,7 @@ func NewRouter() (*mux.Router, error) { router.Use(middleware.ValidateClientVersion) router.HandleFunc("/execute", executionHandler.Post()).Methods("POST") - router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") + router.HandleFunc("/execution/{contextID}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") @@ -73,8 +73,8 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/schedule", scheduleHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.GetAll()).Methods("GET") - router.HandleFunc("/schedule/{scheduleId}", scheduleHandler.Get()).Methods("GET") - router.HandleFunc("/schedule/{scheduleId}", scheduleHandler.Delete()).Methods("DELETE") + router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Get()).Methods("GET") + router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Delete()).Methods("DELETE") return router, nil } diff --git a/internal/pkg/status/status.go b/internal/pkg/status/status.go index 09fc70f5..d7d1a3ed 100644 --- a/internal/pkg/status/status.go +++ b/internal/pkg/status/status.go @@ -14,7 +14,7 @@ const ( ScheduleCronFormatInvalidError HandlerStatus = "Schedule cron format is invalid" ScheduleDuplicateJobNameArgsError HandlerStatus = "Schedule job name and args duplicate is found" - ScheduleIdInvalidError HandlerStatus = "Schedule ID is invalid" + ScheduleIDInvalidError HandlerStatus = "Schedule ID is invalid" ScheduleGroupMissingError HandlerStatus = "Schedule group is missing" ScheduleListNotFoundError HandlerStatus = "Schedule list is not found" ScheduleTagMissingError HandlerStatus = "Schedule tag(s) are missing" From d8f1b9389548b792687037a19d1754c7b620b00e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 24 Jul 2019 17:01:08 +0700 Subject: [PATCH 060/268] [bimo.horizon] Update dependencies --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index b450fe31..bc7a18dd 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,7 @@ github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ= github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= From 5ab307863ca1eb7b88a7879ef9e9e6df97d9b03e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 24 Jul 2019 17:01:47 +0700 Subject: [PATCH 061/268] [bimo.horizon] Change mailer send method to use execution context and schedule model instead --- internal/app/service/infra/mail/mailer.go | 23 +++++++------ .../app/service/infra/mail/mailer_mock.go | 7 ++-- .../app/service/infra/mail/mailer_test.go | 32 ++++++++++++------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/internal/app/service/infra/mail/mailer.go b/internal/app/service/infra/mail/mailer.go index a2953d74..1538aaaa 100644 --- a/internal/app/service/infra/mail/mailer.go +++ b/internal/app/service/infra/mail/mailer.go @@ -1,14 +1,18 @@ package mail import ( + "fmt" "net/smtp" - "proctor/internal/app/service/infra/config" + "strings" + executionContextModel "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/infra/config" + scheduleModel "proctor/internal/app/service/schedule/model" "proctor/internal/pkg/utility" ) type Mailer interface { - Send(string, string, string, map[string]string, []string) error + Send(executionContext executionContextModel.ExecutionContext, schedule scheduleModel.Schedule) error } type mailer struct { @@ -28,18 +32,19 @@ func New(mailServerHost, mailServerPort string) Mailer { } } -func (mailer *mailer) Send(jobName, jobExecutionID, jobExecutionStatus string, jobArgs map[string]string, recipients []string) error { - message := constructMessage(jobName, jobExecutionID, jobExecutionStatus, jobArgs) +func (mailer *mailer) Send(executionContext executionContextModel.ExecutionContext, schedule scheduleModel.Schedule) error { + message := constructMessage(executionContext.JobName, executionContext.ExecutionID, string(executionContext.Status), executionContext.Args) + recipients := strings.Split(schedule.NotificationEmails, ",") return smtp.SendMail(mailer.addr, mailer.auth, mailer.from, recipients, message) } -func constructMessage(jobName, jobExecutionID, jobExecutionStatus string, jobArgs map[string]string) []byte { - subject := "Subject: " + jobName + " | scheduled execution " + jobExecutionStatus +func constructMessage(jobName string, executionID uint64, executionStatus string, executionArgs map[string]string) []byte { + subject := "Subject: " + jobName + " | scheduled execution " + executionStatus body := "Proc execution details:\n" + "\nName:\t" + jobName + - "\nArgs:\t" + utility.MapToString(jobArgs) + - "\nID:\t" + jobExecutionID + - "\nStatus:\t" + jobExecutionStatus + + "\nArgs:\t" + utility.MapToString(executionArgs) + + "\nID:\t" + fmt.Sprint(executionID) + + "\nStatus:\t" + executionStatus + "\n\n\nThis is an auto-generated email" return []byte(subject + "\n\n" + body) diff --git a/internal/app/service/infra/mail/mailer_mock.go b/internal/app/service/infra/mail/mailer_mock.go index 013c91e6..572464e0 100644 --- a/internal/app/service/infra/mail/mailer_mock.go +++ b/internal/app/service/infra/mail/mailer_mock.go @@ -2,13 +2,16 @@ package mail import ( "github.com/stretchr/testify/mock" + + executionContextModel "proctor/internal/app/service/execution/model" + scheduleModel "proctor/internal/app/service/schedule/model" ) type MockMailer struct { mock.Mock } -func (m *MockMailer) Send(jobName, jobExecutionID, jobExecutionStatus string, jobArgs map[string]string, recipients []string) error { - args := m.Called(jobName, jobExecutionID, jobExecutionStatus, jobArgs, recipients) +func (m *MockMailer) Send(executionContext executionContextModel.ExecutionContext, schedule scheduleModel.Schedule) error { + args := m.Called(executionContext, schedule) return args.Error(0) } diff --git a/internal/app/service/infra/mail/mailer_test.go b/internal/app/service/infra/mail/mailer_test.go index 9476b0a3..65d7783e 100644 --- a/internal/app/service/infra/mail/mailer_test.go +++ b/internal/app/service/infra/mail/mailer_test.go @@ -6,10 +6,13 @@ import ( "fmt" "net" "net/textproto" - "proctor/internal/app/service/infra/config" "strings" "testing" + executionContextModel "proctor/internal/app/service/execution/model" + executionStatus "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" + scheduleModel "proctor/internal/app/service/schedule/model" "proctor/internal/pkg/utility" ) @@ -74,12 +77,17 @@ func TestSendMail(t *testing.T) { }(strings.Split(server, "\r\n")) mailer := New(strings.Split(l.Addr().String(), ":")[0], strings.Split(l.Addr().String(), ":")[1]) - jobName := "proc-name" - jobExecutionID := "some-id" - jobExecutionStatus := "SUCCEEDED" - jobArgs := map[string]string{"ARG_ONE": "foo"} - recipients := []string{"foo@bar.com", "goo@bar.com"} - err = mailer.Send(jobName, jobExecutionID, jobExecutionStatus, jobArgs, recipients) + executionContext := executionContextModel.ExecutionContext{ + JobName: "proc-name", + ExecutionID: uint64(1), + Status: executionStatus.Finished, + Args: map[string]string{"ARG_ONE": "foo"}, + } + schedule := scheduleModel.Schedule{ + NotificationEmails: "foo@bar.com,goo@bar.com", + } + recipients := strings.Split(schedule.NotificationEmails, ",") + err = mailer.Send(executionContext, schedule) if err != nil { t.Errorf("%v", err) } @@ -90,21 +98,21 @@ func TestSendMail(t *testing.T) { receivedMail := cmdbuf.String() - stringifiedJobArgs := utility.MapToString(jobArgs) + stringifiedJobArgs := utility.MapToString(executionContext.Args) var sendMailClient = `EHLO localhost HELO localhost MAIL FROM:<` + config.MailUsername() + `> RCPT TO:<` + recipients[0] + `> RCPT TO:<` + recipients[1] + `> DATA -Subject: ` + jobName + ` | scheduled execution ` + jobExecutionStatus + ` +Subject: ` + executionContext.JobName + ` | scheduled execution ` + string(executionContext.Status) + ` Proc execution details: -Name: ` + jobName + ` +Name: ` + executionContext.JobName + ` Args: ` + stringifiedJobArgs + ` -ID: ` + jobExecutionID + ` -Status: ` + jobExecutionStatus + ` +ID: ` + fmt.Sprint(executionContext.ExecutionID) + ` +Status: ` + string(executionContext.Status) + ` This is an auto-generated email From 3cbb3e6db7ffb66fe66c93fe45d6166ae4532006 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 24 Jul 2019 17:02:03 +0700 Subject: [PATCH 062/268] [bimo.horizon] Add worker service --- internal/app/service/worker/worker.go | 107 ++++++++++++ internal/app/service/worker/worker_test.go | 185 +++++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 internal/app/service/worker/worker.go create mode 100644 internal/app/service/worker/worker_test.go diff --git a/internal/app/service/worker/worker.go b/internal/app/service/worker/worker.go new file mode 100644 index 00000000..e0e39373 --- /dev/null +++ b/internal/app/service/worker/worker.go @@ -0,0 +1,107 @@ +package worker + +import ( + "fmt" + "os" + "time" + + "github.com/getsentry/raven-go" + "github.com/robfig/cron" + + executionContextRepository "proctor/internal/app/service/execution/repository" + executionService "proctor/internal/app/service/execution/service" + "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/infra/mail" + scheduleModel "proctor/internal/app/service/schedule/model" + scheduleRepository "proctor/internal/app/service/schedule/repository" + "proctor/internal/pkg/constant" +) + +type worker struct { + executionService executionService.ExecutionService + executionContextRepository executionContextRepository.ExecutionContextRepository + scheduleRepository scheduleRepository.ScheduleRepository + mailer mail.Mailer + inMemorySchedules map[uint64]*cron.Cron +} + +type Worker interface { + Run(<-chan time.Time, <-chan os.Signal) +} + +func NewWorker(executionSvc executionService.ExecutionService, executionContextRepo executionContextRepository.ExecutionContextRepository, scheduleRepo scheduleRepository.ScheduleRepository, mailer mail.Mailer) Worker { + return &worker{ + executionService: executionSvc, + executionContextRepository: executionContextRepo, + scheduleRepository: scheduleRepo, + mailer: mailer, + inMemorySchedules: make(map[uint64]*cron.Cron), + } +} + +func (worker *worker) disableScheduleIfItExists(scheduleID uint64) { + if scheduledCronJob, ok := worker.inMemorySchedules[scheduleID]; ok { + scheduledCronJob.Stop() + delete(worker.inMemorySchedules, scheduleID) + } +} + +func (worker *worker) enableScheduleIfItDoesNotExist(schedule scheduleModel.Schedule) { + if _, ok := worker.inMemorySchedules[schedule.ID]; !ok { + cronJob := cron.New() + err := cronJob.AddFunc(schedule.Cron, func() { + executionContext, _, err := worker.executionService.Execute(schedule.JobName, constant.WorkerEmail, schedule.Args) + if err != nil { + logger.Error(fmt.Sprintf("Error submitting job: %s ", schedule.Tags), schedule.JobName, " for execution: ", err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) + + return + } + + err = worker.mailer.Send(*executionContext, schedule) + + if err != nil { + logger.Error(fmt.Sprintf("Error notifying job: %s `", schedule.Tags), schedule.JobName, "` ID: `", executionContext.ExecutionID, "` execution status: `", executionContext.Status, "` to users: ", err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName, "job_id": fmt.Sprint(executionContext.ExecutionID), "job_execution_status": string(executionContext.Status)}) + return + } + }) + + if err != nil { + logger.Error(fmt.Sprintf("Error adding cron job: %s", schedule.Tags), err.Error()) + raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags}) + return + } + + cronJob.Start() + worker.inMemorySchedules[schedule.ID] = cronJob + } +} + +func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Signal) { + for { + select { + case <-tickerChan: + schedules, err := worker.scheduleRepository.GetAll() + if err != nil { + logger.Error("Error getting scheduled jobs from store: ", err.Error()) + raven.CaptureError(err, nil) + continue + } + + for _, schedule := range schedules { + if schedule.Enabled { + worker.enableScheduleIfItDoesNotExist(schedule) + } else { + worker.disableScheduleIfItExists(schedule.ID) + } + } + case <-signalsChan: + for id := range worker.inMemorySchedules { + worker.disableScheduleIfItExists(id) + } + //TODO: wait for all active executions to complete + return + } + } +} diff --git a/internal/app/service/worker/worker_test.go b/internal/app/service/worker/worker_test.go new file mode 100644 index 00000000..ee6d5082 --- /dev/null +++ b/internal/app/service/worker/worker_test.go @@ -0,0 +1,185 @@ +package worker + +import ( + "os" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + executionContextModel "proctor/internal/app/service/execution/model" + executionContextRepository "proctor/internal/app/service/execution/repository" + executionService "proctor/internal/app/service/execution/service" + "proctor/internal/app/service/infra/mail" + scheduleModel "proctor/internal/app/service/schedule/model" + scheduleRepository "proctor/internal/app/service/schedule/repository" + "proctor/internal/pkg/constant" +) + +type WorkerTestSuite struct { + suite.Suite + mockExecutionService *executionService.MockExecutionService + mockExecutionContextRepository *executionContextRepository.MockExecutionContextRepository + mockScheduleRepository *scheduleRepository.MockScheduleRepository + mockMailer *mail.MockMailer + worker Worker +} + +func (suite *WorkerTestSuite) SetupTest() { + suite.mockExecutionService = &executionService.MockExecutionService{} + suite.mockExecutionContextRepository = &executionContextRepository.MockExecutionContextRepository{} + suite.mockScheduleRepository = &scheduleRepository.MockScheduleRepository{} + suite.mockMailer = &mail.MockMailer{} + suite.worker = NewWorker( + suite.mockExecutionService, + suite.mockExecutionContextRepository, + suite.mockScheduleRepository, + suite.mockMailer, + ) +} + +func (suite *WorkerTestSuite) TestEnableRun() { + t := suite.T() + + jobArgs := map[string]string{"abc": "def"} + enabledJob := "some-job-one" + disabledJob := "some-job-two" + userEmail := "foo@bar.com" + notificationEmails := "foo@bar.com,goo@bar.com" + executionContext := executionContextModel.ExecutionContext{ + ExecutionID: uint64(1), + JobName: enabledJob, + Name: "test", + UserEmail: constant.WorkerEmail, + ImageTag: "test", + Args: jobArgs, + } + scheduledJobs := []scheduleModel.Schedule{ + { + ID: uint64(1), + Enabled: true, + Cron: "*/1 * * * * *", + JobName: enabledJob, + Args: jobArgs, + NotificationEmails: notificationEmails, + UserEmail: userEmail, + Tags: "test", + }, + { + ID: uint64(2), + Enabled: false, + Cron: "*/1 * * * * *", + JobName: disabledJob, + Args: jobArgs, + NotificationEmails: notificationEmails, + UserEmail: userEmail, + Tags: "test", + }, + } + + tickerChan := make(chan time.Time) + signalsChan := make(chan os.Signal, 1) + scheduledJobExecutedChan := make(chan bool) + + suite.mockScheduleRepository.On("GetAll").Return(scheduledJobs, nil) + + suite.mockExecutionService.On("Execute", enabledJob, constant.WorkerEmail, jobArgs).Return(&executionContext, "test", nil) + defer suite.mockExecutionService.AssertExpectations(t) + defer suite.mockExecutionService.AssertNotCalled(t, "Execute", disabledJob, constant.WorkerEmail, jobArgs) + suite.mockMailer.On("Send", executionContext, scheduledJobs[0]).Return(nil).Run( + func(args mock.Arguments) { + scheduledJobExecutedChan <- true + }, + ) + defer suite.mockMailer.AssertExpectations(t) + + go suite.worker.Run(tickerChan, signalsChan) + + tickerChan <- time.Now() + + <-scheduledJobExecutedChan + signalsChan <- syscall.SIGTERM +} + +func (suite *WorkerTestSuite) TestDisableRun() { + t := suite.T() + + jobArgs := map[string]string{"abc": "def"} + enabledJob := "some-job-one" + userEmail := "foo@bar.com" + notificationEmails := "foo@bar.com,goo@bar.com" + executionContext := executionContextModel.ExecutionContext{ + ExecutionID: uint64(1), + JobName: enabledJob, + Name: "test", + UserEmail: constant.WorkerEmail, + ImageTag: "test", + Args: jobArgs, + } + disableExecutionContext := executionContext + enabledScheduledJobs := []scheduleModel.Schedule{ + { + ID: uint64(1), + Enabled: true, + Cron: "*/1 * * * * *", + JobName: enabledJob, + Args: jobArgs, + NotificationEmails: notificationEmails, + UserEmail: userEmail, + Tags: "test", + }, + } + disabledScheduledJobs := []scheduleModel.Schedule{ + { + ID: uint64(1), + Enabled: false, + Cron: "*/1 * * * * *", + JobName: enabledJob, + Args: jobArgs, + NotificationEmails: notificationEmails, + UserEmail: userEmail, + Tags: "test", + }, + } + tickerChan := make(chan time.Time) + signalsChan := make(chan os.Signal, 1) + toggledOffEnabledJobChan := make(chan bool) + + suite.mockScheduleRepository.On("GetAll").Return(enabledScheduledJobs, nil).Once().Run( + func(args mock.Arguments) { + toggledOffEnabledJobChan <- true + }, + ) + suite.mockScheduleRepository.On("GetAll").Return(disabledScheduledJobs, nil).Run( + func(args mock.Arguments) { + toggledOffEnabledJobChan <- true + }, + ) + defer suite.mockScheduleRepository.AssertExpectations(t) + + suite.mockExecutionService.On("Execute", enabledJob, constant.WorkerEmail, jobArgs).Return(&disableExecutionContext, "test", nil) + defer suite.mockExecutionService.AssertExpectations(t) + + suite.mockMailer.On("Send", disableExecutionContext, enabledScheduledJobs[0]).Return(nil).Run( + func(args mock.Arguments) { + toggledOffEnabledJobChan <- true + }, + ) + defer suite.mockMailer.AssertExpectations(t) + + go suite.worker.Run(tickerChan, signalsChan) + + tickerChan <- time.Now() + + <-toggledOffEnabledJobChan + + //Wait for 2 seconds to ensure disabled job isn't executed again + time.Sleep(2 * time.Second) + signalsChan <- syscall.SIGTERM +} + +func TestWorkerTestSuite(t *testing.T) { + suite.Run(t, new(WorkerTestSuite)) +} From fb2a68f161d240b7977dbd5e2dd7224cb8c4f449 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:14:39 +0700 Subject: [PATCH 063/268] [bimo.horizon|mazbergaz] Move scheduler start to worker --- internal/app/service/worker/worker.go | 35 ++++++++++++++++++++++ internal/app/service/worker/worker_test.go | 1 + 2 files changed, 36 insertions(+) diff --git a/internal/app/service/worker/worker.go b/internal/app/service/worker/worker.go index e0e39373..a8edb42d 100644 --- a/internal/app/service/worker/worker.go +++ b/internal/app/service/worker/worker.go @@ -10,10 +10,17 @@ import ( executionContextRepository "proctor/internal/app/service/execution/repository" executionService "proctor/internal/app/service/execution/service" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/db/redis" + "proctor/internal/app/service/infra/kubernetes" + "proctor/internal/app/service/infra/kubernetes/http" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/infra/mail" + metadataRepository "proctor/internal/app/service/metadata/repository" scheduleModel "proctor/internal/app/service/schedule/model" scheduleRepository "proctor/internal/app/service/schedule/repository" + secretRepository "proctor/internal/app/service/secret/repository" "proctor/internal/pkg/constant" ) @@ -105,3 +112,31 @@ func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Sig } } } + +func Start() error { + fmt.Println("started scheduler") + + postgresClient := postgresql.NewClient() + redisClient := redis.NewClient() + + executionContextStore := executionContextRepository.NewExecutionContextRepository(postgresClient) + metadataStore := metadataRepository.NewMetadataRepository(redisClient) + secretStore := secretRepository.NewSecretRepository(redisClient) + scheduleStore := scheduleRepository.NewScheduleRepository(postgresClient) + + httpClient, err := http.NewClient() + if err != nil { + return err + } + kubeClient := kubernetes.NewKubernetesClient(httpClient) + mailer := mail.New(config.MailServerHost(), config.MailServerPort()) + executionSvc := executionService.NewExecutionService(kubeClient, executionContextStore, metadataStore, secretStore) + worker := NewWorker(executionSvc, executionContextStore, scheduleStore, mailer) + ticker := time.NewTicker(time.Duration(config.ScheduledJobsFetchIntervalInMins()) * time.Minute) + signalsChan := make(chan os.Signal, 1) + + worker.Run(ticker.C, signalsChan) + + _ = postgresClient.Close() + return nil +} diff --git a/internal/app/service/worker/worker_test.go b/internal/app/service/worker/worker_test.go index ee6d5082..bb598bc8 100644 --- a/internal/app/service/worker/worker_test.go +++ b/internal/app/service/worker/worker_test.go @@ -143,6 +143,7 @@ func (suite *WorkerTestSuite) TestDisableRun() { Tags: "test", }, } + tickerChan := make(chan time.Time) signalsChan := make(chan os.Signal, 1) toggledOffEnabledJobChan := make(chan bool) From 175f05ca1a510054b6fbb956760ed88607d389bb Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:42:56 +0700 Subject: [PATCH 064/268] [bimo.horizon|mazbergaz] Remove scheduler and move the start function to worker --- cmd/proctord/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/proctord/main.go b/cmd/proctord/main.go index 3b871991..1b0c4e5b 100644 --- a/cmd/proctord/main.go +++ b/cmd/proctord/main.go @@ -3,11 +3,11 @@ package main import ( "github.com/getsentry/raven-go" "os" - "proctor/internal/app/proctord/scheduler" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/migration" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/server" + "proctor/internal/app/service/worker" "github.com/urfave/cli" ) @@ -55,7 +55,7 @@ func main() { Name: "start-scheduler", Usage: "starts scheduler", Action: func(c *cli.Context) error { - return scheduler.Start() + return worker.Start() }, }, } From c7ff333fea9a6b8d3d8b662e51cd11789c2da469 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:43:31 +0700 Subject: [PATCH 065/268] [bimo.horizon|mazbergaz] Swap out audit log persistence on test to execution context instead --- .../infra/db/postgresql/client_test.go | 67 +++++++++---------- 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index 75d5cfc3..bd6176c9 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -2,12 +2,14 @@ package postgresql import ( "fmt" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/config" "testing" "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" + + executionContextModel "proctor/internal/app/service/execution/model" + executionContextStatus "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" ) func TestNamedExec(t *testing.T) { @@ -19,30 +21,28 @@ func TestNamedExec(t *testing.T) { postgresClient := &client{db: db} defer postgresClient.db.Close() - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: "test-job-name", - ImageName: "test-image-name", - ExecutionID: postgres.StringToSQLString("test-submission-name"), - JobArgs: "test-job-args", - JobSubmissionStatus: "test-job-status", - JobExecutionStatus: "test-job-execution-status", + executionContext := &executionContextModel.ExecutionContext{ + JobName: "test-job-name", + ImageTag: "test-image-name", + ExecutionID: uint64(1), + Args: map[string]string{"foo": "bar"}, + Status: executionContextStatus.Finished, } - _, err = postgresClient.NamedExec("INSERT INTO jobs_execution_audit_log (job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", jobsExecutionAuditLog) + _, err = postgresClient.NamedExec("INSERT INTO execution_context (id, job_name, image_tag, args, status) VALUES (:id, :job_name, :image_tag, :args, :status)", executionContext) assert.NoError(t, err) - var persistedJobsExecutionAuditLog postgres.JobsExecutionAuditLog - err = postgresClient.db.Get(&persistedJobsExecutionAuditLog, `SELECT job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status FROM jobs_execution_audit_log WHERE job_name='test-job-name'`) + var persistedExecutionContext executionContextModel.ExecutionContext + err = postgresClient.db.Get(&persistedExecutionContext, `SELECT id, job_name, image_tag, args, status FROM execution_context WHERE job_name='test-job-name'`) assert.NoError(t, err) - assert.Equal(t, jobsExecutionAuditLog.JobName, persistedJobsExecutionAuditLog.JobName) - assert.Equal(t, jobsExecutionAuditLog.ImageName, persistedJobsExecutionAuditLog.ImageName) - assert.Equal(t, jobsExecutionAuditLog.ExecutionID.String, persistedJobsExecutionAuditLog.ExecutionID.String) - assert.Equal(t, jobsExecutionAuditLog.JobArgs, persistedJobsExecutionAuditLog.JobArgs) - assert.Equal(t, jobsExecutionAuditLog.JobSubmissionStatus, persistedJobsExecutionAuditLog.JobSubmissionStatus) - assert.Equal(t, jobsExecutionAuditLog.JobExecutionStatus, persistedJobsExecutionAuditLog.JobExecutionStatus) + assert.Equal(t, executionContext.JobName, persistedExecutionContext.JobName) + assert.Equal(t, executionContext.ImageTag, persistedExecutionContext.ImageTag) + assert.Equal(t, executionContext.ExecutionID, persistedExecutionContext.ExecutionID) + assert.Equal(t, executionContext.Args, persistedExecutionContext.Args) + assert.Equal(t, executionContext.Status, persistedExecutionContext.Status) - _, err = postgresClient.db.Exec("DELETE FROM jobs_execution_audit_log WHERE job_name='test-job-name'") + _, err = postgresClient.db.Exec("DELETE FROM execution_context WHERE job_name='test-job-name'") assert.NoError(t, err) } @@ -56,25 +56,24 @@ func TestSelect(t *testing.T) { defer postgresClient.db.Close() jobName := "test-job-name" - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: jobName, - ImageName: "test-image-name", - ExecutionID: postgres.StringToSQLString("test-submission-name"), - JobArgs: "test-job-args", - JobSubmissionStatus: "test-job-status", - JobExecutionStatus: "test-job-execution-status", + executionContext := &executionContextModel.ExecutionContext{ + JobName: jobName, + ImageTag: "test-image-name", + ExecutionID: uint64(1), + Args: map[string]string{"foo": "bar"}, + Status: executionContextStatus.Finished, } - _, err = postgresClient.NamedExec("INSERT INTO jobs_execution_audit_log (job_name, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", jobsExecutionAuditLog) + _, err = postgresClient.NamedExec("INSERT INTO execution_context (job_name, image_tag, args, status) VALUES (:job_name, :image_tag, :args, :status)", executionContext) assert.NoError(t, err) - jobsExecutionAuditLogResult := []postgres.JobsExecutionAuditLog{} - err = postgresClient.db.Select(&jobsExecutionAuditLogResult, "SELECT job_execution_status from jobs_execution_audit_log where job_name = $1", jobName) + executionContextResult := []executionContextModel.ExecutionContext{} + err = postgresClient.db.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) assert.NoError(t, err) - assert.Equal(t, jobsExecutionAuditLog.JobExecutionStatus, jobsExecutionAuditLogResult[0].JobExecutionStatus) + assert.Equal(t, executionContext.Status, executionContextResult[0].Status) - _, err = postgresClient.db.Exec("DELETE FROM jobs_execution_audit_log WHERE job_name='test-job-name'") + _, err = postgresClient.db.Exec("DELETE FROM execution_context WHERE job_name='test-job-name'") assert.NoError(t, err) } @@ -88,11 +87,11 @@ func TestSelectForNoRows(t *testing.T) { defer postgresClient.db.Close() jobName := "test-job-name" - jobsExecutionAuditLogResult := []postgres.JobsExecutionAuditLog{} - err = postgresClient.db.Select(&jobsExecutionAuditLogResult, "SELECT job_execution_status from jobs_execution_audit_log where job_name = $1", jobName) + executionContextResult := []executionContextModel.ExecutionContext{} + err = postgresClient.db.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) assert.NoError(t, err) - assert.Equal(t, 0, len(jobsExecutionAuditLogResult)) + assert.Equal(t, 0, len(executionContextResult)) assert.NoError(t, err) } From c28db608e0b4e0f9fe21fdf0a87ae961b49b1eac Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:44:45 +0700 Subject: [PATCH 066/268] [bimo.horizon|mazbergaz] Move docs into service --- .../app/{proctord => service}/docs/css/print.css | 0 .../app/{proctord => service}/docs/css/reset.css | 0 .../app/{proctord => service}/docs/css/screen.css | 0 .../app/{proctord => service}/docs/css/style.css | 0 .../{proctord => service}/docs/css/typography.css | 0 .../docs/fonts/DroidSans-Bold.ttf | Bin .../{proctord => service}/docs/fonts/DroidSans.ttf | Bin internal/app/{proctord => service}/docs/handler.go | 0 .../{proctord => service}/docs/images/collapse.gif | Bin .../{proctord => service}/docs/images/expand.gif | Bin .../docs/images/explorer_icons.png | Bin .../docs/images/favicon-16x16.png | Bin .../docs/images/favicon-32x32.png | Bin .../{proctord => service}/docs/images/favicon.ico | Bin .../docs/images/logo_small.png | Bin .../docs/images/pet_store_api.png | Bin .../{proctord => service}/docs/images/throbber.gif | Bin .../docs/images/wordnik_api.png | Bin internal/app/{proctord => service}/docs/index.html | 0 internal/app/{proctord => service}/docs/lang/ca.js | 0 internal/app/{proctord => service}/docs/lang/el.js | 0 internal/app/{proctord => service}/docs/lang/en.js | 0 internal/app/{proctord => service}/docs/lang/es.js | 0 internal/app/{proctord => service}/docs/lang/fr.js | 0 internal/app/{proctord => service}/docs/lang/geo.js | 0 internal/app/{proctord => service}/docs/lang/it.js | 0 internal/app/{proctord => service}/docs/lang/ja.js | 0 .../app/{proctord => service}/docs/lang/ko-kr.js | 0 internal/app/{proctord => service}/docs/lang/pl.js | 0 internal/app/{proctord => service}/docs/lang/pt.js | 0 internal/app/{proctord => service}/docs/lang/ru.js | 0 internal/app/{proctord => service}/docs/lang/tr.js | 0 .../{proctord => service}/docs/lang/translator.js | 0 .../app/{proctord => service}/docs/lang/zh-cn.js | 0 .../{proctord => service}/docs/lib/backbone-min.js | 0 .../app/{proctord => service}/docs/lib/es5-shim.js | 0 .../docs/lib/handlebars-4.0.5.js | 0 .../docs/lib/highlight.9.1.0.pack.js | 0 .../docs/lib/highlight.9.1.0.pack_extended.js | 0 .../docs/lib/jquery-1.8.0.min.js | 0 .../docs/lib/jquery.ba-bbq.min.js | 0 .../docs/lib/jquery.slideto.min.js | 0 .../docs/lib/jquery.wiggle.min.js | 0 .../{proctord => service}/docs/lib/js-yaml.min.js | 0 .../docs/lib/jsoneditor.min.js | 0 .../{proctord => service}/docs/lib/lodash.min.js | 0 .../app/{proctord => service}/docs/lib/marked.js | 0 .../docs/lib/object-assign-pollyfill.js | 0 .../docs/lib/sanitize-html.min.js | 0 .../{proctord => service}/docs/lib/swagger-oauth.js | 0 internal/app/{proctord => service}/docs/o2c.html | 0 .../app/{proctord => service}/docs/swagger-ui.js | 0 .../{proctord => service}/docs/swagger-ui.min.js | 0 internal/app/{proctord => service}/docs/swagger.yml | 0 internal/app/service/server/router.go | 2 +- 55 files changed, 1 insertion(+), 1 deletion(-) rename internal/app/{proctord => service}/docs/css/print.css (100%) rename internal/app/{proctord => service}/docs/css/reset.css (100%) rename internal/app/{proctord => service}/docs/css/screen.css (100%) rename internal/app/{proctord => service}/docs/css/style.css (100%) rename internal/app/{proctord => service}/docs/css/typography.css (100%) rename internal/app/{proctord => service}/docs/fonts/DroidSans-Bold.ttf (100%) rename internal/app/{proctord => service}/docs/fonts/DroidSans.ttf (100%) rename internal/app/{proctord => service}/docs/handler.go (100%) rename internal/app/{proctord => service}/docs/images/collapse.gif (100%) rename internal/app/{proctord => service}/docs/images/expand.gif (100%) rename internal/app/{proctord => service}/docs/images/explorer_icons.png (100%) rename internal/app/{proctord => service}/docs/images/favicon-16x16.png (100%) rename internal/app/{proctord => service}/docs/images/favicon-32x32.png (100%) rename internal/app/{proctord => service}/docs/images/favicon.ico (100%) rename internal/app/{proctord => service}/docs/images/logo_small.png (100%) rename internal/app/{proctord => service}/docs/images/pet_store_api.png (100%) rename internal/app/{proctord => service}/docs/images/throbber.gif (100%) rename internal/app/{proctord => service}/docs/images/wordnik_api.png (100%) rename internal/app/{proctord => service}/docs/index.html (100%) rename internal/app/{proctord => service}/docs/lang/ca.js (100%) rename internal/app/{proctord => service}/docs/lang/el.js (100%) rename internal/app/{proctord => service}/docs/lang/en.js (100%) rename internal/app/{proctord => service}/docs/lang/es.js (100%) rename internal/app/{proctord => service}/docs/lang/fr.js (100%) rename internal/app/{proctord => service}/docs/lang/geo.js (100%) rename internal/app/{proctord => service}/docs/lang/it.js (100%) rename internal/app/{proctord => service}/docs/lang/ja.js (100%) rename internal/app/{proctord => service}/docs/lang/ko-kr.js (100%) rename internal/app/{proctord => service}/docs/lang/pl.js (100%) rename internal/app/{proctord => service}/docs/lang/pt.js (100%) rename internal/app/{proctord => service}/docs/lang/ru.js (100%) rename internal/app/{proctord => service}/docs/lang/tr.js (100%) rename internal/app/{proctord => service}/docs/lang/translator.js (100%) rename internal/app/{proctord => service}/docs/lang/zh-cn.js (100%) rename internal/app/{proctord => service}/docs/lib/backbone-min.js (100%) rename internal/app/{proctord => service}/docs/lib/es5-shim.js (100%) rename internal/app/{proctord => service}/docs/lib/handlebars-4.0.5.js (100%) rename internal/app/{proctord => service}/docs/lib/highlight.9.1.0.pack.js (100%) rename internal/app/{proctord => service}/docs/lib/highlight.9.1.0.pack_extended.js (100%) rename internal/app/{proctord => service}/docs/lib/jquery-1.8.0.min.js (100%) rename internal/app/{proctord => service}/docs/lib/jquery.ba-bbq.min.js (100%) rename internal/app/{proctord => service}/docs/lib/jquery.slideto.min.js (100%) rename internal/app/{proctord => service}/docs/lib/jquery.wiggle.min.js (100%) rename internal/app/{proctord => service}/docs/lib/js-yaml.min.js (100%) rename internal/app/{proctord => service}/docs/lib/jsoneditor.min.js (100%) rename internal/app/{proctord => service}/docs/lib/lodash.min.js (100%) rename internal/app/{proctord => service}/docs/lib/marked.js (100%) rename internal/app/{proctord => service}/docs/lib/object-assign-pollyfill.js (100%) rename internal/app/{proctord => service}/docs/lib/sanitize-html.min.js (100%) rename internal/app/{proctord => service}/docs/lib/swagger-oauth.js (100%) rename internal/app/{proctord => service}/docs/o2c.html (100%) rename internal/app/{proctord => service}/docs/swagger-ui.js (100%) rename internal/app/{proctord => service}/docs/swagger-ui.min.js (100%) rename internal/app/{proctord => service}/docs/swagger.yml (100%) diff --git a/internal/app/proctord/docs/css/print.css b/internal/app/service/docs/css/print.css similarity index 100% rename from internal/app/proctord/docs/css/print.css rename to internal/app/service/docs/css/print.css diff --git a/internal/app/proctord/docs/css/reset.css b/internal/app/service/docs/css/reset.css similarity index 100% rename from internal/app/proctord/docs/css/reset.css rename to internal/app/service/docs/css/reset.css diff --git a/internal/app/proctord/docs/css/screen.css b/internal/app/service/docs/css/screen.css similarity index 100% rename from internal/app/proctord/docs/css/screen.css rename to internal/app/service/docs/css/screen.css diff --git a/internal/app/proctord/docs/css/style.css b/internal/app/service/docs/css/style.css similarity index 100% rename from internal/app/proctord/docs/css/style.css rename to internal/app/service/docs/css/style.css diff --git a/internal/app/proctord/docs/css/typography.css b/internal/app/service/docs/css/typography.css similarity index 100% rename from internal/app/proctord/docs/css/typography.css rename to internal/app/service/docs/css/typography.css diff --git a/internal/app/proctord/docs/fonts/DroidSans-Bold.ttf b/internal/app/service/docs/fonts/DroidSans-Bold.ttf similarity index 100% rename from internal/app/proctord/docs/fonts/DroidSans-Bold.ttf rename to internal/app/service/docs/fonts/DroidSans-Bold.ttf diff --git a/internal/app/proctord/docs/fonts/DroidSans.ttf b/internal/app/service/docs/fonts/DroidSans.ttf similarity index 100% rename from internal/app/proctord/docs/fonts/DroidSans.ttf rename to internal/app/service/docs/fonts/DroidSans.ttf diff --git a/internal/app/proctord/docs/handler.go b/internal/app/service/docs/handler.go similarity index 100% rename from internal/app/proctord/docs/handler.go rename to internal/app/service/docs/handler.go diff --git a/internal/app/proctord/docs/images/collapse.gif b/internal/app/service/docs/images/collapse.gif similarity index 100% rename from internal/app/proctord/docs/images/collapse.gif rename to internal/app/service/docs/images/collapse.gif diff --git a/internal/app/proctord/docs/images/expand.gif b/internal/app/service/docs/images/expand.gif similarity index 100% rename from internal/app/proctord/docs/images/expand.gif rename to internal/app/service/docs/images/expand.gif diff --git a/internal/app/proctord/docs/images/explorer_icons.png b/internal/app/service/docs/images/explorer_icons.png similarity index 100% rename from internal/app/proctord/docs/images/explorer_icons.png rename to internal/app/service/docs/images/explorer_icons.png diff --git a/internal/app/proctord/docs/images/favicon-16x16.png b/internal/app/service/docs/images/favicon-16x16.png similarity index 100% rename from internal/app/proctord/docs/images/favicon-16x16.png rename to internal/app/service/docs/images/favicon-16x16.png diff --git a/internal/app/proctord/docs/images/favicon-32x32.png b/internal/app/service/docs/images/favicon-32x32.png similarity index 100% rename from internal/app/proctord/docs/images/favicon-32x32.png rename to internal/app/service/docs/images/favicon-32x32.png diff --git a/internal/app/proctord/docs/images/favicon.ico b/internal/app/service/docs/images/favicon.ico similarity index 100% rename from internal/app/proctord/docs/images/favicon.ico rename to internal/app/service/docs/images/favicon.ico diff --git a/internal/app/proctord/docs/images/logo_small.png b/internal/app/service/docs/images/logo_small.png similarity index 100% rename from internal/app/proctord/docs/images/logo_small.png rename to internal/app/service/docs/images/logo_small.png diff --git a/internal/app/proctord/docs/images/pet_store_api.png b/internal/app/service/docs/images/pet_store_api.png similarity index 100% rename from internal/app/proctord/docs/images/pet_store_api.png rename to internal/app/service/docs/images/pet_store_api.png diff --git a/internal/app/proctord/docs/images/throbber.gif b/internal/app/service/docs/images/throbber.gif similarity index 100% rename from internal/app/proctord/docs/images/throbber.gif rename to internal/app/service/docs/images/throbber.gif diff --git a/internal/app/proctord/docs/images/wordnik_api.png b/internal/app/service/docs/images/wordnik_api.png similarity index 100% rename from internal/app/proctord/docs/images/wordnik_api.png rename to internal/app/service/docs/images/wordnik_api.png diff --git a/internal/app/proctord/docs/index.html b/internal/app/service/docs/index.html similarity index 100% rename from internal/app/proctord/docs/index.html rename to internal/app/service/docs/index.html diff --git a/internal/app/proctord/docs/lang/ca.js b/internal/app/service/docs/lang/ca.js similarity index 100% rename from internal/app/proctord/docs/lang/ca.js rename to internal/app/service/docs/lang/ca.js diff --git a/internal/app/proctord/docs/lang/el.js b/internal/app/service/docs/lang/el.js similarity index 100% rename from internal/app/proctord/docs/lang/el.js rename to internal/app/service/docs/lang/el.js diff --git a/internal/app/proctord/docs/lang/en.js b/internal/app/service/docs/lang/en.js similarity index 100% rename from internal/app/proctord/docs/lang/en.js rename to internal/app/service/docs/lang/en.js diff --git a/internal/app/proctord/docs/lang/es.js b/internal/app/service/docs/lang/es.js similarity index 100% rename from internal/app/proctord/docs/lang/es.js rename to internal/app/service/docs/lang/es.js diff --git a/internal/app/proctord/docs/lang/fr.js b/internal/app/service/docs/lang/fr.js similarity index 100% rename from internal/app/proctord/docs/lang/fr.js rename to internal/app/service/docs/lang/fr.js diff --git a/internal/app/proctord/docs/lang/geo.js b/internal/app/service/docs/lang/geo.js similarity index 100% rename from internal/app/proctord/docs/lang/geo.js rename to internal/app/service/docs/lang/geo.js diff --git a/internal/app/proctord/docs/lang/it.js b/internal/app/service/docs/lang/it.js similarity index 100% rename from internal/app/proctord/docs/lang/it.js rename to internal/app/service/docs/lang/it.js diff --git a/internal/app/proctord/docs/lang/ja.js b/internal/app/service/docs/lang/ja.js similarity index 100% rename from internal/app/proctord/docs/lang/ja.js rename to internal/app/service/docs/lang/ja.js diff --git a/internal/app/proctord/docs/lang/ko-kr.js b/internal/app/service/docs/lang/ko-kr.js similarity index 100% rename from internal/app/proctord/docs/lang/ko-kr.js rename to internal/app/service/docs/lang/ko-kr.js diff --git a/internal/app/proctord/docs/lang/pl.js b/internal/app/service/docs/lang/pl.js similarity index 100% rename from internal/app/proctord/docs/lang/pl.js rename to internal/app/service/docs/lang/pl.js diff --git a/internal/app/proctord/docs/lang/pt.js b/internal/app/service/docs/lang/pt.js similarity index 100% rename from internal/app/proctord/docs/lang/pt.js rename to internal/app/service/docs/lang/pt.js diff --git a/internal/app/proctord/docs/lang/ru.js b/internal/app/service/docs/lang/ru.js similarity index 100% rename from internal/app/proctord/docs/lang/ru.js rename to internal/app/service/docs/lang/ru.js diff --git a/internal/app/proctord/docs/lang/tr.js b/internal/app/service/docs/lang/tr.js similarity index 100% rename from internal/app/proctord/docs/lang/tr.js rename to internal/app/service/docs/lang/tr.js diff --git a/internal/app/proctord/docs/lang/translator.js b/internal/app/service/docs/lang/translator.js similarity index 100% rename from internal/app/proctord/docs/lang/translator.js rename to internal/app/service/docs/lang/translator.js diff --git a/internal/app/proctord/docs/lang/zh-cn.js b/internal/app/service/docs/lang/zh-cn.js similarity index 100% rename from internal/app/proctord/docs/lang/zh-cn.js rename to internal/app/service/docs/lang/zh-cn.js diff --git a/internal/app/proctord/docs/lib/backbone-min.js b/internal/app/service/docs/lib/backbone-min.js similarity index 100% rename from internal/app/proctord/docs/lib/backbone-min.js rename to internal/app/service/docs/lib/backbone-min.js diff --git a/internal/app/proctord/docs/lib/es5-shim.js b/internal/app/service/docs/lib/es5-shim.js similarity index 100% rename from internal/app/proctord/docs/lib/es5-shim.js rename to internal/app/service/docs/lib/es5-shim.js diff --git a/internal/app/proctord/docs/lib/handlebars-4.0.5.js b/internal/app/service/docs/lib/handlebars-4.0.5.js similarity index 100% rename from internal/app/proctord/docs/lib/handlebars-4.0.5.js rename to internal/app/service/docs/lib/handlebars-4.0.5.js diff --git a/internal/app/proctord/docs/lib/highlight.9.1.0.pack.js b/internal/app/service/docs/lib/highlight.9.1.0.pack.js similarity index 100% rename from internal/app/proctord/docs/lib/highlight.9.1.0.pack.js rename to internal/app/service/docs/lib/highlight.9.1.0.pack.js diff --git a/internal/app/proctord/docs/lib/highlight.9.1.0.pack_extended.js b/internal/app/service/docs/lib/highlight.9.1.0.pack_extended.js similarity index 100% rename from internal/app/proctord/docs/lib/highlight.9.1.0.pack_extended.js rename to internal/app/service/docs/lib/highlight.9.1.0.pack_extended.js diff --git a/internal/app/proctord/docs/lib/jquery-1.8.0.min.js b/internal/app/service/docs/lib/jquery-1.8.0.min.js similarity index 100% rename from internal/app/proctord/docs/lib/jquery-1.8.0.min.js rename to internal/app/service/docs/lib/jquery-1.8.0.min.js diff --git a/internal/app/proctord/docs/lib/jquery.ba-bbq.min.js b/internal/app/service/docs/lib/jquery.ba-bbq.min.js similarity index 100% rename from internal/app/proctord/docs/lib/jquery.ba-bbq.min.js rename to internal/app/service/docs/lib/jquery.ba-bbq.min.js diff --git a/internal/app/proctord/docs/lib/jquery.slideto.min.js b/internal/app/service/docs/lib/jquery.slideto.min.js similarity index 100% rename from internal/app/proctord/docs/lib/jquery.slideto.min.js rename to internal/app/service/docs/lib/jquery.slideto.min.js diff --git a/internal/app/proctord/docs/lib/jquery.wiggle.min.js b/internal/app/service/docs/lib/jquery.wiggle.min.js similarity index 100% rename from internal/app/proctord/docs/lib/jquery.wiggle.min.js rename to internal/app/service/docs/lib/jquery.wiggle.min.js diff --git a/internal/app/proctord/docs/lib/js-yaml.min.js b/internal/app/service/docs/lib/js-yaml.min.js similarity index 100% rename from internal/app/proctord/docs/lib/js-yaml.min.js rename to internal/app/service/docs/lib/js-yaml.min.js diff --git a/internal/app/proctord/docs/lib/jsoneditor.min.js b/internal/app/service/docs/lib/jsoneditor.min.js similarity index 100% rename from internal/app/proctord/docs/lib/jsoneditor.min.js rename to internal/app/service/docs/lib/jsoneditor.min.js diff --git a/internal/app/proctord/docs/lib/lodash.min.js b/internal/app/service/docs/lib/lodash.min.js similarity index 100% rename from internal/app/proctord/docs/lib/lodash.min.js rename to internal/app/service/docs/lib/lodash.min.js diff --git a/internal/app/proctord/docs/lib/marked.js b/internal/app/service/docs/lib/marked.js similarity index 100% rename from internal/app/proctord/docs/lib/marked.js rename to internal/app/service/docs/lib/marked.js diff --git a/internal/app/proctord/docs/lib/object-assign-pollyfill.js b/internal/app/service/docs/lib/object-assign-pollyfill.js similarity index 100% rename from internal/app/proctord/docs/lib/object-assign-pollyfill.js rename to internal/app/service/docs/lib/object-assign-pollyfill.js diff --git a/internal/app/proctord/docs/lib/sanitize-html.min.js b/internal/app/service/docs/lib/sanitize-html.min.js similarity index 100% rename from internal/app/proctord/docs/lib/sanitize-html.min.js rename to internal/app/service/docs/lib/sanitize-html.min.js diff --git a/internal/app/proctord/docs/lib/swagger-oauth.js b/internal/app/service/docs/lib/swagger-oauth.js similarity index 100% rename from internal/app/proctord/docs/lib/swagger-oauth.js rename to internal/app/service/docs/lib/swagger-oauth.js diff --git a/internal/app/proctord/docs/o2c.html b/internal/app/service/docs/o2c.html similarity index 100% rename from internal/app/proctord/docs/o2c.html rename to internal/app/service/docs/o2c.html diff --git a/internal/app/proctord/docs/swagger-ui.js b/internal/app/service/docs/swagger-ui.js similarity index 100% rename from internal/app/proctord/docs/swagger-ui.js rename to internal/app/service/docs/swagger-ui.js diff --git a/internal/app/proctord/docs/swagger-ui.min.js b/internal/app/service/docs/swagger-ui.min.js similarity index 100% rename from internal/app/proctord/docs/swagger-ui.min.js rename to internal/app/service/docs/swagger-ui.min.js diff --git a/internal/app/proctord/docs/swagger.yml b/internal/app/service/docs/swagger.yml similarity index 100% rename from internal/app/proctord/docs/swagger.yml rename to internal/app/service/docs/swagger.yml diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 5b0f62d9..fe310d84 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -7,7 +7,7 @@ import ( "github.com/gorilla/mux" - "proctor/internal/app/proctord/docs" + "proctor/internal/app/service/docs" executionHttpHandler "proctor/internal/app/service/execution/handler" executionContextRepository "proctor/internal/app/service/execution/repository" executionService "proctor/internal/app/service/execution/service" From a3d5e1395050699b3295941b5d1c9a9e347fb05b Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:45:10 +0700 Subject: [PATCH 067/268] [bimo.horizon|mazbergaz] Remove unneeded app proctord --- internal/app/proctord/audit/auditor.go | 63 -- internal/app/proctord/audit/auditor_mock.go | 23 - internal/app/proctord/audit/auditor_test.go | 67 --- .../proctord/jobs/execution/executioner.go | 59 -- .../jobs/execution/executioner_mock.go | 15 - .../jobs/execution/executioner_test.go | 104 ---- .../app/proctord/jobs/execution/handler.go | 153 ----- .../proctord/jobs/execution/handler_test.go | 365 ------------ internal/app/proctord/jobs/logs/handler.go | 67 --- .../app/proctord/jobs/schedule/handler.go | 273 --------- .../proctord/jobs/schedule/handler_test.go | 537 ------------------ .../proctord/jobs/schedule/scheduled_job.go | 37 -- internal/app/proctord/jobs/schedule/worker.go | 132 ----- .../app/proctord/jobs/schedule/worker_test.go | 173 ------ internal/app/proctord/scheduler/scheduler.go | 51 -- .../app/proctord/storage/postgres/schema.go | 50 -- .../app/proctord/storage/postgres/utils.go | 13 - .../proctord/storage/postgres/utils_test.go | 19 - internal/app/proctord/storage/store.go | 113 ---- internal/app/proctord/storage/store_mock.go | 50 -- internal/app/proctord/storage/store_test.go | 255 --------- 21 files changed, 2619 deletions(-) delete mode 100644 internal/app/proctord/audit/auditor.go delete mode 100644 internal/app/proctord/audit/auditor_mock.go delete mode 100644 internal/app/proctord/audit/auditor_test.go delete mode 100644 internal/app/proctord/jobs/execution/executioner.go delete mode 100644 internal/app/proctord/jobs/execution/executioner_mock.go delete mode 100644 internal/app/proctord/jobs/execution/executioner_test.go delete mode 100644 internal/app/proctord/jobs/execution/handler.go delete mode 100644 internal/app/proctord/jobs/execution/handler_test.go delete mode 100644 internal/app/proctord/jobs/logs/handler.go delete mode 100644 internal/app/proctord/jobs/schedule/handler.go delete mode 100644 internal/app/proctord/jobs/schedule/handler_test.go delete mode 100644 internal/app/proctord/jobs/schedule/scheduled_job.go delete mode 100644 internal/app/proctord/jobs/schedule/worker.go delete mode 100644 internal/app/proctord/jobs/schedule/worker_test.go delete mode 100644 internal/app/proctord/scheduler/scheduler.go delete mode 100644 internal/app/proctord/storage/postgres/schema.go delete mode 100644 internal/app/proctord/storage/postgres/utils.go delete mode 100644 internal/app/proctord/storage/postgres/utils_test.go delete mode 100644 internal/app/proctord/storage/store.go delete mode 100644 internal/app/proctord/storage/store_mock.go delete mode 100644 internal/app/proctord/storage/store_test.go diff --git a/internal/app/proctord/audit/auditor.go b/internal/app/proctord/audit/auditor.go deleted file mode 100644 index 8dea6b97..00000000 --- a/internal/app/proctord/audit/auditor.go +++ /dev/null @@ -1,63 +0,0 @@ -package audit - -import ( - "github.com/getsentry/raven-go" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/kubernetes" - "proctor/internal/app/service/infra/logger" - "proctor/internal/pkg/constant" -) - -type Auditor interface { - JobsExecutionAndStatus(*postgres.JobsExecutionAuditLog) - JobsExecution(*postgres.JobsExecutionAuditLog) - JobsExecutionStatus(string) (string, error) -} - -type auditor struct { - store storage.Store - kubeClient kubernetes.KubernetesClient -} - -func New(store storage.Store, kubeClient kubernetes.KubernetesClient) Auditor { - return &auditor{ - store: store, - kubeClient: kubeClient, - } -} - -func (auditor *auditor) JobsExecutionAndStatus(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog) { - auditor.JobsExecution(jobsExecutionAuditLog) - - if jobsExecutionAuditLog.JobSubmissionStatus == constant.JobSubmissionSuccess && - jobsExecutionAuditLog.ExecutionID.Valid { - _, _ = auditor.JobsExecutionStatus(jobsExecutionAuditLog.ExecutionID.String) - } -} - -func (auditor *auditor) JobsExecution(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog) { - err := auditor.store.AuditJobsExecution(jobsExecutionAuditLog) - if err != nil { - logger.Error("Error auditing jobs execution", err) - raven.CaptureError(err, nil) - } -} - -func (auditor *auditor) JobsExecutionStatus(jobExecutionID string) (string, error) { - status, err := auditor.kubeClient.JobExecutionStatus(jobExecutionID) - if err != nil { - logger.Error("Error getting job execution status", err) - raven.CaptureError(err, nil) - return "", err - } - - err = auditor.store.UpdateJobsExecutionAuditLog(jobExecutionID, status) - if err != nil { - logger.Error("Error updating job status", err) - raven.CaptureError(err, nil) - return "", err - } - - return status, err -} diff --git a/internal/app/proctord/audit/auditor_mock.go b/internal/app/proctord/audit/auditor_mock.go deleted file mode 100644 index 69cc507b..00000000 --- a/internal/app/proctord/audit/auditor_mock.go +++ /dev/null @@ -1,23 +0,0 @@ -package audit - -import ( - "github.com/stretchr/testify/mock" - "proctor/internal/app/proctord/storage/postgres" -) - -type MockAuditor struct { - mock.Mock -} - -func (m *MockAuditor) JobsExecution(JobsExecutionAuditLog *postgres.JobsExecutionAuditLog) { - m.Called(JobsExecutionAuditLog) -} - -func (m *MockAuditor) JobsExecutionAndStatus(JobsExecutionAuditLog *postgres.JobsExecutionAuditLog) { - m.Called(JobsExecutionAuditLog) -} - -func (m *MockAuditor) JobsExecutionStatus(jobExecutionID string) (string, error) { - args := m.Called(jobExecutionID) - return args.String(0), args.Error(1) -} diff --git a/internal/app/proctord/audit/auditor_test.go b/internal/app/proctord/audit/auditor_test.go deleted file mode 100644 index db58a5de..00000000 --- a/internal/app/proctord/audit/auditor_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package audit - -import ( - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/kubernetes" - "testing" - - "proctor/internal/pkg/constant" -) - -func TestJobsExecutionAuditing(t *testing.T) { - mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockKubernetesClient{} - testAuditor := New(mockStore, mockKubeClient) - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: "any-job-name", - } - - mockStore.On("AuditJobsExecution", jobsExecutionAuditLog).Return(nil).Once() - - testAuditor.JobsExecution(jobsExecutionAuditLog) - - mockStore.AssertExpectations(t) - mockKubeClient.AssertExpectations(t) -} - -func TestAuditJobsExecutionStatusAuditing(t *testing.T) { - mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockKubernetesClient{} - testAuditor := New(mockStore, mockKubeClient) - - jobExecutionID := "job-execution-id" - jobExecutionStatus := "job-execution-status" - - mockKubeClient.On("JobExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) - mockStore.On("UpdateJobsExecutionAuditLog", jobExecutionID, jobExecutionStatus).Return(nil).Once() - - _,_ = testAuditor.JobsExecutionStatus(jobExecutionID) - - mockStore.AssertExpectations(t) - mockKubeClient.AssertExpectations(t) -} - -func TestAuditJobsExecutionAndStatusAuditing(t *testing.T) { - mockStore := &storage.MockStore{} - mockKubeClient := &kubernetes.MockKubernetesClient{} - testAuditor := New(mockStore, mockKubeClient) - - jobExecutionID := "job-execution-id" - jobExecutionStatus := "job-execution-status" - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: "any-job-name", - ExecutionID: postgres.StringToSQLString(jobExecutionID), - JobSubmissionStatus: constant.JobSubmissionSuccess, - } - - mockStore.On("AuditJobsExecution", jobsExecutionAuditLog).Return(nil).Once() - - mockKubeClient.On("JobExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) - mockStore.On("UpdateJobsExecutionAuditLog", jobExecutionID, jobExecutionStatus).Return(nil).Once() - - testAuditor.JobsExecutionAndStatus(jobsExecutionAuditLog) - - mockStore.AssertExpectations(t) - mockKubeClient.AssertExpectations(t) -} diff --git a/internal/app/proctord/jobs/execution/executioner.go b/internal/app/proctord/jobs/execution/executioner.go deleted file mode 100644 index 2d275fe2..00000000 --- a/internal/app/proctord/jobs/execution/executioner.go +++ /dev/null @@ -1,59 +0,0 @@ -package execution - -import ( - "errors" - "fmt" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/kubernetes" - repository2 "proctor/internal/app/service/metadata/repository" - "proctor/internal/app/service/secret/repository" - - "proctor/internal/pkg/constant" - "proctor/internal/pkg/utility" -) - -type executioner struct { - kubeClient kubernetes.KubernetesClient - metadataStore repository2.MetadataRepository - secretsStore repository.SecretRepository -} - -type Executioner interface { - Execute(*postgres.JobsExecutionAuditLog, string, map[string]string) (string, error) -} - -func NewExecutioner(kubeClient kubernetes.KubernetesClient, metadataStore repository2.MetadataRepository, secretsStore repository.SecretRepository) Executioner { - return &executioner{ - kubeClient: kubeClient, - metadataStore: metadataStore, - secretsStore: secretsStore, - } -} - -func (executioner *executioner) Execute(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog, jobName string, jobArgs map[string]string) (string, error) { - jobsExecutionAuditLog.JobName = jobName - - jobMetadata, err := executioner.metadataStore.GetByName(jobName) - if err != nil { - return "", errors.New(fmt.Sprintf("Error finding image for job: %s. Error: %s", jobName, err.Error())) - } - imageName := jobMetadata.ImageName - jobsExecutionAuditLog.ImageName = imageName - - jobSecrets, err := executioner.secretsStore.GetByJobName(jobName) - if err != nil && err.Error() != "redigo: nil returned" { - return "", errors.New(fmt.Sprintf("Error retrieving secrets for job: %s. Error: %s", jobName, err.Error())) - } - - envVars := utility.MergeMaps(jobArgs, jobSecrets) - jobsExecutionAuditLog.AddJobArgs(envVars) - - jobExecutionID, err := executioner.kubeClient.ExecuteJob(imageName, envVars) - if err != nil { - return "", errors.New(fmt.Sprintf("Error submitting job to kube: %s. Error: %s", jobName, err.Error())) - } - jobsExecutionAuditLog.AddExecutionID(jobExecutionID) - jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionSuccess - - return jobExecutionID, nil -} diff --git a/internal/app/proctord/jobs/execution/executioner_mock.go b/internal/app/proctord/jobs/execution/executioner_mock.go deleted file mode 100644 index 72e920f7..00000000 --- a/internal/app/proctord/jobs/execution/executioner_mock.go +++ /dev/null @@ -1,15 +0,0 @@ -package execution - -import ( - "github.com/stretchr/testify/mock" - "proctor/internal/app/proctord/storage/postgres" -) - -type MockExecutioner struct { - mock.Mock -} - -func (m *MockExecutioner) Execute(jobExecutionAuditLog *postgres.JobsExecutionAuditLog, jobName string, jobArgs map[string]string) (string, error) { - args := m.Called(jobExecutionAuditLog, jobName, jobArgs) - return args.String(0), args.Error(1) -} diff --git a/internal/app/proctord/jobs/execution/executioner_test.go b/internal/app/proctord/jobs/execution/executioner_test.go deleted file mode 100644 index ee204206..00000000 --- a/internal/app/proctord/jobs/execution/executioner_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package execution - -import ( - "errors" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/kubernetes" - metadataRepository "proctor/internal/app/service/metadata/repository" - secretRepository "proctor/internal/app/service/secret/repository" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - modelMetadata "proctor/internal/pkg/model/metadata" - "proctor/internal/pkg/utility" -) - -type ExecutionerTestSuite struct { - suite.Suite - mockKubeClient kubernetes.MockKubernetesClient - mockMetadataStore *metadataRepository.MockMetadataRepository - mockSecretsStore *secretRepository.MockSecretRepository - testExecutioner Executioner -} - -func (suite *ExecutionerTestSuite) SetupTest() { - suite.mockKubeClient = kubernetes.MockKubernetesClient{} - suite.mockMetadataStore = &metadataRepository.MockMetadataRepository{} - suite.mockSecretsStore = &secretRepository.MockSecretRepository{} - suite.testExecutioner = NewExecutioner(&suite.mockKubeClient, suite.mockMetadataStore, suite.mockSecretsStore) -} - -func (suite *ExecutionerTestSuite) TestSuccessfulJobExecution() { - t := suite.T() - - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{} - jobName := "sample-job-name" - jobArgs := map[string]string{ - "argOne": "sample-arg", - } - - jobMetadata := modelMetadata.Metadata{ - ImageName: "img", - } - suite.mockMetadataStore.On("GetByName", jobName).Return(&jobMetadata, nil).Once() - - jobSecrets := map[string]string{ - "secretOne": "sample-secrets", - } - suite.mockSecretsStore.On("GetByJobName", jobName).Return(jobSecrets, nil).Once() - - jobExecutionID := "proctor-ipsum-lorem" - envVarsForJob := utility.MergeMaps(jobArgs, jobSecrets) - suite.mockKubeClient.On("ExecuteJob", jobMetadata.ImageName, envVarsForJob).Return(jobExecutionID, nil).Once() - - executedJobName, err := suite.testExecutioner.Execute(jobsExecutionAuditLog, jobName, jobArgs) - assert.NoError(t, err) - - suite.mockMetadataStore.AssertExpectations(t) - suite.mockSecretsStore.AssertExpectations(t) - suite.mockKubeClient.AssertExpectations(t) - - assert.Equal(t, jobExecutionID, executedJobName) - assert.Equal(t, jobsExecutionAuditLog.JobName, jobName) -} - -func (suite *ExecutionerTestSuite) TestJobExecutionOnImageLookupFailure() { - t := suite.T() - - suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&modelMetadata.Metadata{}, errors.New("image-fetch-error")).Once() - - _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) - assert.EqualError(t, err, "Error finding image for job: any-job. Error: image-fetch-error") -} - -func (suite *ExecutionerTestSuite) TestJobExecutionOnSecretsFetchFailure() { - t := suite.T() - - jobMetadata := modelMetadata.Metadata{ImageName: "img"} - suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&jobMetadata, nil).Once() - - suite.mockSecretsStore.On("GetByJobName", mock.Anything).Return(map[string]string{}, errors.New("secret-store-error")).Once() - - _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) - assert.EqualError(t, err, "Error retrieving secrets for job: any-job. Error: secret-store-error") -} - -func (suite *ExecutionerTestSuite) TestJobExecutionOnKubernetesJobExecutionFailure() { - t := suite.T() - - jobMetadata := modelMetadata.Metadata{ImageName: "img"} - suite.mockMetadataStore.On("GetByName", mock.Anything).Return(&jobMetadata, nil).Once() - - suite.mockSecretsStore.On("GetByJobName", mock.Anything).Return(map[string]string{}, nil).Once() - suite.mockKubeClient.On("ExecuteJob", mock.Anything, mock.Anything).Return("", errors.New("kube-client-error")).Once() - - _, err := suite.testExecutioner.Execute(&postgres.JobsExecutionAuditLog{}, "any-job", map[string]string{}) - - assert.EqualError(t, err, "Error submitting job to kube: any-job. Error: kube-client-error") -} - -func TestExecutionerTestSuite(t *testing.T) { - suite.Run(t, new(ExecutionerTestSuite)) -} diff --git a/internal/app/proctord/jobs/execution/handler.go b/internal/app/proctord/jobs/execution/handler.go deleted file mode 100644 index 7f175711..00000000 --- a/internal/app/proctord/jobs/execution/handler.go +++ /dev/null @@ -1,153 +0,0 @@ -package execution - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/getsentry/raven-go" - "github.com/gorilla/mux" - "net/http" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/execution/handler/parameter" - "proctor/internal/app/service/infra/logger" - "proctor/internal/pkg/constant" - "time" -) - -type executionHandler struct { - auditor audit.Auditor - store storage.Store - executioner Executioner -} - -type ExecutionHandler interface { - Handle() http.HandlerFunc - Status() http.HandlerFunc - sendStatusToCaller(remoteCallerURL, jobExecutionID string) -} - -func NewExecutionHandler(auditor audit.Auditor, store storage.Store, executioner Executioner) ExecutionHandler { - return &executionHandler{ - auditor: auditor, - store: store, - executioner: executioner, - } -} - -func (handler *executionHandler) Status() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - jobExecutionID := mux.Vars(req)["name"] - jobExecutionStatus, err := handler.store.GetJobExecutionStatus(jobExecutionID) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - logger.Error(fmt.Sprintf("Error getting job status for job_id: %s", jobExecutionID), err.Error()) - raven.CaptureError(err, map[string]string{"job_id": jobExecutionID}) - - return - } - - if jobExecutionStatus == "" { - w.WriteHeader(http.StatusNotFound) - return - } - - _, _ = fmt.Fprintf(w, jobExecutionStatus) - } -} - -func (handler *executionHandler) Handle() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobExecutionStatus: "WAITING", - } - - userEmail := req.Header.Get(constant.UserEmailHeaderKey) - jobsExecutionAuditLog.UserEmail = userEmail - - var job parameter.Job - err := json.NewDecoder(req.Body).Decode(&job) - defer req.Body.Close() - if err != nil { - logger.Error(fmt.Sprintf("User: %s: Error parsing request body", userEmail), err.Error()) - raven.CaptureError(err, map[string]string{"user_email": userEmail}) - - jobsExecutionAuditLog.Errors = fmt.Sprintf("Error parsing request body: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionClientError - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.ClientError)) - go handler.auditor.JobsExecutionAndStatus(jobsExecutionAuditLog) - - return - } - jobExecutionID, err := handler.executioner.Execute(jobsExecutionAuditLog, job.Name, job.Args) - if err != nil { - logger.Error(fmt.Sprintf("%s: User %s: Error executing job: ", job.Name, userEmail), err.Error()) - raven.CaptureError(err, map[string]string{"user_email": userEmail, "job_name": job.Name}) - - jobsExecutionAuditLog.Errors = fmt.Sprintf("Error executing job: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionServerError - go handler.auditor.JobsExecutionAndStatus(jobsExecutionAuditLog) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - - return - } - - w.WriteHeader(http.StatusCreated) - _, _ = w.Write([]byte(fmt.Sprintf("{ \"name\":\"%s\" }", jobExecutionID))) - - remoteCallerURL := job.CallbackURL - go handler.postJobExecute(jobsExecutionAuditLog, remoteCallerURL, jobExecutionID) - return - } -} - -func (handler *executionHandler) postJobExecute(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog, remoteCallerURL, jobExecutionID string) { - handler.auditor.JobsExecutionAndStatus(jobsExecutionAuditLog) - if remoteCallerURL != "" { - handler.sendStatusToCaller(remoteCallerURL, jobExecutionID) - } -} - -func (handler *executionHandler) sendStatusToCaller(remoteCallerURL, jobExecutionID string) { - status := constant.JobWaiting - - for { - jobExecutionStatus, _ := handler.store.GetJobExecutionStatus(jobExecutionID) - if jobExecutionStatus == "" { - status = constant.JobNotFound - break - } - - if jobExecutionStatus == constant.JobSucceeded || jobExecutionStatus == constant.JobFailed { - status = jobExecutionStatus - break - } - - time.Sleep(1 * time.Second) - } - - value := map[string]string{"name": jobExecutionID, "status": status} - jsonValue, err := json.Marshal(value) - if err != nil { - logger.Error(fmt.Sprintf("StatusCallback: Error parsing %#v", value), err.Error()) - raven.CaptureError(err, nil) - - return - } - - _, err = http.Post(remoteCallerURL, "application/json", bytes.NewBuffer(jsonValue)) - if err != nil { - logger.Error("StatusCallback: Error sending request to callback url", err.Error()) - raven.CaptureError(err, nil) - - return - } - - return -} diff --git a/internal/app/proctord/jobs/execution/handler_test.go b/internal/app/proctord/jobs/execution/handler_test.go deleted file mode 100644 index 21375d84..00000000 --- a/internal/app/proctord/jobs/execution/handler_test.go +++ /dev/null @@ -1,365 +0,0 @@ -package execution - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/gorilla/mux" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "github.com/urfave/negroni" - "net/http" - "net/http/httptest" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/service/execution/handler/parameter" - "proctor/internal/pkg/constant" - "testing" -) - -type ExecutionHandlerTestSuite struct { - suite.Suite - mockAuditor *audit.MockAuditor - mockStore *storage.MockStore - mockExecutioner *MockExecutioner - testExecutionHandler ExecutionHandler - - Client *http.Client - TestServer *httptest.Server -} - -func (suite *ExecutionHandlerTestSuite) SetupTest() { - suite.mockAuditor = &audit.MockAuditor{} - suite.mockStore = &storage.MockStore{} - suite.mockExecutioner = &MockExecutioner{} - suite.testExecutionHandler = NewExecutionHandler(suite.mockAuditor, suite.mockStore, suite.mockExecutioner) - - suite.Client = &http.Client{} - router := mux.NewRouter() - router.HandleFunc("/jobs/execute/{name}/status", suite.testExecutionHandler.Status()).Methods("GET") - n := negroni.Classic() - n.UseHandler(router) - suite.TestServer = httptest.NewServer(n) -} - -func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandler() { - t := suite.T() - - jobExecutionID := "proctor-ipsum-lorem" - statusChan := make(chan bool) - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - - var value map[string]string - err := json.NewDecoder(req.Body).Decode(&value) - assert.NoError(t, err) - - w.WriteHeader(http.StatusOK) - - assert.Equal(t, jobExecutionID, value["name"]) - assert.Equal(t, constant.JobSucceeded, value["status"]) - - statusChan <- true - })) - defer ts.Close() - - remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) - - userEmail := "mrproctor@example.com" - job := parameter.Job{ - Name: "sample-job-name", - Args: map[string]string{"argOne": "sample-arg"}, - CallbackURL: remoteCallerURL, - } - - requestBody, err := json.Marshal(job) - assert.NoError(t, err) - - req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) - req.Header.Set(constant.UserEmailHeaderKey, userEmail) - responseRecorder := httptest.NewRecorder() - - suite.mockExecutioner.On("Execute", mock.Anything, job.Name, job.Args).Return(jobExecutionID, nil).Once() - - auditingChan := make(chan bool) - - suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( - func(args mock.Arguments) { auditingChan <- true }, - ) - suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(constant.JobSucceeded, nil).Once() - - suite.testExecutionHandler.Handle()(responseRecorder, req) - - <-auditingChan - <-statusChan - suite.mockAuditor.AssertExpectations(t) - suite.mockExecutioner.AssertExpectations(t) - - assert.Equal(t, http.StatusCreated, responseRecorder.Code) - assert.Equal(t, fmt.Sprintf("{ \"name\":\"%s\" }", jobExecutionID), responseRecorder.Body.String()) -} - -func (suite *ExecutionHandlerTestSuite) TestSuccessfulJobExecutionHandlerWithoutCallbackURL() { - t := suite.T() - - jobExecutionID := "proctor-ipsum-lorem" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - assert.Equal(t, "should not call status callback", "called status callback") - })) - defer ts.Close() - - userEmail := "mrproctor@example.com" - job := parameter.Job{ - Name: "sample-job-name", - Args: map[string]string{"argOne": "sample-arg"}, - } - - requestBody, err := json.Marshal(job) - assert.NoError(t, err) - - req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) - req.Header.Set(constant.UserEmailHeaderKey, userEmail) - responseRecorder := httptest.NewRecorder() - - suite.mockExecutioner.On("Execute", mock.Anything, job.Name, job.Args).Return(jobExecutionID, nil).Once() - - auditingChan := make(chan bool) - - suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( - func(args mock.Arguments) { auditingChan <- true }, - ) - suite.mockStore.On("GetJobExecutionStatus", jobExecutionID).Return(constant.JobSucceeded, nil).Once() - - suite.testExecutionHandler.Handle()(responseRecorder, req) - - <-auditingChan - suite.mockAuditor.AssertExpectations(t) - suite.mockExecutioner.AssertExpectations(t) - - assert.Equal(t, http.StatusCreated, responseRecorder.Code) - assert.Equal(t, fmt.Sprintf("{ \"name\":\"%s\" }", jobExecutionID), responseRecorder.Body.String()) -} - -func (suite *ExecutionHandlerTestSuite) TestJobExecutionOnMalformedRequest() { - t := suite.T() - - jobExecutionRequest := fmt.Sprintf("{ some-malformed-request }") - req := httptest.NewRequest("POST", "/execute", bytes.NewReader([]byte(jobExecutionRequest))) - responseRecorder := httptest.NewRecorder() - - auditingChan := make(chan bool) - suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( - func(args mock.Arguments) { auditingChan <- true }, - ) - - suite.testExecutionHandler.Handle()(responseRecorder, req) - - <-auditingChan - suite.mockAuditor.AssertExpectations(t) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - assert.Equal(t, constant.ClientError, responseRecorder.Body.String()) -} - -func (suite *ExecutionHandlerTestSuite) TestJobExecutionServerFailure() { - t := suite.T() - - job := parameter.Job{ - Name: "sample-job-name", - Args: map[string]string{"argOne": "sample-arg"}, - } - - requestBody, err := json.Marshal(job) - assert.NoError(t, err) - - req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) - responseRecorder := httptest.NewRecorder() - - suite.mockExecutioner.On("Execute", mock.Anything, job.Name, job.Args).Return("", errors.New("error executing job")).Once() - - auditingChan := make(chan bool) - suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( - func(args mock.Arguments) { auditingChan <- true }, - ) - suite.mockAuditor.On("JobsExecutionAndStatus", mock.Anything).Return("", nil).Run( - func(args mock.Arguments) { auditingChan <- true }, - ) - - suite.testExecutionHandler.Handle()(responseRecorder, req) - - <-auditingChan - suite.mockAuditor.AssertExpectations(t) - suite.mockExecutioner.AssertExpectations(t) - - assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) -} - -func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200OnSuccess() { - t := suite.T() - - jobName := "sample-job-name" - - url := fmt.Sprintf("%s/jobs/execute/%s/status", suite.TestServer.URL, jobName) - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobSucceeded, nil).Once() - - req, _ := http.NewRequest("GET", url, nil) - - response, _ := suite.Client.Do(req) - suite.mockStore.AssertExpectations(t) - assert.Equal(suite.T(), http.StatusOK, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - jobStatus := buf.String() - assert.Equal(suite.T(), constant.JobSucceeded, jobStatus) -} - -func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn404IfJobStatusIsNotFound() { - t := suite.T() - - jobName := "sample-job-name" - - url := fmt.Sprintf("%s/jobs/execute/%s/status", suite.TestServer.URL, jobName) - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return("", nil).Once() - - req, _ := http.NewRequest("GET", url, nil) - - response, _ := suite.Client.Do(req) - suite.mockStore.AssertExpectations(t) - assert.Equal(suite.T(), http.StatusNotFound, response.StatusCode) -} - -func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn200IfJobStatusIsWaiting() { - t := suite.T() - - jobName := "sample-job-name" - - url := fmt.Sprintf("%s/jobs/execute/%s/status", suite.TestServer.URL, jobName) - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return("WAITING", nil).Once() - - req, _ := http.NewRequest("GET", url, nil) - - response, _ := suite.Client.Do(req) - suite.mockStore.AssertExpectations(t) - assert.Equal(suite.T(), http.StatusOK, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - jobStatus := buf.String() - assert.Equal(suite.T(), constant.JobWaiting, jobStatus) -} - -func (suite *ExecutionHandlerTestSuite) TestJobStatusShouldReturn500OnError() { - t := suite.T() - - jobName := "sample-job-name" - - url := fmt.Sprintf("%s/jobs/execute/%s/status", suite.TestServer.URL, jobName) - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return("", errors.New("error")).Once() - - req, _ := http.NewRequest("GET", url, nil) - - response, _ := suite.Client.Do(req) - suite.mockStore.AssertExpectations(t) - assert.Equal(suite.T(), http.StatusInternalServerError, response.StatusCode) -} - -func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnSuccess() { - t := suite.T() - - jobName := "sample-job-name" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - - var value map[string]string - err := json.NewDecoder(req.Body).Decode(&value) - assert.NoError(t, err) - - w.WriteHeader(http.StatusOK) - - assert.Equal(t, jobName, value["name"]) - assert.Equal(t, constant.JobSucceeded, value["status"]) - })) - defer ts.Close() - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobSucceeded, nil).Once() - - remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) - - suite.testExecutionHandler.sendStatusToCaller(remoteCallerURL, jobName) - suite.mockStore.AssertExpectations(t) -} - -func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnFailure() { - t := suite.T() - - jobName := "sample-job-name" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - - var value map[string]string - err := json.NewDecoder(req.Body).Decode(&value) - assert.NoError(t, err) - - w.WriteHeader(http.StatusOK) - - assert.Equal(t, jobName, value["name"]) - assert.Equal(t, constant.JobFailed, value["status"]) - })) - defer ts.Close() - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobWaiting, nil).Once() - suite.mockStore.On("GetJobExecutionStatus", jobName).Return(constant.JobFailed, nil).Once() - - remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) - - suite.testExecutionHandler.sendStatusToCaller(remoteCallerURL, jobName) - suite.mockStore.AssertExpectations(t) -} - -func (suite *ExecutionHandlerTestSuite) TestSendStatusToCallerOnJobNotFound() { - t := suite.T() - - jobName := "sample-job-name" - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - defer req.Body.Close() - - var value map[string]string - err := json.NewDecoder(req.Body).Decode(&value) - assert.NoError(t, err) - - w.WriteHeader(http.StatusOK) - - assert.Equal(t, jobName, value["name"]) - assert.Equal(t, constant.JobNotFound, value["status"]) - })) - defer ts.Close() - - suite.mockStore.On("GetJobExecutionStatus", jobName).Return("", nil).Once() - - remoteCallerURL := fmt.Sprintf("%s/status", ts.URL) - - suite.testExecutionHandler.sendStatusToCaller(remoteCallerURL, jobName) - suite.mockStore.AssertExpectations(t) -} - -func TestExecutionHandlerTestSuite(t *testing.T) { - suite.Run(t, new(ExecutionHandlerTestSuite)) -} diff --git a/internal/app/proctord/jobs/logs/handler.go b/internal/app/proctord/jobs/logs/handler.go deleted file mode 100644 index 9301d0c9..00000000 --- a/internal/app/proctord/jobs/logs/handler.go +++ /dev/null @@ -1,67 +0,0 @@ -package logs - -import ( - "github.com/getsentry/raven-go" - "net/http" - "proctor/internal/app/service/infra/config" - "proctor/internal/app/service/infra/kubernetes" - _logger "proctor/internal/app/service/infra/logger" - "proctor/internal/pkg/constant" - "strings" - - "github.com/gorilla/websocket" -) - -var upgrader = websocket.Upgrader{ - ReadBufferSize: config.LogsStreamReadBufferSize(), - WriteBufferSize: config.LogsStreamWriteBufferSize(), -} - -type logger struct { - kubeClient kubernetes.KubernetesClient -} - -type Logger interface { - Stream() http.HandlerFunc -} - -func NewLogger(kubeClient kubernetes.KubernetesClient) Logger { - return &logger{ - kubeClient: kubeClient, - } -} - -func CloseWebSocket(message string, conn *websocket.Conn) { - err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, message)) - if err != nil { - _logger.Error("Error closing connection with client after logs are read") - raven.CaptureError(err, nil) - } - return -} - -func (l *logger) Stream() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - conn, err := upgrader.Upgrade(w, req, nil) - if err != nil { - _logger.Error("Error upgrading connection to websocket protocol: ", err) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.ClientError)) - return - } - defer conn.Close() - - jobName := strings.TrimLeft(req.URL.RawQuery, "job_name=") - if jobName == "" { - _logger.Error("No job name provided as part of URL: ", req.URL.RawQuery) - CloseWebSocket("No job name provided while requesting for logs", conn) - return - } - - CloseWebSocket("No job name provided while requesting for logs", conn) - return - - } -} diff --git a/internal/app/proctord/jobs/schedule/handler.go b/internal/app/proctord/jobs/schedule/handler.go deleted file mode 100644 index 7c342393..00000000 --- a/internal/app/proctord/jobs/schedule/handler.go +++ /dev/null @@ -1,273 +0,0 @@ -package schedule - -import ( - "encoding/json" - "fmt" - "github.com/getsentry/raven-go" - "github.com/gorilla/mux" - "net/http" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/service/infra/logger" - "proctor/internal/app/service/metadata/repository" - "strings" - - "github.com/badoux/checkmail" - - "github.com/robfig/cron" - "proctor/internal/pkg/constant" - modelSchedule "proctor/internal/pkg/model/schedule" -) - -type scheduler struct { - store storage.Store - metadataStore repository.MetadataRepository -} - -type Scheduler interface { - Schedule() http.HandlerFunc - GetScheduledJobs() http.HandlerFunc - GetScheduledJob() http.HandlerFunc - RemoveScheduledJob() http.HandlerFunc -} - -func NewScheduler(store storage.Store, metadataStore repository.MetadataRepository) Scheduler { - return &scheduler{ - metadataStore: metadataStore, - store: store, - } -} - -func (scheduler *scheduler) Schedule() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - var scheduledJob modelSchedule.ScheduledJob - err := json.NewDecoder(req.Body).Decode(&scheduledJob) - userEmail := req.Header.Get(constant.UserEmailHeaderKey) - defer req.Body.Close() - if err != nil { - logger.Error("Error parsing request body for scheduling jobs: ", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.ClientError)) - - return - } - - if scheduledJob.Tags == "" { - logger.Error("Tag(s) are missing") - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.InvalidTagError)) - return - } - - _, err = cron.Parse(scheduledJob.Time) - if err != nil { - logger.Error(fmt.Sprintf("Client provided invalid cron expression: %s ", scheduledJob.Tags), scheduledJob.Name, scheduledJob.Time) - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.InvalidCronExpressionClientError)) - return - } - - notificationEmails := strings.Split(scheduledJob.NotificationEmails, ",") - - for _, notificationEmail := range notificationEmails { - err = checkmail.ValidateFormat(notificationEmail) - if err != nil { - logger.Error(fmt.Sprintf("Client provided invalid email address: %s: ", scheduledJob.Tags), scheduledJob.Name, notificationEmail) - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.InvalidEmailIdClientError)) - return - } - } - - if scheduledJob.Group == "" { - logger.Error(fmt.Sprintf("Group Name is missing %s: ", scheduledJob.Tags), scheduledJob.Name) - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte(constant.GroupNameMissingError)) - return - } - - _, err = scheduler.metadataStore.GetByName(scheduledJob.Name) - if err != nil { - if err.Error() == "redigo: nil returned" { - logger.Error(fmt.Sprintf("Client provided non existent proc name: %s ", scheduledJob.Tags), scheduledJob.Name) - - w.WriteHeader(http.StatusNotFound) - _, _ = w.Write([]byte(constant.NonExistentProcClientError)) - } else { - logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", scheduledJob.Tags), scheduledJob.Name, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - } - - return - } - - scheduledJob.Time = fmt.Sprintf("0 %s", scheduledJob.Time) - scheduledJob.ID, err = scheduler.store.InsertScheduledJob(scheduledJob.Name, scheduledJob.Tags, scheduledJob.Time, scheduledJob.NotificationEmails, userEmail, scheduledJob.Group, scheduledJob.Args) - if err != nil { - if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { - logger.Error(fmt.Sprintf("Client provided duplicate combination of scheduled job name and args: %s ", scheduledJob.Tags), scheduledJob.Name, scheduledJob.Args) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - w.WriteHeader(http.StatusConflict) - _, _ = w.Write([]byte(constant.DuplicateJobNameArgsClientError)) - - return - } else { - logger.Error(fmt.Sprintf("Error persisting scheduled job %s ", scheduledJob.Tags), scheduledJob.Name, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - - return - } - } - - responseBody, err := json.Marshal(scheduledJob) - if err != nil { - logger.Error(fmt.Sprintf("Error marshaling response body %s ", scheduledJob.Tags), scheduledJob.Name, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - - return - } - - w.WriteHeader(http.StatusCreated) - _, _ = w.Write(responseBody) - return - } -} - -func (scheduler *scheduler) GetScheduledJobs() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - scheduledJobsStoreFormat, err := scheduler.store.GetEnabledScheduledJobs() - if err != nil { - logger.Error("Error fetching scheduled jobs", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - if len(scheduledJobsStoreFormat) == 0 { - logger.Error(constant.NoScheduledJobsError, nil) - - w.WriteHeader(http.StatusNoContent) - return - } - - scheduledJobs, err := FromStoreToHandler(scheduledJobsStoreFormat) - if err != nil { - logger.Error("Error deserializing scheduled job args to map: ", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - scheduledJobsJson, err := json.Marshal(scheduledJobs) - if err != nil { - logger.Error("Error marshalling scheduled jobs", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - _, _ = w.Write(scheduledJobsJson) - } -} - -func (scheduler *scheduler) GetScheduledJob() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - jobID := mux.Vars(req)["id"] - scheduledJob, err := scheduler.store.GetScheduledJob(jobID) - if err != nil { - if strings.Contains(err.Error(), "invalid input syntax") { - logger.Error(err.Error()) - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("Invalid Job ID")) - return - } - logger.Error("Error fetching scheduled job", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - if len(scheduledJob) == 0 { - logger.Error(constant.JobNotFoundError, nil) - - w.WriteHeader(http.StatusNotFound) - _, _ = w.Write([]byte(constant.JobNotFoundError)) - return - } - - job, err := GetScheduledJob(scheduledJob[0]) - if err != nil { - logger.Error("Error deserializing scheduled job args to map: ", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - scheduledJobJson, err := json.Marshal(job) - if err != nil { - logger.Error("Error marshalling scheduled job", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - _, _ = w.Write(scheduledJobJson) - } -} - -func (scheduler *scheduler) RemoveScheduledJob() http.HandlerFunc { - return func(w http.ResponseWriter, req *http.Request) { - jobID := mux.Vars(req)["id"] - removedJobsCount, err := scheduler.store.RemoveScheduledJob(jobID) - if err != nil { - if strings.Contains(err.Error(), "invalid input syntax") { - logger.Error(err.Error()) - - w.WriteHeader(http.StatusBadRequest) - _, _ = w.Write([]byte("Invalid Job ID")) - return - } - logger.Error("Error fetching scheduled job", err.Error()) - raven.CaptureError(err, nil) - - w.WriteHeader(http.StatusInternalServerError) - _, _ = w.Write([]byte(constant.ServerError)) - return - } - - if removedJobsCount == 0 { - logger.Error(constant.JobNotFoundError, nil) - - w.WriteHeader(http.StatusNotFound) - _, _ = w.Write([]byte(constant.JobNotFoundError)) - return - } - - w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(fmt.Sprintf("Successfully unscheduled Job ID: %s", jobID))) - } -} diff --git a/internal/app/proctord/jobs/schedule/handler_test.go b/internal/app/proctord/jobs/schedule/handler_test.go deleted file mode 100644 index 5e14a116..00000000 --- a/internal/app/proctord/jobs/schedule/handler_test.go +++ /dev/null @@ -1,537 +0,0 @@ -package schedule - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "github.com/gorilla/mux" - "github.com/urfave/negroni" - "io/ioutil" - "net/http" - "net/http/httptest" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/metadata/repository" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - "proctor/internal/pkg/constant" - modelMetadata "proctor/internal/pkg/model/metadata" - modelSchedule "proctor/internal/pkg/model/schedule" -) - -type SchedulerTestSuite struct { - suite.Suite - mockStore *storage.MockStore - mockMetadataStore *repository.MockMetadataRepository - - testScheduler Scheduler - - Client *http.Client - TestServer *httptest.Server -} - -func (suite *SchedulerTestSuite) SetupTest() { - suite.mockMetadataStore = &repository.MockMetadataRepository{} - suite.mockStore = &storage.MockStore{} - suite.testScheduler = NewScheduler(suite.mockStore, suite.mockMetadataStore) - - suite.Client = &http.Client{} - router := mux.NewRouter() - router.HandleFunc("/jobs/schedule/{id}", suite.testScheduler.GetScheduledJob()).Methods("GET") - router.HandleFunc("/jobs/schedule/{id}", suite.testScheduler.RemoveScheduledJob()).Methods("DELETE") - n := negroni.Classic() - n.UseHandler(router) - suite.TestServer = httptest.NewServer(n) -} - -func (suite *SchedulerTestSuite) TestSuccessfulJobScheduling() { - t := suite.T() - - userEmail := "mrproctor@example.com" - scheduledJob := modelSchedule.ScheduledJob{ - Name: "any-job", - Args: map[string]string{}, - Time: "* 2 * * *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Tags: "tag-one,tag-two", - Group: "some-group", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - req.Header.Set(constant.UserEmailHeaderKey, userEmail) - - suite.mockMetadataStore.On("GetByName", scheduledJob.Name).Return(&modelMetadata.Metadata{}, nil) - insertedScheduledJobID := "123" - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, userEmail, scheduledJob.Group, scheduledJob.Args).Return(insertedScheduledJobID, nil) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusCreated, responseRecorder.Code) - - expectedResponse := modelSchedule.ScheduledJob{} - err = json.NewDecoder(responseRecorder.Body).Decode(&expectedResponse) - assert.NoError(t, err) - assert.Equal(t, insertedScheduledJobID, expectedResponse.ID) -} - -func (suite *SchedulerTestSuite) TestBadRequestWhenRequestBodyIsIncorrectForJobScheduling() { - t := suite.T() - - req := httptest.NewRequest("POST", "/schedule", bytes.NewBuffer([]byte("invalid json"))) - responseRecorder := httptest.NewRecorder() - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.ClientError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestInvalidCronExpression() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "2 * invalid *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Tags: "tag-one,tag-two", - Group: "some-group", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.InvalidCronExpressionClientError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestInvalidEmailAddress() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "user-test.com", - Group: "some-group", - Tags: "tag-one,tag-two", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.InvalidEmailIdClientError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestInvalidTag() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - Group: "some-group", - NotificationEmails: "user@proctor.com", - Tags: "", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.InvalidTagError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestInvalidGroupName() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "user@proctor.com", - Tags: "backup", - Group: "", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.GroupNameMissingError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestNonExistentJobScheduling() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Tags: "tag-one,tag-two", - Group: "some-group", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.mockMetadataStore.On("GetByName", scheduledJob.Name).Return(&modelMetadata.Metadata{}, errors.New("redigo: nil returned")) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusNotFound, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.NonExistentProcClientError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestErrorFetchingJobMetadata() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Group: "some-group", - Tags: "tag-one,tag-two", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.mockMetadataStore.On("GetByName", scheduledJob.Name).Return(&modelMetadata.Metadata{}, errors.New("any error")) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.ServerError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestUniqnessConstrainOnJobNameAndArg() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Tags: "tag-one,tag-two", - Group: "group1", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.mockMetadataStore.On("GetByName", scheduledJob.Name).Return(&modelMetadata.Metadata{}, nil) - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "", scheduledJob.Group, scheduledJob.Args).Return("", errors.New("pq: duplicate key value violates unique constraint \"unique_jobs_schedule_name_args\"")) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusConflict, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.DuplicateJobNameArgsClientError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestErrorPersistingScheduledJob() { - t := suite.T() - - scheduledJob := modelSchedule.ScheduledJob{ - Name: "non-existent", - Time: "* 2 * * *", - NotificationEmails: "foo@bar.com,bar@foo.com", - Tags: "tag-one,tag-two", - Group: "group", - } - requestBody, err := json.Marshal(scheduledJob) - assert.NoError(t, err) - - responseRecorder := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(requestBody)) - - suite.mockMetadataStore.On("GetByName", scheduledJob.Name).Return(&modelMetadata.Metadata{}, nil) - suite.mockStore.On("InsertScheduledJob", scheduledJob.Name, scheduledJob.Tags, "0 * 2 * * *", scheduledJob.NotificationEmails, "", scheduledJob.Group, scheduledJob.Args).Return("", errors.New("any-error")) - - suite.testScheduler.Schedule()(responseRecorder, req) - - assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - responseBody, _ := ioutil.ReadAll(responseRecorder.Body) - assert.Equal(t, constant.ServerError, string(responseBody)) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobs() { - t := suite.T() - - req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) - responseRecorder := httptest.NewRecorder() - - scheduledJobsStoreFormat := []postgres.JobsSchedule{ - { - ID: "some-id", - }, - } - suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() - - suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - - suite.mockStore.AssertExpectations(t) - - assert.Equal(t, http.StatusOK, responseRecorder.Code) - - var scheduledJobs []modelSchedule.ScheduledJob - err := json.Unmarshal(responseRecorder.Body.Bytes(), &scheduledJobs) - assert.NoError(t, err) - assert.Equal(t, scheduledJobsStoreFormat[0].ID, scheduledJobs[0].ID) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobsWhenNoJobsFound() { - t := suite.T() - - req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) - responseRecorder := httptest.NewRecorder() - - var scheduledJobsStoreFormat []postgres.JobsSchedule - suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobsStoreFormat, nil).Once() - - suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - - suite.mockStore.AssertExpectations(t) - - assert.Equal(t, http.StatusNoContent, responseRecorder.Code) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobsFailure() { - t := suite.T() - - req := httptest.NewRequest("GET", "/jobs/schedule", bytes.NewReader([]byte{})) - responseRecorder := httptest.NewRecorder() - - var scheduledJobs []postgres.JobsSchedule - suite.mockStore.On("GetEnabledScheduledJobs").Return(scheduledJobs, errors.New("error")).Once() - - suite.testScheduler.GetScheduledJobs()(responseRecorder, req) - - suite.mockStore.AssertExpectations(t) - - assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, constant.ServerError, responseRecorder.Body.String()) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobByID() { - t := suite.T() - jobID := "some-id" - - scheduledJobsStoreFormat := []postgres.JobsSchedule{ - { - ID: jobID, - }, - } - suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("GET", url, nil) - - response, err := suite.Client.Do(req) - - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, response.StatusCode) - - var scheduledJob modelSchedule.ScheduledJob - err = json.NewDecoder(response.Body).Decode(&scheduledJob) - assert.NoError(t, err) - assert.Equal(t, jobID, scheduledJob.ID) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInvalidJobID() { - t := suite.T() - jobID := "invalid-job-id" - - var scheduledJobsStoreFormat []postgres.JobsSchedule - - suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("GET", url, nil) - - response, err := suite.Client.Do(req) - - assert.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Invalid Job ID", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnInternalServerError() { - t := suite.T() - jobID := "job-id" - - var scheduledJobsStoreFormat []postgres.JobsSchedule - - suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, errors.New("some-error")).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("GET", url, nil) - - response, err := suite.Client.Do(req) - assert.NoError(t, err) - - assert.Equal(t, http.StatusInternalServerError, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Something went wrong", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestGetScheduledJobByIDOnJobIDNotFound() { - t := suite.T() - jobID := "absent-job-id" - - var scheduledJobsStoreFormat []postgres.JobsSchedule - - suite.mockStore.On("GetScheduledJob", jobID).Return(scheduledJobsStoreFormat, nil).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("GET", url, nil) - - response, err := suite.Client.Do(req) - assert.NoError(t, err) - - assert.Equal(t, http.StatusNotFound, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Job not found", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestRemoveScheduledJobByID() { - t := suite.T() - jobID := "some-id" - - suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(1), nil).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("DELETE", url, nil) - - response, err := suite.Client.Do(req) - - assert.NoError(t, err) - assert.Equal(t, http.StatusOK, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Successfully unscheduled Job ID: some-id", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInvalidJobID() { - t := suite.T() - jobID := "invalid-job-id" - - suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New(fmt.Sprintf("pq: invalid input syntax for type uuid: \"%s\"", jobID))).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("DELETE", url, nil) - - response, err := suite.Client.Do(req) - - assert.NoError(t, err) - assert.Equal(t, http.StatusBadRequest, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Invalid Job ID", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnInternalServerError() { - t := suite.T() - jobID := "job-id" - - suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), errors.New("some-error")).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("DELETE", url, nil) - - response, err := suite.Client.Do(req) - assert.NoError(t, err) - - assert.Equal(t, http.StatusInternalServerError, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Something went wrong", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func (suite *SchedulerTestSuite) TestRemoveScheduledJobByIDOnJobIDNotFound() { - t := suite.T() - jobID := "absent-job-id" - - suite.mockStore.On("RemoveScheduledJob", jobID).Return(int64(0), nil).Once() - - url := fmt.Sprintf("%s/jobs/schedule/%s", suite.TestServer.URL, jobID) - req, _ := http.NewRequest("DELETE", url, nil) - - response, err := suite.Client.Do(req) - assert.NoError(t, err) - - assert.Equal(t, http.StatusNotFound, response.StatusCode) - - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(response.Body) - responseBody := buf.String() - assert.Equal(t, "Job not found", responseBody) - - suite.mockStore.AssertExpectations(t) -} - -func TestScheduleTestSuite(t *testing.T) { - suite.Run(t, new(SchedulerTestSuite)) -} diff --git a/internal/app/proctord/jobs/schedule/scheduled_job.go b/internal/app/proctord/jobs/schedule/scheduled_job.go deleted file mode 100644 index bf0b6806..00000000 --- a/internal/app/proctord/jobs/schedule/scheduled_job.go +++ /dev/null @@ -1,37 +0,0 @@ -package schedule - -import ( - "proctor/internal/app/proctord/storage/postgres" - modelSchedule "proctor/internal/pkg/model/schedule" - "proctor/internal/pkg/utility" -) - -func FromStoreToHandler(scheduledJobsStoreFormat []postgres.JobsSchedule) ([]modelSchedule.ScheduledJob, error) { - var scheduledJobs []modelSchedule.ScheduledJob - for _, scheduledJobStoreFormat := range scheduledJobsStoreFormat { - scheduledJob, err := GetScheduledJob(scheduledJobStoreFormat) - if err != nil { - return nil, err - } - scheduledJobs = append(scheduledJobs, scheduledJob) - } - return scheduledJobs, nil -} - -func GetScheduledJob(scheduledJobStoreFormat postgres.JobsSchedule) (modelSchedule.ScheduledJob, error) { - args, err := utility.DeserializeMap(scheduledJobStoreFormat.Args) - if err != nil { - return modelSchedule.ScheduledJob{}, err - } - scheduledJob := modelSchedule.ScheduledJob{ - ID: scheduledJobStoreFormat.ID, - Name: scheduledJobStoreFormat.Name, - Args: args, - Tags: scheduledJobStoreFormat.Tags, - Time: scheduledJobStoreFormat.Time, - Group: scheduledJobStoreFormat.Group, - NotificationEmails: scheduledJobStoreFormat.NotificationEmails, - } - return scheduledJob, nil - -} diff --git a/internal/app/proctord/jobs/schedule/worker.go b/internal/app/proctord/jobs/schedule/worker.go deleted file mode 100644 index 597827a2..00000000 --- a/internal/app/proctord/jobs/schedule/worker.go +++ /dev/null @@ -1,132 +0,0 @@ -package schedule - -import ( - "fmt" - "github.com/getsentry/raven-go" - "os" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/logger" - "proctor/internal/app/service/infra/mail" - "strings" - "time" - - "github.com/robfig/cron" - "proctor/internal/pkg/constant" - "proctor/internal/pkg/utility" -) - -type worker struct { - store storage.Store - executioner execution.Executioner - auditor audit.Auditor - mailer mail.Mailer - inMemoryScheduledJobs map[string]*cron.Cron -} - -type Worker interface { - Run(<-chan time.Time, <-chan os.Signal) -} - -func NewWorker(store storage.Store, executioner execution.Executioner, auditor audit.Auditor, mailer mail.Mailer) Worker { - return &worker{ - store: store, - executioner: executioner, - auditor: auditor, - mailer: mailer, - inMemoryScheduledJobs: make(map[string]*cron.Cron), - } -} - -func (worker *worker) disableScheduledJobIfItExists(scheduledJobID string) { - if scheduledCronJob, ok := worker.inMemoryScheduledJobs[scheduledJobID]; ok { - scheduledCronJob.Stop() - delete(worker.inMemoryScheduledJobs, scheduledJobID) - } -} - -func (worker *worker) enableScheduledJobIfItDoesNotExist(scheduledJob postgres.JobsSchedule) { - if _, ok := worker.inMemoryScheduledJobs[scheduledJob.ID]; !ok { - jobArgs, err := utility.DeserializeMap(scheduledJob.Args) - if err != nil { - logger.Error(fmt.Sprintf("Error deserializing job args: %s ", scheduledJob.Tags), scheduledJob.Name, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - return - } - - cronJob := cron.New() - err = cronJob.AddFunc(scheduledJob.Time, func() { - jobsExecutionAuditLog := &postgres.JobsExecutionAuditLog{} - jobsExecutionAuditLog.UserEmail = constant.WorkerEmail - - jobExecutionID, err := worker.executioner.Execute(jobsExecutionAuditLog, scheduledJob.Name, jobArgs) - if err != nil { - logger.Error(fmt.Sprintf("Error submitting job: %s ", scheduledJob.Tags), scheduledJob.Name, " for execution: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - jobsExecutionAuditLog.Errors = fmt.Sprintf("Error executing job: %s", err.Error()) - jobsExecutionAuditLog.JobSubmissionStatus = constant.JobSubmissionServerError - worker.auditor.JobsExecution(jobsExecutionAuditLog) - return - } - - worker.auditor.JobsExecution(jobsExecutionAuditLog) - - jobExecutionStatus, err := worker.auditor.JobsExecutionStatus(jobExecutionID) - if err != nil { - logger.Error(fmt.Sprintf("Error fetching execution status for job: %s ", scheduledJob.Tags), jobExecutionID, ". Error: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name}) - - return - } - - recipients := strings.Split(scheduledJob.NotificationEmails, ",") - err = worker.mailer.Send(scheduledJob.Name, jobExecutionID, jobExecutionStatus, jobArgs, recipients) - - if err != nil { - logger.Error(fmt.Sprintf("Error notifying job: %s `", scheduledJob.Tags), scheduledJob.Name, "` ID: `", jobExecutionID, "` execution status: `", jobExecutionStatus, "` to users: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags, "job_name": scheduledJob.Name, "job_id": jobExecutionID, "job_execution_status": jobExecutionStatus}) - return - } - }) - - if err != nil { - logger.Error(fmt.Sprintf("Error adding cron job: %s", scheduledJob.Tags), err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": scheduledJob.Tags}) - return - } - - cronJob.Start() - worker.inMemoryScheduledJobs[scheduledJob.ID] = cronJob - } -} - -func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Signal) { - for { - select { - case <-tickerChan: - scheduledJobs, err := worker.store.GetScheduledJobs() - if err != nil { - logger.Error("Error getting scheduled jobs from store: ", err.Error()) - raven.CaptureError(err, nil) - continue - } - - for _, scheduledJob := range scheduledJobs { - if scheduledJob.Enabled { - worker.enableScheduledJobIfItDoesNotExist(scheduledJob) - } else { - worker.disableScheduledJobIfItExists(scheduledJob.ID) - } - } - case <-signalsChan: - for id := range worker.inMemoryScheduledJobs { - worker.disableScheduledJobIfItExists(id) - } - //TODO: wait for all active executions to complete - return - } - } -} diff --git a/internal/app/proctord/jobs/schedule/worker_test.go b/internal/app/proctord/jobs/schedule/worker_test.go deleted file mode 100644 index 3e5d3cf3..00000000 --- a/internal/app/proctord/jobs/schedule/worker_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package schedule - -import ( - "encoding/base64" - "encoding/json" - "os" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/mail" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" - "proctor/internal/pkg/constant" -) - -type WorkerTestSuite struct { - suite.Suite - mockStore *storage.MockStore - mockExecutioner *execution.MockExecutioner - mockAuditor *audit.MockAuditor - mockMailer *mail.MockMailer - testWorker Worker -} - -func (suite *WorkerTestSuite) SetupTest() { - suite.mockStore = &storage.MockStore{} - suite.mockExecutioner = &execution.MockExecutioner{} - suite.mockAuditor = &audit.MockAuditor{} - suite.mockMailer = &mail.MockMailer{} - - suite.testWorker = NewWorker(suite.mockStore, suite.mockExecutioner, suite.mockAuditor, suite.mockMailer) -} - -func (suite *WorkerTestSuite) TestCronEnablingForScheduledJobs() { - t := suite.T() - - jobArgs := map[string]string{"abc": "def"} - jsonEncodedJobArgs, err := json.Marshal(jobArgs) - assert.NoError(t, err) - - enabledJob := "some-job-one" - disabledJob := "some-job-two" - notificationEmails := "foo@bar.com,goo@bar.com" - scheduledJobs := []postgres.JobsSchedule{ - { - ID: "some-uuid-one", - Enabled: true, - Time: "*/1 * * * * *", - Name: enabledJob, - Args: base64.StdEncoding.EncodeToString(jsonEncodedJobArgs), - NotificationEmails: notificationEmails, - }, - { - ID: "some-uuid-two", - Enabled: false, - Time: "*/1 * * * * *", - Name: disabledJob, - Args: base64.StdEncoding.EncodeToString(jsonEncodedJobArgs), - NotificationEmails: notificationEmails, - }, - } - - tickerChan := make(chan time.Time) - signalsChan := make(chan os.Signal, 1) - scheduledJobExecutedChan := make(chan bool) - - suite.mockStore.On("GetScheduledJobs").Return(scheduledJobs, nil) - - jobExecutionID := "job-execution-id" - suite.mockExecutioner.On("Execute", mock.Anything, enabledJob, jobArgs).Return(jobExecutionID, nil) - - jobExecutionStatus := constant.JobSucceeded - suite.mockAuditor.On("JobsExecution", mock.Anything).Return() - suite.mockAuditor.On("JobsExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) - - expectedRecipients := strings.Split(notificationEmails, ",") - suite.mockMailer.On("Send", enabledJob, jobExecutionID, jobExecutionStatus, jobArgs, expectedRecipients).Return(nil).Run( - func(args mock.Arguments) { - scheduledJobExecutedChan <- true - }, - ) - - go suite.testWorker.Run(tickerChan, signalsChan) - - tickerChan <- time.Now() - - <-scheduledJobExecutedChan - signalsChan <- syscall.SIGTERM - suite.mockExecutioner.AssertExpectations(t) - suite.mockAuditor.AssertExpectations(t) - suite.mockMailer.AssertExpectations(t) - suite.mockExecutioner.AssertNotCalled(t, "Execute", disabledJob, jobArgs) -} - -func (suite *WorkerTestSuite) TestCronForDisablingEnabledScheduledJobs() { - t := suite.T() - - jobArgs := map[string]string{"abc": "def"} - jsonEncodedJobArgs, err := json.Marshal(jobArgs) - assert.NoError(t, err) - - jobName := "some-job-one" - notificationEmails := "foo@bar.com,goo@bar.com" - scheduledJobs := []postgres.JobsSchedule{ - { - ID: "some-uuid-one", - Enabled: true, - Time: "*/1 * * * * *", - Name: jobName, - Args: base64.StdEncoding.EncodeToString(jsonEncodedJobArgs), - NotificationEmails: notificationEmails, - }, - } - - disabledScheduledJobs := []postgres.JobsSchedule{ - { - ID: "some-uuid-one", - Enabled: false, - Time: "*/1 * * * * *", - Name: jobName, - Args: base64.StdEncoding.EncodeToString(jsonEncodedJobArgs), - }, - } - - tickerChan := make(chan time.Time) - signalsChan := make(chan os.Signal, 1) - toggledOffEnabledJobChan := make(chan bool) - - suite.mockStore.On("GetScheduledJobs").Return(scheduledJobs, nil).Once() - suite.mockStore.On("GetScheduledJobs").Return(disabledScheduledJobs, nil).Run( - func(args mock.Arguments) { - toggledOffEnabledJobChan <- true - }, - ) - - jobExecutionID := "job-execution-id" - suite.mockExecutioner.On("Execute", mock.Anything, jobName, jobArgs).Return(jobExecutionID, nil) - - suite.mockAuditor.On("JobsExecution", mock.Anything).Return() - jobExecutionStatus := constant.JobSucceeded - suite.mockAuditor.On("JobsExecutionStatus", jobExecutionID).Return(jobExecutionStatus, nil) - - expectedRecipients := strings.Split(notificationEmails, ",") - suite.mockMailer.On("Send", jobName, jobExecutionID, jobExecutionStatus, jobArgs, expectedRecipients).Return(nil).Run( - func(args mock.Arguments) { - toggledOffEnabledJobChan <- true - }, - ) - - go suite.testWorker.Run(tickerChan, signalsChan) - - tickerChan <- time.Now() - - <-toggledOffEnabledJobChan - suite.mockExecutioner.AssertExpectations(t) - suite.mockAuditor.AssertExpectations(t) - suite.mockMailer.AssertExpectations(t) - - //Wait for 2 seconds to ensure disabled job isn't executed again - time.Sleep(2 * time.Second) - signalsChan <- syscall.SIGTERM -} - -func TestWorkerTestSuite(t *testing.T) { - suite.Run(t, new(WorkerTestSuite)) -} diff --git a/internal/app/proctord/scheduler/scheduler.go b/internal/app/proctord/scheduler/scheduler.go deleted file mode 100644 index 7148a940..00000000 --- a/internal/app/proctord/scheduler/scheduler.go +++ /dev/null @@ -1,51 +0,0 @@ -package scheduler - -import ( - "fmt" - "os" - "proctor/internal/app/proctord/audit" - "proctor/internal/app/proctord/jobs/execution" - "proctor/internal/app/proctord/jobs/schedule" - "proctor/internal/app/proctord/storage" - "proctor/internal/app/service/infra/config" - "proctor/internal/app/service/infra/db/postgresql" - "proctor/internal/app/service/infra/db/redis" - "proctor/internal/app/service/infra/kubernetes" - "proctor/internal/app/service/infra/kubernetes/http" - "proctor/internal/app/service/infra/mail" - metadataRepository "proctor/internal/app/service/metadata/repository" - secretRepository "proctor/internal/app/service/secret/repository" - "time" -) - -func Start() error { - fmt.Println("started scheduler") - - postgresClient := postgresql.NewClient() - redisClient := redis.NewClient() - - store := storage.New(postgresClient) - metadataStore := metadataRepository.NewMetadataRepository(redisClient) - secretsStore := secretRepository.NewSecretRepository(redisClient) - - httpClient, err := http.NewClient() - if err != nil { - return err - } - kubeClient := kubernetes.NewKubernetesClient(httpClient) - - jobExecutioner := execution.NewExecutioner(kubeClient, metadataStore, secretsStore) - - auditor := audit.New(store, kubeClient) - - mailer := mail.New(config.MailServerHost(), config.MailServerPort()) - - worker := schedule.NewWorker(store, jobExecutioner, auditor, mailer) - - ticker := time.NewTicker(time.Duration(config.ScheduledJobsFetchIntervalInMins()) * time.Minute) - signalsChan := make(chan os.Signal, 1) - worker.Run(ticker.C, signalsChan) - - _ = postgresClient.Close() - return nil -} diff --git a/internal/app/proctord/storage/postgres/schema.go b/internal/app/proctord/storage/postgres/schema.go deleted file mode 100644 index 30ef0854..00000000 --- a/internal/app/proctord/storage/postgres/schema.go +++ /dev/null @@ -1,50 +0,0 @@ -package postgres - -import ( - "database/sql" - "encoding/base64" - "encoding/json" - "proctor/internal/app/service/infra/logger" - "time" -) - -type JobsExecutionAuditLog struct { - JobName string `db:"job_name"` - UserEmail string `db:"user_email"` - ImageName string `db:"image_name"` - ExecutionID sql.NullString `db:"job_name_submitted_for_execution"` - JobArgs string `db:"job_args"` - JobSubmissionStatus string `db:"job_submission_status"` - Errors string `db:"errors"` - JobExecutionStatus string `db:"job_execution_status"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` -} - -func (j *JobsExecutionAuditLog) AddJobArgs(jobArgs map[string]string) { - jsonEncodedArgs, err := json.Marshal(jobArgs) - if err != nil { - logger.Error("Error marshaling job args: ", err.Error()) - return - } - - j.JobArgs = base64.StdEncoding.EncodeToString(jsonEncodedArgs) -} - -func (j *JobsExecutionAuditLog) AddExecutionID(jobExecutionID string) { - j.ExecutionID = StringToSQLString(jobExecutionID) -} - -type JobsSchedule struct { - ID string `db:"id"` - Name string `db:"name"` - Args string `db:"args"` - Tags string `db:"tags"` - Time string `db:"time"` - NotificationEmails string `db:"notification_emails"` - UserEmail string `db:"user_email"` - Group string `db:"group_name"` - Enabled bool `db:"enabled"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` -} diff --git a/internal/app/proctord/storage/postgres/utils.go b/internal/app/proctord/storage/postgres/utils.go deleted file mode 100644 index ced778e4..00000000 --- a/internal/app/proctord/storage/postgres/utils.go +++ /dev/null @@ -1,13 +0,0 @@ -package postgres - -import "database/sql" - -func StringToSQLString(str string) sql.NullString { - if len(str) == 0 { - return sql.NullString{} - } - return sql.NullString{ - String: str, - Valid: true, - } -} diff --git a/internal/app/proctord/storage/postgres/utils_test.go b/internal/app/proctord/storage/postgres/utils_test.go deleted file mode 100644 index d6029b18..00000000 --- a/internal/app/proctord/storage/postgres/utils_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package postgres - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestStringToSQLStringForEmptyString(t *testing.T) { - sqlString := StringToSQLString("") - assert.Equal(t, false, sqlString.Valid) -} - -func TestStringToSQLStringForNonEmptyString(t *testing.T) { - sqlString := StringToSQLString("any") - - assert.Equal(t, true, sqlString.Valid) - assert.Equal(t, "any", sqlString.String) -} diff --git a/internal/app/proctord/storage/store.go b/internal/app/proctord/storage/store.go deleted file mode 100644 index 6043e61e..00000000 --- a/internal/app/proctord/storage/store.go +++ /dev/null @@ -1,113 +0,0 @@ -package storage - -import ( - "encoding/base64" - "encoding/json" - uuid "github.com/satori/go.uuid" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/db/postgresql" - "time" -) - -type Store interface { - AuditJobsExecution(*postgres.JobsExecutionAuditLog) error - UpdateJobsExecutionAuditLog(string, string) error - GetJobExecutionStatus(string) (string, error) - InsertScheduledJob(string, string, string, string, string, string, map[string]string) (string, error) - GetScheduledJobs() ([]postgres.JobsSchedule, error) - GetEnabledScheduledJobs() ([]postgres.JobsSchedule, error) - GetScheduledJob(string) ([]postgres.JobsSchedule, error) - RemoveScheduledJob(string) (int64, error) -} - -type store struct { - postgresClient postgresql.Client -} - -func New(postgresClient postgresql.Client) Store { - return &store{ - postgresClient: postgresClient, - } -} - -func (store *store) AuditJobsExecution(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog) error { - _, err := store.postgresClient.NamedExec("INSERT INTO jobs_execution_audit_log (job_name, user_email, image_name, job_name_submitted_for_execution, job_args, job_submission_status,"+ - " job_execution_status) VALUES (:job_name, :user_email, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", - &jobsExecutionAuditLog) - return err -} - -func (store *store) UpdateJobsExecutionAuditLog(jobExecutionID, jobExecutionStatus string) error { - jobsExecutionAuditLog := postgres.JobsExecutionAuditLog{ - JobExecutionStatus: jobExecutionStatus, - ExecutionID: postgres.StringToSQLString(jobExecutionID), - UpdatedAt: time.Now(), - } - - _, err := store.postgresClient.NamedExec("UPDATE jobs_execution_audit_log SET job_execution_status = :job_execution_status, updated_at = :updated_at where job_name_submitted_for_execution = "+ - ":job_name_submitted_for_execution", &jobsExecutionAuditLog) - return err -} - -func (store *store) GetJobExecutionStatus(JobNameSubmittedForExecution string) (string, error) { - jobsExecutionAuditLogResult := []postgres.JobsExecutionAuditLog{} - err := store.postgresClient.Select(&jobsExecutionAuditLogResult, "SELECT job_execution_status from jobs_execution_audit_log where job_name_submitted_for_execution = $1", JobNameSubmittedForExecution) - if err != nil { - return "", err - } - - if len(jobsExecutionAuditLogResult) == 0 { - return "", nil - } - - return jobsExecutionAuditLogResult[0].JobExecutionStatus, nil -} - -func (store *store) InsertScheduledJob(name, tags, time, notificationEmails, userEmail, groupName string, args map[string]string) (string, error) { - jsonEncodedArgs, err := json.Marshal(args) - if err != nil { - return "", err - } - - jobsSchedule := postgres.JobsSchedule{ - ID: uuid.NewV4().String(), - Name: name, - Args: base64.StdEncoding.EncodeToString(jsonEncodedArgs), - Tags: tags, - Time: time, - NotificationEmails: notificationEmails, - UserEmail: userEmail, - Group: groupName, - Enabled: true, - } - _, err = store.postgresClient.NamedExec("INSERT INTO jobs_schedule (id, name, tags, time, notification_emails, user_email, group_name, args, enabled) "+ - "VALUES (:id, :name, :tags, :time, :notification_emails, :user_email, :group_name, :args, :enabled)", &jobsSchedule) - return jobsSchedule.ID, err -} - -func (store *store) GetScheduledJobs() ([]postgres.JobsSchedule, error) { - scheduledJobs := []postgres.JobsSchedule{} - err := store.postgresClient.Select(&scheduledJobs, "SELECT id, name, args, time, notification_emails, group_name, enabled from jobs_schedule") - return scheduledJobs, err -} - -func (store *store) GetEnabledScheduledJobs() ([]postgres.JobsSchedule, error) { - scheduledJobs := []postgres.JobsSchedule{} - err := store.postgresClient.Select(&scheduledJobs, "SELECT id, name, args, time, tags, notification_emails,group_name from jobs_schedule where enabled = 't'") - return scheduledJobs, err -} - -func (store *store) GetScheduledJob(jobID string) ([]postgres.JobsSchedule, error) { - scheduledJob := []postgres.JobsSchedule{} - err := store.postgresClient.Select(&scheduledJob, "SELECT id, name, args, time, tags, notification_emails,group_name from jobs_schedule where id = $1 and enabled = 't'", jobID) - return scheduledJob, err -} - -func (store *store) RemoveScheduledJob(jobID string) (int64, error) { - job := postgres.JobsSchedule{ - ID: jobID, - UpdatedAt: time.Now(), - } - rowsAffected, err := store.postgresClient.NamedExec("UPDATE jobs_schedule set enabled = 'f', updated_at = :updated_at where id = :id and enabled = 't'", &job) - return rowsAffected, err -} diff --git a/internal/app/proctord/storage/store_mock.go b/internal/app/proctord/storage/store_mock.go deleted file mode 100644 index eea9b7ca..00000000 --- a/internal/app/proctord/storage/store_mock.go +++ /dev/null @@ -1,50 +0,0 @@ -package storage - -import ( - "github.com/stretchr/testify/mock" - "proctor/internal/app/proctord/storage/postgres" -) - -type MockStore struct { - mock.Mock -} - -func (m *MockStore) AuditJobsExecution(jobsExecutionAuditLog *postgres.JobsExecutionAuditLog) error { - args := m.Called(jobsExecutionAuditLog) - return args.Error(0) -} - -func (m *MockStore) UpdateJobsExecutionAuditLog(JobNameSubmittedForExecution, status string) error { - args := m.Called(JobNameSubmittedForExecution, status) - return args.Error(0) -} - -func (m *MockStore) GetJobExecutionStatus(jobName string) (string, error) { - args := m.Called(jobName) - return args.String(0), args.Error(1) -} - -func (m *MockStore) InsertScheduledJob(jobName, tags, time, notificationEmails, userEmail, groupName string, jobArgs map[string]string) (string, error) { - args := m.Called(jobName, tags, time, notificationEmails, userEmail, groupName, jobArgs) - return args.String(0), args.Error(1) -} - -func (m *MockStore) GetScheduledJobs() ([]postgres.JobsSchedule, error) { - args := m.Called() - return args.Get(0).([]postgres.JobsSchedule), args.Error(1) -} - -func (m *MockStore) GetEnabledScheduledJobs() ([]postgres.JobsSchedule, error) { - args := m.Called() - return args.Get(0).([]postgres.JobsSchedule), args.Error(1) -} - -func (m *MockStore) GetScheduledJob(jobID string) ([]postgres.JobsSchedule, error) { - args := m.Called(jobID) - return args.Get(0).([]postgres.JobsSchedule), args.Error(1) -} - -func (m *MockStore) RemoveScheduledJob(jobID string) (int64, error) { - args := m.Called(jobID) - return args.Get(0).(int64), args.Error(1) -} diff --git a/internal/app/proctord/storage/store_test.go b/internal/app/proctord/storage/store_test.go deleted file mode 100644 index d8af6254..00000000 --- a/internal/app/proctord/storage/store_test.go +++ /dev/null @@ -1,255 +0,0 @@ -package storage - -import ( - "bytes" - "encoding/gob" - "errors" - uuid "github.com/satori/go.uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "proctor/internal/app/proctord/storage/postgres" - "proctor/internal/app/service/infra/db/postgresql" - "proctor/internal/pkg/constant" - "testing" -) - -func TestJobsExecutionAuditLog(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - - jobExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: "sample-job", - ImageName: "any-image", - UserEmail: "mrproctor@example.com", - } - - var encodedJobArgs bytes.Buffer - enc := gob.NewEncoder(&encodedJobArgs) - err := enc.Encode(map[string]string{}) - assert.NoError(t, err) - - mockPostgresClient.On("NamedExec", - "INSERT INTO jobs_execution_audit_log (job_name, user_email, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :user_email, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", mock.Anything).Run(func(args mock.Arguments) { - }).Return(int64(1), nil).Once() - - err = testStore.AuditJobsExecution(jobExecutionAuditLog) - - assert.NoError(t, err) - mockPostgresClient.AssertExpectations(t) -} - -func TestJobsExecutionAuditLogPostgresClientFailure(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - - jobExecutionAuditLog := &postgres.JobsExecutionAuditLog{ - JobName: "sample-job", - } - - var encodedJobArgs bytes.Buffer - enc := gob.NewEncoder(&encodedJobArgs) - err := enc.Encode(map[string]string{}) - assert.NoError(t, err) - - mockPostgresClient.On("NamedExec", - "INSERT INTO jobs_execution_audit_log (job_name, user_email, image_name, job_name_submitted_for_execution, job_args, job_submission_status, job_execution_status) VALUES (:job_name, :user_email, :image_name, :job_name_submitted_for_execution, :job_args, :job_submission_status, :job_execution_status)", - mock.Anything). - Return(int64(0), errors.New("error")). - Once() - - err = testStore.AuditJobsExecution(jobExecutionAuditLog) - - assert.Error(t, err) - mockPostgresClient.AssertExpectations(t) -} - -func TestUpdateJobsExecutionAuditLog(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - - executionID := "any-submission" - jobExecutionStatus := "updated-status" - - mockPostgresClient.On("NamedExec", - "UPDATE jobs_execution_audit_log SET job_execution_status = :job_execution_status, updated_at = :updated_at where job_name_submitted_for_execution = :job_name_submitted_for_execution", - mock.Anything). - Run(func(args mock.Arguments) { - data := args.Get(1).(*postgres.JobsExecutionAuditLog) - - assert.Equal(t, postgres.StringToSQLString(executionID), data.ExecutionID) - assert.Equal(t, jobExecutionStatus, data.JobExecutionStatus) - }). - Return(int64(1), nil). - Once() - - err := testStore.UpdateJobsExecutionAuditLog(executionID, jobExecutionStatus) - - assert.NoError(t, err) - mockPostgresClient.AssertExpectations(t) -} - -func TestGetJobsStatusWhenJobIsPresent(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - jobName := "any-job" - - dest := []postgres.JobsExecutionAuditLog{} - - mockPostgresClient.On("Select", - &dest, - "SELECT job_execution_status from jobs_execution_audit_log where job_name_submitted_for_execution = $1", - jobName). - Return(nil). - Run(func(args mock.Arguments) { - jobsExecutionAuditLogResult := args.Get(0).(*[]postgres.JobsExecutionAuditLog) - *jobsExecutionAuditLogResult = append(*jobsExecutionAuditLogResult, postgres.JobsExecutionAuditLog{ - JobExecutionStatus: constant.JobSucceeded, - }) - }). - Once() - - status, err := testStore.GetJobExecutionStatus(jobName) - assert.NoError(t, err) - - assert.Equal(t, constant.JobSucceeded, status) - - mockPostgresClient.AssertExpectations(t) -} - -func TestGetJobsStatusWhenJobIsNotPresent(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - jobName := "any-job" - - dest := []postgres.JobsExecutionAuditLog{} - - mockPostgresClient.On("Select", - &dest, - "SELECT job_execution_status from jobs_execution_audit_log where job_name_submitted_for_execution = $1", - jobName). - Return(nil). - Once() - - status, err := testStore.GetJobExecutionStatus(jobName) - assert.NoError(t, err) - - assert.Equal(t, "", status) - - mockPostgresClient.AssertExpectations(t) -} - -func TestGetJobsStatusWhenError(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - jobName := "any-job" - - dest := []postgres.JobsExecutionAuditLog{} - - mockPostgresClient.On("Select", - &dest, - "SELECT job_execution_status from jobs_execution_audit_log where job_name_submitted_for_execution = $1", - jobName). - Return(errors.New("error")). - Once() - - _, err := testStore.GetJobExecutionStatus(jobName) - assert.Error(t, err, "error") -} - -func TestJobsScheduleInsertionSuccessfull(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - scheduledJobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) - assert.NoError(t, err) - _, err = uuid.FromString(scheduledJobID) - assert.NoError(t, err) - - _, err = postgresClient.GetDB().Exec("truncate table jobs_schedule;") - assert.NoError(t, err) -} - -func TestJobsScheduleInsertionFailed(t *testing.T) { - mockPostgresClient := &postgresql.ClientMock{} - testStore := New(mockPostgresClient) - - jobName := "job-name" - tag := "tag-one1" - time := "* * 3 * *" - notificationEmail := "foo@bar.com" - userEmail := "ms@proctor.com" - groupName := "group1" - - mockPostgresClient.On("NamedExec", - "INSERT INTO jobs_schedule (id, name, tags, time, notification_emails, user_email, group_name, args, enabled) "+ - "VALUES (:id, :name, :tags, :time, :notification_emails, :user_email, :group_name, :args, :enabled)", - mock.AnythingOfType("*postgres.JobsSchedule")).Run(func(args mock.Arguments) { - }).Return(int64(0), errors.New("any-error")). - Once() - - _, err := testStore.InsertScheduledJob(jobName, tag, time, notificationEmail, userEmail, groupName, map[string]string{}) - - assert.Error(t, err) - - mockPostgresClient.AssertExpectations(t) -} - -func TestGetScheduledJobByID(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) - assert.NoError(t, err) - - resultJob, err := testStore.GetScheduledJob(jobID) - assert.NoError(t, err) - assert.Equal(t, "job-name", resultJob[0].Name) - assert.Equal(t, "tag-one", resultJob[0].Tags) - assert.Equal(t, "* * 3 * *", resultJob[0].Time) - - _, err = postgresClient.GetDB().Exec("truncate table jobs_schedule;") - assert.NoError(t, err) -} - -func TestGetScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - resultJob, err := testStore.GetScheduledJob("86A7963B-3621-492D-8D6C-33076242256B") - assert.NoError(t, err) - assert.Equal(t, []postgres.JobsSchedule{}, resultJob) -} - -func TestRemoveScheduledJobByID(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - jobID, err := testStore.InsertScheduledJob("job-name", "tag-one", "* * 3 * *", "foo@bar.com", "ms@proctor.com", "group1", map[string]string{}) - assert.NoError(t, err) - - removedJobsCount, err := testStore.RemoveScheduledJob(jobID) - assert.NoError(t, err) - assert.Equal(t, int64(1), removedJobsCount) - - _, err = postgresClient.GetDB().Exec("truncate table jobs_schedule;") - assert.NoError(t, err) -} - -func TestRemoveScheduledJobByIDReturnErrorIfIDnotFound(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - removedJobsCount, err := testStore.RemoveScheduledJob("86A7963B-3621-492D-8D6C-33076242256B") - assert.NoError(t, err) - assert.Equal(t, int64(0), removedJobsCount) -} - -func TestRemoveScheduledJobByIDReturnErrorIfIDIsInvalid(t *testing.T) { - postgresClient := postgresql.NewClient() - testStore := New(postgresClient) - - removedJobsCount, err := testStore.RemoveScheduledJob("86A7963B") - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid input syntax") - assert.Equal(t, int64(0), removedJobsCount) -} From 81b740e2bda382af5d74d5a0e9ed01ceaa6dff03 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 11:48:52 +0700 Subject: [PATCH 068/268] [bimo.horizon|mazbergaz] Move worker email to worker --- internal/app/service/worker/worker.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/app/service/worker/worker.go b/internal/app/service/worker/worker.go index a8edb42d..dc8187b0 100644 --- a/internal/app/service/worker/worker.go +++ b/internal/app/service/worker/worker.go @@ -21,9 +21,10 @@ import ( scheduleModel "proctor/internal/app/service/schedule/model" scheduleRepository "proctor/internal/app/service/schedule/repository" secretRepository "proctor/internal/app/service/secret/repository" - "proctor/internal/pkg/constant" ) +const WorkerEmail = "worker@proctor" + type worker struct { executionService executionService.ExecutionService executionContextRepository executionContextRepository.ExecutionContextRepository @@ -57,7 +58,7 @@ func (worker *worker) enableScheduleIfItDoesNotExist(schedule scheduleModel.Sche if _, ok := worker.inMemorySchedules[schedule.ID]; !ok { cronJob := cron.New() err := cronJob.AddFunc(schedule.Cron, func() { - executionContext, _, err := worker.executionService.Execute(schedule.JobName, constant.WorkerEmail, schedule.Args) + executionContext, _, err := worker.executionService.Execute(schedule.JobName, WorkerEmail, schedule.Args) if err != nil { logger.Error(fmt.Sprintf("Error submitting job: %s ", schedule.Tags), schedule.JobName, " for execution: ", err.Error()) raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) From dcbb2ea5d1381fa23871d892a2e03a8afaf53ed0 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 25 Jul 2019 12:54:18 +0700 Subject: [PATCH 069/268] [bimo.horizon|mazbergaz] Change Http instance to HTTP to follow proper naming convention --- internal/app/cli/daemon/client.go | 4 +- .../app/service/execution/handler/http.go | 18 ++++---- .../service/execution/handler/http_test.go | 44 +++++++++---------- .../kubernetes/client_integration_test.go | 7 ++- .../service/infra/kubernetes/client_test.go | 6 +-- internal/app/service/metadata/handler/http.go | 12 ++--- .../app/service/metadata/handler/http_test.go | 14 +++--- internal/app/service/schedule/handler/http.go | 16 +++---- .../app/service/schedule/handler/http_test.go | 44 +++++++++---------- internal/app/service/secret/handler/http.go | 4 +- .../app/service/secret/handler/http_test.go | 4 +- .../validate_client_version_test.go | 6 +-- internal/app/service/server/router.go | 18 ++++---- 13 files changed, 98 insertions(+), 99 deletions(-) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 77cabacb..0246d1c4 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -403,7 +403,7 @@ func buildHTTPError(c *client, resp *http.Response) error { } if resp.StatusCode == http.StatusBadRequest { - return getHttpResponseError(resp.Body) + return getHTTPResponseError(resp.Body) } if resp.StatusCode == http.StatusNoContent { @@ -421,7 +421,7 @@ func buildHTTPError(c *client, resp *http.Response) error { return fmt.Errorf("%s\nStatus Code: %d, %s", constant.GenericResponseErrorHeader, resp.StatusCode, http.StatusText(resp.StatusCode)) } -func getHttpResponseError(response ioReader.ReadCloser) error { +func getHTTPResponseError(response ioReader.ReadCloser) error { body, _ := ioutil.ReadAll(response) bodyString := string(body) return fmt.Errorf(bodyString) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 3864db5a..64e48f0d 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -21,13 +21,13 @@ import ( "github.com/gorilla/websocket" ) -type ExecutionHttpHandler interface { +type ExecutionHTTPHandler interface { Post() http.HandlerFunc GetStatus() http.HandlerFunc GetLogs() http.HandlerFunc } -type executionHttpHandler struct { +type executionHTTPHandler struct { service service.ExecutionService repository repository.ExecutionContextRepository } @@ -37,23 +37,23 @@ var upgrader = websocket.Upgrader{ WriteBufferSize: config.LogsStreamWriteBufferSize(), } -func NewExecutionHttpHandler( +func NewExecutionHTTPHandler( executionService service.ExecutionService, repository repository.ExecutionContextRepository, -) ExecutionHttpHandler { - return &executionHttpHandler{ +) ExecutionHTTPHandler { + return &executionHTTPHandler{ service: executionService, repository: repository, } } -func (httpHandler *executionHttpHandler) closeWebSocket(message string, conn *websocket.Conn) { +func (httpHandler *executionHTTPHandler) closeWebSocket(message string, conn *websocket.Conn) { err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, message)) logger.LogErrors(err, "close WebSocket") return } -func (httpHandler *executionHttpHandler) GetLogs() http.HandlerFunc { +func (httpHandler *executionHTTPHandler) GetLogs() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { conn, err := upgrader.Upgrade(response, request, nil) logger.LogErrors(err, "upgrade http server connection to WebSocket") @@ -115,7 +115,7 @@ func (httpHandler *executionHttpHandler) GetLogs() http.HandlerFunc { } } -func (httpHandler *executionHttpHandler) GetStatus() http.HandlerFunc { +func (httpHandler *executionHTTPHandler) GetStatus() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { contextId := mux.Vars(request)["contextId"] executionContextId, err := strconv.ParseUint(contextId, 10, 64) @@ -153,7 +153,7 @@ func (httpHandler *executionHttpHandler) GetStatus() http.HandlerFunc { } } -func (httpHandler *executionHttpHandler) Post() http.HandlerFunc { +func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { userEmail := request.Header.Get(parameter.UserEmailHeader) diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index c99e5d03..023ed06e 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -28,19 +28,19 @@ import ( "proctor/internal/pkg/constant" ) -type ExecutionHttpHandlerTestSuite struct { +type ExecutionHTTPHandlerTestSuite struct { suite.Suite mockExecutionerService *service.MockExecutionService mockExecutionerContextRepository *repository.MockExecutionContextRepository mockKubernetesClient kubernetes.MockKubernetesClient - testExecutionHttpHandler ExecutionHttpHandler + testExecutionHTTPHandler ExecutionHTTPHandler } -func (suite *ExecutionHttpHandlerTestSuite) SetupTest() { +func (suite *ExecutionHTTPHandlerTestSuite) SetupTest() { suite.mockExecutionerService = &service.MockExecutionService{} suite.mockExecutionerContextRepository = &repository.MockExecutionContextRepository{} suite.mockKubernetesClient = kubernetes.MockKubernetesClient{} - suite.testExecutionHttpHandler = NewExecutionHttpHandler(suite.mockExecutionerService, suite.mockExecutionerContextRepository) + suite.testExecutionHTTPHandler = NewExecutionHTTPHandler(suite.mockExecutionerService, suite.mockExecutionerContextRepository) } type logsHandlerServer struct { @@ -57,9 +57,9 @@ const ( logsHandlerRequestURI = "/execution/logs" ) -func (suite *ExecutionHttpHandlerTestSuite) newServer() *logsHandlerServer { +func (suite *ExecutionHTTPHandlerTestSuite) newServer() *logsHandlerServer { var s logsHandlerServer - s.Server = httptest.NewServer(suite.testExecutionHttpHandler.GetLogs()) + s.Server = httptest.NewServer(suite.testExecutionHTTPHandler.GetLogs()) s.Server.URL += logsHandlerRequestURI s.URL = makeWsProto(s.Server.URL) return &s @@ -69,7 +69,7 @@ func makeWsProto(s string) string { return "ws" + strings.TrimPrefix(s, "http") } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenFinishedHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenFinishedHTTPHandler() { t := suite.T() s := suite.newServer() @@ -104,7 +104,7 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhe assert.Equal(t, "test", string(firstMessage)) } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenReadyHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhenReadyHTTPHandler() { t := suite.T() s := suite.newServer() @@ -152,7 +152,7 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhe assert.Equal(t, "test3", string(thirdMessage)) } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetStatusHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetStatusHTTPHandler() { t := suite.T() executionContextId := uint64(1) @@ -188,13 +188,13 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionGetStatusH suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) - suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) + suite.testExecutionHTTPHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionGetStatusHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestMalformedRequestforJobExecutionGetStatusHTTPHandler() { t := suite.T() executionContextId := uint64(1) @@ -203,13 +203,13 @@ func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestforJobExecutionG req = mux.SetURLVars(req, map[string]string{"contextId": "notfound"}) responseRecorder := httptest.NewRecorder() - suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) + suite.testExecutionHTTPHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) assert.Equal(t, string(handlerStatus.PathParameterError), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionGetStatusHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestNotFoundJobExecutionGetStatusHTTPHandler() { t := suite.T() executionContextId := uint64(1) @@ -236,13 +236,13 @@ func (suite *ExecutionHttpHandlerTestSuite) TestNotFoundJobExecutionGetStatusHtt suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, notFoundErr).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) - suite.testExecutionHttpHandler.GetStatus()(responseRecorder, req) + suite.testExecutionHTTPHandler.GetStatus()(responseRecorder, req) assert.Equal(t, http.StatusNotFound, responseRecorder.Code) assert.Equal(t, string(handlerStatus.ExecutionContextNotFound), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionPostHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHandler() { t := suite.T() userEmail := "mrproctor@example.com" @@ -278,25 +278,25 @@ func (suite *ExecutionHttpHandlerTestSuite) TestSuccessfulJobExecutionPostHttpHa suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", nil).Once() defer suite.mockExecutionerService.AssertExpectations(t) - suite.testExecutionHttpHandler.Post()(responseRecorder, req) + suite.testExecutionHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusCreated, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestMalformedRequestJobExecutionPostHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestMalformedRequestJobExecutionPostHTTPHandler() { t := suite.T() req := httptest.NewRequest("POST", "/execute", bytes.NewReader([]byte("test"))) responseRecorder := httptest.NewRecorder() - suite.testExecutionHttpHandler.Post()(responseRecorder, req) + suite.testExecutionHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusBadRequest, responseRecorder.Code) assert.Equal(t, string(handlerStatus.MalformedRequest), responseRecorder.Body.String()) } -func (suite *ExecutionHttpHandlerTestSuite) TestGenericErrorJobExecutionPostHttpHandler() { +func (suite *ExecutionHTTPHandlerTestSuite) TestGenericErrorJobExecutionPostHTTPHandler() { t := suite.T() userEmail := "mrproctor@example.com" @@ -322,12 +322,12 @@ func (suite *ExecutionHttpHandlerTestSuite) TestGenericErrorJobExecutionPostHttp suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", genericError).Once() defer suite.mockExecutionerService.AssertExpectations(t) - suite.testExecutionHttpHandler.Post()(responseRecorder, req) + suite.testExecutionHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) assert.Equal(t, fmt.Sprintf("%s , Errors Detail %s", handlerStatus.JobExecutionError, genericError), responseRecorder.Body.String()) } -func TestExecutionHttpHandlerTestSuite(t *testing.T) { - suite.Run(t, new(ExecutionHttpHandlerTestSuite)) +func TestExecutionHTTPHandlerTestSuite(t *testing.T) { + suite.Run(t, new(ExecutionHTTPHandlerTestSuite)) } diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 114f80b5..205c5bb8 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -8,7 +8,7 @@ import ( "k8s.io/client-go/kubernetes" "os" "proctor/internal/app/service/infra/config" - kubeHttpClient "proctor/internal/app/service/infra/kubernetes/http" + kubeHTTPClient "proctor/internal/app/service/infra/kubernetes/http" "proctor/internal/pkg/constant" "testing" ) @@ -21,9 +21,9 @@ type IntegrationTestSuite struct { func (suite *IntegrationTestSuite) SetupTest() { t := suite.T() - kubeHttpClient, err := kubeHttpClient.NewClient() + kubeHTTPClient, err := kubeHTTPClient.NewClient() assert.NoError(t, err) - suite.testClient = NewKubernetesClient(kubeHttpClient) + suite.testClient = NewKubernetesClient(kubeHTTPClient) suite.clientSet, err = NewClientSet() assert.NoError(t, err) } @@ -86,7 +86,6 @@ func (suite *IntegrationTestSuite) TestJobExecutionStatus() { assert.Equal(t, status, constant.JobSucceeded) } - func TestIntegrationTestSuite(t *testing.T) { value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") if available == true && value == "true" { diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index cf6dfa29..5e47bec7 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -27,7 +27,7 @@ type ClientTestSuite struct { jobName string podName string fakeClientSetStreaming *fakeclientset.Clientset - fakeHttpClient *http.Client + fakeHTTPClient *http.Client testClientStreaming KubernetesClient } @@ -57,10 +57,10 @@ func (suite *ClientTestSuite) SetupTest() { }, }) - suite.fakeHttpClient = &http.Client{} + suite.fakeHTTPClient = &http.Client{} suite.testClientStreaming = &kubernetesClient{ clientSet: suite.fakeClientSetStreaming, - httpClient: suite.fakeHttpClient, + httpClient: suite.fakeHTTPClient, } } diff --git a/internal/app/service/metadata/handler/http.go b/internal/app/service/metadata/handler/http.go index cd8245c2..17521f43 100644 --- a/internal/app/service/metadata/handler/http.go +++ b/internal/app/service/metadata/handler/http.go @@ -10,22 +10,22 @@ import ( modelMetadata "proctor/internal/pkg/model/metadata" ) -type metadataHttpHandler struct { +type metadataHTTPHandler struct { repository repository.MetadataRepository } -type MetadataHttpHandler interface { +type MetadataHTTPHandler interface { Post() http.HandlerFunc GetAll() http.HandlerFunc } -func NewMetadataHttpHandler(repository repository.MetadataRepository) MetadataHttpHandler { - return &metadataHttpHandler{ +func NewMetadataHTTPHandler(repository repository.MetadataRepository) MetadataHTTPHandler { + return &metadataHTTPHandler{ repository: repository, } } -func (handler *metadataHttpHandler) Post() http.HandlerFunc { +func (handler *metadataHTTPHandler) Post() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { var metadata []modelMetadata.Metadata err := json.NewDecoder(request.Body).Decode(&metadata) @@ -54,7 +54,7 @@ func (handler *metadataHttpHandler) Post() http.HandlerFunc { } } -func (handler *metadataHttpHandler) GetAll() http.HandlerFunc { +func (handler *metadataHTTPHandler) GetAll() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { metadataSlice, err := handler.repository.GetAll() diff --git a/internal/app/service/metadata/handler/http_test.go b/internal/app/service/metadata/handler/http_test.go index 0baa6b5b..2987f86c 100644 --- a/internal/app/service/metadata/handler/http_test.go +++ b/internal/app/service/metadata/handler/http_test.go @@ -22,14 +22,14 @@ import ( type MetadataHandlerTestSuite struct { suite.Suite mockRepository *metadataRepository.MockMetadataRepository - metadataHttpHandler MetadataHttpHandler + metadataHTTPHandler MetadataHTTPHandler serverError string } func (s *MetadataHandlerTestSuite) SetupTest() { s.mockRepository = &metadataRepository.MockMetadataRepository{} - s.metadataHttpHandler = NewMetadataHttpHandler(s.mockRepository) + s.metadataHTTPHandler = NewMetadataHTTPHandler(s.mockRepository) s.serverError = "Something went wrong" } @@ -73,7 +73,7 @@ func (s *MetadataHandlerTestSuite) TestSuccessfulMetadataSubmission() { s.mockRepository.On("Save", metadata).Return(nil).Once() - s.metadataHttpHandler.Post()(responseRecorder, req) + s.metadataHTTPHandler.Post()(responseRecorder, req) s.mockRepository.AssertExpectations(t) @@ -87,7 +87,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionMalformedRequest() { req := httptest.NewRequest("PUT", "/metadata", bytes.NewReader([]byte(jobMetadataSubmissionRequest))) responseRecorder := httptest.NewRecorder() - s.metadataHttpHandler.Post()(responseRecorder, req) + s.metadataHTTPHandler.Post()(responseRecorder, req) s.mockRepository.AssertNotCalled(t, "Save", mock.Anything) @@ -109,7 +109,7 @@ func (s *MetadataHandlerTestSuite) TestJobMetadataSubmissionForStoreFailure() { s.mockRepository.On("Save", metadata).Return(errors.New("error")).Once() - s.metadataHttpHandler.Post()(responseRecorder, req) + s.metadataHTTPHandler.Post()(responseRecorder, req) s.mockRepository.AssertExpectations(t) @@ -126,7 +126,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { jobsMetadata := []modelMetadata.Metadata{} s.mockRepository.On("GetAll").Return(jobsMetadata, nil).Once() - s.metadataHttpHandler.GetAll()(responseRecorder, req) + s.metadataHTTPHandler.GetAll()(responseRecorder, req) s.mockRepository.AssertExpectations(t) @@ -146,7 +146,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { jobsMetadata := []modelMetadata.Metadata{} s.mockRepository.On("GetAll").Return(jobsMetadata, errors.New("error")).Once() - s.metadataHttpHandler.GetAll()(responseRecorder, req) + s.metadataHTTPHandler.GetAll()(responseRecorder, req) s.mockRepository.AssertExpectations(t) diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 7a699ecd..311007ba 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -19,26 +19,26 @@ import ( "proctor/internal/pkg/status" ) -type scheduleHttpHandler struct { +type scheduleHTTPHandler struct { repository scheduleRepository.ScheduleRepository metadataRepository metadataRepository.MetadataRepository } -type ScheduleHttpHandler interface { +type ScheduleHTTPHandler interface { Post() http.HandlerFunc GetAll() http.HandlerFunc Get() http.HandlerFunc Delete() http.HandlerFunc } -func NewScheduleHttpHandler(repository scheduleRepository.ScheduleRepository, metadataRepository metadataRepository.MetadataRepository) ScheduleHttpHandler { - return &scheduleHttpHandler{ +func NewScheduleHTTPHandler(repository scheduleRepository.ScheduleRepository, metadataRepository metadataRepository.MetadataRepository) ScheduleHTTPHandler { + return &scheduleHTTPHandler{ repository: repository, metadataRepository: metadataRepository, } } -func (httpHandler *scheduleHttpHandler) Post() http.HandlerFunc { +func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { var schedule modelSchedule.Schedule err := json.NewDecoder(request.Body).Decode(&schedule) @@ -145,7 +145,7 @@ func (httpHandler *scheduleHttpHandler) Post() http.HandlerFunc { } } -func (httpHandler *scheduleHttpHandler) GetAll() http.HandlerFunc { +func (httpHandler *scheduleHTTPHandler) GetAll() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { scheduleList, err := httpHandler.repository.GetAllEnabled() if err != nil { @@ -178,7 +178,7 @@ func (httpHandler *scheduleHttpHandler) GetAll() http.HandlerFunc { } } -func (httpHandler *scheduleHttpHandler) Get() http.HandlerFunc { +func (httpHandler *scheduleHTTPHandler) Get() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { scheduleIDParam := mux.Vars(request)["scheduleID"] scheduleID, err := strconv.ParseUint(scheduleIDParam, 10, 64) @@ -219,7 +219,7 @@ func (httpHandler *scheduleHttpHandler) Get() http.HandlerFunc { } } -func (httpHandler *scheduleHttpHandler) Delete() http.HandlerFunc { +func (httpHandler *scheduleHTTPHandler) Delete() http.HandlerFunc { return func(response http.ResponseWriter, request *http.Request) { scheduleIDParam := mux.Vars(request)["scheduleID"] scheduleID, err := strconv.ParseUint(scheduleIDParam, 10, 64) diff --git a/internal/app/service/schedule/handler/http_test.go b/internal/app/service/schedule/handler/http_test.go index 203292ed..8415a7f7 100644 --- a/internal/app/service/schedule/handler/http_test.go +++ b/internal/app/service/schedule/handler/http_test.go @@ -21,20 +21,20 @@ import ( "proctor/internal/pkg/status" ) -type ScheduleHttpHandlerTestSuite struct { +type ScheduleHTTPHandlerTestSuite struct { suite.Suite mockScheduleRepository *scheduleRepository.MockScheduleRepository mockMetadataRepository *metadataRepository.MockMetadataRepository - testScheduleHttpHandler ScheduleHttpHandler + testScheduleHTTPHandler ScheduleHTTPHandler } -func (suite *ScheduleHttpHandlerTestSuite) SetupTest() { +func (suite *ScheduleHTTPHandlerTestSuite) SetupTest() { suite.mockScheduleRepository = &scheduleRepository.MockScheduleRepository{} suite.mockMetadataRepository = &metadataRepository.MockMetadataRepository{} - suite.testScheduleHttpHandler = NewScheduleHttpHandler(suite.mockScheduleRepository, suite.mockMetadataRepository) + suite.testScheduleHTTPHandler = NewScheduleHTTPHandler(suite.mockScheduleRepository, suite.mockMetadataRepository) } -func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulSchedulePostHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestSuccessfulSchedulePostHTTPHandler() { t := suite.T() scheduleID := uint64(0) @@ -70,13 +70,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulSchedulePostHttpHandler suite.mockScheduleRepository.On("Insert", &requestSchedule).Return(0, nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) - suite.testScheduleHttpHandler.Post()(responseRecorder, req) + suite.testScheduleHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusCreated, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ScheduleHttpHandlerTestSuite) TestErrorSchedulePostHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestErrorSchedulePostHTTPHandler() { t := suite.T() argsMap := types.Base64Map{ @@ -143,14 +143,14 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorSchedulePostHttpHandler() { req := httptest.NewRequest("POST", "/schedule", bytes.NewReader(errorTest.requestBody)) responseRecorder := httptest.NewRecorder() - suite.testScheduleHttpHandler.Post()(responseRecorder, req) + suite.testScheduleHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) } } -func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetAllHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestSuccessfulScheduleGetAllHTTPHandler() { t := suite.T() argsMap := types.Base64Map{ @@ -187,13 +187,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetAllHttpHandl suite.mockScheduleRepository.On("GetAllEnabled").Return(responseScheduleList, nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) - suite.testScheduleHttpHandler.GetAll()(responseRecorder, req) + suite.testScheduleHTTPHandler.GetAll()(responseRecorder, req) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetAllHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestErrorScheduleGetAllHTTPHandler() { t := suite.T() scheduleGetAllErrorTests := []struct { @@ -214,14 +214,14 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetAllHttpHandler() req := httptest.NewRequest("GET", "/schedule", bytes.NewReader([]byte(""))) responseRecorder := httptest.NewRecorder() - suite.testScheduleHttpHandler.GetAll()(responseRecorder, req) + suite.testScheduleHTTPHandler.GetAll()(responseRecorder, req) assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) } } -func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestSuccessfulScheduleGetHTTPHandler() { t := suite.T() scheduleID := uint64(1) @@ -249,13 +249,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleGetHttpHandler( suite.mockScheduleRepository.On("GetByID", scheduleID).Return(&responseSchedule, nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) - suite.testScheduleHttpHandler.Get()(responseRecorder, req) + suite.testScheduleHTTPHandler.Get()(responseRecorder, req) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(responseBody), responseRecorder.Body.String()) } -func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestErrorScheduleGetHTTPHandler() { t := suite.T() scheduleID := uint64(1) @@ -280,14 +280,14 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleGetHttpHandler() { req = mux.SetURLVars(req, map[string]string{"scheduleID": errorTest.requestParams}) responseRecorder := httptest.NewRecorder() - suite.testScheduleHttpHandler.Get()(responseRecorder, req) + suite.testScheduleHTTPHandler.Get()(responseRecorder, req) assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) } } -func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleDeleteHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestSuccessfulScheduleDeleteHTTPHandler() { t := suite.T() scheduleID := uint64(1) @@ -299,13 +299,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestSuccessfulScheduleDeleteHttpHandl suite.mockScheduleRepository.On("Delete", scheduleID).Return(nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) - suite.testScheduleHttpHandler.Delete()(responseRecorder, req) + suite.testScheduleHTTPHandler.Delete()(responseRecorder, req) assert.Equal(t, http.StatusOK, responseRecorder.Code) assert.Equal(t, string(status.ScheduleDeleteSuccess), responseRecorder.Body.String()) } -func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleDeleteHttpHandler() { +func (suite *ScheduleHTTPHandlerTestSuite) TestErrorScheduleDeleteHTTPHandler() { t := suite.T() scheduleID := uint64(1) @@ -330,13 +330,13 @@ func (suite *ScheduleHttpHandlerTestSuite) TestErrorScheduleDeleteHttpHandler() req = mux.SetURLVars(req, map[string]string{"scheduleID": errorTest.requestParams}) responseRecorder := httptest.NewRecorder() - suite.testScheduleHttpHandler.Delete()(responseRecorder, req) + suite.testScheduleHTTPHandler.Delete()(responseRecorder, req) assert.Equal(t, errorTest.responseStatus, responseRecorder.Code) assert.Equal(t, string(errorTest.responseBody), responseRecorder.Body.String()) } } -func TestScheduleHttpHandlerTestSuite(t *testing.T) { - suite.Run(t, new(ScheduleHttpHandlerTestSuite)) +func TestScheduleHTTPHandlerTestSuite(t *testing.T) { + suite.Run(t, new(ScheduleHTTPHandlerTestSuite)) } diff --git a/internal/app/service/secret/handler/http.go b/internal/app/service/secret/handler/http.go index f1343665..af586029 100644 --- a/internal/app/service/secret/handler/http.go +++ b/internal/app/service/secret/handler/http.go @@ -15,11 +15,11 @@ type handler struct { repository repository.SecretRepository } -type SecretHttpHandler interface { +type SecretHTTPHandler interface { Post() http.HandlerFunc } -func NewSecretHttpHandler(repository repository.SecretRepository) SecretHttpHandler { +func NewSecretHTTPHandler(repository repository.SecretRepository) SecretHTTPHandler { return &handler{ repository: repository, } diff --git a/internal/app/service/secret/handler/http_test.go b/internal/app/service/secret/handler/http_test.go index ecde3fb0..80f3afbc 100644 --- a/internal/app/service/secret/handler/http_test.go +++ b/internal/app/service/secret/handler/http_test.go @@ -20,13 +20,13 @@ import ( type SecretsHandlerTestSuite struct { suite.Suite mockSecretRepository *repository.MockSecretRepository - secretHandler SecretHttpHandler + secretHandler SecretHTTPHandler } func (suite *SecretsHandlerTestSuite) SetupTest() { suite.mockSecretRepository = &repository.MockSecretRepository{} - suite.secretHandler = NewSecretHttpHandler(suite.mockSecretRepository) + suite.secretHandler = NewSecretHTTPHandler(suite.mockSecretRepository) } func (suite *SecretsHandlerTestSuite) TestPostSecretSuccess() { diff --git a/internal/app/service/server/middleware/validate_client_version_test.go b/internal/app/service/server/middleware/validate_client_version_test.go index dfdef9a6..19195fae 100644 --- a/internal/app/service/server/middleware/validate_client_version_test.go +++ b/internal/app/service/server/middleware/validate_client_version_test.go @@ -16,7 +16,7 @@ func getTestHandler() http.HandlerFunc { return http.HandlerFunc(fn) } -func TestValidClientVersionHttpHeader(t *testing.T) { +func TestValidClientVersionHTTPHeader(t *testing.T) { _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") @@ -34,7 +34,7 @@ func TestValidClientVersionHttpHeader(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } -func TestEmptyClientVersionHttpHeader(t *testing.T) { +func TestEmptyClientVersionHTTPHeader(t *testing.T) { _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") @@ -51,7 +51,7 @@ func TestEmptyClientVersionHttpHeader(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } -func TestInvalidClientVersionHttpHeader(t *testing.T) { +func TestInvalidClientVersionHTTPHeader(t *testing.T) { _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.3.0") diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index fe310d84..608ed60b 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -8,19 +8,19 @@ import ( "github.com/gorilla/mux" "proctor/internal/app/service/docs" - executionHttpHandler "proctor/internal/app/service/execution/handler" + executionHTTPHandler "proctor/internal/app/service/execution/handler" executionContextRepository "proctor/internal/app/service/execution/repository" executionService "proctor/internal/app/service/execution/service" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/postgresql" "proctor/internal/app/service/infra/db/redis" "proctor/internal/app/service/infra/kubernetes" - kubernetesHttpClient "proctor/internal/app/service/infra/kubernetes/http" + kubernetesHTTPClient "proctor/internal/app/service/infra/kubernetes/http" metadataHandler "proctor/internal/app/service/metadata/handler" metadataRepository "proctor/internal/app/service/metadata/repository" - scheduleHttpHandler "proctor/internal/app/service/schedule/handler" + scheduleHTTPHandler "proctor/internal/app/service/schedule/handler" scheduleRepository "proctor/internal/app/service/schedule/repository" - secretHttpHandler "proctor/internal/app/service/secret/handler" + secretHTTPHandler "proctor/internal/app/service/secret/handler" secretRepository "proctor/internal/app/service/secret/repository" "proctor/internal/app/service/server/middleware" ) @@ -32,7 +32,7 @@ func NewRouter() (*mux.Router, error) { redisClient := redis.NewClient() postgresClient = postgresql.NewClient() - httpClient, err := kubernetesHttpClient.NewClient() + httpClient, err := kubernetesHTTPClient.NewClient() if err != nil { return router, err } @@ -45,10 +45,10 @@ func NewRouter() (*mux.Router, error) { _executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) - executionHandler := executionHttpHandler.NewExecutionHttpHandler(_executionService, executionStore) - jobMetadataHandler := metadataHandler.NewMetadataHttpHandler(metadataStore) - jobSecretsHandler := secretHttpHandler.NewSecretHttpHandler(secretsStore) - scheduleHandler := scheduleHttpHandler.NewScheduleHttpHandler(scheduleStore, metadataStore) + executionHandler := executionHTTPHandler.NewExecutionHTTPHandler(_executionService, executionStore) + jobMetadataHandler := metadataHandler.NewMetadataHTTPHandler(metadataStore) + jobSecretsHandler := secretHTTPHandler.NewSecretHTTPHandler(secretsStore) + scheduleHandler := scheduleHTTPHandler.NewScheduleHTTPHandler(scheduleStore, metadataStore) router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { _, _ = fmt.Fprintf(w, "pong") From bc5f9e98dfe7f052d7dfec593452cbb47f421190 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 25 Jul 2019 14:33:45 +0700 Subject: [PATCH 070/268] [Jasoet] Add wait time on integration test --- .../app/service/execution/service/execution_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 159ae7f1..99c07a42 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -80,7 +80,7 @@ func (suite *TestExecutionIntegrationSuite) TestExecuteJobSuccess() { assert.NoError(t, err) assert.NotNil(t, context) - time.Sleep(30 * time.Second) + time.Sleep(40 * time.Second) expectedContext, err := suite.repository.GetById(context.ExecutionID) assert.NoError(t, err) assert.NotNil(t, expectedContext) From d639d1effa8978b4bdf1414ab17e527dc4505dbd Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Fri, 26 Jul 2019 14:03:46 +0700 Subject: [PATCH 071/268] [bimo.horizon] Fix incorrect routes --- internal/app/cli/daemon/client.go | 24 ++++++---- internal/app/cli/daemon/client_test.go | 66 +++++++++++++------------- internal/app/service/server/router.go | 4 +- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 0246d1c4..51022d63 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -25,6 +25,14 @@ import ( modelSchedule "proctor/internal/pkg/model/schedule" ) +const ( + ExecutionRoute string = "/execution" + ExecutionLogsRoute string = "/execution/logs" + MetadataRoute string = "/metadata" + SecretRoute string = "/secret" + ScheduleRoute string = "/schedule" +) + type Client interface { ListProcs() ([]modelMetadata.Metadata, error) ExecuteProc(string, map[string]string) (string, error) @@ -90,7 +98,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, } client := &http.Client{} - req, err := http.NewRequest("POST", "http://"+c.proctordHost+"/jobs/schedule", bytes.NewReader(requestBody)) + req, err := http.NewRequest("POST", "http://"+c.proctordHost+ScheduleRoute, bytes.NewReader(requestBody)) req.Header.Add("Content-Type", "application/json") req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) @@ -138,7 +146,7 @@ func (c *client) ListProcs() ([]modelMetadata.Metadata, error) { client := &http.Client{ Timeout: c.connectionTimeoutSecs, } - req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/metadata", nil) + req, err := http.NewRequest("GET", "http://"+c.proctordHost+MetadataRoute, nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) @@ -167,7 +175,7 @@ func (c *client) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { client := &http.Client{ Timeout: c.connectionTimeoutSecs, } - req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/schedule", nil) + req, err := http.NewRequest("GET", "http://"+c.proctordHost+ScheduleRoute, nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) @@ -196,7 +204,7 @@ func (c *client) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob client := &http.Client{ Timeout: c.connectionTimeoutSecs, } - url := fmt.Sprintf("http://"+c.proctordHost+"/jobs/schedule/%s", jobID) + url := fmt.Sprintf("http://"+c.proctordHost+ScheduleRoute+"/%s", jobID) req, err := http.NewRequest("GET", url, nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) @@ -226,7 +234,7 @@ func (c *client) RemoveScheduledProc(jobID string) error { client := &http.Client{ Timeout: c.connectionTimeoutSecs, } - url := fmt.Sprintf("http://"+c.proctordHost+"/jobs/schedule/%s", jobID) + url := fmt.Sprintf("http://"+c.proctordHost+ScheduleRoute+"/%s", jobID) req, err := http.NewRequest("DELETE", url, nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) @@ -262,7 +270,7 @@ func (c *client) ExecuteProc(name string, args map[string]string) (string, error } client := &http.Client{} - req, err := http.NewRequest("POST", "http://"+c.proctordHost+"/jobs/execute", bytes.NewReader(requestBody)) + req, err := http.NewRequest("POST", "http://"+c.proctordHost+ExecutionRoute, bytes.NewReader(requestBody)) req.Header.Add("Content-Type", "application/json") req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) @@ -296,7 +304,7 @@ func (c *client) StreamProcLogs(name string) error { interrupt := make(chan os.Signal, 1) signal.Notify(interrupt, os.Interrupt) - proctodWebsocketURL := url.URL{Scheme: "ws", Host: c.proctordHost, Path: "/jobs/logs"} + proctodWebsocketURL := url.URL{Scheme: "ws", Host: c.proctordHost, Path: ExecutionLogsRoute} proctodWebsocketURLWithProcName := proctodWebsocketURL.String() + "?" + "job_name=" + name headers := make(map[string][]string) @@ -357,7 +365,7 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro Timeout: c.connectionTimeoutSecs, } - req, err := http.NewRequest("GET", "http://"+c.proctordHost+"/jobs/execute/"+procName+"/status", nil) + req, err := http.NewRequest("GET", "http://"+c.proctordHost+ExecutionRoute+"/"+procName+"/status", nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 567b49d4..765824fa 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -71,7 +71,7 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, body), nil }, @@ -104,7 +104,7 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, `{}`), nil }, @@ -138,7 +138,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} }, @@ -155,7 +155,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { procList, err := s.testClient.ListProcs() - assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/jobs/metadata: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) + assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/metadata: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) assert.Equal(t, []modelMetadata.Metadata{}, procList) s.mockConfigLoader.AssertExpectations(t) } @@ -171,7 +171,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return nil, TestConnectionError{message: "Unknown Error", timeout: false} }, @@ -188,7 +188,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { procList, err := s.testClient.ListProcs() - assert.Equal(t, errors.New("Network Error!!!\nGet http://proctor.example.com/jobs/metadata: Unknown Error"), err) + assert.Equal(t, errors.New("Network Error!!!\nGet http://proctor.example.com/metadata: Unknown Error"), err) assert.Equal(t, []modelMetadata.Metadata{}, procList) s.mockConfigLoader.AssertExpectations(t) } @@ -204,7 +204,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedUser() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(401, `{}`), nil }, @@ -236,7 +236,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/metadata", + "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(401, `{}`), nil }, @@ -272,7 +272,7 @@ func (s *ClientTestSuite) TestExecuteProc() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/execute", + "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(201, body), nil }, @@ -314,7 +314,7 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/schedule", + "http://"+proctorConfig.Host+ScheduleRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(201, body), nil }, @@ -353,7 +353,7 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/schedule", + "http://"+proctorConfig.Host+ScheduleRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(409, "Server Error!!!\nStatus Code: 409, Conflict"), nil }, @@ -386,7 +386,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/execute", + "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, @@ -417,7 +417,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/execute", + "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(401, ""), nil }, @@ -449,7 +449,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/execute", + "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(401, ""), nil }, @@ -481,7 +481,7 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", - "http://"+proctorConfig.Host+"/jobs/execute", + "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { return nil, TestConnectionError{message: "Unknown Error", timeout: false} }, @@ -499,7 +499,7 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { response, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) assert.Equal(t, "", response) - assert.Equal(t, errors.New("Network Error!!!\nPost http://proctor.example.com/jobs/execute: Unknown Error"), err) + assert.Equal(t, errors.New("Network Error!!!\nPost http://proctor.example.com/execution: Unknown Error"), err) s.mockConfigLoader.AssertExpectations(t) } @@ -579,7 +579,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/execute/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, responseBody), nil }, @@ -615,7 +615,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/execute/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, responseBody), nil }, @@ -648,7 +648,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/execute/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", func(req *http.Request) (*http.Response, error) { return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} }, @@ -665,7 +665,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") - assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/jobs/execute/some-proc-name/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) + assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/some-proc-name/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, "", procExecutionStatus) } @@ -681,7 +681,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/execute/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, @@ -720,7 +720,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+"/jobs/execute/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", func(req *http.Request) (*http.Response, error) { requestsToProctorDCount += 1 return httpmock.NewStringResponse(200, responseBody), nil @@ -757,7 +757,7 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, body), nil }, @@ -793,7 +793,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(400, body), nil }, @@ -826,7 +826,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWhenJobIDNotFound() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(404, "Job not found"), nil }, @@ -859,7 +859,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, @@ -893,7 +893,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule"), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, body), nil }, @@ -928,7 +928,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenNoJobsScheduled() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule"), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(204, body), nil }, @@ -960,7 +960,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule"), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, @@ -994,7 +994,7 @@ func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, body), nil }, @@ -1027,7 +1027,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWithInvalidJobID() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(400, "Invalid Job ID"), nil }, @@ -1060,7 +1060,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWhenJobIDNotFound() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(404, "Job not found"), nil }, @@ -1093,7 +1093,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+"/jobs/schedule/%s", jobID), + fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 608ed60b..1cf7d505 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -63,13 +63,13 @@ func NewRouter() (*mux.Router, error) { router = middleware.InstrumentNewRelic(router) router.Use(middleware.ValidateClientVersion) - router.HandleFunc("/execute", executionHandler.Post()).Methods("POST") + router.HandleFunc("/execution", executionHandler.Post()).Methods("POST") router.HandleFunc("/execution/{contextID}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") - router.HandleFunc("/secrets", jobSecretsHandler.Post()).Methods("POST") + router.HandleFunc("/secret", jobSecretsHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.GetAll()).Methods("GET") From 22863babedba9aa36444bc328936341c5e7aaea8 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Fri, 26 Jul 2019 16:40:31 +0700 Subject: [PATCH 072/268] [bimo.horizon] Change binary name --- cmd/{cli => proctor}/main.go | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cmd/{cli => proctor}/main.go (100%) diff --git a/cmd/cli/main.go b/cmd/proctor/main.go similarity index 100% rename from cmd/cli/main.go rename to cmd/proctor/main.go From 53c643e026bb4b7e2a13503d6e08349e1bb74e81 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 12:10:14 +0700 Subject: [PATCH 073/268] [Jasoet] Fix cmd folder name and Makefile --- Makefile | 2 +- cmd/{proctor => cli}/main.go | 0 cmd/{proctord => server}/main.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename cmd/{proctor => cli}/main.go (100%) rename cmd/{proctord => server}/main.go (100%) diff --git a/Makefile b/Makefile index e0a38364..03903df1 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ itest: .PHONY: server server: - go build -o $(BIN_DIR)/server ./cmd/proctord/main.go + go build -o $(BIN_DIR)/server ./cmd/server/main.go .PHONY: start-server start-server: diff --git a/cmd/proctor/main.go b/cmd/cli/main.go similarity index 100% rename from cmd/proctor/main.go rename to cmd/cli/main.go diff --git a/cmd/proctord/main.go b/cmd/server/main.go similarity index 100% rename from cmd/proctord/main.go rename to cmd/server/main.go From e934aed76d6a6d0fcd5440562a3098034f5add15 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 15:42:50 +0700 Subject: [PATCH 074/268] [Jasoet] Merge utility helper to particular app --- internal/app/cli/command/list/lister.go | 2 +- .../{pkg/utility => app/cli}/sort/sort.go | 0 .../utility => app/cli}/sort/sort_test.go | 0 internal/app/service/infra/mail/mailer.go | 12 ++++- .../app/service/infra/mail/mailer_test.go | 3 +- internal/pkg/utility/buffer.go | 43 ------------------ internal/pkg/utility/utils.go | 44 ------------------- 7 files changed, 12 insertions(+), 92 deletions(-) rename internal/{pkg/utility => app/cli}/sort/sort.go (100%) rename internal/{pkg/utility => app/cli}/sort/sort_test.go (100%) delete mode 100644 internal/pkg/utility/buffer.go delete mode 100644 internal/pkg/utility/utils.go diff --git a/internal/app/cli/command/list/lister.go b/internal/app/cli/command/list/lister.go index 533cc1a6..7d55d4ec 100644 --- a/internal/app/cli/command/list/lister.go +++ b/internal/app/cli/command/list/lister.go @@ -7,7 +7,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/internal/pkg/utility/sort" + "proctor/internal/app/cli/sort" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/internal/pkg/utility/sort/sort.go b/internal/app/cli/sort/sort.go similarity index 100% rename from internal/pkg/utility/sort/sort.go rename to internal/app/cli/sort/sort.go diff --git a/internal/pkg/utility/sort/sort_test.go b/internal/app/cli/sort/sort_test.go similarity index 100% rename from internal/pkg/utility/sort/sort_test.go rename to internal/app/cli/sort/sort_test.go diff --git a/internal/app/service/infra/mail/mailer.go b/internal/app/service/infra/mail/mailer.go index 1538aaaa..c75426a9 100644 --- a/internal/app/service/infra/mail/mailer.go +++ b/internal/app/service/infra/mail/mailer.go @@ -1,6 +1,7 @@ package mail import ( + "bytes" "fmt" "net/smtp" "strings" @@ -8,7 +9,6 @@ import ( executionContextModel "proctor/internal/app/service/execution/model" "proctor/internal/app/service/infra/config" scheduleModel "proctor/internal/app/service/schedule/model" - "proctor/internal/pkg/utility" ) type Mailer interface { @@ -42,10 +42,18 @@ func constructMessage(jobName string, executionID uint64, executionStatus string subject := "Subject: " + jobName + " | scheduled execution " + executionStatus body := "Proc execution details:\n" + "\nName:\t" + jobName + - "\nArgs:\t" + utility.MapToString(executionArgs) + + "\nArgs:\t" + MapToString(executionArgs) + "\nID:\t" + fmt.Sprint(executionID) + "\nStatus:\t" + executionStatus + "\n\n\nThis is an auto-generated email" return []byte(subject + "\n\n" + body) } + +func MapToString(someMap map[string]string) string { + b := new(bytes.Buffer) + for key, value := range someMap { + _, _ = fmt.Fprintf(b, "%s=\"%s\",", key, value) + } + return strings.TrimRight(b.String(), ",") +} diff --git a/internal/app/service/infra/mail/mailer_test.go b/internal/app/service/infra/mail/mailer_test.go index 65d7783e..6a66b492 100644 --- a/internal/app/service/infra/mail/mailer_test.go +++ b/internal/app/service/infra/mail/mailer_test.go @@ -13,7 +13,6 @@ import ( executionStatus "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/config" scheduleModel "proctor/internal/app/service/schedule/model" - "proctor/internal/pkg/utility" ) func TestSendMail(t *testing.T) { @@ -98,7 +97,7 @@ func TestSendMail(t *testing.T) { receivedMail := cmdbuf.String() - stringifiedJobArgs := utility.MapToString(executionContext.Args) + stringifiedJobArgs := MapToString(executionContext.Args) var sendMailClient = `EHLO localhost HELO localhost MAIL FROM:<` + config.MailUsername() + `> diff --git a/internal/pkg/utility/buffer.go b/internal/pkg/utility/buffer.go deleted file mode 100644 index 94fc3c24..00000000 --- a/internal/pkg/utility/buffer.go +++ /dev/null @@ -1,43 +0,0 @@ -package utility - -import ( - "bytes" - "sync" -) - -type Buffer struct { - mutex sync.Mutex - wasClosed bool - buffer bytes.Buffer -} - -func NewBuffer() *Buffer { - return &Buffer{ - wasClosed: false, - } -} - -func (b *Buffer) Close() error { - b.mutex.Lock() - defer b.mutex.Unlock() - b.wasClosed = true - return nil -} - -func (b *Buffer) WasClosed() bool { - b.mutex.Lock() - defer b.mutex.Unlock() - return b.wasClosed -} - -func (b *Buffer) Read(p []byte) (n int, err error) { - b.mutex.Lock() - defer b.mutex.Unlock() - return b.buffer.Read(p) -} - -func (b *Buffer) Write(p []byte) (n int, err error) { - b.mutex.Lock() - defer b.mutex.Unlock() - return b.buffer.Write(p) -} diff --git a/internal/pkg/utility/utils.go b/internal/pkg/utility/utils.go deleted file mode 100644 index 69e94911..00000000 --- a/internal/pkg/utility/utils.go +++ /dev/null @@ -1,44 +0,0 @@ -package utility - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "strings" -) - -func MergeMaps(mapOne, mapTwo map[string]string) map[string]string { - result := make(map[string]string) - - for k, v := range mapOne { - result[k] = v - } - for k, v := range mapTwo { - result[k] = v - } - return result -} - -func MapToString(someMap map[string]string) string { - b := new(bytes.Buffer) - for key, value := range someMap { - _, _ = fmt.Fprintf(b, "%s=\"%s\",", key, value) - } - return strings.TrimRight(b.String(), ",") -} - -func DeserializeMap(encodedMap string) (map[string]string, error) { - var mapStringToString map[string]string - if encodedMap == "" { - return mapStringToString, nil - } - - decodedMap, err := base64.StdEncoding.DecodeString(encodedMap) - if err != nil { - return mapStringToString, err - } - - err = json.Unmarshal(decodedMap, &mapStringToString) - return mapStringToString, err -} From 1ce5136804026778b9b65aada66c53bef2417cf7 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 15:48:52 +0700 Subject: [PATCH 075/268] [Jasoet] Fix goreleaser build config --- .gitignore | 3 ++- .goreleaser.yml | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3a0f0003..001dc7f5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ _output/* *.swo .idea .DS_Store -.vscode/ \ No newline at end of file +.vscode/ +dist/ \ No newline at end of file diff --git a/.goreleaser.yml b/.goreleaser.yml index 8490ef83..7b25cea4 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,8 +1,18 @@ # This is an example goreleaser.yaml file with some sane defaults. # Make sure to check the documentation at http://goreleaser.com builds: -- env: - - CGO_ENABLED=0 + - + id: "proctor-server" + main: ./cmd/server/main.go + binary: proctor-server + env: + - CGO_ENABLED=0 + - + id: "proctor-cli" + main: ./cmd/cli/main.go + binary: proctor + env: + - CGO_ENABLED=0 archive: replacements: darwin: Darwin From 58fec0181569ef514d436e06de62b1f73e091c1e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 19:08:21 +0700 Subject: [PATCH 076/268] [Jasoet|Bimo.horizon] Functional Test: add script to package procs and update metadata --- .env.test | 2 +- Makefile | 14 ++++++ test/package_procs.rb | 45 +++++++++++++++++ test/{procs_example => procs}/README.md | 0 .../say-hello-world/Dockerfile | 0 .../say-hello-world/metadata.json | 1 + .../say-hello-world/say_hello_world.sh | 0 test/update_metadata.rb | 50 +++++++++++++++++++ 8 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 test/package_procs.rb rename test/{procs_example => procs}/README.md (100%) rename test/{procs_example => procs}/say-hello-world/Dockerfile (100%) rename test/{procs_example => procs}/say-hello-world/metadata.json (94%) rename test/{procs_example => procs}/say-hello-world/say_hello_world.sh (100%) create mode 100644 test/update_metadata.rb diff --git a/.env.test b/.env.test index 94079980..739c1b74 100644 --- a/.env.test +++ b/.env.test @@ -23,7 +23,7 @@ export PROCTOR_POSTGRES_DATABASE=proctord_test export PROCTOR_POSTGRES_MAX_CONNECTIONS=50 export PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 export PROCTOR_NEW_RELIC_APP_NAME="PROCTORD" -export PROCTOR_NEW_RELIC_LICENCE_KEY="nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr" +export PROCTOR_NEW_RELIC_LICENCE_KEY=0123456789012345678901234567890123456789 export PROCTOR_MIN_CLIENT_VERSION="v0.2.0" export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS="5" export PROCTOR_MAIL_USERNAME="user@mail.com" diff --git a/Makefile b/Makefile index 03903df1..751f6a7c 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,7 @@ export $(shell sed 's/=.*//' .env.test) SRC_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) OUT_DIR := $(SRC_DIR)/_output BIN_DIR := $(OUT_DIR)/bin +FTEST_DIR := test/procs GOPROXY ?= https://proxy.golang.org GO111MODULE := on @@ -60,3 +61,16 @@ db.rollback: server db.teardown: -PGPASSWORD=$(PROCTOR_POSTGRES_PASSWORD) psql -h $(PROCTOR_POSTGRES_HOST) -p $(PROCTOR_POSTGRES_PORT) -c 'drop database $(PROCTOR_POSTGRES_DATABASE);' -U $(PROCTOR_POSTGRES_USER) + +.PHONY: ftest.package.procs +ftest.package.procs: + PROCTOR_JOBS_PATH=$(FTEST_DIR) \ + ruby ./test/package_procs.rb + +ftest.update.metadata: + $(BIN_DIR)/server s & \ + PROCTOR_JOBS_PATH=$(FTEST_DIR) \ + PROCTOR_URI=http://localhost:$(PROCTOR_APP_PORT)/metadata \ + ruby ./test/update_metadata.rb + + diff --git a/test/package_procs.rb b/test/package_procs.rb new file mode 100644 index 00000000..77591e2d --- /dev/null +++ b/test/package_procs.rb @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby + +require 'json' + +hub_username = ENV['DOCKERHUB_USERNAME'] +hub_password = ENV['DOCKERHUB_PASSWORD'] +jobs_path = ENV['PROCTOR_JOBS_PATH'] +team_name = ENV['PROCTOR_JOB_TEAM_NAME'] || "test" +container_registry = ENV["PROCTOR_CONTAINER_REGISTRY"] || "docker.io/proctorscripts" +metadata_file_name = ENV['PROCTOR_METADATA_FILE_NAME'] || "metadata.json" + +def run_cmd(cmd) + puts cmd + result = system(cmd) + if !result + puts "#{cmd} exited with non-zero code" + exit 1 + end +end + +def login(username, password) + run_cmd("docker login -u #{username} -p #{password}") +end + +for dir in Dir["#{jobs_path}/*/"] + metadata_file = dir + '/' + metadata_file_name + + login(hub_username, hub_password) + + if File.exist?(metadata_file) + file = File.read(metadata_file) + data_hash = JSON.parse(file) + image_name = "#{container_registry}/#{team_name}-#{data_hash['name']}:latest" + + Dir.chdir(dir) { + puts "===== build and push image =====" + run_cmd("docker build -t #{image_name} .") + run_cmd("docker push #{image_name}") + } + else + puts "#{dir} doesn't have metadata_file" + end + +end + diff --git a/test/procs_example/README.md b/test/procs/README.md similarity index 100% rename from test/procs_example/README.md rename to test/procs/README.md diff --git a/test/procs_example/say-hello-world/Dockerfile b/test/procs/say-hello-world/Dockerfile similarity index 100% rename from test/procs_example/say-hello-world/Dockerfile rename to test/procs/say-hello-world/Dockerfile diff --git a/test/procs_example/say-hello-world/metadata.json b/test/procs/say-hello-world/metadata.json similarity index 94% rename from test/procs_example/say-hello-world/metadata.json rename to test/procs/say-hello-world/metadata.json index 36ff8423..0b7598df 100644 --- a/test/procs_example/say-hello-world/metadata.json +++ b/test/procs/say-hello-world/metadata.json @@ -1,4 +1,5 @@ { + "name": "say-hello-world", "description":"This proc says hello world", "env_vars":{ "secrets": [ diff --git a/test/procs_example/say-hello-world/say_hello_world.sh b/test/procs/say-hello-world/say_hello_world.sh similarity index 100% rename from test/procs_example/say-hello-world/say_hello_world.sh rename to test/procs/say-hello-world/say_hello_world.sh diff --git a/test/update_metadata.rb b/test/update_metadata.rb new file mode 100644 index 00000000..b6940f49 --- /dev/null +++ b/test/update_metadata.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby + +require 'json' +require 'net/http' +require 'uri' + +proctor_uri = ENV['PROCTOR_URI'] + +jobs_path = ENV['PROCTOR_JOBS_PATH'] +team_name = ENV['PROCTOR_JOB_TEAM_NAME'] || "test" +container_registry = ENV["PROCTOR_CONTAINER_REGISTRY"] || "docker.io/proctorscripts" +metadata_file_name = ENV['PROCTOR_METADATA_FILE_NAME'] || "metadata.json" + +sleep(2) + +jobs = [] + +for dir in Dir["#{jobs_path}/*/"] + metadata_file = dir + metadata_file_name + puts "Processing #{metadata_file}" + + if File.exist?(metadata_file) + file = File.read(metadata_file) + data_hash = JSON.parse(file) + image_name = "#{container_registry}/#{team_name}-#{data_hash['name']}:latest" + data_hash['image_name'] = image_name + jobs << data_hash + end +end + +uri = URI.parse(proctor_uri) +header = {"Content-Type" => "application/json"} + +# Create the HTTP objects +http = Net::HTTP.new(uri.host, uri.port) +request = Net::HTTP::Post.new(uri.request_uri, header) +request.body = jobs.to_json + +# Send the request +puts "making req with body #{request.body}" +response = http.request(request) + +if response.code == "201" + puts 'Updated proctor metadata' +else + puts 'Something went wrong while updating proctor metadata! Response from proctor:' + puts response + exit 1 +end + From 5abcaa74478fca2c3d2ecf50be1e9b8cb602b1c1 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 19:52:44 +0700 Subject: [PATCH 077/268] [Jasoet|Bimo.horizon] Add Gracefull Shutdown for Proctor Server --- Makefile | 3 ++ .../execution/handler/parameter/parameter.go | 3 +- internal/app/service/server/api.go | 35 ++++++++++++++++--- .../server/middleware/parameter/parameter.go | 1 - .../server/middleware/status/status.go | 1 - 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 751f6a7c..11c7f1f5 100644 --- a/Makefile +++ b/Makefile @@ -73,4 +73,7 @@ ftest.update.metadata: PROCTOR_URI=http://localhost:$(PROCTOR_APP_PORT)/metadata \ ruby ./test/update_metadata.rb +ftest.proctor.list: + $(BIN_DIR)/cli list + diff --git a/internal/app/service/execution/handler/parameter/parameter.go b/internal/app/service/execution/handler/parameter/parameter.go index b0c9ff75..d76c5e26 100644 --- a/internal/app/service/execution/handler/parameter/parameter.go +++ b/internal/app/service/execution/handler/parameter/parameter.go @@ -1,6 +1,5 @@ package parameter - const ( - UserEmailHeader string = "Email-Id" + UserEmailHeader string = "Email-Id" ) diff --git a/internal/app/service/server/api.go b/internal/app/service/server/api.go index e71efb28..d9ff7daf 100644 --- a/internal/app/service/server/api.go +++ b/internal/app/service/server/api.go @@ -1,13 +1,15 @@ package server import ( + "context" + "github.com/urfave/negroni" + "net/http" + "os" + "os/signal" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/server/middleware" - "time" - - "github.com/tylerb/graceful" - "github.com/urfave/negroni" + "syscall" ) func Start() error { @@ -25,8 +27,31 @@ func Start() error { server.UseHandler(router) logger.Info("Starting server on port", appPort) + httpServer := &http.Server{ + Addr: appPort, + Handler: server, + } + + idleConnsClosed := make(chan struct{}) + go func() { + sigint := make(chan os.Signal, 1) + + signal.Notify(sigint, os.Interrupt) + signal.Notify(sigint, syscall.SIGTERM) + + <-sigint + + if shutdownErr := httpServer.Shutdown(context.Background()); shutdownErr != nil { + logger.Error("Received an Interrupt Signal", shutdownErr) + } + close(idleConnsClosed) + }() + + if err = httpServer.ListenAndServe(); err != http.ErrServerClosed { + logger.Error("HTTP Server Failed ", err) + } - graceful.Run(appPort, 2*time.Second, server) + <-idleConnsClosed _ = postgresClient.Close() logger.Info("Stopped server gracefully") diff --git a/internal/app/service/server/middleware/parameter/parameter.go b/internal/app/service/server/middleware/parameter/parameter.go index b4f54009..11b50e18 100644 --- a/internal/app/service/server/middleware/parameter/parameter.go +++ b/internal/app/service/server/middleware/parameter/parameter.go @@ -1,6 +1,5 @@ package parameter - const ( AccessTokenHeader string = "Access-Token" ClientVersionHeader string = "Client-Version" diff --git a/internal/app/service/server/middleware/status/status.go b/internal/app/service/server/middleware/status/status.go index 41c9c962..5c47b6c5 100644 --- a/internal/app/service/server/middleware/status/status.go +++ b/internal/app/service/server/middleware/status/status.go @@ -1,6 +1,5 @@ package status - const ( ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s . To continue using proctor, please upgrade to latest version." ) From 61f83dcc6ee12c3c46fde2891edfb7e93e07ea3d Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 5 Aug 2019 19:54:36 +0700 Subject: [PATCH 078/268] [Jasoet|Bimo.horizon] Tyding Go Mod --- go.mod | 4 ---- go.sum | 10 ---------- 2 files changed, 14 deletions(-) diff --git a/go.mod b/go.mod index 85b0fe21..db928a88 100644 --- a/go.mod +++ b/go.mod @@ -25,14 +25,11 @@ require ( github.com/goware/urlx v0.2.0 // indirect github.com/hashicorp/go-version v1.2.0 github.com/imdario/mergo v0.3.7 // indirect - github.com/jarcoal/httpmock v1.0.4 github.com/jmoiron/sqlx v1.2.0 github.com/json-iterator/go v1.1.6 // indirect - github.com/k0kubun/pp v3.0.1+incompatible // indirect github.com/lib/pq v1.1.1 github.com/mattes/migrate v3.0.1+incompatible github.com/mattn/go-colorable v0.1.2 // indirect - github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f // indirect github.com/modern-go/reflect2 v1.0.1 // indirect github.com/newrelic/go-agent v2.9.0+incompatible github.com/opencontainers/go-digest v1.0.0-rc1 // indirect @@ -45,7 +42,6 @@ require ( github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.3.0 github.com/thingful/httpmock v0.0.2 - github.com/tylerb/graceful v1.2.15 github.com/urfave/cli v1.20.0 github.com/urfave/negroni v1.0.0 gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index bc7a18dd..222c7f46 100644 --- a/go.sum +++ b/go.sum @@ -112,8 +112,6 @@ github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= -github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -121,8 +119,6 @@ github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBv github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40= -github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -148,8 +144,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f h1:ee+twVCignaZjt7jpbMSLxAeTN/Nfq9W/nm91E7QO1A= -github.com/mdempsky/gocode v0.0.0-20190203001940-7fb65232883f/go.mod h1:hltEC42XzfMNgg0S1v6JTywwra2Mu6F6cLR03debVQ8= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -186,8 +180,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/robfig/cron v1.1.0 h1:jk4/Hud3TTdcrJgUOBgsqrZBarcxl6ADIjSC2iniwLY= -github.com/robfig/cron v1.1.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -227,8 +219,6 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/thingful/httpmock v0.0.2 h1:1eWdzHkrygIk0LAcvX3M36Ol3IoIOwHWIqxUyGLsEdw= github.com/thingful/httpmock v0.0.2/go.mod h1:7l+awGvIFiugIInunvwUQYHNg5U0KXLPbNQshUfqAIk= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tylerb/graceful v1.2.15 h1:B0x01Y8fsJpogzZTkDg6BDi6eMf03s01lEKGdrv83oA= -github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= From 563569e4b17a20e577ebc9e560fab04553660f0f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 6 Aug 2019 11:56:44 +0700 Subject: [PATCH 079/268] [Jasoet] fix validate client middleware bug --- .env.test | 20 +++++++++---------- Makefile | 3 ++- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/config/config.go | 5 ++++- .../server/middleware/status/status.go | 4 +++- .../middleware/validate_client_version.go | 11 ++++++++-- test/config/proctor.yaml | 3 +++ 7 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 test/config/proctor.yaml diff --git a/.env.test b/.env.test index 739c1b74..74b69458 100644 --- a/.env.test +++ b/.env.test @@ -22,14 +22,14 @@ export PROCTOR_POSTGRES_PORT=5432 export PROCTOR_POSTGRES_DATABASE=proctord_test export PROCTOR_POSTGRES_MAX_CONNECTIONS=50 export PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 -export PROCTOR_NEW_RELIC_APP_NAME="PROCTORD" +export PROCTOR_NEW_RELIC_APP_NAME=PROCTORD export PROCTOR_NEW_RELIC_LICENCE_KEY=0123456789012345678901234567890123456789 -export PROCTOR_MIN_CLIENT_VERSION="v0.2.0" -export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS="5" -export PROCTOR_MAIL_USERNAME="user@mail.com" -export PROCTOR_MAIL_PASSWORD="password" -export PROCTOR_MAIL_SERVER_HOST="smtp.mail.com" -export PROCTOR_MAIL_SERVER_PORT="123" -export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" -export PROCTOR_SENTRY_DSN="foo" -export PROCTOR_DOCS_PATH="/path/to/docs/dir" +export PROCTOR_MIN_CLIENT_VERSION=v2.0.0 +export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS=5 +export PROCTOR_MAIL_USERNAME=user@mail.com +export PROCTOR_MAIL_PASSWORD=password +export PROCTOR_MAIL_SERVER_HOST=smtp.mail.com +export PROCTOR_MAIL_SERVER_PORT=123 +export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} +export PROCTOR_SENTRY_DSN=foo +export PROCTOR_DOCS_PATH=/path/to/docs/dir diff --git a/Makefile b/Makefile index 11c7f1f5..bc950d1c 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ SRC_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) OUT_DIR := $(SRC_DIR)/_output BIN_DIR := $(OUT_DIR)/bin FTEST_DIR := test/procs +CONFIG_DIR := test/config GOPROXY ?= https://proxy.golang.org GO111MODULE := on @@ -74,6 +75,6 @@ ftest.update.metadata: ruby ./test/update_metadata.rb ftest.proctor.list: - $(BIN_DIR)/cli list + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli list diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index 19d2ab62..2cc1ce5e 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v0.6.0" +const ClientVersion = "v2.0.0" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/config/config.go b/internal/app/cli/config/config.go index fd70c929..e9b4a306 100644 --- a/internal/app/cli/config/config.go +++ b/internal/app/cli/config/config.go @@ -86,7 +86,10 @@ func (loader *loader) Load() (ProctorConfig, ConfigError) { // Returns Config file directory // This allows to test on dev environment without conflicting with installed proctor config file func ConfigFileDir() string { - if os.Getenv(Environment) == "test" { + localConfigDir, localConfigAvailable := os.LookupEnv("LOCAL_CONFIG_DIR") + if localConfigAvailable { + return localConfigDir + } else if os.Getenv(Environment) == "test" { return "/tmp" } else { return fmt.Sprintf("%s/.proctor", os.Getenv("HOME")) diff --git a/internal/app/service/server/middleware/status/status.go b/internal/app/service/server/middleware/status/status.go index 5c47b6c5..828909e2 100644 --- a/internal/app/service/server/middleware/status/status.go +++ b/internal/app/service/server/middleware/status/status.go @@ -1,5 +1,7 @@ package status const ( - ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s . To continue using proctor, please upgrade to latest version." + ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s. To continue using proctor, please upgrade to latest version." + ClientVersionInvalidMessage = "Your Proctor client config is Invalid. Please contact your Proctor Administrator. Error: %s" + ServerConfigErrorMessage = "Your Proctor server configuration is invalid. Please contact your Proctor Administrator. Error: %s" ) diff --git a/internal/app/service/server/middleware/validate_client_version.go b/internal/app/service/server/middleware/validate_client_version.go index 08d98ba3..3a10c712 100644 --- a/internal/app/service/server/middleware/validate_client_version.go +++ b/internal/app/service/server/middleware/validate_client_version.go @@ -19,18 +19,25 @@ func ValidateClientVersion(next http.Handler) http.Handler { clientVersion, err := version.NewVersion(requestHeaderClientVersion) if err != nil { logger.Error("Error while creating requestHeaderClientVersion", err.Error()) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(fmt.Sprintf(status.ClientVersionInvalidMessage, "Minimum Client Version Config is Missing"))) + return } minClientVersion, err := version.NewVersion(config.MinClientVersion()) if err != nil { - logger.Error("Error while creating minClientVersion", err.Error()) + logger.Error("Error while creating minClientVersion ", err.Error()) + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(fmt.Sprintf(status.ServerConfigErrorMessage, "Minimum Client Version Config is Missing"))) + return } if clientVersion.LessThan(minClientVersion) { - w.WriteHeader(400) + w.WriteHeader(http.StatusForbidden) _, _ = w.Write([]byte(fmt.Sprintf(status.ClientOutdatedErrorMessage, clientVersion))) return } + next.ServeHTTP(w, r) } else { next.ServeHTTP(w, r) diff --git a/test/config/proctor.yaml b/test/config/proctor.yaml new file mode 100644 index 00000000..0e0d200d --- /dev/null +++ b/test/config/proctor.yaml @@ -0,0 +1,3 @@ +PROCTOR_HOST: localhost:5000 +EMAIL_ID: deny.prasetyo@go-pay.co.id +ACCESS_TOKEN: PROCTORACCESSTOKENHERE From 2c060613cc6b520cb4a49c956e76d8175bfc2c5b Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 6 Aug 2019 16:48:18 +0700 Subject: [PATCH 080/268] [Jasoet|Bimo.horizon] Fixing Proctor CLI --- Makefile | 12 ++++++- .../app/cli/command/execution/executioner.go | 6 ++-- .../app/cli/command/version/version_test.go | 2 +- internal/app/cli/daemon/client.go | 34 +++++++++--------- internal/app/cli/daemon/client_mock.go | 13 +++---- internal/app/cli/daemon/client_test.go | 16 ++++----- .../app/service/execution/handler/http.go | 36 ++++++++++--------- .../service/execution/handler/http_test.go | 35 ++++++++++-------- .../service/execution/service/execution.go | 3 +- internal/app/service/server/api.go | 4 +-- .../middleware/validate_client_version.go | 4 +-- .../validate_client_version_test.go | 2 +- internal/pkg/model/execution/result.go | 10 ++++++ 13 files changed, 102 insertions(+), 75 deletions(-) create mode 100644 internal/pkg/model/execution/result.go diff --git a/Makefile b/Makefile index bc950d1c..8ccc2001 100644 --- a/Makefile +++ b/Makefile @@ -62,6 +62,7 @@ db.rollback: server db.teardown: -PGPASSWORD=$(PROCTOR_POSTGRES_PASSWORD) psql -h $(PROCTOR_POSTGRES_HOST) -p $(PROCTOR_POSTGRES_PORT) -c 'drop database $(PROCTOR_POSTGRES_DATABASE);' -U $(PROCTOR_POSTGRES_USER) + redis-cli FLUSHALL .PHONY: ftest.package.procs ftest.package.procs: @@ -69,12 +70,21 @@ ftest.package.procs: ruby ./test/package_procs.rb ftest.update.metadata: - $(BIN_DIR)/server s & \ PROCTOR_JOBS_PATH=$(FTEST_DIR) \ PROCTOR_URI=http://localhost:$(PROCTOR_APP_PORT)/metadata \ ruby ./test/update_metadata.rb +ftest.update.secret: + curl -X POST \ + http://localhost:5000/secret \ + -H 'Content-Type: application/json' \ + -d '{"job_name": "say-hello-world","secrets": {"SAMPLE_SECRET_ONE": "Secret One :*","SAMPLE_SECRET_TWO": "Secret Two :V"}}' + ftest.proctor.list: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli list +ftest.proctor.describe: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli describe say-hello-world +ftest.proctor.execute: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli execute say-hello-world SAMPLE_ARG_ONE=foo SAMPLE_ARG_TWO=bar diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index 513d16d3..741ec52d 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -43,7 +43,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in printer.Println("With No Variables", color.FgRed) } - executedProcName, err := proctorDClient.ExecuteProc(procName, procArgs) + executionResult, err := proctorDClient.ExecuteProc(procName, procArgs) if err != nil { printer.Println(err.Error(), color.FgRed) print() @@ -52,7 +52,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in } printer.Println("Proc submitted for execution. \nStreaming logs:", color.FgGreen) - err = proctorDClient.StreamProcLogs(executedProcName) + err = proctorDClient.StreamProcLogs(executionResult.ExecutionId) if err != nil { printer.Println("Error Streaming Logs", color.FgRed) osExitFunc(1) @@ -61,7 +61,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in printer.Println("Log stream of proc completed.", color.FgGreen) - procExecutionStatus, err := proctorDClient.GetDefinitiveProcExecutionStatus(executedProcName) + procExecutionStatus, err := proctorDClient.GetDefinitiveProcExecutionStatus(executionResult.ExecutionId) if err != nil { printer.Println("Error Fetching Proc execution status", color.FgRed) osExitFunc(1) diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 524a6789..72726e7b 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v0.6.0" + version := "v2.0.0" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gojektech", "proctor").Return(version, nil) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 51022d63..a6157143 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -16,6 +16,7 @@ import ( "proctor/internal/app/cli/config" "proctor/internal/pkg/constant" "proctor/internal/pkg/io" + "proctor/internal/pkg/model/execution" "time" "github.com/briandowns/spinner" @@ -29,15 +30,14 @@ const ( ExecutionRoute string = "/execution" ExecutionLogsRoute string = "/execution/logs" MetadataRoute string = "/metadata" - SecretRoute string = "/secret" ScheduleRoute string = "/schedule" ) type Client interface { ListProcs() ([]modelMetadata.Metadata, error) - ExecuteProc(string, map[string]string) (string, error) - StreamProcLogs(string) error - GetDefinitiveProcExecutionStatus(string) (string, error) + ExecuteProc(string, map[string]string) (*execution.ExecutionResult, error) + StreamProcLogs(executionId uint64) error + GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) ScheduleJob(string, string, string, string, string, map[string]string) (string, error) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) DescribeScheduledProc(string) (modelSchedule.ScheduledJob, error) @@ -253,10 +253,10 @@ func (c *client) RemoveScheduledProc(jobID string) error { return nil } -func (c *client) ExecuteProc(name string, args map[string]string) (string, error) { +func (c *client) ExecuteProc(name string, args map[string]string) (*execution.ExecutionResult, error) { err := c.loadProctorConfig() if err != nil { - return "", err + return nil, err } procToExecute := ProcToExecute{ @@ -266,7 +266,7 @@ func (c *client) ExecuteProc(name string, args map[string]string) (string, error requestBody, err := json.Marshal(procToExecute) if err != nil { - return "", err + return nil, err } client := &http.Client{} @@ -277,21 +277,21 @@ func (c *client) ExecuteProc(name string, args map[string]string) (string, error req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) resp, err := client.Do(req) if err != nil { - return "", buildNetworkError(err) + return nil, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { - return "", buildHTTPError(c, resp) + return nil, buildHTTPError(c, resp) } - var executedProc ProcToExecute - err = json.NewDecoder(resp.Body).Decode(&executedProc) + var executionResult execution.ExecutionResult + err = json.NewDecoder(resp.Body).Decode(&executionResult) - return executedProc.Name, err + return &executionResult, err } -func (c *client) StreamProcLogs(name string) error { +func (c *client) StreamProcLogs(executionId uint64) error { err := c.loadProctorConfig() if err != nil { return err @@ -305,7 +305,7 @@ func (c *client) StreamProcLogs(name string) error { signal.Notify(interrupt, os.Interrupt) proctodWebsocketURL := url.URL{Scheme: "ws", Host: c.proctordHost, Path: ExecutionLogsRoute} - proctodWebsocketURLWithProcName := proctodWebsocketURL.String() + "?" + "job_name=" + name + proctodWebsocketURLWithProcName := fmt.Sprintf("%s?context_id=%v", proctodWebsocketURL.String(), executionId) headers := make(map[string][]string) token := []string{c.accessToken} @@ -354,7 +354,7 @@ func (c *client) StreamProcLogs(name string) error { } } -func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, error) { +func (c *client) GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) { err := c.loadProctorConfig() if err != nil { return "", err @@ -365,7 +365,7 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro Timeout: c.connectionTimeoutSecs, } - req, err := http.NewRequest("GET", "http://"+c.proctordHost+ExecutionRoute+"/"+procName+"/status", nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s%s/%v/status", c.proctordHost, ExecutionRoute, executionId), nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) @@ -392,7 +392,7 @@ func (c *client) GetDefinitiveProcExecutionStatus(procName string) (string, erro time.Sleep(time.Duration(count) * 100 * time.Millisecond) } - return "", errors.New(fmt.Sprintf("No definitive status received for proc name %s from proctord", procName)) + return "", errors.New(fmt.Sprintf("No definitive status received for execution with id %v from proctord", executionId)) } func buildNetworkError(err error) error { diff --git a/internal/app/cli/daemon/client_mock.go b/internal/app/cli/daemon/client_mock.go index d9be1356..2246101c 100644 --- a/internal/app/cli/daemon/client_mock.go +++ b/internal/app/cli/daemon/client_mock.go @@ -2,6 +2,7 @@ package daemon import ( "github.com/stretchr/testify/mock" + "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" modelSchedule "proctor/internal/pkg/model/schedule" ) @@ -20,18 +21,18 @@ func (m *MockClient) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) return args.Get(0).([]modelSchedule.ScheduledJob), args.Error(1) } -func (m *MockClient) ExecuteProc(name string, procArgs map[string]string) (string, error) { +func (m *MockClient) ExecuteProc(name string, procArgs map[string]string) (*execution.ExecutionResult, error) { args := m.Called(name, procArgs) - return args.Get(0).(string), args.Error(1) + return args.Get(0).(*execution.ExecutionResult), args.Error(1) } -func (m *MockClient) StreamProcLogs(name string) error { - args := m.Called(name) +func (m *MockClient) StreamProcLogs(executionId uint64) error { + args := m.Called(executionId) return args.Error(0) } -func (m *MockClient) GetDefinitiveProcExecutionStatus(name string) (string, error) { - args := m.Called(name) +func (m *MockClient) GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) { + args := m.Called(executionId) return args.Get(0).(string), args.Error(1) } diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 765824fa..7d1df862 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -525,7 +525,7 @@ func (s *ClientTestSuite) TestLogStreamForAuthorizedUser() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - err := s.testClient.StreamProcLogs("test-job-id") + err := s.testClient.StreamProcLogs(uint64(42)) assert.NoError(t, err) s.mockConfigLoader.AssertExpectations(t) } @@ -541,7 +541,7 @@ func (s *ClientTestSuite) TestLogStreamForBadWebSocketHandshake() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - errStreamLogs := s.testClient.StreamProcLogs("test-job-id") + errStreamLogs := s.testClient.StreamProcLogs(uint64(42)) assert.Equal(t, errors.New("websocket: bad handshake"), errStreamLogs) s.mockConfigLoader.AssertExpectations(t) } @@ -559,7 +559,7 @@ func (s *ClientTestSuite) TestLogStreamForUnauthorizedUser() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - errStreamLogs := s.testClient.StreamProcLogs("test-job-id") + errStreamLogs := s.testClient.StreamProcLogs(uint64(42)) assert.Error(t, errors.New(http.StatusText(http.StatusUnauthorized)), errStreamLogs) s.mockConfigLoader.AssertExpectations(t) @@ -594,7 +594,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") + procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) assert.NoError(t, err) s.mockConfigLoader.AssertExpectations(t) @@ -630,7 +630,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") + procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) assert.NoError(t, err) s.mockConfigLoader.AssertExpectations(t) @@ -663,7 +663,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") + procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/some-proc-name/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) s.mockConfigLoader.AssertExpectations(t) @@ -696,7 +696,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") + procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) assert.Equal(t, errors.New("Server Error!!!\nStatus Code: 500, Internal Server Error"), err) s.mockConfigLoader.AssertExpectations(t) @@ -736,7 +736,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus("some-proc-name") + procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) assert.Equal(t, errors.New("No definitive status received for proc name some-proc-name from proctord"), err) s.mockConfigLoader.AssertExpectations(t) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 64e48f0d..2e41452e 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "proctor/internal/pkg/model/execution" "strconv" "strings" "time" @@ -135,18 +136,19 @@ func (httpHandler *executionHTTPHandler) GetStatus() http.HandlerFunc { return } - responseMap := map[string]string{ - "ExecutionId": fmt.Sprint(context.ExecutionID), - "JobName": context.JobName, - "ImageTag": context.ImageTag, - "CreatedAt": context.CreatedAt.String(), - "Status": string(context.Status), + responseBody := &execution.ExecutionResult{ + ExecutionId: context.ExecutionID, + JobName: context.JobName, + ExecutionName: context.Name, + ImageTag: context.ImageTag, + CreatedAt: context.CreatedAt.String(), + Status: string(context.Status), } response.WriteHeader(http.StatusOK) - responseJson, err := json.Marshal(responseMap) - logger.LogErrors(err, "marshal json from: ", responseMap) + responseJson, err := json.Marshal(responseBody) + logger.LogErrors(err, "marshal json from: ", responseBody) _, _ = response.Write(responseJson) @@ -180,19 +182,19 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { return } - responseMap := map[string]string{ - "ExecutionId": fmt.Sprint(context.ExecutionID), - "JobName": context.JobName, - "ExecutionName": executionName, - "ImageTag": context.ImageTag, - "CreatedAt": context.CreatedAt.String(), - "Status": string(context.Status), + responseBody := &execution.ExecutionResult{ + ExecutionId: context.ExecutionID, + JobName: context.JobName, + ExecutionName: executionName, + ImageTag: context.ImageTag, + CreatedAt: context.CreatedAt.String(), + Status: string(context.Status), } response.WriteHeader(http.StatusCreated) - responseJson, err := json.Marshal(responseMap) - logger.LogErrors(err, "marshal json from: ", responseMap) + responseJson, err := json.Marshal(responseBody) + logger.LogErrors(err, "marshal json from: ", responseBody) _, _ = response.Write(responseJson) return diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 023ed06e..6185950b 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "net/http/httptest" + "proctor/internal/pkg/model/execution" "strings" "testing" "time" @@ -165,20 +166,23 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetStatusH ExecutionID: executionContextId, UserEmail: userEmail, JobName: job.Name, + Name: "execution_name", ImageTag: "test", Args: job.Args, CreatedAt: time.Now(), Status: status.Finished, } - responseMap := map[string]string{ - "ExecutionId": fmt.Sprint(executionContextId), - "JobName": context.JobName, - "ImageTag": context.ImageTag, - "CreatedAt": context.CreatedAt.String(), - "Status": string(context.Status), + + expectedResponse := &execution.ExecutionResult{ + ExecutionId: context.ExecutionID, + JobName: context.JobName, + ExecutionName: context.Name, + ImageTag: context.ImageTag, + CreatedAt: context.CreatedAt.String(), + Status: string(context.Status), } - responseBody, err := json.Marshal(responseMap) + responseBody, err := json.Marshal(expectedResponse) assert.NoError(t, err) req := httptest.NewRequest("GET", fmt.Sprintf("/execution/%s/status", fmt.Sprint(executionContextId)), bytes.NewReader([]byte(""))) @@ -253,22 +257,23 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHa context := &model.ExecutionContext{ UserEmail: userEmail, JobName: job.Name, + Name: "test", Args: job.Args, Status: status.Finished, } - responseMap := map[string]string{ - "CreatedAt": context.CreatedAt.String(), - "ExecutionId": "0", - "ExecutionName": "test", - "ImageTag": "", - "JobName": context.JobName, - "Status": string(context.Status), + expectedResponse := &execution.ExecutionResult{ + ExecutionId: context.ExecutionID, + JobName: context.JobName, + ExecutionName: context.Name, + ImageTag: context.ImageTag, + CreatedAt: context.CreatedAt.String(), + Status: string(context.Status), } requestBody, err := json.Marshal(job) assert.NoError(t, err) - responseBody, err := json.Marshal(responseMap) + responseBody, err := json.Marshal(expectedResponse) assert.NoError(t, err) req := httptest.NewRequest("POST", "/execute", bytes.NewReader(requestBody)) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 06e5d75d..c8f94afb 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -102,8 +102,6 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st Status: status.Created, } - defer service.save(context) - metadata, err := service.metadataRepository.GetByName(jobName) if err != nil { context.Status = status.RequirementNotMet @@ -139,6 +137,7 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st go service.watchProcess(executionName, context) + defer service.save(context) return &context, executionName, nil } diff --git a/internal/app/service/server/api.go b/internal/app/service/server/api.go index d9ff7daf..08f7720d 100644 --- a/internal/app/service/server/api.go +++ b/internal/app/service/server/api.go @@ -44,11 +44,11 @@ func Start() error { if shutdownErr := httpServer.Shutdown(context.Background()); shutdownErr != nil { logger.Error("Received an Interrupt Signal", shutdownErr) } - close(idleConnsClosed) }() - if err = httpServer.ListenAndServe(); err != http.ErrServerClosed { + if err = httpServer.ListenAndServe(); err != nil { logger.Error("HTTP Server Failed ", err) + close(idleConnsClosed) } <-idleConnsClosed diff --git a/internal/app/service/server/middleware/validate_client_version.go b/internal/app/service/server/middleware/validate_client_version.go index 3a10c712..9fc9d7b0 100644 --- a/internal/app/service/server/middleware/validate_client_version.go +++ b/internal/app/service/server/middleware/validate_client_version.go @@ -19,7 +19,7 @@ func ValidateClientVersion(next http.Handler) http.Handler { clientVersion, err := version.NewVersion(requestHeaderClientVersion) if err != nil { logger.Error("Error while creating requestHeaderClientVersion", err.Error()) - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(fmt.Sprintf(status.ClientVersionInvalidMessage, "Minimum Client Version Config is Missing"))) return } @@ -33,7 +33,7 @@ func ValidateClientVersion(next http.Handler) http.Handler { } if clientVersion.LessThan(minClientVersion) { - w.WriteHeader(http.StatusForbidden) + w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(fmt.Sprintf(status.ClientOutdatedErrorMessage, clientVersion))) return } diff --git a/internal/app/service/server/middleware/validate_client_version_test.go b/internal/app/service/server/middleware/validate_client_version_test.go index 19195fae..e0a6d141 100644 --- a/internal/app/service/server/middleware/validate_client_version_test.go +++ b/internal/app/service/server/middleware/validate_client_version_test.go @@ -69,5 +69,5 @@ func TestInvalidClientVersionHTTPHeader(t *testing.T) { assert.Equal(t, http.StatusBadRequest, resp.StatusCode) body, _ := ioutil.ReadAll(resp.Body) bodyString := string(body) - assert.Equal(t, bodyString, "Your Proctor client is using an outdated version: 0.1.0 . To continue using proctor, please upgrade to latest version.") + assert.Equal(t, bodyString, "Your Proctor client is using an outdated version: 0.1.0. To continue using proctor, please upgrade to latest version.") } diff --git a/internal/pkg/model/execution/result.go b/internal/pkg/model/execution/result.go new file mode 100644 index 00000000..ca5dfa36 --- /dev/null +++ b/internal/pkg/model/execution/result.go @@ -0,0 +1,10 @@ +package execution + +type ExecutionResult struct { + ExecutionId uint64 `json:"id"` + JobName string `json:"job_name"` + ExecutionName string `json:"name"` + ImageTag string `json:"image_tag"` + CreatedAt string `json:"created_at"` + Status string `json:"status"` +} From 551408a62d53bb72e2aef6b6f36fdd8ea7ed5a1d Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 7 Aug 2019 10:56:57 +0700 Subject: [PATCH 081/268] Fix some failing tests --- internal/app/cli/daemon/client_test.go | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 7d1df862..361de362 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -5,11 +5,13 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" + "testing" + "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/config" "proctor/internal/pkg/io" - "strings" - "testing" + "proctor/internal/pkg/model/execution" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" @@ -260,8 +262,16 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { func (s *ClientTestSuite) TestExecuteProc() { t := s.T() + executionName := "proctor-777b1dfb-ea27-46d9-b02c-839b75a542e2" proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - expectedProcResponse := "proctor-777b1dfb-ea27-46d9-b02c-839b75a542e2" + expectedProcResponse := &execution.ExecutionResult{ + ExecutionId: uint64(0), + JobName: "", + ExecutionName: executionName, + ImageTag: "", + CreatedAt: "", + Status: "", + } body := `{ "name": "proctor-777b1dfb-ea27-46d9-b02c-839b75a542e2"}` procName := "run-sample" procArgs := map[string]string{"SAMPLE_ARG1": "sample-value"} @@ -376,7 +386,6 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { func (s *ClientTestSuite) TestExecuteProcInternalServerError() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - expectedProcResponse := "" procName := "run-sample" procArgs := map[string]string{"SAMPLE_ARG1": "sample-value"} @@ -402,6 +411,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) + var expectedProcResponse *execution.ExecutionResult assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) assert.Equal(t, expectedProcResponse, executeProcResponse) s.mockConfigLoader.AssertExpectations(t) @@ -434,7 +444,8 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { executeProcResponse, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - assert.Equal(t, "", executeProcResponse) + var expectedProcResponse *execution.ExecutionResult + assert.Equal(t, expectedProcResponse, executeProcResponse) assert.Equal(t, "Unauthorized Access!!!\nPlease check the EMAIL_ID and ACCESS_TOKEN validity in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -466,7 +477,8 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS executeProcResponse, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - assert.Equal(t, "", executeProcResponse) + var expectedProcResponse *execution.ExecutionResult + assert.Equal(t, expectedProcResponse, executeProcResponse) assert.Equal(t, "Unauthorized Access!!!\nEMAIL_ID or ACCESS_TOKEN is not present in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -498,7 +510,8 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { response, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - assert.Equal(t, "", response) + var expectedProcResponse *execution.ExecutionResult + assert.Equal(t, expectedProcResponse, response) assert.Equal(t, errors.New("Network Error!!!\nPost http://proctor.example.com/execution: Unknown Error"), err) s.mockConfigLoader.AssertExpectations(t) } From c662fd1f5da286f9f68c68563ab29e3f7cd85957 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 7 Aug 2019 11:49:24 +0700 Subject: [PATCH 082/268] [Jasoet|Bimo.horizon] Fix Failed Test --- .../cli/command/execution/executioner_test.go | 61 +++++++++++++------ internal/app/cli/daemon/client_test.go | 14 ++--- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index 1eb096e8..fb8de6ff 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/mock" "proctor/internal/app/cli/daemon" "proctor/internal/pkg/io" + "proctor/internal/pkg/model/execution" "testing" "github.com/fatih/color" @@ -49,14 +50,18 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmd() { s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "SAMPLE_ARG_ONE", "any"), color.Reset).Once() s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "SAMPLE_ARG_TWO", "variable"), color.Reset).Once() - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } + + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -71,15 +76,19 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForNoProcVariables() { s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() s.mockPrinter.On("Println", "With No Variables", color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } + procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -95,15 +104,19 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForIncorrectVariableFormat() { s.mockPrinter.On("Println", "With Variables", color.FgMagenta).Once() s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "\nIncorrect variable format\n", "incorrect-format"), color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } + procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobSucceeded, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -118,8 +131,11 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDExecutionFailure() { s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() s.mockPrinter.On("Println", "With No Variables", color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("", errors.New("test error")).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, errors.New("test error")).Once() s.mockPrinter.On("Println", mock.Anything, color.FgRed).Once() @@ -139,12 +155,15 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDLogStreamingFailure() s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() s.mockPrinter.On("Println", "With No Variables", color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(errors.New("error")).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(errors.New("error")).Once() s.mockPrinter.On("Println", "Error Streaming Logs", color.FgRed).Once() osExitFunc := func(exitCode int) { @@ -163,15 +182,18 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() s.mockPrinter.On("Println", "With No Variables", color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return("", errors.New("some error")).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return("", errors.New("some error")).Once() s.mockPrinter.On("Println", "Error Fetching Proc execution status", color.FgRed).Once() osExitFunc := func(exitCode int) { @@ -190,15 +212,18 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() s.mockPrinter.On("Println", "With No Variables", color.FgRed).Once() + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + } procArgs := make(map[string]string) - s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return("executed-proc-name", nil).Once() + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", "executed-proc-name").Return(nil).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", "executed-proc-name").Return(constant.JobFailed, nil).Once() + s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobFailed, nil).Once() s.mockPrinter.On("Println", "Proc execution failed", color.FgRed).Once() osExitFunc := func(exitCode int) { diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 361de362..0350d955 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -592,7 +592,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, responseBody), nil }, @@ -628,7 +628,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(200, responseBody), nil }, @@ -661,7 +661,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} }, @@ -678,7 +678,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) - assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/some-proc-name/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) + assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/42/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, "", procExecutionStatus) } @@ -694,7 +694,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { return httpmock.NewStringResponse(500, ""), nil }, @@ -733,7 +733,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/some-proc-name/status", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { requestsToProctorDCount += 1 return httpmock.NewStringResponse(200, responseBody), nil @@ -751,7 +751,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) - assert.Equal(t, errors.New("No definitive status received for proc name some-proc-name from proctord"), err) + assert.Equal(t, errors.New("No definitive status received for execution with id 42 from proctord"), err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, expectedRequestsToProctorDCount, requestsToProctorDCount) assert.Equal(t, "", procExecutionStatus) From fcffb87cc690387601b5dea302c5e2ca19c0abb3 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 7 Aug 2019 13:28:58 +0700 Subject: [PATCH 083/268] [Jasoet|Bimo.horizon] Make CLI message more descriptive --- .../app/cli/command/execution/executioner.go | 26 +++----- .../cli/command/execution/executioner_test.go | 61 +++++++++++-------- internal/app/cli/daemon/client.go | 4 ++ internal/app/cli/daemon/client_test.go | 24 ++++---- .../app/service/execution/handler/http.go | 14 +++-- .../service/execution/handler/http_test.go | 4 +- .../execution/handler/status/status.go | 2 +- internal/app/service/server/router.go | 2 +- test/procs/say-hello-world/say_hello_world.sh | 14 +---- 9 files changed, 73 insertions(+), 78 deletions(-) diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index 741ec52d..f315fa8a 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -8,7 +8,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/internal/pkg/constant" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { @@ -51,30 +50,19 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in return } - printer.Println("Proc submitted for execution. \nStreaming logs:", color.FgGreen) - err = proctorDClient.StreamProcLogs(executionResult.ExecutionId) - if err != nil { - printer.Println("Error Streaming Logs", color.FgRed) - osExitFunc(1) - return - } - - printer.Println("Log stream of proc completed.", color.FgGreen) + printer.Println("\nExecution Created", color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100s", "Name", executionResult.ExecutionName), color.FgGreen) - procExecutionStatus, err := proctorDClient.GetDefinitiveProcExecutionStatus(executionResult.ExecutionId) + printer.Println("\nStreaming logs", color.FgGreen) + err = proctorDClient.StreamProcLogs(executionResult.ExecutionId) if err != nil { - printer.Println("Error Fetching Proc execution status", color.FgRed) - osExitFunc(1) - return - } - - if procExecutionStatus != constant.JobSucceeded { - printer.Println("Proc execution failed", color.FgRed) + printer.Println("Error while Streaming Log.", color.FgRed) osExitFunc(1) return } - printer.Println("Proc execution successful", color.FgGreen) + printer.Println("Execution completed.", color.FgGreen) }, } } diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index fb8de6ff..6864e4fb 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -13,7 +13,6 @@ import ( "github.com/spf13/cobra" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" - "proctor/internal/pkg/constant" ) type ExecutionCmdTestSuite struct { @@ -52,17 +51,18 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmd() { executionResult := &execution.ExecutionResult{ ExecutionId: uint64(42), + ExecutionName: "Mantoel", } s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() - s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() - s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -83,13 +83,14 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForNoProcVariables() { procArgs := make(map[string]string) s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() - s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() - s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -111,13 +112,14 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForIncorrectVariableFormat() { procArgs := make(map[string]string) s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() - s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobSucceeded, nil).Once() - s.mockPrinter.On("Println", "Proc execution successful", color.FgGreen).Once() + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() s.testExecutionCmd.Run(&cobra.Command{}, args) @@ -161,10 +163,14 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDLogStreamingFailure() procArgs := make(map[string]string) s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(errors.New("error")).Once() - s.mockPrinter.On("Println", "Error Streaming Logs", color.FgRed).Once() + + s.mockPrinter.On("Println", "Error while Streaming Log.", color.FgRed).Once() osExitFunc := func(exitCode int) { assert.Equal(s.T(), 1, exitCode) @@ -185,16 +191,18 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec executionResult := &execution.ExecutionResult{ ExecutionId: uint64(42), } + procArgs := make(map[string]string) s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() - s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(errors.New("error")).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return("", errors.New("some error")).Once() - s.mockPrinter.On("Println", "Error Fetching Proc execution status", color.FgRed).Once() + s.mockPrinter.On("Println", "Error while Streaming Log.", color.FgRed).Once() osExitFunc := func(exitCode int) { assert.Equal(s.T(), 1, exitCode) @@ -218,13 +226,14 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec procArgs := make(map[string]string) s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() - s.mockPrinter.On("Println", "Proc submitted for execution. \nStreaming logs:", color.FgGreen).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() - s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() - s.mockPrinter.On("Println", "Log stream of proc completed.", color.FgGreen).Once() + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(errors.New("error")).Once() - s.mockProctorDClient.On("GetDefinitiveProcExecutionStatus", executionResult.ExecutionId).Return(constant.JobFailed, nil).Once() - s.mockPrinter.On("Println", "Proc execution failed", color.FgRed).Once() + s.mockPrinter.On("Println", "Error while Streaming Log.", color.FgRed).Once() osExitFunc := func(exitCode int) { assert.Equal(s.T(), 1, exitCode) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index a6157143..6e86061a 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -426,6 +426,10 @@ func buildHTTPError(c *client, resp *http.Response) error { return fmt.Errorf(constant.JobForbiddenErrorHeader) } + if resp.StatusCode == http.StatusInternalServerError { + return getHTTPResponseError(resp.Body) + } + return fmt.Errorf("%s\nStatus Code: %d, %s", constant.GenericResponseErrorHeader, resp.StatusCode, http.StatusText(resp.StatusCode)) } diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 0350d955..9a22d5b8 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -108,7 +108,7 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { "GET", "http://"+proctorConfig.Host+MetadataRoute, func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, `{}`), nil + return httpmock.NewStringResponse(500, "list proc error"), nil }, ).WithHeader( &http.Header{ @@ -126,7 +126,7 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { assert.Equal(t, []modelMetadata.Metadata{}, procList) assert.Error(t, err) s.mockConfigLoader.AssertExpectations(t) - assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) + assert.Equal(t, "list proc error", err.Error()) } func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { @@ -397,7 +397,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { "POST", "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, ""), nil + return httpmock.NewStringResponse(500, "Execute Error"), nil }, ).WithHeader( &http.Header{ @@ -412,7 +412,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) var expectedProcResponse *execution.ExecutionResult - assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) + assert.Equal(t, "Execute Error", err.Error()) assert.Equal(t, expectedProcResponse, executeProcResponse) s.mockConfigLoader.AssertExpectations(t) } @@ -696,7 +696,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, ""), nil + return httpmock.NewStringResponse(500, "execute Error"), nil }, ).WithHeader( &http.Header{ @@ -711,7 +711,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) - assert.Equal(t, errors.New("Server Error!!!\nStatus Code: 500, Internal Server Error"), err) + assert.Equal(t, errors.New("execute Error"), err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, "", procExecutionStatus) } @@ -874,7 +874,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, ""), nil + return httpmock.NewStringResponse(500, "Schedule Failed"), nil }, ).WithHeader( &http.Header{ @@ -889,7 +889,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { _, err := s.testClient.DescribeScheduledProc(jobID) - assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) + assert.Equal(t, "Schedule Failed", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -975,7 +975,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, ""), nil + return httpmock.NewStringResponse(500, "Schedule Error"), nil }, ).WithHeader( &http.Header{ @@ -990,7 +990,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal _, err := s.testClient.ListScheduledProcs() - assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) + assert.Equal(t, "Schedule Error", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -1108,7 +1108,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, ""), nil + return httpmock.NewStringResponse(500, "Schedule Error"), nil }, ).WithHeader( &http.Header{ @@ -1123,6 +1123,6 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { err := s.testClient.RemoveScheduledProc(jobID) - assert.Equal(t, "Server Error!!!\nStatus Code: 500, Internal Server Error", err.Error()) + assert.Equal(t, "Schedule Error", err.Error()) s.mockConfigLoader.AssertExpectations(t) } diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 2e41452e..c46841cf 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -50,7 +50,7 @@ func NewExecutionHTTPHandler( func (httpHandler *executionHTTPHandler) closeWebSocket(message string, conn *websocket.Conn) { err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, message)) - logger.LogErrors(err, "close WebSocket") + logger.LogErrors(err, "close WebSocket ", message) return } @@ -74,7 +74,7 @@ func (httpHandler *executionHTTPHandler) GetLogs() http.HandlerFunc { } context, err := httpHandler.repository.GetById(executionContextId) - + logger.LogErrors(err, "fetch context from repository ", *context) if err != nil { logger.Error("No execution context id found ", executionContextId) httpHandler.closeWebSocket(fmt.Sprintf("No execution context found with id %v", executionContextId), conn) @@ -82,13 +82,15 @@ func (httpHandler *executionHTTPHandler) GetLogs() http.HandlerFunc { } if context.Status == executionStatus.Finished { + logger.Debug("Execution is Finished, return output from repository") err = conn.WriteMessage(websocket.TextMessage, context.Output) logger.LogErrors(err, "write output to socket:", string(context.Output)) - httpHandler.closeWebSocket("Finished Streaming log", conn) + httpHandler.closeWebSocket("Finished Streaming log from repository", conn) return } - if context.Status == executionStatus.PodReady { + if context.Status == executionStatus.Created { + logger.Debug("Execution is Created, Stream output from pod") waitTime := config.KubeLogProcessWaitTime() * time.Second podLog, _err := httpHandler.service.StreamJobLogs(context.Name, waitTime) @@ -107,7 +109,7 @@ func (httpHandler *executionHTTPHandler) GetLogs() http.HandlerFunc { for scanner.Scan() { _ = conn.WriteMessage(websocket.TextMessage, scanner.Bytes()) } - httpHandler.closeWebSocket("Finished Streaming log", conn) + httpHandler.closeWebSocket("Finished Streaming log from pod", conn) return } @@ -178,7 +180,7 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { if err != nil { response.WriteHeader(http.StatusInternalServerError) - _, _ = response.Write([]byte(fmt.Sprintf("%s , Errors Detail %s", status.JobExecutionError, err.Error()))) + _, _ = response.Write([]byte(fmt.Sprintf("%s, Errors Detail %s", status.JobExecutionError, err.Error()))) return } diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 6185950b..12036eef 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -125,7 +125,7 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhe ImageTag: "test", Args: job.Args, CreatedAt: time.Now(), - Status: status.PodReady, + Status: status.Created, Output: types.GzippedText("test"), } @@ -330,7 +330,7 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestGenericErrorJobExecutionPostHTTP suite.testExecutionHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusInternalServerError, responseRecorder.Code) - assert.Equal(t, fmt.Sprintf("%s , Errors Detail %s", handlerStatus.JobExecutionError, genericError), responseRecorder.Body.String()) + assert.Equal(t, fmt.Sprintf("%s, Errors Detail %s", handlerStatus.JobExecutionError, genericError), responseRecorder.Body.String()) } func TestExecutionHTTPHandlerTestSuite(t *testing.T) { diff --git a/internal/app/service/execution/handler/status/status.go b/internal/app/service/execution/handler/status/status.go index d1a1e7f8..2c4aff2e 100644 --- a/internal/app/service/execution/handler/status/status.go +++ b/internal/app/service/execution/handler/status/status.go @@ -4,7 +4,7 @@ type ExecutionHandlerStatus string const ( MalformedRequest ExecutionHandlerStatus = "Failed to parse request body from CLI" - JobExecutionError ExecutionHandlerStatus = "Failed to execute Job into Executor" + JobExecutionError ExecutionHandlerStatus = "Failed to execute Job" PathParameterError ExecutionHandlerStatus = "Failed to translate path parameter to uint64" WebSocketInitError ExecutionHandlerStatus = "WebSocket Initializer Error" ExecutionContextNotFound ExecutionHandlerStatus = "Execution context not Found" diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 1cf7d505..8d79b87f 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -64,7 +64,7 @@ func NewRouter() (*mux.Router, error) { router.Use(middleware.ValidateClientVersion) router.HandleFunc("/execution", executionHandler.Post()).Methods("POST") - router.HandleFunc("/execution/{contextID}/status", executionHandler.GetStatus()).Methods("GET") + router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") diff --git a/test/procs/say-hello-world/say_hello_world.sh b/test/procs/say-hello-world/say_hello_world.sh index f46f570c..4322d212 100755 --- a/test/procs/say-hello-world/say_hello_world.sh +++ b/test/procs/say-hello-world/say_hello_world.sh @@ -1,13 +1,5 @@ #!/usr/bin/env bash -set -euxo pipefail - -main() { - - echo "Hello World! -I received secrets: $SAMPLE_SECRET_ONE and $SAMPLE_SECRET_TWO -I received arguments: $SAMPLE_ARG_ONE and $SAMPLE_ARG_TWO" - -} - -main +echo "Hello World! + I received secrets: $SAMPLE_SECRET_ONE and $SAMPLE_SECRET_TWO + I received arguments: $SAMPLE_ARG_ONE and $SAMPLE_ARG_TWO" From aa9deb3d056926933803cb672391b60e56ad7c7f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 7 Aug 2019 13:35:40 +0700 Subject: [PATCH 084/268] [Jasoet|Bimo.horizon] Store image tag from metadata --- internal/app/service/execution/service/execution.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index c8f94afb..3709faa0 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -108,6 +108,7 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st return &context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", jobName, err.Error())) } + context.ImageTag = metadata.ImageName secret, err := service.secretRepository.GetByJobName(jobName) if err != nil { context.Status = status.RequirementNotMet From 6a6f1539b2ee2b146df0c3d883b33428ccae0247 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 7 Aug 2019 16:54:10 +0700 Subject: [PATCH 085/268] Add log command --- internal/app/cli/command/log/log.go | 43 ++++++++++++ internal/app/cli/command/log/log_test.go | 87 ++++++++++++++++++++++++ internal/app/cli/command/root.go | 9 ++- internal/app/cli/command/root_test.go | 1 + 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 internal/app/cli/command/log/log.go create mode 100644 internal/app/cli/command/log/log_test.go diff --git a/internal/app/cli/command/log/log.go b/internal/app/cli/command/log/log.go new file mode 100644 index 00000000..b752d39b --- /dev/null +++ b/internal/app/cli/command/log/log.go @@ -0,0 +1,43 @@ +package log + +import ( + "fmt" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" + "strconv" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { + return &cobra.Command{ + Use: "logs", + Short: "Get logs of an execution context", + Long: "To get a log of execution context, this command helps retrieve logs from previous execution", + Example: "proctor logs 123", + Args: cobra.MinimumNArgs(1), + + Run: func(cmd *cobra.Command, args []string) { + executionIDParam := args[0] + executionID, err := strconv.ParseUint(executionIDParam, 10, 64) + if executionIDParam == "" || err != nil { + printer.Println("No valid execution context id provided as argument", color.FgRed) + return + } + + printer.Println("Getting logs", color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen) + + printer.Println("\nStreaming logs", color.FgGreen) + err = proctorDClient.StreamProcLogs(executionID) + if err != nil { + printer.Println("Error while Streaming Log.", color.FgRed) + osExitFunc(1) + return + } + + printer.Println("Execution completed.", color.FgGreen) + }, + } +} diff --git a/internal/app/cli/command/log/log_test.go b/internal/app/cli/command/log/log_test.go new file mode 100644 index 00000000..e1b680e2 --- /dev/null +++ b/internal/app/cli/command/log/log_test.go @@ -0,0 +1,87 @@ +package log + +import ( + "errors" + "fmt" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" + "testing" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type LogCmdTestSuite struct { + suite.Suite + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testLogCmd *cobra.Command +} + +func (s *LogCmdTestSuite) SetupTest() { + s.mockPrinter = &io.MockPrinter{} + s.mockProctorDClient = &daemon.MockClient{} + s.testLogCmd = NewCmd(s.mockPrinter, s.mockProctorDClient, func(exitCode int) {}) +} + +func (s *LogCmdTestSuite) TestLogCmdUsage() { + assert.Equal(s.T(), "logs", s.testLogCmd.Use) +} + +func (s *LogCmdTestSuite) TestLogCmdHelp() { + assert.Equal(s.T(), "Get logs of an execution context", s.testLogCmd.Short) + assert.Equal(s.T(), "To get a log of execution context, this command helps retrieve logs from previous execution", s.testLogCmd.Long) + assert.Equal(s.T(), "proctor logs 123", s.testLogCmd.Example) +} + +func (s *LogCmdTestSuite) TestLogCmd() { + executionID := uint64(42) + + s.mockPrinter.On("Println", "Getting logs", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() + + s.mockProctorDClient.On("StreamProcLogs", executionID).Return(nil).Once() + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() + + s.testLogCmd.Run(&cobra.Command{}, []string{"42"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + +func (s *LogCmdTestSuite) TestLogCmdInvalidExecutionIDError() { + t := s.T() + + s.mockPrinter.On("Println", "No valid execution context id provided as argument", color.FgRed).Once() + + s.testLogCmd.Run(&cobra.Command{}, []string{"foo"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) + s.mockPrinter.AssertNotCalled(t, "Println", "Execution completed.", color.FgGreen) +} + +func (s *LogCmdTestSuite) TestLogCmdInvalidStreamProcLogsError() { + t := s.T() + + executionID := uint64(42) + + s.mockPrinter.On("Println", "Getting logs", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() + + s.mockProctorDClient.On("StreamProcLogs", executionID).Return(errors.New("test")).Once() + s.mockPrinter.On("Println", "Error while Streaming Log.", color.FgRed).Once() + s.testLogCmd.Run(&cobra.Command{}, []string{"42"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) + s.mockPrinter.AssertNotCalled(t, "Println", "Execution completed.", color.FgGreen) +} + +func TestLogCmdTestSuite(t *testing.T) { + suite.Run(t, new(LogCmdTestSuite)) +} diff --git a/internal/app/cli/command/root.go b/internal/app/cli/command/root.go index 0401b325..edc6f94c 100644 --- a/internal/app/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -3,11 +3,15 @@ package command import ( "fmt" "os" + + "github.com/spf13/cobra" + "proctor/internal/app/cli/command/config" "proctor/internal/app/cli/command/config/view" "proctor/internal/app/cli/command/description" "proctor/internal/app/cli/command/execution" "proctor/internal/app/cli/command/list" + "proctor/internal/app/cli/command/log" "proctor/internal/app/cli/command/schedule" scheduleDescribe "proctor/internal/app/cli/command/schedule/describe" scheduleList "proctor/internal/app/cli/command/schedule/list" @@ -16,8 +20,6 @@ import ( "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/daemon" "proctor/internal/pkg/io" - - "github.com/spf13/cobra" ) var ( @@ -39,6 +41,9 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith executionCmd := execution.NewCmd(printer, proctorDClient, os.Exit) rootCmd.AddCommand(executionCmd) + logCmd := log.NewCmd(printer, proctorDClient, os.Exit) + rootCmd.AddCommand(logCmd) + listCmd := list.NewCmd(printer, proctorDClient) rootCmd.AddCommand(listCmd) diff --git a/internal/app/cli/command/root_test.go b/internal/app/cli/command/root_test.go index 1fb8f835..9da0edf2 100644 --- a/internal/app/cli/command/root_test.go +++ b/internal/app/cli/command/root_test.go @@ -32,6 +32,7 @@ func TestRootCmdSubCommands(t *testing.T) { assert.True(t, contains(rootCmd.Commands(), "describe")) assert.True(t, contains(rootCmd.Commands(), "execute")) + assert.True(t, contains(rootCmd.Commands(), "logs")) assert.True(t, contains(rootCmd.Commands(), "help")) assert.True(t, contains(rootCmd.Commands(), "list")) assert.True(t, contains(rootCmd.Commands(), "config")) From 22d6c2f3d57f0a6ba4127127ff1bf0e614d991d6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 7 Aug 2019 16:54:26 +0700 Subject: [PATCH 086/268] Change test mock value --- internal/app/cli/command/execution/executioner_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index 6864e4fb..22031cd4 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -50,8 +50,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmd() { s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "SAMPLE_ARG_TWO", "variable"), color.Reset).Once() executionResult := &execution.ExecutionResult{ - ExecutionId: uint64(42), - ExecutionName: "Mantoel", + ExecutionId: uint64(42), + ExecutionName: "Test", } s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() From 3dec7b2d4975e6206f093e5159881e0fe286cb0e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 8 Aug 2019 14:11:09 +0700 Subject: [PATCH 087/268] [Jasoet] Create new table for ScheduleContext --- .../infra/db/postgresql/client_test.go | 6 ++- internal/app/service/schedule/handler/http.go | 2 +- .../app/service/schedule/handler/http_test.go | 4 +- .../schedule/model/schedule_context.go | 11 ++++ .../service/schedule/repository/schedule.go | 4 +- .../schedule/repository/schedule_context.go | 53 +++++++++++++++++++ .../schedule/repository/schedule_mock.go | 2 +- .../schedule/repository/schedule_test.go | 10 ++-- .../server/middleware/status/status.go | 4 +- .../10_CreateExecutionContextTable.up.sql | 2 +- migrations/11_CreateScheduleTable.up.sql | 2 +- .../13_CreateScheduleContextTable.down.sql | 1 + .../13_CreateScheduleContextTable.up.sql | 13 +++++ 13 files changed, 97 insertions(+), 17 deletions(-) create mode 100644 internal/app/service/schedule/model/schedule_context.go create mode 100644 internal/app/service/schedule/repository/schedule_context.go create mode 100644 migrations/13_CreateScheduleContextTable.down.sql create mode 100644 migrations/13_CreateScheduleContextTable.up.sql diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index bd6176c9..d530550b 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -2,6 +2,7 @@ package postgresql import ( "fmt" + "proctor/internal/app/service/infra/id" "testing" "github.com/jmoiron/sqlx" @@ -56,15 +57,16 @@ func TestSelect(t *testing.T) { defer postgresClient.db.Close() jobName := "test-job-name" + snowflakeID, _ := id.NextID() executionContext := &executionContextModel.ExecutionContext{ + ExecutionID: snowflakeID, JobName: jobName, ImageTag: "test-image-name", - ExecutionID: uint64(1), Args: map[string]string{"foo": "bar"}, Status: executionContextStatus.Finished, } - _, err = postgresClient.NamedExec("INSERT INTO execution_context (job_name, image_tag, args, status) VALUES (:job_name, :image_tag, :args, :status)", executionContext) + _, err = postgresClient.NamedExec("INSERT INTO execution_context (id,job_name, image_tag, args, status) VALUES (:id, :job_name, :image_tag, :args, :status)", executionContext) assert.NoError(t, err) executionContextResult := []executionContextModel.ExecutionContext{} diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 311007ba..6185ce0d 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -107,7 +107,7 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { } schedule.Cron = fmt.Sprintf("0 %s", schedule.Cron) - schedule.ID, err = httpHandler.repository.Insert(&schedule) + schedule.ID, err = httpHandler.repository.Insert(schedule) if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { logger.Error(fmt.Sprintf("Duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) diff --git a/internal/app/service/schedule/handler/http_test.go b/internal/app/service/schedule/handler/http_test.go index 8415a7f7..9d83d99d 100644 --- a/internal/app/service/schedule/handler/http_test.go +++ b/internal/app/service/schedule/handler/http_test.go @@ -67,7 +67,7 @@ func (suite *ScheduleHTTPHandlerTestSuite) TestSuccessfulSchedulePostHTTPHandler defer suite.mockMetadataRepository.AssertExpectations(t) requestSchedule.Cron = "0 * * * * *" - suite.mockScheduleRepository.On("Insert", &requestSchedule).Return(0, nil).Once() + suite.mockScheduleRepository.On("Insert", requestSchedule).Return(0, nil).Once() defer suite.mockScheduleRepository.AssertExpectations(t) suite.testScheduleHTTPHandler.Post()(responseRecorder, req) @@ -136,7 +136,7 @@ func (suite *ScheduleHTTPHandlerTestSuite) TestErrorSchedulePostHTTPHandler() { suite.mockMetadataRepository.On("GetByName", requestSchedule.JobName).Return(&metadataModel.Metadata{}, nil).Once() defer suite.mockMetadataRepository.AssertExpectations(t) // Schedule duplicate job name and args error - suite.mockScheduleRepository.On("Insert", &requestSchedule).Return(0, errors.New("duplicate key value violates unique constraint")).Once() + suite.mockScheduleRepository.On("Insert", requestSchedule).Return(0, errors.New("duplicate key value violates unique constraint")).Once() defer suite.mockScheduleRepository.AssertExpectations(t) for _, errorTest := range schedulePostErrorTests { diff --git a/internal/app/service/schedule/model/schedule_context.go b/internal/app/service/schedule/model/schedule_context.go new file mode 100644 index 00000000..f1dd86e1 --- /dev/null +++ b/internal/app/service/schedule/model/schedule_context.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type ScheduleContext struct { + ID uint64 `json:"id" db:"id"` + ScheduleId uint64 `json:"scheduleId" db:"schedule_id"` + ExecutionContextId uint64 `json:"executionContextId" db:"execution_context_id"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` + UpdatedAt time.Time `json:"updatedAt" db:"updated_at"` +} diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go index 7275b7fb..5d1d45c3 100644 --- a/internal/app/service/schedule/repository/schedule.go +++ b/internal/app/service/schedule/repository/schedule.go @@ -9,7 +9,7 @@ import ( ) type ScheduleRepository interface { - Insert(context *model.Schedule) (uint64, error) + Insert(context model.Schedule) (uint64, error) Delete(id uint64) error GetByID(id uint64) (*model.Schedule, error) Disable(id uint64) error @@ -54,7 +54,7 @@ func (repository *scheduleRepository) Disable(id uint64) error { return err } -func (repository *scheduleRepository) Insert(context *model.Schedule) (uint64, error) { +func (repository *scheduleRepository) Insert(context model.Schedule) (uint64, error) { snowflakeID, _ := id.NextID() context.ID = snowflakeID sql := "INSERT INTO schedule (id, job_name, args,cron,notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :cron, :notification_emails, :user_email, :group, :enabled)" diff --git a/internal/app/service/schedule/repository/schedule_context.go b/internal/app/service/schedule/repository/schedule_context.go new file mode 100644 index 00000000..bfdf3c35 --- /dev/null +++ b/internal/app/service/schedule/repository/schedule_context.go @@ -0,0 +1,53 @@ +package repository + +import ( + executionModel "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/schedule/model" +) + +type ScheduleContextRepository interface { + Insert(context *model.ScheduleContext) (uint64, error) + Delete(id uint64) error + GetByID(id uint64) (*model.ScheduleContext, error) + GetContextByScheduleId(scheduleId uint64) ([]executionModel.ExecutionContext, error) + GetScheduleByContextId(contextId uint64) (*model.Schedule, error) + deleteAll() error +} + +type scheduleContextRepository struct { + postgresqlClient postgresql.Client +} + +func NewScheduleContextRepository(client postgresql.Client) ScheduleContextRepository { + return &scheduleContextRepository{ + postgresqlClient: client, + } +} + +func (repository *scheduleContextRepository) Insert(context *model.ScheduleContext) (uint64, error) { + panic("implement me") +} + +func (repository *scheduleContextRepository) Delete(id uint64) error { + panic("implement me") +} + +func (repository *scheduleContextRepository) GetByID(id uint64) (*model.ScheduleContext, error) { + panic("implement me") +} + +func (repository *scheduleContextRepository) GetContextByScheduleId(scheduleId uint64) ([]executionModel.ExecutionContext, error) { + panic("implement me") +} + +func (repository *scheduleContextRepository) GetScheduleByContextId(contextId uint64) (*model.Schedule, error) { + panic("implement me") +} + +func (repository *scheduleContextRepository) deleteAll() error { + sql := "DELETE FROM schedule_context" + schedules := model.ScheduleContext{} + _, err := repository.postgresqlClient.NamedExec(sql, schedules) + return err +} diff --git a/internal/app/service/schedule/repository/schedule_mock.go b/internal/app/service/schedule/repository/schedule_mock.go index 7d33c1bc..6dabef1e 100644 --- a/internal/app/service/schedule/repository/schedule_mock.go +++ b/internal/app/service/schedule/repository/schedule_mock.go @@ -9,7 +9,7 @@ type MockScheduleRepository struct { mock.Mock } -func (repository *MockScheduleRepository) Insert(context *model.Schedule) (uint64, error) { +func (repository *MockScheduleRepository) Insert(context model.Schedule) (uint64, error) { args := repository.Called(context) return uint64(args.Int(0)), args.Error(1) } diff --git a/internal/app/service/schedule/repository/schedule_test.go b/internal/app/service/schedule/repository/schedule_test.go index 143797a6..09d3ae26 100644 --- a/internal/app/service/schedule/repository/schedule_test.go +++ b/internal/app/service/schedule/repository/schedule_test.go @@ -33,7 +33,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Insert() { t := suite.T() mapKey := fake.FirstName() mapValue := fake.LastName() - schedule := &model.Schedule{ + schedule := model.Schedule{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), Args: map[string]string{ @@ -64,7 +64,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Delete() { t := suite.T() mapKey := fake.FirstName() mapValue := fake.LastName() - schedule := &model.Schedule{ + schedule := model.Schedule{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), Args: map[string]string{ @@ -172,7 +172,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledByID() { mapKey := fake.FirstName() mapValue := fake.LastName() - schedule := &model.Schedule{ + schedule := model.Schedule{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), Args: map[string]string{ @@ -205,7 +205,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { t := suite.T() mapKey := fake.FirstName() mapValue := fake.LastName() - schedule := &model.Schedule{ + schedule := model.Schedule{ JobName: fake.BuzzWord(), UserEmail: fake.Email(), Args: map[string]string{ @@ -263,7 +263,7 @@ func populateSeedDataForTest(repository ScheduleRepository, count int, seedField enabled, _ = strconv.ParseBool(val) } - schedule := &model.Schedule{ + schedule := model.Schedule{ JobName: jobName, UserEmail: email, Args: map[string]string{ diff --git a/internal/app/service/server/middleware/status/status.go b/internal/app/service/server/middleware/status/status.go index 828909e2..141eb978 100644 --- a/internal/app/service/server/middleware/status/status.go +++ b/internal/app/service/server/middleware/status/status.go @@ -1,7 +1,7 @@ package status const ( - ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s. To continue using proctor, please upgrade to latest version." + ClientOutdatedErrorMessage = "Your Proctor client is using an outdated version: %s. To continue using proctor, please upgrade to latest version." ClientVersionInvalidMessage = "Your Proctor client config is Invalid. Please contact your Proctor Administrator. Error: %s" - ServerConfigErrorMessage = "Your Proctor server configuration is invalid. Please contact your Proctor Administrator. Error: %s" + ServerConfigErrorMessage = "Your Proctor server configuration is invalid. Please contact your Proctor Administrator. Error: %s" ) diff --git a/migrations/10_CreateExecutionContextTable.up.sql b/migrations/10_CreateExecutionContextTable.up.sql index 8ec479a0..9d46edec 100644 --- a/migrations/10_CreateExecutionContextTable.up.sql +++ b/migrations/10_CreateExecutionContextTable.up.sql @@ -1,6 +1,6 @@ CREATE TABLE execution_context ( - id bigint, + id bigint not null primary key, job_name varchar(255), user_email varchar(255), image_tag text, diff --git a/migrations/11_CreateScheduleTable.up.sql b/migrations/11_CreateScheduleTable.up.sql index a37d1a3f..7a07b429 100644 --- a/migrations/11_CreateScheduleTable.up.sql +++ b/migrations/11_CreateScheduleTable.up.sql @@ -1,6 +1,6 @@ CREATE TABLE schedule ( - id bigint, + id bigint not null primary key, job_name varchar(255), args text, cron varchar(255), diff --git a/migrations/13_CreateScheduleContextTable.down.sql b/migrations/13_CreateScheduleContextTable.down.sql new file mode 100644 index 00000000..772ec83b --- /dev/null +++ b/migrations/13_CreateScheduleContextTable.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS schedule_context; diff --git a/migrations/13_CreateScheduleContextTable.up.sql b/migrations/13_CreateScheduleContextTable.up.sql new file mode 100644 index 00000000..6027e934 --- /dev/null +++ b/migrations/13_CreateScheduleContextTable.up.sql @@ -0,0 +1,13 @@ +CREATE TABLE schedule_context +( + id bigint not null primary key, + schedule_id bigint, + context_id bigint unique, + created_at timestamp default now(), + updated_at timestamp default now(), + UNIQUE (context_id), + FOREIGN KEY (context_id) references execution_context (id), + FOREIGN KEY (schedule_id) references schedule (id) +); + + From 30b7dac798a4c08182c89e341840be66fe46fc1d Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 9 Aug 2019 11:07:20 +0700 Subject: [PATCH 088/268] [Jasoet] Add test for Schedule Context Table --- .../execution/repository/execution_context.go | 4 +- .../repository/execution_context_mock.go | 2 +- .../repository/execution_context_test.go | 14 +- .../service/schedule/repository/schedule.go | 2 +- .../schedule/repository/schedule_context.go | 57 +++++- .../repository/schedule_context_mock.go | 41 ++++ .../repository/schedule_context_test.go | 190 ++++++++++++++++++ .../13_CreateScheduleContextTable.up.sql | 8 +- 8 files changed, 296 insertions(+), 22 deletions(-) create mode 100644 internal/app/service/schedule/repository/schedule_context_mock.go create mode 100644 internal/app/service/schedule/repository/schedule_context_test.go diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 0d927631..55d45ef3 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -19,7 +19,7 @@ type ExecutionContextRepository interface { GetByEmail(userEmail string) ([]model.ExecutionContext, error) GetByJobName(jobName string) ([]model.ExecutionContext, error) GetByStatus(status string) ([]model.ExecutionContext, error) - deleteAll() error + DeleteAll() error } type executionContextRepository struct { @@ -122,7 +122,7 @@ func (repository *executionContextRepository) GetByStatus(status string) ([]mode return contexts, nil } -func (repository *executionContextRepository) deleteAll() error { +func (repository *executionContextRepository) DeleteAll() error { sql := "DELETE FROM execution_context" context := model.ExecutionContext{} _, err := repository.postgresqlClient.NamedExec(sql, context) diff --git a/internal/app/service/execution/repository/execution_context_mock.go b/internal/app/service/execution/repository/execution_context_mock.go index 1908dfd0..4133b92f 100644 --- a/internal/app/service/execution/repository/execution_context_mock.go +++ b/internal/app/service/execution/repository/execution_context_mock.go @@ -51,7 +51,7 @@ func (mockRepository *MockExecutionContextRepository) GetByStatus(status string) return args.Get(0).([]model.ExecutionContext), args.Error(1) } -func (mockRepository *MockExecutionContextRepository) deleteAll() error { +func (mockRepository *MockExecutionContextRepository) DeleteAll() error { args := mockRepository.Called() return args.Error(0) } diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index 7289b023..7ca6f7b5 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -14,7 +14,7 @@ func TestExecutionContextRepository_Insert(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() fake.Seed(0) mapKey := fake.FirstName() @@ -47,7 +47,7 @@ func TestExecutionContextRepository_Delete(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() fake.Seed(0) context := model.ExecutionContext{ @@ -76,7 +76,7 @@ func TestExecutionContextRepository_UpdateStatus(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() fake.Seed(0) context := model.ExecutionContext{ @@ -107,7 +107,7 @@ func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() fake.Seed(0) context := model.ExecutionContext{ @@ -197,7 +197,7 @@ func TestExecutionContextRepository_GetByEmail(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() recordCount := 15 userEmail := "bimo.horizon@go-pay.co.id" @@ -214,7 +214,7 @@ func TestExecutionContextRepository_GetByJobName(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() recordCount := 15 jobName := "some_job_that_only_exists_in_your_past" @@ -231,7 +231,7 @@ func TestExecutionContextRepository_GetByStatus(t *testing.T) { postgresqlClient := postgresql.NewClient() defer postgresqlClient.Close() repository := NewExecutionContextRepository(postgresqlClient) - defer repository.deleteAll() + defer repository.DeleteAll() recordCount := 15 status := "well_execution_status_here_must_be_cool" diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go index 5d1d45c3..f37b8f8a 100644 --- a/internal/app/service/schedule/repository/schedule.go +++ b/internal/app/service/schedule/repository/schedule.go @@ -60,7 +60,7 @@ func (repository *scheduleRepository) Insert(context model.Schedule) (uint64, er sql := "INSERT INTO schedule (id, job_name, args,cron,notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :cron, :notification_emails, :user_email, :group, :enabled)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { - return 0, nil + return 0, err } return snowflakeID, nil } diff --git a/internal/app/service/schedule/repository/schedule_context.go b/internal/app/service/schedule/repository/schedule_context.go index bfdf3c35..64cdfba9 100644 --- a/internal/app/service/schedule/repository/schedule_context.go +++ b/internal/app/service/schedule/repository/schedule_context.go @@ -1,13 +1,15 @@ package repository import ( + "github.com/pkg/errors" executionModel "proctor/internal/app/service/execution/model" "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/id" "proctor/internal/app/service/schedule/model" ) type ScheduleContextRepository interface { - Insert(context *model.ScheduleContext) (uint64, error) + Insert(context model.ScheduleContext) (*model.ScheduleContext, error) Delete(id uint64) error GetByID(id uint64) (*model.ScheduleContext, error) GetContextByScheduleId(scheduleId uint64) ([]executionModel.ExecutionContext, error) @@ -25,24 +27,65 @@ func NewScheduleContextRepository(client postgresql.Client) ScheduleContextRepos } } -func (repository *scheduleContextRepository) Insert(context *model.ScheduleContext) (uint64, error) { - panic("implement me") +func (repository *scheduleContextRepository) Insert(context model.ScheduleContext) (*model.ScheduleContext, error) { + snowflakeID, _ := id.NextID() + context.ID = snowflakeID + sql := "INSERT INTO schedule_context (id,schedule_id, execution_context_id) VALUES (:id, :schedule_id, :execution_context_id)" + _, err := repository.postgresqlClient.NamedExec(sql, &context) + if err != nil { + return nil, err + } + return &context, nil } func (repository *scheduleContextRepository) Delete(id uint64) error { - panic("implement me") + sql := "DELETE FROM schedule_context WHERE id = :id" + schedule := model.Schedule{ + ID: id, + } + _, err := repository.postgresqlClient.NamedExec(sql, &schedule) + return err } func (repository *scheduleContextRepository) GetByID(id uint64) (*model.ScheduleContext, error) { - panic("implement me") + sql := "SELECT id, schedule_id, execution_context_id, created_at, updated_at FROM schedule_context WHERE id=$1 " + var schedules []model.ScheduleContext + err := repository.postgresqlClient.Select(&schedules, sql, id) + if err != nil { + return nil, err + } + + if len(schedules) == 0 { + return nil, errors.Errorf("Execution context with id %v is not found!", id) + } + + return &schedules[0], nil } func (repository *scheduleContextRepository) GetContextByScheduleId(scheduleId uint64) ([]executionModel.ExecutionContext, error) { - panic("implement me") + sql := "SELECT e.id, e.job_name, e.name, e.user_email, e.image_tag, e.args, e.output, e.status, e.created_at, e.updated_at FROM execution_context e INNER JOIN schedule_context sc ON e.id = sc.execution_context_id WHERE sc.schedule_id=$1" + var contexts []executionModel.ExecutionContext + err := repository.postgresqlClient.Select(&contexts, sql, scheduleId) + if err != nil { + return nil, err + } + + return contexts, nil } func (repository *scheduleContextRepository) GetScheduleByContextId(contextId uint64) (*model.Schedule, error) { - panic("implement me") + sql := "SELECT s.id, s.job_name, s.args, s.cron, s.notification_emails, s.user_email,s.\"group\", s.enabled, s.created_at, s.updated_at FROM schedule s INNER JOIN schedule_context sc ON s.id = sc.schedule_id WHERE sc.execution_context_id=$1 " + var schedules []model.Schedule + err := repository.postgresqlClient.Select(&schedules, sql, contextId) + if err != nil { + return nil, err + } + + if len(schedules) == 0 { + return nil, errors.Errorf("Execution context with id %v is not found!", contextId) + } + + return &schedules[0], nil } func (repository *scheduleContextRepository) deleteAll() error { diff --git a/internal/app/service/schedule/repository/schedule_context_mock.go b/internal/app/service/schedule/repository/schedule_context_mock.go new file mode 100644 index 00000000..995d9726 --- /dev/null +++ b/internal/app/service/schedule/repository/schedule_context_mock.go @@ -0,0 +1,41 @@ +package repository + +import ( + "github.com/stretchr/testify/mock" + executionModel "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/schedule/model" +) + +type MockScheduleContextRepository struct { + mock.Mock +} + +func (repository *MockScheduleContextRepository) Insert(context model.ScheduleContext) (*model.ScheduleContext, error) { + args := repository.Called(context) + return args.Get(0).(*model.ScheduleContext), args.Error(1) +} + +func (repository *MockScheduleContextRepository) Delete(id uint64) error { + args := repository.Called(id) + return args.Error(0) +} + +func (repository *MockScheduleContextRepository) GetByID(id uint64) (*model.ScheduleContext, error) { + args := repository.Called(id) + return args.Get(0).(*model.ScheduleContext), args.Error(1) +} + +func (repository *MockScheduleContextRepository) GetContextByScheduleId(scheduleId uint64) ([]executionModel.ExecutionContext, error) { + args := repository.Called(scheduleId) + return args.Get(0).([]executionModel.ExecutionContext), args.Error(1) +} + +func (repository *MockScheduleContextRepository) GetScheduleByContextId(contextId uint64) (*model.Schedule, error) { + args := repository.Called(contextId) + return args.Get(0).(*model.Schedule), args.Error(1) +} + +func (repository *MockScheduleContextRepository) deleteAll() error { + args := repository.Called() + return args.Error(0) +} diff --git a/internal/app/service/schedule/repository/schedule_context_test.go b/internal/app/service/schedule/repository/schedule_context_test.go new file mode 100644 index 00000000..a1fd627e --- /dev/null +++ b/internal/app/service/schedule/repository/schedule_context_test.go @@ -0,0 +1,190 @@ +package repository + +import ( + fake "github.com/brianvoe/gofakeit" + "github.com/stretchr/testify/assert" + executionModel "proctor/internal/app/service/execution/model" + executionRepository "proctor/internal/app/service/execution/repository" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/schedule/model" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + postgresqlClient postgresql.Client + repository ScheduleContextRepository + executionContextRepository executionRepository.ExecutionContextRepository + scheduleRepository ScheduleRepository +} + +func (context *testContext) setUp(t *testing.T) { + context.postgresqlClient = postgresql.NewClient() + context.repository = NewScheduleContextRepository(context.postgresqlClient) + context.executionContextRepository = executionRepository.NewExecutionContextRepository(context.postgresqlClient) + context.scheduleRepository = NewScheduleRepository(context.postgresqlClient) + err := context.repository.deleteAll() + assert.NoError(t, err) + err = context.executionContextRepository.DeleteAll() + assert.NoError(t, err) + err = context.scheduleRepository.deleteAll() + assert.NoError(t, err) + fake.Seed(0) +} + +func (context *testContext) tearDown() { + context.postgresqlClient.Close() +} +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + ctx := &testContext{} + return ctx +} + +func TestScheduleContextRepository_InsertConstraintFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + scheduleId := fake.Uint64() + executionContextId := fake.Uint64() + + scheduleContext := model.ScheduleContext{ + ScheduleId: scheduleId, + ExecutionContextId: executionContextId, + } + + updatedContext, err := ctx.instance().repository.Insert(scheduleContext) + assert.Error(t, err) + assert.Nil(t, updatedContext) + + ctx.tearDown() +} + +func TestScheduleContextRepository_InsertSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + schedule := model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: fake.Bool(), + } + + scheduleId, err := ctx.instance().scheduleRepository.Insert(schedule) + assert.NoError(t, err) + assert.NotNil(t, scheduleId) + + fake.Seed(0) + executionContext := executionModel.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: status.Received, + } + + executionContextId, err := ctx.instance().executionContextRepository.Insert(executionContext) + assert.NoError(t, err) + assert.NotNil(t, executionContextId) + + scheduleContext := model.ScheduleContext{ + ScheduleId: scheduleId, + ExecutionContextId: executionContextId, + } + + updatedContext, err := ctx.instance().repository.Insert(scheduleContext) + assert.NoError(t, err) + assert.NotNil(t, updatedContext) + + expectedContext, err := ctx.instance().repository.GetByID(updatedContext.ID) + assert.NoError(t, err) + assert.NotNil(t, expectedContext) + + assert.Equal(t, updatedContext.ID, expectedContext.ID) + assert.NotNil(t, expectedContext.CreatedAt) + assert.NotNil(t, expectedContext.UpdatedAt) + + ctx.tearDown() +} + +func TestScheduleContextRepository_GetContextAndSchedule(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + schedule := model.Schedule{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Cron: "5 * * * *", + Tags: fake.BeerMalt(), + NotificationEmails: fake.Email(), + Group: fake.HackerIngverb(), + Enabled: fake.Bool(), + } + + scheduleId, err := ctx.instance().scheduleRepository.Insert(schedule) + assert.NoError(t, err) + assert.NotNil(t, scheduleId) + + contextCount := 4 + for i := 1; i <= contextCount; i++ { + fake.Seed(0) + executionContext := executionModel.ExecutionContext{ + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), + Args: map[string]string{ + fake.FirstName(): fake.LastName(), + }, + Status: status.Received, + } + + executionContextId, err := ctx.instance().executionContextRepository.Insert(executionContext) + assert.NoError(t, err) + assert.NotNil(t, executionContextId) + + scheduleContext := model.ScheduleContext{ + ScheduleId: scheduleId, + ExecutionContextId: executionContextId, + } + + updatedContext, err := ctx.instance().repository.Insert(scheduleContext) + assert.NoError(t, err) + assert.NotNil(t, updatedContext) + } + + contexts, err := ctx.instance().repository.GetContextByScheduleId(scheduleId) + assert.NoError(t, err) + assert.NotEmpty(t, contexts) + assert.Equal(t, contextCount, len(contexts)) + + firstExecutionId := contexts[0].ExecutionID + assert.NotNil(t, firstExecutionId) + + scheduleFromContext, err := ctx.instance().repository.GetScheduleByContextId(firstExecutionId) + assert.NoError(t, err) + assert.NotNil(t, scheduleFromContext) + assert.Equal(t, scheduleId, scheduleFromContext.ID) + + ctx.tearDown() +} diff --git a/migrations/13_CreateScheduleContextTable.up.sql b/migrations/13_CreateScheduleContextTable.up.sql index 6027e934..18cd79ef 100644 --- a/migrations/13_CreateScheduleContextTable.up.sql +++ b/migrations/13_CreateScheduleContextTable.up.sql @@ -2,12 +2,12 @@ CREATE TABLE schedule_context ( id bigint not null primary key, schedule_id bigint, - context_id bigint unique, + execution_context_id bigint unique, created_at timestamp default now(), updated_at timestamp default now(), - UNIQUE (context_id), - FOREIGN KEY (context_id) references execution_context (id), - FOREIGN KEY (schedule_id) references schedule (id) + UNIQUE (execution_context_id), + FOREIGN KEY (execution_context_id) references execution_context (id) ON DELETE CASCADE ON UPDATE CASCADE, + FOREIGN KEY (schedule_id) references schedule (id) ON DELETE CASCADE ON UPDATE CASCADE ); From 91c73877401c7a55bffd904279bffc045abb7010 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 9 Aug 2019 13:20:36 +0700 Subject: [PATCH 089/268] [Jasoet] Fix test for Schedule Worker --- internal/app/service/metadata/handler/http.go | 4 ---- internal/app/service/schedule/handler/http.go | 11 ----------- internal/app/service/secret/handler/http.go | 2 -- internal/app/service/worker/worker.go | 18 ++++++++++-------- internal/app/service/worker/worker_test.go | 6 ++++++ 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/internal/app/service/metadata/handler/http.go b/internal/app/service/metadata/handler/http.go index 17521f43..0d129dc9 100644 --- a/internal/app/service/metadata/handler/http.go +++ b/internal/app/service/metadata/handler/http.go @@ -2,7 +2,6 @@ package handler import ( "encoding/json" - "github.com/getsentry/raven-go" "net/http" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" @@ -42,7 +41,6 @@ func (handler *metadataHTTPHandler) Post() http.HandlerFunc { err = handler.repository.Save(metadata) if err != nil { logger.Error("updating metadata to storage, failed", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(constant.ServerError)) @@ -60,7 +58,6 @@ func (handler *metadataHTTPHandler) GetAll() http.HandlerFunc { metadataSlice, err := handler.repository.GetAll() if err != nil { logger.Error("Error fetching metadata", err.Error()) - raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(constant.ServerError)) @@ -70,7 +67,6 @@ func (handler *metadataHTTPHandler) GetAll() http.HandlerFunc { metadataByte, err := json.Marshal(metadataSlice) if err != nil { logger.Error("Error marshalling jobs metadata in json", err.Error()) - raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(constant.ServerError)) diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 6185ce0d..669b0ca8 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -8,7 +8,6 @@ import ( "strings" "github.com/badoux/checkmail" - "github.com/getsentry/raven-go" "github.com/gorilla/mux" "github.com/robfig/cron" @@ -45,7 +44,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { defer request.Body.Close() if err != nil { logger.Error("Error parsing request body for schedule: ", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.MalformedRequestError)) @@ -97,7 +95,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { _, _ = response.Write([]byte(status.MetadataNotFoundError)) } else { logger.Error(fmt.Sprintf("Error fetching metadata for proc %s ", schedule.Tags), schedule.JobName, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -111,7 +108,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { logger.Error(fmt.Sprintf("Duplicate combination of scheduled job name and args: %s ", schedule.Tags), schedule.JobName, schedule.Args) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusConflict) _, _ = response.Write([]byte(status.ScheduleDuplicateJobNameArgsError)) @@ -119,7 +115,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { return } else { logger.Error(fmt.Sprintf("Error persisting scheduled job %s ", schedule.Tags), schedule.JobName, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -131,7 +126,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { responseBody, err := json.Marshal(schedule) if err != nil { logger.Error(fmt.Sprintf("Error marshaling response body %s ", schedule.Tags), schedule.JobName, err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -150,7 +144,6 @@ func (httpHandler *scheduleHTTPHandler) GetAll() http.HandlerFunc { scheduleList, err := httpHandler.repository.GetAllEnabled() if err != nil { logger.Error("Error fetching scheduled jobs", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -167,7 +160,6 @@ func (httpHandler *scheduleHTTPHandler) GetAll() http.HandlerFunc { scheduleListJson, err := json.Marshal(scheduleList) if err != nil { logger.Error("Error marshalling schedule list", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -198,7 +190,6 @@ func (httpHandler *scheduleHTTPHandler) Get() http.HandlerFunc { return } logger.Error("Error fetching scheduled job", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -208,7 +199,6 @@ func (httpHandler *scheduleHTTPHandler) Get() http.HandlerFunc { scheduleJson, err := json.Marshal(schedule) if err != nil { logger.Error("Error marshalling scheduled job", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) @@ -240,7 +230,6 @@ func (httpHandler *scheduleHTTPHandler) Delete() http.HandlerFunc { return } logger.Error("Error fetching schedule", err.Error()) - raven.CaptureError(err, nil) response.WriteHeader(http.StatusInternalServerError) _, _ = response.Write([]byte(status.GenericServerError)) diff --git a/internal/app/service/secret/handler/http.go b/internal/app/service/secret/handler/http.go index af586029..6076af40 100644 --- a/internal/app/service/secret/handler/http.go +++ b/internal/app/service/secret/handler/http.go @@ -2,7 +2,6 @@ package handler import ( "encoding/json" - "github.com/getsentry/raven-go" "net/http" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/secret/model" @@ -41,7 +40,6 @@ func (handler *handler) Post() http.HandlerFunc { err = handler.repository.Save(secret) if err != nil { logger.Error("saving secret to storage, failed", err.Error()) - raven.CaptureError(err, nil) w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(constant.ServerError)) diff --git a/internal/app/service/worker/worker.go b/internal/app/service/worker/worker.go index dc8187b0..65ae72ff 100644 --- a/internal/app/service/worker/worker.go +++ b/internal/app/service/worker/worker.go @@ -5,7 +5,6 @@ import ( "os" "time" - "github.com/getsentry/raven-go" "github.com/robfig/cron" executionContextRepository "proctor/internal/app/service/execution/repository" @@ -29,6 +28,7 @@ type worker struct { executionService executionService.ExecutionService executionContextRepository executionContextRepository.ExecutionContextRepository scheduleRepository scheduleRepository.ScheduleRepository + scheduleContextRepository scheduleRepository.ScheduleContextRepository mailer mail.Mailer inMemorySchedules map[uint64]*cron.Cron } @@ -37,11 +37,17 @@ type Worker interface { Run(<-chan time.Time, <-chan os.Signal) } -func NewWorker(executionSvc executionService.ExecutionService, executionContextRepo executionContextRepository.ExecutionContextRepository, scheduleRepo scheduleRepository.ScheduleRepository, mailer mail.Mailer) Worker { +func NewWorker(executionSvc executionService.ExecutionService, + executionContextRepo executionContextRepository.ExecutionContextRepository, + scheduleRepo scheduleRepository.ScheduleRepository, + scheduleContextRepository scheduleRepository.ScheduleContextRepository, + mailer mail.Mailer, +) Worker { return &worker{ executionService: executionSvc, executionContextRepository: executionContextRepo, scheduleRepository: scheduleRepo, + scheduleContextRepository: scheduleContextRepository, mailer: mailer, inMemorySchedules: make(map[uint64]*cron.Cron), } @@ -61,8 +67,6 @@ func (worker *worker) enableScheduleIfItDoesNotExist(schedule scheduleModel.Sche executionContext, _, err := worker.executionService.Execute(schedule.JobName, WorkerEmail, schedule.Args) if err != nil { logger.Error(fmt.Sprintf("Error submitting job: %s ", schedule.Tags), schedule.JobName, " for execution: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName}) - return } @@ -70,14 +74,12 @@ func (worker *worker) enableScheduleIfItDoesNotExist(schedule scheduleModel.Sche if err != nil { logger.Error(fmt.Sprintf("Error notifying job: %s `", schedule.Tags), schedule.JobName, "` ID: `", executionContext.ExecutionID, "` execution status: `", executionContext.Status, "` to users: ", err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags, "job_name": schedule.JobName, "job_id": fmt.Sprint(executionContext.ExecutionID), "job_execution_status": string(executionContext.Status)}) return } }) if err != nil { logger.Error(fmt.Sprintf("Error adding cron job: %s", schedule.Tags), err.Error()) - raven.CaptureError(err, map[string]string{"job_tags": schedule.Tags}) return } @@ -93,7 +95,6 @@ func (worker *worker) Run(tickerChan <-chan time.Time, signalsChan <-chan os.Sig schedules, err := worker.scheduleRepository.GetAll() if err != nil { logger.Error("Error getting scheduled jobs from store: ", err.Error()) - raven.CaptureError(err, nil) continue } @@ -124,6 +125,7 @@ func Start() error { metadataStore := metadataRepository.NewMetadataRepository(redisClient) secretStore := secretRepository.NewSecretRepository(redisClient) scheduleStore := scheduleRepository.NewScheduleRepository(postgresClient) + scheduleContextStore := scheduleRepository.NewScheduleContextRepository(postgresClient) httpClient, err := http.NewClient() if err != nil { @@ -132,7 +134,7 @@ func Start() error { kubeClient := kubernetes.NewKubernetesClient(httpClient) mailer := mail.New(config.MailServerHost(), config.MailServerPort()) executionSvc := executionService.NewExecutionService(kubeClient, executionContextStore, metadataStore, secretStore) - worker := NewWorker(executionSvc, executionContextStore, scheduleStore, mailer) + worker := NewWorker(executionSvc, executionContextStore, scheduleStore, scheduleContextStore, mailer) ticker := time.NewTicker(time.Duration(config.ScheduledJobsFetchIntervalInMins()) * time.Minute) signalsChan := make(chan os.Signal, 1) diff --git a/internal/app/service/worker/worker_test.go b/internal/app/service/worker/worker_test.go index bb598bc8..32c06d19 100644 --- a/internal/app/service/worker/worker_test.go +++ b/internal/app/service/worker/worker_test.go @@ -23,6 +23,7 @@ type WorkerTestSuite struct { mockExecutionService *executionService.MockExecutionService mockExecutionContextRepository *executionContextRepository.MockExecutionContextRepository mockScheduleRepository *scheduleRepository.MockScheduleRepository + mockScheduleContextRepository *scheduleRepository.MockScheduleContextRepository mockMailer *mail.MockMailer worker Worker } @@ -31,11 +32,13 @@ func (suite *WorkerTestSuite) SetupTest() { suite.mockExecutionService = &executionService.MockExecutionService{} suite.mockExecutionContextRepository = &executionContextRepository.MockExecutionContextRepository{} suite.mockScheduleRepository = &scheduleRepository.MockScheduleRepository{} + suite.mockScheduleContextRepository = &scheduleRepository.MockScheduleContextRepository{} suite.mockMailer = &mail.MockMailer{} suite.worker = NewWorker( suite.mockExecutionService, suite.mockExecutionContextRepository, suite.mockScheduleRepository, + suite.mockScheduleContextRepository, suite.mockMailer, ) } @@ -88,6 +91,8 @@ func (suite *WorkerTestSuite) TestEnableRun() { suite.mockExecutionService.On("Execute", enabledJob, constant.WorkerEmail, jobArgs).Return(&executionContext, "test", nil) defer suite.mockExecutionService.AssertExpectations(t) defer suite.mockExecutionService.AssertNotCalled(t, "Execute", disabledJob, constant.WorkerEmail, jobArgs) + + suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(scheduleModel.ScheduleContext{}, nil) suite.mockMailer.On("Send", executionContext, scheduledJobs[0]).Return(nil).Run( func(args mock.Arguments) { scheduledJobExecutedChan <- true @@ -163,6 +168,7 @@ func (suite *WorkerTestSuite) TestDisableRun() { suite.mockExecutionService.On("Execute", enabledJob, constant.WorkerEmail, jobArgs).Return(&disableExecutionContext, "test", nil) defer suite.mockExecutionService.AssertExpectations(t) + suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(scheduleModel.ScheduleContext{}, nil) suite.mockMailer.On("Send", disableExecutionContext, enabledScheduledJobs[0]).Return(nil).Run( func(args mock.Arguments) { toggledOffEnabledJobChan <- true From ceea1e100490049b9b58b40efa1707f8d4ab8a45 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 9 Aug 2019 13:22:08 +0700 Subject: [PATCH 090/268] [Jasoet] Move Worker into Schedule --- cmd/server/main.go | 2 +- internal/app/service/{ => schedule}/worker/worker.go | 0 internal/app/service/{ => schedule}/worker/worker_test.go | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename internal/app/service/{ => schedule}/worker/worker.go (100%) rename internal/app/service/{ => schedule}/worker/worker_test.go (100%) diff --git a/cmd/server/main.go b/cmd/server/main.go index 1b0c4e5b..2e680bf5 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -6,8 +6,8 @@ import ( "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/migration" "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/schedule/worker" "proctor/internal/app/service/server" - "proctor/internal/app/service/worker" "github.com/urfave/cli" ) diff --git a/internal/app/service/worker/worker.go b/internal/app/service/schedule/worker/worker.go similarity index 100% rename from internal/app/service/worker/worker.go rename to internal/app/service/schedule/worker/worker.go diff --git a/internal/app/service/worker/worker_test.go b/internal/app/service/schedule/worker/worker_test.go similarity index 100% rename from internal/app/service/worker/worker_test.go rename to internal/app/service/schedule/worker/worker_test.go From f6c0604ae3d5bc88b1e3164b3bd800d1df761d41 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 9 Aug 2019 13:45:50 +0700 Subject: [PATCH 091/268] [Jasoet] Add Insert Schedule Context on Worker --- internal/app/service/schedule/worker/worker.go | 8 ++++++++ internal/app/service/schedule/worker/worker_test.go | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/internal/app/service/schedule/worker/worker.go b/internal/app/service/schedule/worker/worker.go index 65ae72ff..44363ab1 100644 --- a/internal/app/service/schedule/worker/worker.go +++ b/internal/app/service/schedule/worker/worker.go @@ -70,6 +70,14 @@ func (worker *worker) enableScheduleIfItDoesNotExist(schedule scheduleModel.Sche return } + scheduleContext := scheduleModel.ScheduleContext{ + ScheduleId: schedule.ID, + ExecutionContextId: executionContext.ExecutionID, + } + + _, err = worker.scheduleContextRepository.Insert(scheduleContext) + logger.LogErrors(err, "saving schedule context", scheduleContext) + err = worker.mailer.Send(*executionContext, schedule) if err != nil { diff --git a/internal/app/service/schedule/worker/worker_test.go b/internal/app/service/schedule/worker/worker_test.go index 32c06d19..fe81d5a4 100644 --- a/internal/app/service/schedule/worker/worker_test.go +++ b/internal/app/service/schedule/worker/worker_test.go @@ -92,7 +92,7 @@ func (suite *WorkerTestSuite) TestEnableRun() { defer suite.mockExecutionService.AssertExpectations(t) defer suite.mockExecutionService.AssertNotCalled(t, "Execute", disabledJob, constant.WorkerEmail, jobArgs) - suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(scheduleModel.ScheduleContext{}, nil) + suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(&scheduleModel.ScheduleContext{}, nil) suite.mockMailer.On("Send", executionContext, scheduledJobs[0]).Return(nil).Run( func(args mock.Arguments) { scheduledJobExecutedChan <- true @@ -168,7 +168,7 @@ func (suite *WorkerTestSuite) TestDisableRun() { suite.mockExecutionService.On("Execute", enabledJob, constant.WorkerEmail, jobArgs).Return(&disableExecutionContext, "test", nil) defer suite.mockExecutionService.AssertExpectations(t) - suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(scheduleModel.ScheduleContext{}, nil) + suite.mockScheduleContextRepository.On("Insert", mock.Anything).Return(&scheduleModel.ScheduleContext{}, nil) suite.mockMailer.On("Send", disableExecutionContext, enabledScheduledJobs[0]).Return(nil).Run( func(args mock.Arguments) { toggledOffEnabledJobChan <- true From 5215b60dd1b01f2406b7d1893ec760598ff677cb Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 12 Aug 2019 12:08:56 +0700 Subject: [PATCH 092/268] [Jasoet|Bimo.horizon] Change mechanism to persist when execute job --- .env.test | 4 +- .../service/execution/handler/http_test.go | 2 +- .../execution/repository/execution_context.go | 5 +- .../service/execution/service/execution.go | 70 ++++++++----------- .../app/service/execution/status/execution.go | 1 - 5 files changed, 33 insertions(+), 49 deletions(-) diff --git a/.env.test b/.env.test index 74b69458..ad63ee91 100644 --- a/.env.test +++ b/.env.test @@ -11,8 +11,8 @@ export PROCTOR_KUBE_JOB_RETRIES=0 export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 -export PROCTOR_KUBE_POD_LIST_WAIT_TIME=30 -export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=30 +export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 +export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60 export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== export PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK export PROCTOR_POSTGRES_USER=postgres diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 12036eef..16c7c1b2 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -131,7 +131,7 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetLogsWhe readCloser := ioutil.NopCloser(bytes.NewReader([]byte("test1\ntest2\ntest3\n"))) defer readCloser.Close() - suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(30)*time.Second).Return(readCloser, nil).Once() + suite.mockExecutionerService.On("StreamJobLogs", "1", time.Duration(60)*time.Second).Return(readCloser, nil).Once() defer suite.mockExecutionerService.AssertExpectations(t) suite.mockExecutionerContextRepository.On("GetById", executionContextId).Return(context, nil).Once() defer suite.mockExecutionerContextRepository.AssertExpectations(t) diff --git a/internal/app/service/execution/repository/execution_context.go b/internal/app/service/execution/repository/execution_context.go index 55d45ef3..0c59eeea 100644 --- a/internal/app/service/execution/repository/execution_context.go +++ b/internal/app/service/execution/repository/execution_context.go @@ -6,7 +6,6 @@ import ( "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/db/postgresql" - "proctor/internal/app/service/infra/id" "time" ) @@ -33,14 +32,12 @@ func NewExecutionContextRepository(client postgresql.Client) ExecutionContextRep } func (repository *executionContextRepository) Insert(context model.ExecutionContext) (uint64, error) { - snowflakeId, _ := id.NextID() - context.ExecutionID = snowflakeId sql := "INSERT INTO execution_context (id, job_name,name, user_email, image_tag, args, output, status) VALUES (:id, :job_name, :name, :user_email, :image_tag, :args, :output, :status)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { return 0, nil } - return snowflakeId, nil + return context.ExecutionID, nil } func (repository *executionContextRepository) UpdateJobOutput(executionId uint64, output types.GzippedText) error { diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 3709faa0..5ef3119f 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -12,6 +12,7 @@ import ( "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/id" "proctor/internal/app/service/infra/kubernetes" "proctor/internal/app/service/infra/logger" svcMetadataRepository "proctor/internal/app/service/metadata/repository" @@ -23,7 +24,7 @@ type ExecutionService interface { Execute(jobName string, userEmail string, args map[string]string) (*model.ExecutionContext, string, error) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) - save(executionContext model.ExecutionContext) error + update(executionContext model.ExecutionContext) } type executionService struct { @@ -47,27 +48,13 @@ func NewExecutionService( } } -func (service *executionService) save(executionContext model.ExecutionContext) error { - var err error - if executionContext.ExecutionID == 0 { - _, err = service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db", executionContext) - } else { - context, _err := service.repository.GetById(executionContext.ExecutionID) - logger.LogErrors(_err, "get context from db by execution id", executionContext) - if _err != nil || context == nil { - _, err = service.repository.Insert(executionContext) - logger.LogErrors(err, "save execution context to db", executionContext) - } else { - err = service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) - logger.LogErrors(err, "update execution context status", executionContext) - if len(executionContext.Output) > 0 { - err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) - logger.LogErrors(err, "update execution context output", executionContext) - } - } - } - return err +func (service *executionService) update(executionContext model.ExecutionContext) { + err := service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) + logger.LogErrors(err, "update execution context status", executionContext) + if len(executionContext.Output) > 0 { + err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) + logger.LogErrors(err, "update execution context output", executionContext) + } } func (service *executionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { @@ -95,16 +82,19 @@ func (service *executionService) Execute(jobName string, userEmail string, args } func (service *executionService) ExecuteWithCommand(jobName string, userEmail string, args map[string]string, commands []string) (*model.ExecutionContext, string, error) { + executionID, _ := id.NextID() context := model.ExecutionContext{ - UserEmail: userEmail, - JobName: jobName, - Args: args, - Status: status.Created, + ExecutionID: executionID, + UserEmail: userEmail, + JobName: jobName, + Args: args, + Status: status.Created, } metadata, err := service.metadataRepository.GetByName(jobName) if err != nil { context.Status = status.RequirementNotMet + service.insertContext(context) return &context, "", errors.New(fmt.Sprintf("metadata not found for %v, throws error %v", jobName, err.Error())) } @@ -112,6 +102,7 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st secret, err := service.secretRepository.GetByJobName(jobName) if err != nil { context.Status = status.RequirementNotMet + service.insertContext(context) return &context, "", errors.New(fmt.Sprintf("secret not found for %v, throws error %v", jobName, err.Error())) } @@ -122,30 +113,27 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st logger.Info("Executed Job on Kubernetes got ", executionName, " execution jobName and ", err, "errors") if err != nil { context.Status = status.CreationFailed + service.insertContext(context) return &context, "", errors.New(fmt.Sprintf("error when executing image %v with args %v, throws error %v", jobName, args, err.Error())) } context.Name = executionName - contextId, err := service.repository.Insert(context) - logger.LogErrors(err, "save execution context to db", context) - if err != nil { - context.Status = status.ContextSavingFailed - return &context, "", errors.New(fmt.Sprintf("error when saving execution context %v with errors %v", context, err.Error())) - } - - context.ExecutionID = contextId + service.insertContext(context) + go service.watchProcess(context) - go service.watchProcess(executionName, context) - - defer service.save(context) return &context, executionName, nil } -func (service *executionService) watchProcess(executionName string, context model.ExecutionContext) { +func (service *executionService) insertContext(context model.ExecutionContext) { + _, err := service.repository.Insert(context) + logger.LogErrors(err, "save execution context to db", context) +} + +func (service *executionService) watchProcess(context model.ExecutionContext) { waitTime := config.KubeLogProcessWaitTime() * time.Second - err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) + err := service.kubernetesClient.WaitForReadyJob(context.Name, waitTime) if err != nil { context.Status = status.JobCreationFailed @@ -155,7 +143,7 @@ func (service *executionService) watchProcess(executionName string, context mode context.Status = status.JobReady logger.Info("Job Ready for ", context.ExecutionID) - pod, err := service.kubernetesClient.WaitForReadyPod(executionName, waitTime) + pod, err := service.kubernetesClient.WaitForReadyPod(context.Name, waitTime) if err != nil { context.Status = status.PodCreationFailed return @@ -192,7 +180,7 @@ func (service *executionService) watchProcess(executionName string, context mode context.Status = status.Finished } - defer service.save(context) + defer service.update(context) return } diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go index 28d6a25c..f50ec2d6 100644 --- a/internal/app/service/execution/status/execution.go +++ b/internal/app/service/execution/status/execution.go @@ -7,7 +7,6 @@ const ( RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" Created ExecutionStatus = "CREATED" CreationFailed ExecutionStatus = "CREATION_FAILED" - ContextSavingFailed ExecutionStatus = "CONTEXT_SAVING_FAILED" JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" JobReady ExecutionStatus = "JOB_READY" PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" From d92642fcce0feaa3380d85c8cce0f43f4285996c Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:16:35 +0700 Subject: [PATCH 093/268] [bimo.horizon] Add updated at on execution result model --- internal/app/cli/daemon/client_test.go | 7 +++++++ internal/app/service/execution/handler/http.go | 6 +++--- internal/app/service/execution/handler/http_test.go | 2 ++ internal/pkg/model/execution/result.go | 1 + 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 9a22d5b8..401568c1 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -588,6 +588,13 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( expectedProcExecutionStatus := constant.JobSucceeded responseBody := expectedProcExecutionStatus + ExecutionId: uint64(0), + JobName: "", + ExecutionName: "", + ImageTag: "", + CreatedAt: "", + UpdatedAt: "", + Status: constant.JobSucceeded, httpmock.RegisterStubRequest( httpmock.NewStubRequest( diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index c46841cf..6eb3ed28 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -17,9 +17,7 @@ import ( executionStatus "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" - - "github.com/gorilla/mux" - "github.com/gorilla/websocket" + "proctor/internal/pkg/model/execution" ) type ExecutionHTTPHandler interface { @@ -144,6 +142,7 @@ func (httpHandler *executionHTTPHandler) GetStatus() http.HandlerFunc { ExecutionName: context.Name, ImageTag: context.ImageTag, CreatedAt: context.CreatedAt.String(), + UpdatedAt: context.UpdatedAt.String(), Status: string(context.Status), } @@ -190,6 +189,7 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { ExecutionName: executionName, ImageTag: context.ImageTag, CreatedAt: context.CreatedAt.String(), + UpdatedAt: context.UpdatedAt.String(), Status: string(context.Status), } diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 12036eef..1397f81d 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -179,6 +179,7 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionGetStatusH ExecutionName: context.Name, ImageTag: context.ImageTag, CreatedAt: context.CreatedAt.String(), + UpdatedAt: context.UpdatedAt.String(), Status: string(context.Status), } @@ -267,6 +268,7 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHa ExecutionName: context.Name, ImageTag: context.ImageTag, CreatedAt: context.CreatedAt.String(), + UpdatedAt: context.UpdatedAt.String(), Status: string(context.Status), } diff --git a/internal/pkg/model/execution/result.go b/internal/pkg/model/execution/result.go index ca5dfa36..f39f766a 100644 --- a/internal/pkg/model/execution/result.go +++ b/internal/pkg/model/execution/result.go @@ -6,5 +6,6 @@ type ExecutionResult struct { ExecutionName string `json:"name"` ImageTag string `json:"image_tag"` CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` Status string `json:"status"` } From c0ced64146c7e1fb8b30048ef756a751a8282d0e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:18:52 +0700 Subject: [PATCH 094/268] [bimo.horizon] Split get status method into two method with polling and without --- internal/app/cli/daemon/client.go | 90 ++++++++------ internal/app/cli/daemon/client_mock.go | 16 ++- internal/app/cli/daemon/client_test.go | 164 ++++++++++++++++++++----- 3 files changed, 198 insertions(+), 72 deletions(-) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 6e86061a..93edd6a4 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -12,16 +12,17 @@ import ( "net/url" "os" "os/signal" - "proctor/internal/app/cli/command/version" - "proctor/internal/app/cli/config" - "proctor/internal/pkg/constant" - "proctor/internal/pkg/io" - "proctor/internal/pkg/model/execution" "time" "github.com/briandowns/spinner" "github.com/fatih/color" "github.com/gorilla/websocket" + + "proctor/internal/app/cli/command/version" + "proctor/internal/app/cli/config" + "proctor/internal/pkg/constant" + "proctor/internal/pkg/io" + modelExecution "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" modelSchedule "proctor/internal/pkg/model/schedule" ) @@ -35,9 +36,10 @@ const ( type Client interface { ListProcs() ([]modelMetadata.Metadata, error) - ExecuteProc(string, map[string]string) (*execution.ExecutionResult, error) + ExecuteProc(string, map[string]string) (*modelExecution.ExecutionResult, error) StreamProcLogs(executionId uint64) error - GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) + GetExecutionContextStatusWithPolling(executionId uint64) (*modelExecution.ExecutionResult, error) + GetExecutionContextStatus(executionId uint64) (*modelExecution.ExecutionResult, error) ScheduleJob(string, string, string, string, string, map[string]string) (string, error) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) DescribeScheduledProc(string) (modelSchedule.ScheduledJob, error) @@ -253,7 +255,7 @@ func (c *client) RemoveScheduledProc(jobID string) error { return nil } -func (c *client) ExecuteProc(name string, args map[string]string) (*execution.ExecutionResult, error) { +func (c *client) ExecuteProc(name string, args map[string]string) (*modelExecution.ExecutionResult, error) { err := c.loadProctorConfig() if err != nil { return nil, err @@ -285,7 +287,7 @@ func (c *client) ExecuteProc(name string, args map[string]string) (*execution.Ex return nil, buildHTTPError(c, resp) } - var executionResult execution.ExecutionResult + var executionResult modelExecution.ExecutionResult err = json.NewDecoder(resp.Body).Decode(&executionResult) return &executionResult, err @@ -354,45 +356,63 @@ func (c *client) StreamProcLogs(executionId uint64) error { } } -func (c *client) GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) { +func (c *client) GetExecutionContextStatusWithPolling(executionId uint64) (*modelExecution.ExecutionResult, error) { err := c.loadProctorConfig() if err != nil { - return "", err + return nil, err } for count := 0; count < c.procExecutionStatusPollCount; count += 1 { - httpClient := &http.Client{ - Timeout: c.connectionTimeoutSecs, + executionContextStatus, err := c.GetExecutionContextStatus(executionId) + if err != nil { + return nil, err + } + if executionContextStatus.Status == constant.JobSucceeded || executionContextStatus.Status == constant.JobFailed { + return executionContextStatus, nil } - req, err := http.NewRequest("GET", fmt.Sprintf("http://%s%s/%v/status", c.proctordHost, ExecutionRoute, executionId), nil) - req.Header.Add(constant.UserEmailHeaderKey, c.emailId) - req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) - req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) + time.Sleep(time.Duration(count) * 100 * time.Millisecond) + } + return nil, errors.New(fmt.Sprintf("No definitive status received for execution with id %v from proctord", executionId)) +} - resp, err := httpClient.Do(req) - if err != nil { - return "", buildNetworkError(err) - } +func (c *client) GetExecutionContextStatus(executionId uint64) (*modelExecution.ExecutionResult, error) { + err := c.loadProctorConfig() + if err != nil { + return nil, err + } - if resp.StatusCode != http.StatusOK { - return "", buildHTTPError(c, resp) - } + httpClient := &http.Client{ + Timeout: c.connectionTimeoutSecs, + } - body, err := ioutil.ReadAll(resp.Body) - defer resp.Body.Close() - if err != nil { - return "", err - } + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s%s/%v/status", c.proctordHost, ExecutionRoute, executionId), nil) + req.Header.Add(constant.UserEmailHeaderKey, c.emailId) + req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) + req.Header.Add(constant.ClientVersionHeaderKey, c.clientVersion) - procExecutionStatus := string(body) - if procExecutionStatus == constant.JobSucceeded || procExecutionStatus == constant.JobFailed { - return procExecutionStatus, nil - } + resp, err := httpClient.Do(req) + if err != nil { + return nil, buildNetworkError(err) + } - time.Sleep(time.Duration(count) * 100 * time.Millisecond) + if resp.StatusCode != http.StatusOK { + return nil, buildHTTPError(c, resp) + } + + body, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + return nil, err } - return "", errors.New(fmt.Sprintf("No definitive status received for execution with id %v from proctord", executionId)) + + var executionResult modelExecution.ExecutionResult + err = json.Unmarshal(body, &executionResult) + if err != nil { + return nil, err + } + + return &executionResult, nil } func buildNetworkError(err error) error { diff --git a/internal/app/cli/daemon/client_mock.go b/internal/app/cli/daemon/client_mock.go index 2246101c..9e33f79b 100644 --- a/internal/app/cli/daemon/client_mock.go +++ b/internal/app/cli/daemon/client_mock.go @@ -2,7 +2,8 @@ package daemon import ( "github.com/stretchr/testify/mock" - "proctor/internal/pkg/model/execution" + + modelExecution "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" modelSchedule "proctor/internal/pkg/model/schedule" ) @@ -21,9 +22,9 @@ func (m *MockClient) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) return args.Get(0).([]modelSchedule.ScheduledJob), args.Error(1) } -func (m *MockClient) ExecuteProc(name string, procArgs map[string]string) (*execution.ExecutionResult, error) { +func (m *MockClient) ExecuteProc(name string, procArgs map[string]string) (*modelExecution.ExecutionResult, error) { args := m.Called(name, procArgs) - return args.Get(0).(*execution.ExecutionResult), args.Error(1) + return args.Get(0).(*modelExecution.ExecutionResult), args.Error(1) } func (m *MockClient) StreamProcLogs(executionId uint64) error { @@ -31,9 +32,14 @@ func (m *MockClient) StreamProcLogs(executionId uint64) error { return args.Error(0) } -func (m *MockClient) GetDefinitiveProcExecutionStatus(executionId uint64) (string, error) { +func (m *MockClient) GetExecutionContextStatusWithPolling(executionId uint64) (*modelExecution.ExecutionResult, error) { args := m.Called(executionId) - return args.Get(0).(string), args.Error(1) + return args.Get(0).(*modelExecution.ExecutionResult), args.Error(1) +} + +func (m *MockClient) GetExecutionContextStatus(executionId uint64) (*modelExecution.ExecutionResult, error) { + args := m.Called(executionId) + return args.Get(0).(*modelExecution.ExecutionResult), args.Error(1) } func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, group string, jobArgs map[string]string) (string, error) { diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 401568c1..ea066a29 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -8,16 +8,16 @@ import ( "strings" "testing" - "proctor/internal/app/cli/command/version" - "proctor/internal/app/cli/config" - "proctor/internal/pkg/io" - "proctor/internal/pkg/model/execution" - "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "github.com/thingful/httpmock" + + "proctor/internal/app/cli/command/version" + "proctor/internal/app/cli/config" "proctor/internal/pkg/constant" + "proctor/internal/pkg/io" + modelExecution "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" "proctor/internal/pkg/model/metadata/env" ) @@ -264,12 +264,13 @@ func (s *ClientTestSuite) TestExecuteProc() { executionName := "proctor-777b1dfb-ea27-46d9-b02c-839b75a542e2" proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - expectedProcResponse := &execution.ExecutionResult{ + expectedProcResponse := &modelExecution.ExecutionResult{ ExecutionId: uint64(0), JobName: "", ExecutionName: executionName, ImageTag: "", CreatedAt: "", + UpdatedAt: "", Status: "", } body := `{ "name": "proctor-777b1dfb-ea27-46d9-b02c-839b75a542e2"}` @@ -411,7 +412,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) - var expectedProcResponse *execution.ExecutionResult + var expectedProcResponse *modelExecution.ExecutionResult assert.Equal(t, "Execute Error", err.Error()) assert.Equal(t, expectedProcResponse, executeProcResponse) s.mockConfigLoader.AssertExpectations(t) @@ -444,7 +445,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { executeProcResponse, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - var expectedProcResponse *execution.ExecutionResult + var expectedProcResponse *modelExecution.ExecutionResult assert.Equal(t, expectedProcResponse, executeProcResponse) assert.Equal(t, "Unauthorized Access!!!\nPlease check the EMAIL_ID and ACCESS_TOKEN validity in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) @@ -477,7 +478,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS executeProcResponse, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - var expectedProcResponse *execution.ExecutionResult + var expectedProcResponse *modelExecution.ExecutionResult assert.Equal(t, expectedProcResponse, executeProcResponse) assert.Equal(t, "Unauthorized Access!!!\nEMAIL_ID or ACCESS_TOKEN is not present in proctor config file.", err.Error()) s.mockConfigLoader.AssertExpectations(t) @@ -510,7 +511,7 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { response, err := s.testClient.ExecuteProc("run-sample", map[string]string{"SAMPLE_ARG1": "sample-value"}) - var expectedProcResponse *execution.ExecutionResult + var expectedProcResponse *modelExecution.ExecutionResult assert.Equal(t, expectedProcResponse, response) assert.Equal(t, errors.New("Network Error!!!\nPost http://proctor.example.com/execution: Unknown Error"), err) s.mockConfigLoader.AssertExpectations(t) @@ -578,7 +579,7 @@ func (s *ClientTestSuite) TestLogStreamForUnauthorizedUser() { } -func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs() { +func (s *ClientTestSuite) TestGetExecutionContextStatusForSucceededProcs() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} @@ -586,8 +587,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( httpmock.Activate() defer httpmock.DeactivateAndReset() - expectedProcExecutionStatus := constant.JobSucceeded - responseBody := expectedProcExecutionStatus + expectedExecutionContextStatus := &modelExecution.ExecutionResult{ ExecutionId: uint64(0), JobName: "", ExecutionName: "", @@ -595,6 +595,8 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( CreatedAt: "", UpdatedAt: "", Status: constant.JobSucceeded, + } + responseBody := fmt.Sprintf(`{ "status": "%s" }`, constant.JobSucceeded) httpmock.RegisterStubRequest( httpmock.NewStubRequest( @@ -614,14 +616,14 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForSucceededProcs( s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) + executionContextStatus, err := s.testClient.GetExecutionContextStatus(uint64(42)) assert.NoError(t, err) s.mockConfigLoader.AssertExpectations(t) - assert.Equal(t, expectedProcExecutionStatus, procExecutionStatus) + assert.Equal(t, expectedExecutionContextStatus, executionContextStatus) } -func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { +func (s *ClientTestSuite) TestGetExecutionContextStatusForFailedProcs() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} @@ -629,8 +631,16 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { httpmock.Activate() defer httpmock.DeactivateAndReset() - expectedProcExecutionStatus := constant.JobFailed - responseBody := expectedProcExecutionStatus + expectedExecutionContextStatus := &modelExecution.ExecutionResult{ + ExecutionId: uint64(0), + JobName: "", + ExecutionName: "", + ImageTag: "", + CreatedAt: "", + UpdatedAt: "", + Status: constant.JobFailed, + } + responseBody := fmt.Sprintf(`{ "status": "%s" }`, constant.JobFailed) httpmock.RegisterStubRequest( httpmock.NewStubRequest( @@ -650,14 +660,14 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForFailedProcs() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) + executionContextStatus, err := s.testClient.GetExecutionContextStatus(uint64(42)) assert.NoError(t, err) s.mockConfigLoader.AssertExpectations(t) - assert.Equal(t, expectedProcExecutionStatus, procExecutionStatus) + assert.Equal(t, expectedExecutionContextStatus, executionContextStatus) } -func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFailure() { +func (s *ClientTestSuite) TestGetExecutionContextStatusForHTTPRequestFailure() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} @@ -683,14 +693,15 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForHTTPRequestFail s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) + executionContextStatus, err := s.testClient.GetExecutionContextStatus(uint64(42)) assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/42/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) s.mockConfigLoader.AssertExpectations(t) - assert.Equal(t, "", procExecutionStatus) + var executionResult *modelExecution.ExecutionResult + assert.Equal(t, executionResult, executionContextStatus) } -func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() { +func (s *ClientTestSuite) TestGetExecutionContextStatusForNonOKResponse() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} @@ -716,14 +727,103 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusForNonOKResponse() s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) + executionContextStatus, err := s.testClient.GetExecutionContextStatus(uint64(42)) assert.Equal(t, errors.New("execute Error"), err) s.mockConfigLoader.AssertExpectations(t) - assert.Equal(t, "", procExecutionStatus) + var executionResult *modelExecution.ExecutionResult + assert.Equal(t, executionResult, executionContextStatus) +} + +func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForCompletedProcs() { + t := s.T() + + proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + completedProcs := []struct { + expectedExecutionContextStatus string + executionID uint64 + }{ + {constant.JobSucceeded, uint64(42)}, + {constant.JobFailed, uint64(43)}, + } + + for _, proc := range completedProcs { + expectedExecutionContextStatus := &modelExecution.ExecutionResult{ + ExecutionId: proc.executionID, + JobName: "", + ExecutionName: "", + ImageTag: "", + CreatedAt: "", + UpdatedAt: "", + Status: proc.expectedExecutionContextStatus, + } + responseBody := fmt.Sprintf(`{ "id": %v, "status": "%s" }`, fmt.Sprint(proc.executionID), proc.expectedExecutionContextStatus) + + httpmock.RegisterStubRequest( + httpmock.NewStubRequest( + "GET", + "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(proc.executionID)+"/status", + func(req *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(200, responseBody), nil + }, + ).WithHeader( + &http.Header{ + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, + }, + ), + ) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() + + executionContextStatus, err := s.testClient.GetExecutionContextStatusWithPolling(proc.executionID) + + assert.NoError(t, err) + s.mockConfigLoader.AssertExpectations(t) + assert.Equal(t, expectedExecutionContextStatus, executionContextStatus) + } +} + +func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForGetError() { + t := s.T() + + proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: 1} + + httpmock.Activate() + defer httpmock.DeactivateAndReset() + + httpmock.RegisterStubRequest( + httpmock.NewStubRequest( + "GET", + "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", + func(req *http.Request) (*http.Response, error) { + return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} + }, + ).WithHeader( + &http.Header{ + constant.UserEmailHeaderKey: []string{"proctor@example.com"}, + constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, + }, + ), + ) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() + + executionContextStatus, err := s.testClient.GetExecutionContextStatusWithPolling(uint64(42)) + + assert.Equal(t, errors.New("Connection Timeout!!!\nGet http://proctor.example.com/execution/42/status: Unable to reach http://proctor.example.com/\nPlease check your Internet/VPN connection for connectivity to ProctorD."), err) + s.mockConfigLoader.AssertExpectations(t) + var executionResult *modelExecution.ExecutionResult + assert.Equal(t, executionResult, executionContextStatus) } -func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReached() { +func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingWhenPollCountReached() { t := s.T() expectedRequestsToProctorDCount := 2 @@ -731,8 +831,7 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token", ProcExecutionStatusPollCount: expectedRequestsToProctorDCount} - expectedProcExecutionStatus := constant.JobWaiting - responseBody := expectedProcExecutionStatus + responseBody := fmt.Sprintf(`{ "status": "%s" }`, constant.JobWaiting) httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -754,14 +853,15 @@ func (s *ClientTestSuite) TestGetDefinitiveProcExecutionStatusWhenPollCountReach ), ) - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Times(3) - procExecutionStatus, err := s.testClient.GetDefinitiveProcExecutionStatus(uint64(42)) + executionContextStatus, err := s.testClient.GetExecutionContextStatusWithPolling(uint64(42)) assert.Equal(t, errors.New("No definitive status received for execution with id 42 from proctord"), err) s.mockConfigLoader.AssertExpectations(t) assert.Equal(t, expectedRequestsToProctorDCount, requestsToProctorDCount) - assert.Equal(t, "", procExecutionStatus) + var executionResult *modelExecution.ExecutionResult + assert.Equal(t, executionResult, executionContextStatus) } func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { From 2ed331d7f0e90d0b3265fb7af20fc19555557d9a Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:19:13 +0700 Subject: [PATCH 095/268] [bimo.horizon] Tidy import order --- internal/app/service/execution/handler/http.go | 4 +++- .../service/execution/repository/execution_context_test.go | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 6eb3ed28..67422a46 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" "net/http" - "proctor/internal/pkg/model/execution" "strconv" "strings" "time" + "github.com/gorilla/mux" + "github.com/gorilla/websocket" + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/execution/handler/status" "proctor/internal/app/service/execution/repository" diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index 7ca6f7b5..2fff42f9 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -1,13 +1,16 @@ package repository import ( + "testing" + fake "github.com/brianvoe/gofakeit" "github.com/jmoiron/sqlx/types" "github.com/stretchr/testify/assert" + "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/db/postgresql" - "testing" + "proctor/internal/app/service/infra/id" ) func TestExecutionContextRepository_Insert(t *testing.T) { From ba140653324f58ddbfaced2ee9079b7d3425922e Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:19:41 +0700 Subject: [PATCH 096/268] [bimo.horizon] Add id generation on insert and delete --- .../service/execution/repository/execution_context_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index 2fff42f9..ee56d7fc 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -63,6 +63,8 @@ func TestExecutionContextRepository_Delete(t *testing.T) { Status: status.Received, } + expectedContextID, _ := id.NextID() + context.ExecutionID = expectedContextID id, err := repository.Insert(context) assert.Nil(t, err) assert.NotZero(t, id) @@ -92,6 +94,8 @@ func TestExecutionContextRepository_UpdateStatus(t *testing.T) { Status: status.Received, } + expectedContextID, _ := id.NextID() + context.ExecutionID = expectedContextID id, err := repository.Insert(context) assert.Nil(t, err) assert.NotZero(t, id) From a32994e5580b36db853b72723f57cbc62a1f5d1f Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:20:24 +0700 Subject: [PATCH 097/268] [bimo.horizon] Add status cmd --- internal/app/cli/command/root.go | 4 + internal/app/cli/command/root_test.go | 1 + internal/app/cli/command/status/status.go | 45 ++++++++ .../app/cli/command/status/status_test.go | 101 ++++++++++++++++++ 4 files changed, 151 insertions(+) create mode 100644 internal/app/cli/command/status/status.go create mode 100644 internal/app/cli/command/status/status_test.go diff --git a/internal/app/cli/command/root.go b/internal/app/cli/command/root.go index edc6f94c..57294b2e 100644 --- a/internal/app/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -16,6 +16,7 @@ import ( scheduleDescribe "proctor/internal/app/cli/command/schedule/describe" scheduleList "proctor/internal/app/cli/command/schedule/list" "proctor/internal/app/cli/command/schedule/remove" + "proctor/internal/app/cli/command/status" "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/daemon" @@ -44,6 +45,9 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith logCmd := log.NewCmd(printer, proctorDClient, os.Exit) rootCmd.AddCommand(logCmd) + statusCmd := status.NewCmd(printer, proctorDClient, os.Exit) + rootCmd.AddCommand(statusCmd) + listCmd := list.NewCmd(printer, proctorDClient) rootCmd.AddCommand(listCmd) diff --git a/internal/app/cli/command/root_test.go b/internal/app/cli/command/root_test.go index 9da0edf2..2816b267 100644 --- a/internal/app/cli/command/root_test.go +++ b/internal/app/cli/command/root_test.go @@ -33,6 +33,7 @@ func TestRootCmdSubCommands(t *testing.T) { assert.True(t, contains(rootCmd.Commands(), "describe")) assert.True(t, contains(rootCmd.Commands(), "execute")) assert.True(t, contains(rootCmd.Commands(), "logs")) + assert.True(t, contains(rootCmd.Commands(), "status")) assert.True(t, contains(rootCmd.Commands(), "help")) assert.True(t, contains(rootCmd.Commands(), "list")) assert.True(t, contains(rootCmd.Commands(), "config")) diff --git a/internal/app/cli/command/status/status.go b/internal/app/cli/command/status/status.go new file mode 100644 index 00000000..3139f4ce --- /dev/null +++ b/internal/app/cli/command/status/status.go @@ -0,0 +1,45 @@ +package status + +import ( + "fmt" + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/io" + "strconv" + + "github.com/fatih/color" + "github.com/spf13/cobra" +) + +func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "Get status of an execution context", + Long: "To get status of an execution context, this command retrieve status from previous execution", + Example: "proctor status 123", + Args: cobra.MinimumNArgs(1), + + Run: func(cmd *cobra.Command, args []string) { + executionIDParam := args[0] + executionID, err := strconv.ParseUint(executionIDParam, 10, 64) + if executionIDParam == "" || err != nil { + printer.Println("No valid execution context id provided as argument", color.FgRed) + return + } + + printer.Println("Getting status", color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen) + + executionContextStatus, err := proctorDClient.GetExecutionContextStatus(executionID) + if err != nil { + printer.Println(fmt.Sprintf("%-40s %-100v", "Error while Getting Status:", err.Error()), color.FgRed) + osExitFunc(1) + return + } + + printer.Println(fmt.Sprintf("%-40s %-100v", "Job Name", executionContextStatus.JobName), color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100v", "Status", executionContextStatus.Status), color.FgGreen) + printer.Println(fmt.Sprintf("%-40s %-100v", "Updated At", executionContextStatus.UpdatedAt), color.FgGreen) + printer.Println("Execution completed.", color.FgGreen) + }, + } +} diff --git a/internal/app/cli/command/status/status_test.go b/internal/app/cli/command/status/status_test.go new file mode 100644 index 00000000..38a08f87 --- /dev/null +++ b/internal/app/cli/command/status/status_test.go @@ -0,0 +1,101 @@ +package status + +import ( + "errors" + "fmt" + "testing" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + "proctor/internal/app/cli/daemon" + "proctor/internal/pkg/constant" + "proctor/internal/pkg/io" + modelExecution "proctor/internal/pkg/model/execution" +) + +type StatusCmdTestSuite struct { + suite.Suite + mockPrinter *io.MockPrinter + mockProctorDClient *daemon.MockClient + testStatusCmd *cobra.Command +} + +func (s *StatusCmdTestSuite) SetupTest() { + s.mockPrinter = &io.MockPrinter{} + s.mockProctorDClient = &daemon.MockClient{} + s.testStatusCmd = NewCmd(s.mockPrinter, s.mockProctorDClient, func(exitCode int) {}) +} + +func (s *StatusCmdTestSuite) TestStatusCmdUsage() { + assert.Equal(s.T(), "status", s.testStatusCmd.Use) +} + +func (s *StatusCmdTestSuite) TestStatusCmdHelp() { + assert.Equal(s.T(), "Get status of an execution context", s.testStatusCmd.Short) + assert.Equal(s.T(), "To get status of an execution context, this command retrieve status from previous execution", s.testStatusCmd.Long) + assert.Equal(s.T(), "proctor status 123", s.testStatusCmd.Example) +} + +func (s *StatusCmdTestSuite) TestStatusCmd() { + executionID := uint64(42) + + s.mockPrinter.On("Println", "Getting status", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Job Name", "foo"), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Status", constant.JobSucceeded), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Updated At", ""), color.FgGreen).Once() + + executionResult := &modelExecution.ExecutionResult{ + ExecutionId: uint64(0), + JobName: "foo", + ExecutionName: "", + ImageTag: "", + CreatedAt: "", + UpdatedAt: "", + Status: constant.JobSucceeded, + } + s.mockProctorDClient.On("GetExecutionContextStatus", executionID).Return(executionResult, nil).Once() + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() + + s.testStatusCmd.Run(&cobra.Command{}, []string{"42"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + +func (s *StatusCmdTestSuite) TestStatusCmdInvalidExecutionIDError() { + t := s.T() + + s.mockPrinter.On("Println", "No valid execution context id provided as argument", color.FgRed).Once() + + s.testStatusCmd.Run(&cobra.Command{}, []string{"foo"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) + s.mockPrinter.AssertNotCalled(t, "Println", "Execution completed.", color.FgGreen) +} + +func (s *StatusCmdTestSuite) TestStatusCmdGetExecutionStatusError() { + t := s.T() + + executionID := uint64(42) + + s.mockPrinter.On("Println", "Getting status", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionID), color.FgGreen).Once() + + s.mockProctorDClient.On("GetExecutionContextStatus", executionID).Return(&modelExecution.ExecutionResult{}, errors.New("test")).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Error while Getting Status:", "test"), color.FgRed).Once() + s.testStatusCmd.Run(&cobra.Command{}, []string{"42"}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) + s.mockPrinter.AssertNotCalled(t, "Println", "Execution completed.", color.FgGreen) + s.mockPrinter.AssertNotCalled(t, "Println", fmt.Sprintf("%-40s %-100v", "Status", constant.JobSucceeded), color.FgGreen) +} + +func TestStatusCmdTestSuite(t *testing.T) { + suite.Run(t, new(StatusCmdTestSuite)) +} From 0e660608ad18fe102cce424dd060301b716f2491 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 17:20:39 +0700 Subject: [PATCH 098/268] [bimo.horizon] Add missing make command --- Makefile | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Makefile b/Makefile index 8ccc2001..429bf1f5 100644 --- a/Makefile +++ b/Makefile @@ -69,22 +69,35 @@ ftest.package.procs: PROCTOR_JOBS_PATH=$(FTEST_DIR) \ ruby ./test/package_procs.rb +.PHONY: ftest.update.metadata ftest.update.metadata: PROCTOR_JOBS_PATH=$(FTEST_DIR) \ PROCTOR_URI=http://localhost:$(PROCTOR_APP_PORT)/metadata \ ruby ./test/update_metadata.rb +.PHONY: ftest.update.secret ftest.update.secret: curl -X POST \ http://localhost:5000/secret \ -H 'Content-Type: application/json' \ -d '{"job_name": "say-hello-world","secrets": {"SAMPLE_SECRET_ONE": "Secret One :*","SAMPLE_SECRET_TWO": "Secret Two :V"}}' +.PHONY: ftest.proctor.list ftest.proctor.list: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli list +.PHONY: ftest.proctor.describe ftest.proctor.describe: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli describe say-hello-world +.PHONY: ftest.proctor.execute ftest.proctor.execute: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli execute say-hello-world SAMPLE_ARG_ONE=foo SAMPLE_ARG_TWO=bar + +.PHONY: ftest.proctor.logs +ftest.proctor.logs: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli logs $(EXECUTION_ID) + +.PHONY: ftest.proctor.status +ftest.proctor.status: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli status $(EXECUTION_ID) From 5f42a290f82e343b32b3906a90a8dea8d3ecb309 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 12 Aug 2019 21:06:24 +0700 Subject: [PATCH 099/268] [bimo.horizon] Remove unneeded tests --- .../execution/service/execution_test.go | 50 ++----------------- 1 file changed, 5 insertions(+), 45 deletions(-) diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 54c54791..196900f1 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -1,24 +1,23 @@ package service import ( + "io/ioutil" + "strings" + "testing" + fake "github.com/brianvoe/gofakeit" "github.com/docker/docker/pkg/testutil/assert" - "github.com/jmoiron/sqlx/types" "github.com/pkg/errors" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" - "io/ioutil" v1 "k8s.io/api/core/v1" - "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" - "proctor/internal/app/service/infra/id" "proctor/internal/app/service/infra/kubernetes" svcMetadataRepository "proctor/internal/app/service/metadata/repository" svcSecretRepository "proctor/internal/app/service/secret/repository" "proctor/internal/pkg/model/metadata" - "strings" - "testing" ) type TestExecutionServiceSuite struct { @@ -43,45 +42,6 @@ func (suite *TestExecutionServiceSuite) SetupTest() { ) } -func (suite *TestExecutionServiceSuite) TestSaveNoExecutionId() { - t := suite.T() - context := model.ExecutionContext{} - - suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() - err := suite.service.save(context) - assert.Error(t, err, "Insert Failed") - - suite.mockRepository.On("Insert", context).Return(0, nil).Once() - err = suite.service.save(context) - assert.NilError(t, err) -} - -func (suite *TestExecutionServiceSuite) TestSaveWithExecutionId() { - t := suite.T() - _id, _ := id.NextID() - context := model.ExecutionContext{ - ExecutionID: _id, - Status: status.Created, - } - - suite.mockRepository.On("GetById", _id).Return(&context, errors.New("Get By Id Error")).Once() - suite.mockRepository.On("Insert", context).Return(0, errors.New("Insert Failed")).Once() - err := suite.service.save(context) - assert.Error(t, err, "Insert Failed") - - suite.mockRepository.On("GetById", _id).Return(&context, nil).Once() - suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(errors.New("Update Status Failed")).Once() - err = suite.service.save(context) - assert.Error(t, err, "Update Status Failed") - - context.Output = types.GzippedText("This is some output") - suite.mockRepository.On("GetById", _id).Return(&context, nil).Once() - suite.mockRepository.On("UpdateStatus", context.ExecutionID, context.Status).Return(nil).Once() - suite.mockRepository.On("UpdateJobOutput", context.ExecutionID, context.Output).Return(errors.New("Update Output Failed")).Once() - err = suite.service.save(context) - assert.Error(t, err, "Update Output Failed") -} - func (suite *TestExecutionServiceSuite) TestExecuteMetadataNotFound() { t := suite.T() jobName := fake.Username() From b387375cf7ce655623e2b8985ba492d74eae1d88 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 13 Aug 2019 11:30:30 +0700 Subject: [PATCH 100/268] [bimo.horizon] Fix failing test caused by moved execution id generation --- .../repository/execution_context_test.go | 47 ++++++++++--------- .../repository/schedule_context_test.go | 20 ++++---- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/internal/app/service/execution/repository/execution_context_test.go b/internal/app/service/execution/repository/execution_context_test.go index ee56d7fc..18016b86 100644 --- a/internal/app/service/execution/repository/execution_context_test.go +++ b/internal/app/service/execution/repository/execution_context_test.go @@ -23,10 +23,12 @@ func TestExecutionContextRepository_Insert(t *testing.T) { mapKey := fake.FirstName() mapValue := fake.LastName() + executionID, _ := id.NextID() context := model.ExecutionContext{ - JobName: fake.BuzzWord(), - UserEmail: fake.Email(), - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), Args: map[string]string{ mapKey: mapValue, }, @@ -53,18 +55,18 @@ func TestExecutionContextRepository_Delete(t *testing.T) { defer repository.DeleteAll() fake.Seed(0) + executionID, _ := id.NextID() context := model.ExecutionContext{ - JobName: fake.BuzzWord(), - UserEmail: fake.Email(), - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), Args: map[string]string{ fake.FirstName(): fake.LastName(), }, Status: status.Received, } - expectedContextID, _ := id.NextID() - context.ExecutionID = expectedContextID id, err := repository.Insert(context) assert.Nil(t, err) assert.NotZero(t, id) @@ -84,18 +86,18 @@ func TestExecutionContextRepository_UpdateStatus(t *testing.T) { defer repository.DeleteAll() fake.Seed(0) + executionID, _ := id.NextID() context := model.ExecutionContext{ - JobName: fake.BuzzWord(), - UserEmail: fake.Email(), - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), Args: map[string]string{ fake.FirstName(): fake.LastName(), }, Status: status.Received, } - expectedContextID, _ := id.NextID() - context.ExecutionID = expectedContextID id, err := repository.Insert(context) assert.Nil(t, err) assert.NotZero(t, id) @@ -117,11 +119,12 @@ func TestExecutionContextRepository_UpdateJobOutput(t *testing.T) { defer repository.DeleteAll() fake.Seed(0) + executionID, _ := id.NextID() context := model.ExecutionContext{ - JobName: fake.BuzzWord(), - Name: fake.HackerAdjective(), - UserEmail: fake.Email(), - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), Args: map[string]string{ fake.FirstName(): fake.LastName(), }, @@ -180,11 +183,13 @@ func populateSeedDataForTest(repository ExecutionContextRepository, count int, s defaultStatus = status.ExecutionStatus(val) } + executionID, _ := id.NextID() context := model.ExecutionContext{ - JobName: jobName, - Name: fake.HackerAdjective(), - UserEmail: email, - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: jobName, + Name: fake.HackerAdjective(), + UserEmail: email, + ImageTag: fake.BeerStyle(), Args: map[string]string{ fake.FirstName(): fake.LastName(), }, diff --git a/internal/app/service/schedule/repository/schedule_context_test.go b/internal/app/service/schedule/repository/schedule_context_test.go index a1fd627e..5aa49e4d 100644 --- a/internal/app/service/schedule/repository/schedule_context_test.go +++ b/internal/app/service/schedule/repository/schedule_context_test.go @@ -1,14 +1,16 @@ package repository import ( - fake "github.com/brianvoe/gofakeit" - "github.com/stretchr/testify/assert" executionModel "proctor/internal/app/service/execution/model" executionRepository "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/db/postgresql" + "proctor/internal/app/service/infra/id" "proctor/internal/app/service/schedule/model" "testing" + + fake "github.com/brianvoe/gofakeit" + "github.com/stretchr/testify/assert" ) type context interface { @@ -149,23 +151,25 @@ func TestScheduleContextRepository_GetContextAndSchedule(t *testing.T) { contextCount := 4 for i := 1; i <= contextCount; i++ { fake.Seed(0) + executionID, _ := id.NextID() executionContext := executionModel.ExecutionContext{ - JobName: fake.BuzzWord(), - UserEmail: fake.Email(), - ImageTag: fake.BeerStyle(), + ExecutionID: executionID, + JobName: fake.BuzzWord(), + UserEmail: fake.Email(), + ImageTag: fake.BeerStyle(), Args: map[string]string{ fake.FirstName(): fake.LastName(), }, Status: status.Received, } - executionContextId, err := ctx.instance().executionContextRepository.Insert(executionContext) + persistedExecutionID, err := ctx.instance().executionContextRepository.Insert(executionContext) assert.NoError(t, err) - assert.NotNil(t, executionContextId) + assert.NotNil(t, persistedExecutionID) scheduleContext := model.ScheduleContext{ ScheduleId: scheduleId, - ExecutionContextId: executionContextId, + ExecutionContextId: persistedExecutionID, } updatedContext, err := ctx.instance().repository.Insert(scheduleContext) From 0e849f5baf6c1382ebc9bc11fe3bd7b6a70918d6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 13 Aug 2019 11:30:51 +0700 Subject: [PATCH 101/268] [bimo.horizon] Add missing mocks --- .../app/service/execution/service/execution.go | 18 +++++++++--------- .../execution/service/execution_mock.go | 9 ++++++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 5ef3119f..9a7b1259 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -48,15 +48,6 @@ func NewExecutionService( } } -func (service *executionService) update(executionContext model.ExecutionContext) { - err := service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) - logger.LogErrors(err, "update execution context status", executionContext) - if len(executionContext.Output) > 0 { - err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) - logger.LogErrors(err, "update execution context output", executionContext) - } -} - func (service *executionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { err := service.kubernetesClient.WaitForReadyJob(executionName, waitTime) if err != nil { @@ -130,6 +121,15 @@ func (service *executionService) insertContext(context model.ExecutionContext) { logger.LogErrors(err, "save execution context to db", context) } +func (service *executionService) update(executionContext model.ExecutionContext) { + err := service.repository.UpdateStatus(executionContext.ExecutionID, executionContext.Status) + logger.LogErrors(err, "update execution context status", executionContext) + if len(executionContext.Output) > 0 { + err = service.repository.UpdateJobOutput(executionContext.ExecutionID, executionContext.Output) + logger.LogErrors(err, "update execution context output", executionContext) + } +} + func (service *executionService) watchProcess(context model.ExecutionContext) { waitTime := config.KubeLogProcessWaitTime() * time.Second diff --git a/internal/app/service/execution/service/execution_mock.go b/internal/app/service/execution/service/execution_mock.go index c7afaebb..60c7b86d 100644 --- a/internal/app/service/execution/service/execution_mock.go +++ b/internal/app/service/execution/service/execution_mock.go @@ -21,9 +21,12 @@ func (mockService *MockExecutionService) ExecuteWithCommand(jobName string, user return arguments.Get(0).(*model.ExecutionContext), arguments.String(1), arguments.Error(2) } -func (mockService *MockExecutionService) save(executionContext model.ExecutionContext) error { - args := mockService.Called(executionContext) - return args.Error(0) +func (mockService *MockExecutionService) update(executionContext model.ExecutionContext) { + mockService.Called(executionContext) +} + +func (mockService *MockExecutionService) insertContext(executionContext model.ExecutionContext) { + mockService.Called(executionContext) } func (mockService *MockExecutionService) StreamJobLogs(executionName string, waitTime time.Duration) (io.ReadCloser, error) { From 2c7006b4fe6a98ae404f27f0da435afc22d0ebfc Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 13 Aug 2019 11:31:36 +0700 Subject: [PATCH 102/268] [bimo.horizon] Fix failing test because weird log behavior --- .../app/service/execution/service/execution_test.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index 196900f1..cf86b72a 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -1,12 +1,14 @@ package service import ( + "bytes" "io/ioutil" "strings" "testing" fake "github.com/brianvoe/gofakeit" "github.com/docker/docker/pkg/testutil/assert" + "github.com/jmoiron/sqlx/types" "github.com/pkg/errors" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" @@ -127,9 +129,17 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { ImageName: imageName, } + // This is needed because #NopCloser adds additional foreign character + // at the end of the input string + logBuf := new(bytes.Buffer) + mockLog := ioutil.NopCloser(strings.NewReader("hello world")) + logBuf.ReadFrom(mockLog) + suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Times(3) + suite.mockRepository.On("UpdateStatus", mock.Anything, status.Finished).Return(nil).Once() + suite.mockRepository.On("UpdateJobOutput", mock.Anything, types.GzippedText(logBuf.String())).Return(nil).Once() suite.mockRepository.On("GetById", mock.Anything).Return(0, nil).Times(3) executionName := "execution-name" @@ -137,7 +147,6 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { suite.mockKubernetesClient.On("WaitForReadyJob", executionName, mock.Anything).Return(nil) podDetail := &v1.Pod{} suite.mockKubernetesClient.On("WaitForReadyPod", executionName, mock.Anything).Return(podDetail, nil) - mockLog := ioutil.NopCloser(strings.NewReader("hello world")) suite.mockKubernetesClient.On("GetPodLogs", podDetail).Return(mockLog, nil) context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) From e7de4cdcd57233e71688260b02bb781e197dc574 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 13 Aug 2019 15:19:24 +0700 Subject: [PATCH 103/268] [Jasoet] Set integration to disable parallel build --- Makefile | 2 +- internal/app/service/execution/service/execution.go | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 429bf1f5..6f4afdc7 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ test: .PHONY: itest itest: ENABLE_INTEGRATION_TEST=true \ - go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... + go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: server server: diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 9a7b1259..6841ec78 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -117,6 +117,7 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st } func (service *executionService) insertContext(context model.ExecutionContext) { + logger.Info("Insert Execution Context with ID: ", context.ExecutionID, ", name: ", context.Name, ", and Status: "+context.Status) _, err := service.repository.Insert(context) logger.LogErrors(err, "save execution context to db", context) } @@ -131,6 +132,7 @@ func (service *executionService) update(executionContext model.ExecutionContext) } func (service *executionService) watchProcess(context model.ExecutionContext) { + logger.Info("Start Watch Process for Job With Context ID: ", context.ExecutionID, ", name: ", context.Name, ", and Status: "+context.Status) waitTime := config.KubeLogProcessWaitTime() * time.Second err := service.kubernetesClient.WaitForReadyJob(context.Name, waitTime) From 7915d05ec6c7d805b8b5ed67414b38d5cae76fe5 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 13 Aug 2019 15:24:05 +0700 Subject: [PATCH 104/268] [Jasoet] Move printer to cli --- cmd/cli/main.go | 2 +- internal/app/cli/command/config/manager.go | 2 +- internal/app/cli/command/config/view/view.go | 2 +- internal/app/cli/command/description/descriptor.go | 2 +- internal/app/cli/command/description/descriptor_test.go | 2 +- internal/app/cli/command/execution/executioner.go | 2 +- internal/app/cli/command/execution/executioner_test.go | 2 +- internal/app/cli/command/list/lister.go | 4 ++-- internal/app/cli/command/list/lister_test.go | 2 +- internal/app/cli/command/log/log.go | 2 +- internal/app/cli/command/log/log_test.go | 2 +- internal/app/cli/command/root.go | 2 +- internal/app/cli/command/root_test.go | 2 +- internal/app/cli/command/schedule/describe/describe.go | 2 +- internal/app/cli/command/schedule/describe/describe_test.go | 2 +- internal/app/cli/command/schedule/list/list.go | 2 +- internal/app/cli/command/schedule/list/list_test.go | 2 +- internal/app/cli/command/schedule/remove/remove.go | 2 +- internal/app/cli/command/schedule/remove/remove_test.go | 2 +- internal/app/cli/command/schedule/schedule.go | 2 +- internal/app/cli/command/schedule/schedule_test.go | 2 +- internal/app/cli/command/status/status.go | 2 +- internal/app/cli/command/status/status_test.go | 2 +- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- internal/app/cli/daemon/client.go | 2 +- internal/app/cli/daemon/client_test.go | 2 +- internal/{pkg => app/cli/utility}/io/printer.go | 0 internal/{pkg => app/cli/utility}/io/printer_mock.go | 0 internal/app/cli/{ => utility}/sort/sort.go | 0 internal/app/cli/{ => utility}/sort/sort_test.go | 0 31 files changed, 28 insertions(+), 28 deletions(-) rename internal/{pkg => app/cli/utility}/io/printer.go (100%) rename internal/{pkg => app/cli/utility}/io/printer_mock.go (100%) rename internal/app/cli/{ => utility}/sort/sort.go (100%) rename internal/app/cli/{ => utility}/sort/sort_test.go (100%) diff --git a/cmd/cli/main.go b/cmd/cli/main.go index a889e403..29e4d2a6 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -5,7 +5,7 @@ import ( "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/config" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" ) func main() { diff --git a/internal/app/cli/command/config/manager.go b/internal/app/cli/command/config/manager.go index 21986a1e..822d241a 100644 --- a/internal/app/cli/command/config/manager.go +++ b/internal/app/cli/command/config/manager.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" "proctor/internal/app/cli/config" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strings" "github.com/fatih/color" diff --git a/internal/app/cli/command/config/view/view.go b/internal/app/cli/command/config/view/view.go index 4b37d0e4..03aaeb32 100644 --- a/internal/app/cli/command/config/view/view.go +++ b/internal/app/cli/command/config/view/view.go @@ -9,7 +9,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" ) func NewCmd(printer io.Printer) *cobra.Command { diff --git a/internal/app/cli/command/description/descriptor.go b/internal/app/cli/command/description/descriptor.go index 7fc563ec..d4540646 100644 --- a/internal/app/cli/command/description/descriptor.go +++ b/internal/app/cli/command/description/descriptor.go @@ -3,7 +3,7 @@ package description import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strings" "github.com/fatih/color" diff --git a/internal/app/cli/command/description/descriptor_test.go b/internal/app/cli/command/description/descriptor_test.go index fbc4af16..cc11145c 100644 --- a/internal/app/cli/command/description/descriptor_test.go +++ b/internal/app/cli/command/description/descriptor_test.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" daemon2 "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strings" "testing" diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index f315fa8a..66d10c06 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -3,7 +3,7 @@ package execution import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strings" "github.com/fatih/color" diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index 22031cd4..fceee22f 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/stretchr/testify/mock" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "proctor/internal/pkg/model/execution" "testing" diff --git a/internal/app/cli/command/list/lister.go b/internal/app/cli/command/list/lister.go index 7d55d4ec..c1e222ef 100644 --- a/internal/app/cli/command/list/lister.go +++ b/internal/app/cli/command/list/lister.go @@ -3,11 +3,11 @@ package list import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "github.com/fatih/color" "github.com/spf13/cobra" - "proctor/internal/app/cli/sort" + "proctor/internal/app/cli/utility/sort" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/internal/app/cli/command/list/lister_test.go b/internal/app/cli/command/list/lister_test.go index 14d51df5..efac7918 100644 --- a/internal/app/cli/command/list/lister_test.go +++ b/internal/app/cli/command/list/lister_test.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" "github.com/fatih/color" diff --git a/internal/app/cli/command/log/log.go b/internal/app/cli/command/log/log.go index b752d39b..ccfe7940 100644 --- a/internal/app/cli/command/log/log.go +++ b/internal/app/cli/command/log/log.go @@ -3,7 +3,7 @@ package log import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strconv" "github.com/fatih/color" diff --git a/internal/app/cli/command/log/log_test.go b/internal/app/cli/command/log/log_test.go index e1b680e2..9fc3949d 100644 --- a/internal/app/cli/command/log/log_test.go +++ b/internal/app/cli/command/log/log_test.go @@ -4,7 +4,7 @@ import ( "errors" "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" "github.com/fatih/color" diff --git a/internal/app/cli/command/root.go b/internal/app/cli/command/root.go index 57294b2e..15078ced 100644 --- a/internal/app/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -20,7 +20,7 @@ import ( "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" ) var ( diff --git a/internal/app/cli/command/root_test.go b/internal/app/cli/command/root_test.go index 2816b267..67ac520b 100644 --- a/internal/app/cli/command/root_test.go +++ b/internal/app/cli/command/root_test.go @@ -3,7 +3,7 @@ package command import ( "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" "github.com/spf13/cobra" diff --git a/internal/app/cli/command/schedule/describe/describe.go b/internal/app/cli/command/schedule/describe/describe.go index d9a81522..1616d559 100644 --- a/internal/app/cli/command/schedule/describe/describe.go +++ b/internal/app/cli/command/schedule/describe/describe.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/internal/app/cli/command/schedule/describe/describe_test.go b/internal/app/cli/command/schedule/describe/describe_test.go index 80246000..aa7324bd 100644 --- a/internal/app/cli/command/schedule/describe/describe_test.go +++ b/internal/app/cli/command/schedule/describe/describe_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" ) diff --git a/internal/app/cli/command/schedule/list/list.go b/internal/app/cli/command/schedule/list/list.go index 5587efac..8d6d9b8f 100644 --- a/internal/app/cli/command/schedule/list/list.go +++ b/internal/app/cli/command/schedule/list/list.go @@ -3,7 +3,7 @@ package list import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "github.com/fatih/color" "github.com/spf13/cobra" diff --git a/internal/app/cli/command/schedule/list/list_test.go b/internal/app/cli/command/schedule/list/list_test.go index 0ebbd9dc..654656a8 100644 --- a/internal/app/cli/command/schedule/list/list_test.go +++ b/internal/app/cli/command/schedule/list/list_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" ) diff --git a/internal/app/cli/command/schedule/remove/remove.go b/internal/app/cli/command/schedule/remove/remove.go index f13c1fbc..7d1135f9 100644 --- a/internal/app/cli/command/schedule/remove/remove.go +++ b/internal/app/cli/command/schedule/remove/remove.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { diff --git a/internal/app/cli/command/schedule/remove/remove_test.go b/internal/app/cli/command/schedule/remove/remove_test.go index 8176df85..e121162d 100644 --- a/internal/app/cli/command/schedule/remove/remove_test.go +++ b/internal/app/cli/command/schedule/remove/remove_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" ) diff --git a/internal/app/cli/command/schedule/schedule.go b/internal/app/cli/command/schedule/schedule.go index 036f0037..0407766c 100644 --- a/internal/app/cli/command/schedule/schedule.go +++ b/internal/app/cli/command/schedule/schedule.go @@ -5,7 +5,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strings" ) diff --git a/internal/app/cli/command/schedule/schedule_test.go b/internal/app/cli/command/schedule/schedule_test.go index 018d5015..3f637aeb 100644 --- a/internal/app/cli/command/schedule/schedule_test.go +++ b/internal/app/cli/command/schedule/schedule_test.go @@ -5,7 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" ) diff --git a/internal/app/cli/command/status/status.go b/internal/app/cli/command/status/status.go index 3139f4ce..7cf5d999 100644 --- a/internal/app/cli/command/status/status.go +++ b/internal/app/cli/command/status/status.go @@ -3,7 +3,7 @@ package status import ( "fmt" "proctor/internal/app/cli/daemon" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "strconv" "github.com/fatih/color" diff --git a/internal/app/cli/command/status/status_test.go b/internal/app/cli/command/status/status_test.go index 38a08f87..e2692108 100644 --- a/internal/app/cli/command/status/status_test.go +++ b/internal/app/cli/command/status/status_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/suite" "proctor/internal/app/cli/daemon" + "proctor/internal/app/cli/utility/io" "proctor/internal/pkg/constant" - "proctor/internal/pkg/io" modelExecution "proctor/internal/pkg/model/execution" ) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index 2cc1ce5e..f47f6704 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -3,7 +3,7 @@ package version import ( "fmt" "proctor/internal/app/cli/command/version/github" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "github.com/fatih/color" "github.com/spf13/cobra" diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 72726e7b..6e535f3f 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -3,7 +3,7 @@ package version import ( "fmt" "proctor/internal/app/cli/command/version/github" - "proctor/internal/pkg/io" + "proctor/internal/app/cli/utility/io" "testing" "github.com/fatih/color" diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 93edd6a4..e4c74844 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -20,8 +20,8 @@ import ( "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/config" + "proctor/internal/app/cli/utility/io" "proctor/internal/pkg/constant" - "proctor/internal/pkg/io" modelExecution "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" modelSchedule "proctor/internal/pkg/model/schedule" diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index ea066a29..70139bcb 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -15,8 +15,8 @@ import ( "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/config" + "proctor/internal/app/cli/utility/io" "proctor/internal/pkg/constant" - "proctor/internal/pkg/io" modelExecution "proctor/internal/pkg/model/execution" modelMetadata "proctor/internal/pkg/model/metadata" "proctor/internal/pkg/model/metadata/env" diff --git a/internal/pkg/io/printer.go b/internal/app/cli/utility/io/printer.go similarity index 100% rename from internal/pkg/io/printer.go rename to internal/app/cli/utility/io/printer.go diff --git a/internal/pkg/io/printer_mock.go b/internal/app/cli/utility/io/printer_mock.go similarity index 100% rename from internal/pkg/io/printer_mock.go rename to internal/app/cli/utility/io/printer_mock.go diff --git a/internal/app/cli/sort/sort.go b/internal/app/cli/utility/sort/sort.go similarity index 100% rename from internal/app/cli/sort/sort.go rename to internal/app/cli/utility/sort/sort.go diff --git a/internal/app/cli/sort/sort_test.go b/internal/app/cli/utility/sort/sort_test.go similarity index 100% rename from internal/app/cli/sort/sort_test.go rename to internal/app/cli/utility/sort/sort_test.go From c4d8fefeb7d7357cf6fc10ce083f942ec9a9c4cb Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Tue, 13 Aug 2019 15:27:21 +0700 Subject: [PATCH 105/268] [bimo.horizon] Reduce integration test waiting time --- .../app/service/execution/service/execution_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 99c07a42..159ae7f1 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -80,7 +80,7 @@ func (suite *TestExecutionIntegrationSuite) TestExecuteJobSuccess() { assert.NoError(t, err) assert.NotNil(t, context) - time.Sleep(40 * time.Second) + time.Sleep(30 * time.Second) expectedContext, err := suite.repository.GetById(context.ExecutionID) assert.NoError(t, err) assert.NotNil(t, expectedContext) From dfc5bf3346655b72179862843470ce2f48704d72 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 14 Aug 2019 13:07:52 +0700 Subject: [PATCH 106/268] [Jasoet] Add Security Service --- .env.test | 2 + internal/app/service/infra/config/config.go | 16 ++ internal/app/service/infra/plugin/plugin.go | 33 +++++ .../infra/plugin/plugin_integration_test.go | 74 ++++++++++ .../app/service/infra/plugin/plugin_mock.go | 15 ++ .../app/service/security/service/security.go | 66 +++++++++ .../service/security/service/security_mock.go | 25 ++++ .../service/security/service/security_test.go | 137 ++++++++++++++++++ internal/app/service/server/router.go | 3 +- pkg/auth/auth.go | 13 ++ pkg/auth/auth_mock.go | 19 +++ plugins/gate-auth-plugin/auth.go | 3 + 12 files changed, 405 insertions(+), 1 deletion(-) create mode 100644 internal/app/service/infra/plugin/plugin.go create mode 100644 internal/app/service/infra/plugin/plugin_integration_test.go create mode 100644 internal/app/service/infra/plugin/plugin_mock.go create mode 100644 internal/app/service/security/service/security.go create mode 100644 internal/app/service/security/service/security_mock.go create mode 100644 internal/app/service/security/service/security_test.go create mode 100644 pkg/auth/auth.go create mode 100644 pkg/auth/auth_mock.go create mode 100644 plugins/gate-auth-plugin/auth.go diff --git a/.env.test b/.env.test index ad63ee91..234112d0 100644 --- a/.env.test +++ b/.env.test @@ -33,3 +33,5 @@ export PROCTOR_MAIL_SERVER_PORT=123 export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir +export PROCTOR_AUTH_PLUGIN_BINARY= +export PROCTOR_AUTH_PLUGIN_EXPORTED=Auth diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 71c00f9d..dee747c1 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -37,14 +37,17 @@ func RedisAddress() string { return viper.GetString("REDIS_ADDRESS") } +//TODO remove func KubeClusterHostName() string { return viper.GetString("KUBE_CLUSTER_HOST_NAME") } +//TODO Should Not be Used since we use ~/.kube/config func KubeCACertEncoded() string { return viper.GetString("KUBE_CA_CERT_ENCODED") } +//TODO remove func KubeBasicAuthEncoded() string { return viper.GetString("KUBE_BASIC_AUTH_ENCODED") } @@ -61,10 +64,12 @@ func LogsStreamWriteBufferSize() int { return viper.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE") } +//TODO test func KubePodsListWaitTime() time.Duration { return time.Duration(viper.GetInt("KUBE_POD_LIST_WAIT_TIME")) } +//TODO test func KubeLogProcessWaitTime() time.Duration { return time.Duration(viper.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")) } @@ -158,3 +163,14 @@ func SentryDSN() string { func DocsPath() string { return viper.GetString("DOCS_PATH") } + +//TODO test +func AuthPluginBinary() string { + return viper.GetString("AUTH_PLUGIN_BINARY") +} + +//TODO test +func AuthPluginExported() string { + viper.SetDefault("AUTH_PLUGIN_EXPORTED", "Auth") + return viper.GetString("AUTH_PLUGIN_EXPORTED") +} diff --git a/internal/app/service/infra/plugin/plugin.go b/internal/app/service/infra/plugin/plugin.go new file mode 100644 index 00000000..4671344b --- /dev/null +++ b/internal/app/service/infra/plugin/plugin.go @@ -0,0 +1,33 @@ +package plugin + +import ( + "fmt" + "plugin" + "proctor/internal/app/service/infra/logger" +) + +type GoPlugin interface { + Load(pluginBinary string, exportedName string) (plugin.Symbol, error) +} + +type goPlugin struct{} + +func (g *goPlugin) Load(pluginBinary string, exportedName string) (plugin.Symbol, error) { + binary, err := plugin.Open(pluginBinary) + logger.LogErrors(err, "load auth plugin binary from location: ", pluginBinary) + if err != nil { + return nil, fmt.Errorf("failed to load plugin binary from location: %s", pluginBinary) + } + + raw, err := binary.Lookup(exportedName) + logger.LogErrors(err, "Lookup ", pluginBinary, " for ", exportedName) + if err != nil { + return nil, fmt.Errorf("failed to Lookup plugin binary from location: %s with Exported Name: %s", pluginBinary, exportedName) + } + return raw, nil +} + +func NewGoPlugin() GoPlugin { + return &goPlugin{} +} + diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go new file mode 100644 index 00000000..b2cc04e3 --- /dev/null +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -0,0 +1,74 @@ +package plugin + +import ( + "github.com/stretchr/testify/assert" + "os" + "proctor/internal/app/service/infra/config" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + pluginBinary string + exportedName string + goPlugin GoPlugin +} + +func (context *testContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + context.pluginBinary = config.AuthPluginBinary() + assert.NotEmpty(t, context.pluginBinary) + context.exportedName = config.AuthPluginExported() + assert.NotEmpty(t, context.exportedName) + context.goPlugin = NewGoPlugin() + assert.NotNil(t, context.goPlugin) +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + ctx := &testContext{} + return ctx +} + +func TestGoPlugin_LoadSuccessfully(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + raw, err := ctx.instance().goPlugin.Load(ctx.instance().pluginBinary, ctx.instance().exportedName) + assert.NoError(t, err) + assert.NotNil(t, raw) +} + +func TestGoPlugin_LoadPluginFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + binary := "non-existing-binary" + raw, err := ctx.instance().goPlugin.Load(binary, ctx.instance().exportedName) + assert.EqualErrorf(t, err, "failed to load plugin binary from location: %s", binary) + assert.Nil(t, raw) +} + +func TestGoPlugin_LoadExportedFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + exportedName := "non-existing-exported" + raw, err := ctx.instance().goPlugin.Load(ctx.instance().pluginBinary, exportedName) + assert.EqualErrorf(t, err, "failed to Lookup plugin binary from location: %s with Exported Name: %s", ctx.instance().pluginBinary, exportedName) + assert.Nil(t, raw) +} diff --git a/internal/app/service/infra/plugin/plugin_mock.go b/internal/app/service/infra/plugin/plugin_mock.go new file mode 100644 index 00000000..31efb947 --- /dev/null +++ b/internal/app/service/infra/plugin/plugin_mock.go @@ -0,0 +1,15 @@ +package plugin + +import ( + "github.com/stretchr/testify/mock" + "plugin" +) + +type GoPluginMock struct { + mock.Mock +} + +func (g *GoPluginMock) Load(pluginBinary string, exportedName string) (plugin.Symbol, error) { + args := g.Called(pluginBinary, exportedName) + return args.Get(0).(plugin.Symbol), args.Error(1) +} diff --git a/internal/app/service/security/service/security.go b/internal/app/service/security/service/security.go new file mode 100644 index 00000000..3e8f7f0f --- /dev/null +++ b/internal/app/service/security/service/security.go @@ -0,0 +1,66 @@ +package service + +import ( + "fmt" + "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/infra/plugin" + "proctor/pkg/auth" + "sync" +) + +type SecurityService interface { + initializePlugin() error + auth.Auth +} + +type securityService struct { + goPlugin plugin.GoPlugin + pluginBinary string + pluginExportedName string + authInstance auth.Auth + once sync.Once +} + +func (s *securityService) Verify(userDetail auth.UserDetail, group []string) (bool, error) { + return s.authInstance.Verify(userDetail, group) +} + +func (s *securityService) Auth(email string, token string) (*auth.UserDetail, error) { + err := s.initializePlugin() + logger.LogErrors(err, "initialize plugin") + if err != nil { + return nil, err + } + return s.authInstance.Auth(email, token) +} + +func (s *securityService) initializePlugin() error { + s.once.Do(func() { + raw, err := s.goPlugin.Load(s.pluginBinary, s.pluginExportedName) + logger.LogErrors(err, "Load GoPlugin binary") + if err != nil { + return + } + authInstance, ok := raw.(auth.Auth) + if !ok { + logger.Error("Failed to convert exported plugin to *auth.Auth type") + return + } + s.authInstance = authInstance + }) + + if s.authInstance == nil { + return fmt.Errorf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", s.pluginBinary, s.pluginExportedName) + } else { + return nil + } +} + +func NewSecurityService(pluginBinary string, exportedName string, goPlugin plugin.GoPlugin) SecurityService { + return &securityService{ + pluginBinary: pluginBinary, + pluginExportedName: exportedName, + goPlugin: goPlugin, + once: sync.Once{}, + } +} diff --git a/internal/app/service/security/service/security_mock.go b/internal/app/service/security/service/security_mock.go new file mode 100644 index 00000000..209df278 --- /dev/null +++ b/internal/app/service/security/service/security_mock.go @@ -0,0 +1,25 @@ +package service + +import ( + "github.com/stretchr/testify/mock" + "proctor/pkg/auth" +) + +type SecurityServiceMock struct { + mock.Mock +} + +func (m *SecurityServiceMock) initializePlugin() error { + args := m.Called() + return args.Error(0) +} + +func (m *SecurityServiceMock) Auth(email string, token string) (auth.UserDetail, error) { + args := m.Called(email, token) + return args.Get(0).(auth.UserDetail), args.Error(1) +} + +func (m *SecurityServiceMock) Verify(userDetail auth.UserDetail, group []string) (bool, error) { + args := m.Called(userDetail, group) + return args.Get(0).(bool), args.Error(1) +} diff --git a/internal/app/service/security/service/security_test.go b/internal/app/service/security/service/security_test.go new file mode 100644 index 00000000..ad153ec4 --- /dev/null +++ b/internal/app/service/security/service/security_test.go @@ -0,0 +1,137 @@ +package service + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "proctor/internal/app/service/infra/plugin" + "proctor/pkg/auth" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + pluginBinary string + exportedName string + goPlugin *plugin.GoPluginMock + securityService SecurityService + auth *auth.AuthMock +} + +func (context *testContext) setUp(t *testing.T) { + context.pluginBinary = "plugin-binary" + assert.NotEmpty(t, context.pluginBinary) + context.exportedName = "exported-name" + assert.NotEmpty(t, context.exportedName) + context.goPlugin = &plugin.GoPluginMock{} + assert.NotNil(t, context.goPlugin) + context.securityService = NewSecurityService(context.pluginBinary, context.exportedName, context.goPlugin) + assert.NotNil(t, context.securityService) + context.auth = &auth.AuthMock{} + assert.NotNil(t, context.auth) +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + ctx := &testContext{} + return ctx +} + +func TestSecurityService_AuthSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "jasoet@go-jek.com" + token := "randomtokenforthesakeofopensource" + + userDetail := &auth.UserDetail{ + Name: "Deny Prasetyo", + Email: "jasoet87@gmail.com", + Active: true, + Group: []string{"system", "proctor_executor"}, + } + ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) + + ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) + + expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) + assert.NoError(t, err) + assert.NotNil(t, expectedUserDetail) + assert.Equal(t, userDetail, expectedUserDetail) +} + +func TestSecurityService_AuthPluginLoadFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "jasoet@go-jek.com" + token := "randomtokenforthesakeofopensource" + + ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, fmt.Errorf("load error bro")) + + expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) + assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", ctx.instance().pluginBinary, ctx.instance().exportedName)) + assert.Nil(t, expectedUserDetail) +} + +func TestSecurityService_AuthPluginFailedToCast(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "jasoet@go-jek.com" + token := "randomtokenforthesakeofopensource" + + userDetail := &auth.UserDetail{ + Name: "Deny Prasetyo", + Email: "jasoet87@gmail.com", + Active: true, + Group: []string{"system", "proctor_executor"}, + } + + ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(userDetail, nil) + + expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) + assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", ctx.instance().pluginBinary, ctx.instance().exportedName)) + assert.Nil(t, expectedUserDetail) +} + +func TestSecurityService_VerifySuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "jasoet@go-jek.com" + token := "randomtokenforthesakeofopensource" + + userDetail := &auth.UserDetail{ + Name: "Deny Prasetyo", + Email: "jasoet87@gmail.com", + Active: true, + Group: []string{"system", "proctor_executor"}, + } + + ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) + + ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) + + expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) + assert.NoError(t, err) + assert.NotNil(t, expectedUserDetail) + assert.Equal(t, userDetail, expectedUserDetail) + + group := []string{"system", "proctor_executor"} + ctx.instance().auth.On("Verify", *expectedUserDetail, group).Return(true, nil) + verified, err := ctx.instance().securityService.Verify(*expectedUserDetail, group) + assert.True(t, verified) + assert.NoError(t, err) +} + diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 8d79b87f..32d7541e 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -67,8 +67,9 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") - router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") + + router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") router.HandleFunc("/secret", jobSecretsHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.Post()).Methods("POST") diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go new file mode 100644 index 00000000..0cf51ef9 --- /dev/null +++ b/pkg/auth/auth.go @@ -0,0 +1,13 @@ +package auth + +type Auth interface { + Auth(email string, token string) (*UserDetail, error) + Verify(userDetail UserDetail, group []string) (bool, error) +} + +type UserDetail struct { + Name string + Email string + Active bool + Group []string +} diff --git a/pkg/auth/auth_mock.go b/pkg/auth/auth_mock.go new file mode 100644 index 00000000..6e59527a --- /dev/null +++ b/pkg/auth/auth_mock.go @@ -0,0 +1,19 @@ +package auth + +import ( + "github.com/stretchr/testify/mock" +) + +type AuthMock struct { + mock.Mock +} + +func (m *AuthMock) Auth(email string, token string) (*UserDetail, error) { + args := m.Called(email, token) + return args.Get(0).(*UserDetail), args.Error(1) +} + +func (m *AuthMock) Verify(userDetail UserDetail, group []string) (bool, error) { + args := m.Called(userDetail, group) + return args.Get(0).(bool), args.Error(1) +} diff --git a/plugins/gate-auth-plugin/auth.go b/plugins/gate-auth-plugin/auth.go new file mode 100644 index 00000000..1110061a --- /dev/null +++ b/plugins/gate-auth-plugin/auth.go @@ -0,0 +1,3 @@ +package main + + From 1cca83a999d2585bd62ed2f9274b0d0c0e21efa6 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 14 Aug 2019 13:16:51 +0700 Subject: [PATCH 107/268] [Jasoet] Add test for Verify Auth --- .../app/service/security/service/security.go | 5 +++ .../service/security/service/security_test.go | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/internal/app/service/security/service/security.go b/internal/app/service/security/service/security.go index 3e8f7f0f..57339cf8 100644 --- a/internal/app/service/security/service/security.go +++ b/internal/app/service/security/service/security.go @@ -22,6 +22,11 @@ type securityService struct { } func (s *securityService) Verify(userDetail auth.UserDetail, group []string) (bool, error) { + err := s.initializePlugin() + logger.LogErrors(err, "initialize plugin") + if err != nil { + return false, err + } return s.authInstance.Verify(userDetail, group) } diff --git a/internal/app/service/security/service/security_test.go b/internal/app/service/security/service/security_test.go index ad153ec4..7ed7ac63 100644 --- a/internal/app/service/security/service/security_test.go +++ b/internal/app/service/security/service/security_test.go @@ -135,3 +135,34 @@ func TestSecurityService_VerifySuccess(t *testing.T) { assert.NoError(t, err) } +func TestSecurityService_VerifyFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "jasoet@go-jek.com" + token := "randomtokenforthesakeofopensource" + + userDetail := &auth.UserDetail{ + Name: "Deny Prasetyo", + Email: "jasoet87@gmail.com", + Active: true, + Group: []string{"system", "proctor_executor"}, + } + + ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) + + ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) + + expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) + assert.NoError(t, err) + assert.NotNil(t, expectedUserDetail) + assert.Equal(t, userDetail, expectedUserDetail) + + group := []string{"system", "proctor_executor"} + ctx.instance().auth.On("Verify", *expectedUserDetail, group).Return(false, fmt.Errorf("verify error")) + verified, err := ctx.instance().securityService.Verify(*expectedUserDetail, group) + assert.False(t, verified) + assert.EqualError(t, err, "verify error") +} + + From efde1376da6fb0868ae1cd3092ec92e4afbb7a93 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:36:36 +0700 Subject: [PATCH 108/268] [bimo.horizon] Add yaml dependency --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index db928a88..535481ff 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/urfave/cli v1.20.0 github.com/urfave/negroni v1.0.0 gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.2.2 k8s.io/api v0.0.0-20190409021203-6e4e0e4f393b k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible From 621b36409de7ac3abe4f5262e63e31a357734d8b Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:38:40 +0700 Subject: [PATCH 109/268] [bimo.horizon] Move schedule flags declaration from root to its own context --- internal/app/cli/command/root.go | 11 ----------- internal/app/cli/command/schedule/schedule.go | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/internal/app/cli/command/root.go b/internal/app/cli/command/root.go index 15078ced..cb54e3dc 100644 --- a/internal/app/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -65,17 +65,6 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith scheduleRemoveCmd := remove.NewCmd(printer, proctorDClient) scheduleCmd.AddCommand(scheduleRemoveCmd) - var Time, NotifyEmails, Tags, Group string - - scheduleCmd.PersistentFlags().StringVarP(&Time, "time", "t", "", "Schedule time") - _ = scheduleCmd.MarkFlagRequired("time") - scheduleCmd.PersistentFlags().StringVarP(&Group, "group", "g", "", "Group Name") - _ = scheduleCmd.MarkFlagRequired("group") - scheduleCmd.PersistentFlags().StringVarP(&NotifyEmails, "notify", "n", "", "Notifier Email ID's") - _ = scheduleCmd.MarkFlagRequired("notify") - scheduleCmd.PersistentFlags().StringVarP(&Tags, "tags", "T", "", "Tags") - _ = scheduleCmd.MarkFlagRequired("tags") - if err := rootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) diff --git a/internal/app/cli/command/schedule/schedule.go b/internal/app/cli/command/schedule/schedule.go index 0407766c..049630fe 100644 --- a/internal/app/cli/command/schedule/schedule.go +++ b/internal/app/cli/command/schedule/schedule.go @@ -10,7 +10,7 @@ import ( ) func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { - return &cobra.Command{ + scheduleCmd := &cobra.Command{ Use: "schedule", Short: "Create scheduled jobs", Long: "This command helps to create scheduled jobs", @@ -69,4 +69,17 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { printer.Println(fmt.Sprintf("Scheduled Job UUID : %s", scheduledJobID), color.FgGreen) }, } + + var Time, NotifyEmails, Tags, Group string + + scheduleCmd.PersistentFlags().StringVarP(&Time, "time", "t", "", "Schedule time") + _ = scheduleCmd.MarkFlagRequired("time") + scheduleCmd.PersistentFlags().StringVarP(&Group, "group", "g", "", "Group Name") + _ = scheduleCmd.MarkFlagRequired("group") + scheduleCmd.PersistentFlags().StringVarP(&NotifyEmails, "notify", "n", "", "Notifier Email ID's") + _ = scheduleCmd.MarkFlagRequired("notify") + scheduleCmd.PersistentFlags().StringVarP(&Tags, "tags", "T", "", "Tags") + _ = scheduleCmd.MarkFlagRequired("tags") + + return scheduleCmd } From 98f7d06a2ac37c95a41efcb7fde239636e25d55c Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:39:18 +0700 Subject: [PATCH 110/268] [bimo.horizon] Fix executioner help grammar --- internal/app/cli/command/execution/executioner.go | 2 +- internal/app/cli/command/execution/executioner_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index 66d10c06..f4580013 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -14,7 +14,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in return &cobra.Command{ Use: "execute", Short: "Execute a proc with given arguments", - Long: "To execute a proc, this command helps communicate with `proctord` and streams to logs of proc in execution", + Long: "To execute a proc, this command helps to communicate with `proctord` and streams to logs of proc in execution", Example: "proctor execute proc-one SOME_VAR=foo ANOTHER_VAR=bar\nproctor execute proc-two ANY_VAR=baz", Args: cobra.MinimumNArgs(1), diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index fceee22f..fb2ab80a 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -34,7 +34,7 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdUsage() { func (s *ExecutionCmdTestSuite) TestExecutionCmdHelp() { assert.Equal(s.T(), "Execute a proc with given arguments", s.testExecutionCmd.Short) - assert.Equal(s.T(), "To execute a proc, this command helps communicate with `proctord` and streams to logs of proc in execution", s.testExecutionCmd.Long) + assert.Equal(s.T(), "To execute a proc, this command helps to communicate with `proctord` and streams to logs of proc in execution", s.testExecutionCmd.Long) assert.Equal(s.T(), "proctor execute proc-one SOME_VAR=foo ANOTHER_VAR=bar\nproctor execute proc-two ANY_VAR=baz", s.testExecutionCmd.Example) } From 6c0e1be958c4c3f32936f7cb175dc22787784c92 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:42:37 +0700 Subject: [PATCH 111/268] [bimo.horizon] Move args parsing to its own utility --- .../app/cli/command/execution/executioner.go | 11 ++------ internal/app/cli/utility/args/args.go | 22 +++++++++++++++ internal/app/cli/utility/args/args_test.go | 28 +++++++++++++++++++ 3 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 internal/app/cli/utility/args/args.go create mode 100644 internal/app/cli/utility/args/args_test.go diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index f4580013..a4cac6b1 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -8,6 +8,7 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" + utilArgs "proctor/internal/app/cli/utility/args" ) func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { @@ -26,15 +27,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in if len(args) > 1 { printer.Println("With Variables", color.FgMagenta) for _, v := range args[1:] { - arg := strings.Split(v, "=") - - if len(arg) < 2 { - printer.Println(fmt.Sprintf("%-40s %-100s", "\nIncorrect variable format\n", v), color.FgRed) - continue - } - - combinedArgValue := strings.Join(arg[1:], "=") - procArgs[arg[0]] = combinedArgValue + utilArgs.ParseArg(printer, procArgs, v) printer.Println(fmt.Sprintf("%-40s %-100s", arg[0], combinedArgValue), color.Reset) } diff --git a/internal/app/cli/utility/args/args.go b/internal/app/cli/utility/args/args.go new file mode 100644 index 00000000..fb5c276e --- /dev/null +++ b/internal/app/cli/utility/args/args.go @@ -0,0 +1,22 @@ +package args + +import ( + "fmt" + "strings" + + "github.com/fatih/color" + + "proctor/internal/app/cli/utility/io" +) + +func ParseArg(printer io.Printer, procArgs map[string]string, arg string) { + parsedArg := strings.Split(arg, "=") + + if len(parsedArg) < 2 { + printer.Println(fmt.Sprintf("%-40s %-100s", "\nIncorrect variable format\n", arg), color.FgRed) + return + } + + combinedArgValue := strings.Join(parsedArg[1:], "=") + procArgs[parsedArg[0]] = combinedArgValue +} diff --git a/internal/app/cli/utility/args/args_test.go b/internal/app/cli/utility/args/args_test.go new file mode 100644 index 00000000..908f805d --- /dev/null +++ b/internal/app/cli/utility/args/args_test.go @@ -0,0 +1,28 @@ +package args + +import ( + "fmt" + "testing" + + "github.com/fatih/color" + "github.com/stretchr/testify/assert" + + "proctor/internal/app/cli/utility/io" +) + +func TestParseArg(t *testing.T) { + procArgs := make(map[string]string) + mockPrinter := &io.MockPrinter{} + ParseArg(mockPrinter, procArgs, "foo=moo") + assert.Equal(t, procArgs["foo"], "moo") +} + +func TestParseArgError(t *testing.T) { + procArgs := make(map[string]string) + mockPrinter := &io.MockPrinter{} + + mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "\nIncorrect variable format\n", "foo"), color.FgRed) + defer mockPrinter.AssertExpectations(t) + + ParseArg(mockPrinter, procArgs, "foo") +} From b073ff362e0b1d5b123e98d5173b7fa53222b7d6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:43:38 +0700 Subject: [PATCH 112/268] [bimo.horizon] Add yaml parsing support on executioner command --- .../app/cli/command/execution/executioner.go | 33 +++++++-- .../cli/command/execution/executioner_test.go | 70 ++++++++++++++++--- internal/app/cli/utility/file/file.go | 25 +++++++ internal/app/cli/utility/file/file_test.go | 46 ++++++++++++ 4 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 internal/app/cli/utility/file/file.go create mode 100644 internal/app/cli/utility/file/file_test.go diff --git a/internal/app/cli/command/execution/executioner.go b/internal/app/cli/command/execution/executioner.go index a4cac6b1..9027c9c7 100644 --- a/internal/app/cli/command/execution/executioner.go +++ b/internal/app/cli/command/execution/executioner.go @@ -2,17 +2,19 @@ package execution import ( "fmt" - "proctor/internal/app/cli/daemon" - "proctor/internal/app/cli/utility/io" "strings" "github.com/fatih/color" "github.com/spf13/cobra" + + "proctor/internal/app/cli/daemon" utilArgs "proctor/internal/app/cli/utility/args" + utilFile "proctor/internal/app/cli/utility/file" + utilIO "proctor/internal/app/cli/utility/io" ) -func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { - return &cobra.Command{ +func NewCmd(printer utilIO.Printer, proctorDClient daemon.Client, osExitFunc func(int)) *cobra.Command { + executionCmd := &cobra.Command{ Use: "execute", Short: "Execute a proc with given arguments", Long: "To execute a proc, this command helps to communicate with `proctord` and streams to logs of proc in execution", @@ -23,13 +25,26 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in procName := args[0] printer.Println(fmt.Sprintf("%-40s %-100s", "Executing Proc", procName), color.Reset) + filename, err := cmd.Flags().GetString("filename") + if err != nil && !strings.Contains(err.Error(), "flag accessed but not defined") { + printer.Println(err.Error(), color.FgRed) + } + procArgs := make(map[string]string) - if len(args) > 1 { + if filename != "" { + parseErr := utilFile.ParseYAML(filename, procArgs) + if err != nil { + printer.Println(parseErr.Error(), color.FgRed) + } + } + if len(procArgs) > 1 || len(args) > 1 { printer.Println("With Variables", color.FgMagenta) for _, v := range args[1:] { utilArgs.ParseArg(printer, procArgs, v) + } - printer.Println(fmt.Sprintf("%-40s %-100s", arg[0], combinedArgValue), color.Reset) + for field, value := range procArgs { + printer.Println(fmt.Sprintf("%-40s %-100s", field, value), color.Reset) } } else { printer.Println("With No Variables", color.FgRed) @@ -58,4 +73,10 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client, osExitFunc func(in printer.Println("Execution completed.", color.FgGreen) }, } + var Filename string + + executionCmd.Flags().StringVarP(&Filename, "filename", "f", "", "Filename") + executionCmd.MarkFlagFilename("filename") + + return executionCmd } diff --git a/internal/app/cli/command/execution/executioner_test.go b/internal/app/cli/command/execution/executioner_test.go index fb2ab80a..7c108d42 100644 --- a/internal/app/cli/command/execution/executioner_test.go +++ b/internal/app/cli/command/execution/executioner_test.go @@ -3,16 +3,19 @@ package execution import ( "errors" "fmt" - "github.com/stretchr/testify/mock" - "proctor/internal/app/cli/daemon" - "proctor/internal/app/cli/utility/io" - "proctor/internal/pkg/model/execution" + "io/ioutil" + "os" "testing" "github.com/fatih/color" "github.com/spf13/cobra" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" + + "proctor/internal/app/cli/daemon" + "proctor/internal/app/cli/utility/io" + "proctor/internal/pkg/model/execution" ) type ExecutionCmdTestSuite struct { @@ -70,6 +73,47 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmd() { s.mockPrinter.AssertExpectations(s.T()) } +func (s *ExecutionCmdTestSuite) TestExecutionCmdForYAMLInput() { + t := s.T() + + filename := "/tmp/yaml-input-test" + testYAML := []byte("SAMPLE_ARG_ONE: any\nSAMPLE_ARG_TWO: variable") + err := ioutil.WriteFile(filename, testYAML, 0644) + defer os.Remove(filename) + assert.NoError(t, err) + + args := []string{"say-hello-world", "-f", filename} + procArgs := make(map[string]string) + procArgs["SAMPLE_ARG_ONE"] = "any" + procArgs["SAMPLE_ARG_TWO"] = "variable" + + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "Executing Proc", "say-hello-world"), color.Reset).Once() + s.mockPrinter.On("Println", "With Variables", color.FgMagenta).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "SAMPLE_ARG_ONE", "any"), color.Reset).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", "SAMPLE_ARG_TWO", "variable"), color.Reset).Once() + + executionResult := &execution.ExecutionResult{ + ExecutionId: uint64(42), + ExecutionName: "Test", + } + + s.mockProctorDClient.On("ExecuteProc", "say-hello-world", procArgs).Return(executionResult, nil).Once() + s.mockPrinter.On("Println", "\nExecution Created", color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "ID", executionResult.ExecutionId), color.FgGreen).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100v", "Name", executionResult.ExecutionName), color.FgGreen).Once() + s.mockPrinter.On("Println", "\nStreaming logs", color.FgGreen).Once() + + s.mockProctorDClient.On("StreamProcLogs", executionResult.ExecutionId).Return(nil).Once() + + s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() + + s.testExecutionCmd.SetArgs(args) + s.testExecutionCmd.Execute() + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + func (s *ExecutionCmdTestSuite) TestExecutionCmdForNoProcVariables() { args := []string{"say-hello-world"} @@ -92,7 +136,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForNoProcVariables() { s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() - s.testExecutionCmd.Run(&cobra.Command{}, args) + s.testExecutionCmd.SetArgs(args) + s.testExecutionCmd.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) @@ -121,7 +166,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForIncorrectVariableFormat() { s.mockPrinter.On("Println", "Execution completed.", color.FgGreen).Once() - s.testExecutionCmd.Run(&cobra.Command{}, args) + s.testExecutionCmd.SetArgs(args) + s.testExecutionCmd.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) @@ -145,7 +191,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDExecutionFailure() { assert.Equal(s.T(), 1, exitCode) } testExecutionCmdOSExit := NewCmd(s.mockPrinter, s.mockProctorDClient, osExitFunc) - testExecutionCmdOSExit.Run(&cobra.Command{}, args) + testExecutionCmdOSExit.SetArgs(args) + testExecutionCmdOSExit.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) @@ -176,7 +223,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDLogStreamingFailure() assert.Equal(s.T(), 1, exitCode) } testExecutionCmdOSExit := NewCmd(s.mockPrinter, s.mockProctorDClient, osExitFunc) - testExecutionCmdOSExit.Run(&cobra.Command{}, args) + testExecutionCmdOSExit.SetArgs(args) + testExecutionCmdOSExit.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) @@ -208,7 +256,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec assert.Equal(s.T(), 1, exitCode) } testExecutionCmdOSExit := NewCmd(s.mockPrinter, s.mockProctorDClient, osExitFunc) - testExecutionCmdOSExit.Run(&cobra.Command{}, args) + testExecutionCmdOSExit.SetArgs(args) + testExecutionCmdOSExit.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) @@ -239,7 +288,8 @@ func (s *ExecutionCmdTestSuite) TestExecutionCmdForProctorDGetDefinitiveProcExec assert.Equal(s.T(), 1, exitCode) } testExecutionCmdOSExit := NewCmd(s.mockPrinter, s.mockProctorDClient, osExitFunc) - testExecutionCmdOSExit.Run(&cobra.Command{}, args) + testExecutionCmdOSExit.SetArgs(args) + testExecutionCmdOSExit.Execute() s.mockProctorDClient.AssertExpectations(s.T()) s.mockPrinter.AssertExpectations(s.T()) diff --git a/internal/app/cli/utility/file/file.go b/internal/app/cli/utility/file/file.go new file mode 100644 index 00000000..2386583d --- /dev/null +++ b/internal/app/cli/utility/file/file.go @@ -0,0 +1,25 @@ +package file + +import ( + "io/ioutil" + "os" + + "gopkg.in/yaml.v2" +) + +func ParseYAML(filename string, procArgs map[string]string) error { + file, err := os.Open(filename) + if err != nil { + return err + } + defer file.Close() + + buffer, err := ioutil.ReadAll(file) + + err = yaml.Unmarshal(buffer, &procArgs) + if err != nil { + return err + } + + return nil +} diff --git a/internal/app/cli/utility/file/file_test.go b/internal/app/cli/utility/file/file_test.go new file mode 100644 index 00000000..240b0bad --- /dev/null +++ b/internal/app/cli/utility/file/file_test.go @@ -0,0 +1,46 @@ +package file + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseYAML(t *testing.T) { + filename := "/tmp/yaml-test-parse" + testYAML := []byte("foo: bar\nmoo: zoo") + err := ioutil.WriteFile(filename, testYAML, 0644) + defer os.Remove(filename) + assert.NoError(t, err) + + procArgs := make(map[string]string) + err = ParseYAML(filename, procArgs) + assert.NoError(t, err) + assert.Equal(t, procArgs["foo"], "bar") + assert.Equal(t, procArgs["moo"], "zoo") +} + +func TestParseYAMLError(t *testing.T) { + + errorTests := []struct { + Filename string + ErrorMessage string + }{ + {"/tmp/foo", "no such file or directory"}, + {"/tmp/yaml-test-parse-error", "cannot unmarshal"}, + } + + filename := "/tmp/yaml-test-parse-error" + testYAML := []byte("foo bar") + err := ioutil.WriteFile(filename, testYAML, 0644) + defer os.Remove(filename) + assert.NoError(t, err) + + for _, errorTest := range errorTests { + procArgs := make(map[string]string) + err = ParseYAML(errorTest.Filename, procArgs) + assert.Contains(t, err.Error(), errorTest.ErrorMessage) + } +} From f00fb02da99366aa69ff2c561e8246ecb9eb7ae7 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:44:22 +0700 Subject: [PATCH 113/268] [bimo.horizon] Add example yaml file for testing --- test/procs/say-hello-world/say_hello_world.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 test/procs/say-hello-world/say_hello_world.yaml diff --git a/test/procs/say-hello-world/say_hello_world.yaml b/test/procs/say-hello-world/say_hello_world.yaml new file mode 100644 index 00000000..99d012ef --- /dev/null +++ b/test/procs/say-hello-world/say_hello_world.yaml @@ -0,0 +1,2 @@ +SAMPLE_ARG_ONE: foo +SAMPLE_ARG_TWO: bar From 6e3b9d8bcb80f2def2d2d44d1593dc0fa0653f61 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 14 Aug 2019 15:48:03 +0700 Subject: [PATCH 114/268] [bimo.horizon] Add make entry of executing with yaml --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 6f4afdc7..c8d8986a 100644 --- a/Makefile +++ b/Makefile @@ -94,6 +94,10 @@ ftest.proctor.describe: ftest.proctor.execute: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli execute say-hello-world SAMPLE_ARG_ONE=foo SAMPLE_ARG_TWO=bar +.PHONY: ftest.proctor.execute-with-yaml +ftest.proctor.execute-with-yaml: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli execute say-hello-world -f $(FTEST_DIR)/say-hello-world/say_hello_world.yaml + .PHONY: ftest.proctor.logs ftest.proctor.logs: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli logs $(EXECUTION_ID) From 8cc23d8c5f57ecc80b869bffa842528378252227 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 14 Aug 2019 15:59:38 +0700 Subject: [PATCH 115/268] [Jasoet] Enable Integration test for plugin --- .env.test | 1 - Makefile | 22 +++-- cmd/server/main.go | 3 - go.mod | 3 +- go.sum | 8 +- .../infra/plugin/plugin_integration_test.go | 8 +- .../app/service/security/service/security.go | 4 +- .../service/security/service/security_mock.go | 4 +- .../service/security/service/security_test.go | 10 +-- pkg/auth/auth.go | 4 +- pkg/auth/auth_mock.go | 4 +- plugins/gate-auth-plugin/auth.go | 50 +++++++++++ plugins/gate-auth-plugin/gate/client.go | 84 +++++++++++++++++++ plugins/gate-auth-plugin/gate/config.go | 23 +++++ 14 files changed, 194 insertions(+), 34 deletions(-) create mode 100644 plugins/gate-auth-plugin/gate/client.go create mode 100644 plugins/gate-auth-plugin/gate/config.go diff --git a/.env.test b/.env.test index 234112d0..7c07a062 100644 --- a/.env.test +++ b/.env.test @@ -33,5 +33,4 @@ export PROCTOR_MAIL_SERVER_PORT=123 export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir -export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=Auth diff --git a/Makefile b/Makefile index 6f4afdc7..1d4c91c2 100644 --- a/Makefile +++ b/Makefile @@ -9,12 +9,13 @@ export $(shell sed 's/=.*//' .env.test) SRC_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) OUT_DIR := $(SRC_DIR)/_output BIN_DIR := $(OUT_DIR)/bin +PLUGIN_DIR := $(BIN_DIR)/plugin FTEST_DIR := test/procs CONFIG_DIR := test/config GOPROXY ?= https://proxy.golang.org GO111MODULE := on -$(@info $(shell mkdir -p $(OUT_DIR) $(BIN_DIR))) +$(@info $(shell mkdir -p $(OUT_DIR) $(BIN_DIR) $(PLUGIN_DIR)) .PHONY: build build: test-with-race server cli @@ -29,21 +30,28 @@ test: go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: itest -itest: +itest: plugin.auth + PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ ENABLE_INTEGRATION_TEST=true \ go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: server server: - go build -o $(BIN_DIR)/server ./cmd/server/main.go + go build -race -o $(BIN_DIR)/server ./cmd/server/main.go -.PHONY: start-server -start-server: - $(BIN_DIR)/server s +.PHONY: plugin.auth +plugin.auth: + go build -race -buildmode=plugin -o $(PLUGIN_DIR)/auth.so ./plugins/gate-auth-plugin/auth.go .PHONY: cli cli: - go build -o $(BIN_DIR)/cli ./cmd/cli/main.go + go build -race -o $(BIN_DIR)/cli ./cmd/cli/main.go + +build-all: server cli plugin.auth + +.PHONY: start-server +start-server: + $(BIN_DIR)/server s generate: go get -u github.com/go-bindata/go-bindata/... diff --git a/cmd/server/main.go b/cmd/server/main.go index 2e680bf5..6d3f608e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -1,9 +1,7 @@ package main import ( - "github.com/getsentry/raven-go" "os" - "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/db/migration" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/schedule/worker" @@ -14,7 +12,6 @@ import ( func main() { logger.Setup() - _ = raven.SetDSN(config.SentryDSN()) proctord := cli.NewApp() proctord.Name = "proctord" diff --git a/go.mod b/go.mod index db928a88..d9217d47 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/badoux/checkmail v0.0.0-20181210160741-9661bd69e9ad github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 github.com/brianvoe/gofakeit v3.18.0+incompatible - github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 // indirect @@ -15,7 +14,7 @@ require ( github.com/evanphx/json-patch v4.4.0+incompatible // indirect github.com/fatih/color v1.7.0 github.com/garyburd/redigo v1.6.0 - github.com/getsentry/raven-go v0.2.0 + github.com/go-resty/resty/v2 v2.0.0 github.com/google/go-github v17.0.0+incompatible github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.0.0 // indirect diff --git a/go.sum b/go.sum index 222c7f46..db7ac3a8 100644 --- a/go.sum +++ b/go.sum @@ -23,8 +23,6 @@ github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 h1:GMmnK0dvr0Sf github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= github.com/brianvoe/gofakeit v3.18.0+incompatible h1:wDOmHc9DLG4nRjUVVaxA+CEglKOW72Y5+4WNxUIkjM8= github.com/brianvoe/gofakeit v3.18.0+incompatible/go.mod h1:kfwdRA90vvNhPutZWfH7WPaDzUjz+CZFqG+rPkOjGOc= -github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc= -github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -59,8 +57,6 @@ github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -70,6 +66,8 @@ github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+ github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-resty/resty/v2 v2.0.0 h1:9Nq/U+V4xsoDnDa/iTrABDWUCuk3Ne92XFHPe6dKWUc= +github.com/go-resty/resty/v2 v2.0.0/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -245,6 +243,8 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index b2cc04e3..12e968e3 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -1,6 +1,7 @@ package plugin import ( + "fmt" "github.com/stretchr/testify/assert" "os" "proctor/internal/app/service/infra/config" @@ -27,7 +28,6 @@ func (context *testContext) setUp(t *testing.T) { context.pluginBinary = config.AuthPluginBinary() assert.NotEmpty(t, context.pluginBinary) context.exportedName = config.AuthPluginExported() - assert.NotEmpty(t, context.exportedName) context.goPlugin = NewGoPlugin() assert.NotNil(t, context.goPlugin) } @@ -54,21 +54,23 @@ func TestGoPlugin_LoadSuccessfully(t *testing.T) { } func TestGoPlugin_LoadPluginFailed(t *testing.T) { + t.SkipNow() ctx := newContext() ctx.setUp(t) binary := "non-existing-binary" raw, err := ctx.instance().goPlugin.Load(binary, ctx.instance().exportedName) - assert.EqualErrorf(t, err, "failed to load plugin binary from location: %s", binary) + assert.EqualError(t, err, fmt.Sprintf("failed to load plugin binary from location: %s", binary)) assert.Nil(t, raw) } func TestGoPlugin_LoadExportedFailed(t *testing.T) { + t.SkipNow() ctx := newContext() ctx.setUp(t) exportedName := "non-existing-exported" raw, err := ctx.instance().goPlugin.Load(ctx.instance().pluginBinary, exportedName) - assert.EqualErrorf(t, err, "failed to Lookup plugin binary from location: %s with Exported Name: %s", ctx.instance().pluginBinary, exportedName) + assert.EqualError(t, err, fmt.Sprintf("failed to Lookup plugin binary from location: %s with Exported Name: %s", ctx.instance().pluginBinary, exportedName)) assert.Nil(t, raw) } diff --git a/internal/app/service/security/service/security.go b/internal/app/service/security/service/security.go index 57339cf8..25f851d8 100644 --- a/internal/app/service/security/service/security.go +++ b/internal/app/service/security/service/security.go @@ -21,13 +21,13 @@ type securityService struct { once sync.Once } -func (s *securityService) Verify(userDetail auth.UserDetail, group []string) (bool, error) { +func (s *securityService) Verify(userDetail auth.UserDetail, requiredGroups []string) (bool, error) { err := s.initializePlugin() logger.LogErrors(err, "initialize plugin") if err != nil { return false, err } - return s.authInstance.Verify(userDetail, group) + return s.authInstance.Verify(userDetail, requiredGroups) } func (s *securityService) Auth(email string, token string) (*auth.UserDetail, error) { diff --git a/internal/app/service/security/service/security_mock.go b/internal/app/service/security/service/security_mock.go index 209df278..656f1aa3 100644 --- a/internal/app/service/security/service/security_mock.go +++ b/internal/app/service/security/service/security_mock.go @@ -19,7 +19,7 @@ func (m *SecurityServiceMock) Auth(email string, token string) (auth.UserDetail, return args.Get(0).(auth.UserDetail), args.Error(1) } -func (m *SecurityServiceMock) Verify(userDetail auth.UserDetail, group []string) (bool, error) { - args := m.Called(userDetail, group) +func (m *SecurityServiceMock) Verify(userDetail auth.UserDetail, requiredGroups []string) (bool, error) { + args := m.Called(userDetail, requiredGroups) return args.Get(0).(bool), args.Error(1) } diff --git a/internal/app/service/security/service/security_test.go b/internal/app/service/security/service/security_test.go index 7ed7ac63..52db5ff5 100644 --- a/internal/app/service/security/service/security_test.go +++ b/internal/app/service/security/service/security_test.go @@ -58,7 +58,7 @@ func TestSecurityService_AuthSuccess(t *testing.T) { Name: "Deny Prasetyo", Email: "jasoet87@gmail.com", Active: true, - Group: []string{"system", "proctor_executor"}, + Groups: []string{"system", "proctor_executor"}, } ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) @@ -95,7 +95,7 @@ func TestSecurityService_AuthPluginFailedToCast(t *testing.T) { Name: "Deny Prasetyo", Email: "jasoet87@gmail.com", Active: true, - Group: []string{"system", "proctor_executor"}, + Groups: []string{"system", "proctor_executor"}, } ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(userDetail, nil) @@ -116,7 +116,7 @@ func TestSecurityService_VerifySuccess(t *testing.T) { Name: "Deny Prasetyo", Email: "jasoet87@gmail.com", Active: true, - Group: []string{"system", "proctor_executor"}, + Groups: []string{"system", "proctor_executor"}, } ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) @@ -146,7 +146,7 @@ func TestSecurityService_VerifyFailed(t *testing.T) { Name: "Deny Prasetyo", Email: "jasoet87@gmail.com", Active: true, - Group: []string{"system", "proctor_executor"}, + Groups: []string{"system", "proctor_executor"}, } ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) @@ -164,5 +164,3 @@ func TestSecurityService_VerifyFailed(t *testing.T) { assert.False(t, verified) assert.EqualError(t, err, "verify error") } - - diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index 0cf51ef9..cc12609e 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -2,12 +2,12 @@ package auth type Auth interface { Auth(email string, token string) (*UserDetail, error) - Verify(userDetail UserDetail, group []string) (bool, error) + Verify(userDetail UserDetail, requiredGroups []string) (bool, error) } type UserDetail struct { Name string Email string Active bool - Group []string + Groups []string } diff --git a/pkg/auth/auth_mock.go b/pkg/auth/auth_mock.go index 6e59527a..f0802525 100644 --- a/pkg/auth/auth_mock.go +++ b/pkg/auth/auth_mock.go @@ -13,7 +13,7 @@ func (m *AuthMock) Auth(email string, token string) (*UserDetail, error) { return args.Get(0).(*UserDetail), args.Error(1) } -func (m *AuthMock) Verify(userDetail UserDetail, group []string) (bool, error) { - args := m.Called(userDetail, group) +func (m *AuthMock) Verify(userDetail UserDetail, requiredGroups []string) (bool, error) { + args := m.Called(userDetail, requiredGroups) return args.Get(0).(bool), args.Error(1) } diff --git a/plugins/gate-auth-plugin/auth.go b/plugins/gate-auth-plugin/auth.go index 1110061a..65e53113 100644 --- a/plugins/gate-auth-plugin/auth.go +++ b/plugins/gate-auth-plugin/auth.go @@ -1,3 +1,53 @@ package main +import ( + "proctor/internal/app/service/infra/logger" + "proctor/pkg/auth" + "proctor/plugins/gate-auth-plugin/gate" +) +type gateAuth struct { + gateClient gate.GateClient +} + +func (g *gateAuth) Auth(email string, token string) (*auth.UserDetail, error) { + userDetail, err := g.gateClient.GetUserProfile(email, token) + logger.LogErrors(err, "get user detail with email: ", email) + if err != nil { + return nil, err + } + + return userDetail, nil +} + +//Verify check whether user has all required groups +func (g *gateAuth) Verify(userDetail auth.UserDetail, requiredGroups []string) (bool, error) { + if !userDetail.Active { + return false, nil + } + + for _, requiredGroup := range requiredGroups { + if !contains(userDetail.Groups, requiredGroup) { + return false, nil + } + } + + return true, nil +} + +func contains(groups []string, value string) bool { + for _, group := range groups { + if group == value { + return true + } + } + return false +} + +func newGateAuth() auth.Auth { + return &gateAuth{ + gateClient: gate.NewGateClient(), + } +} + +var Auth = newGateAuth() diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go new file mode 100644 index 00000000..d2c05411 --- /dev/null +++ b/plugins/gate-auth-plugin/gate/client.go @@ -0,0 +1,84 @@ +package gate + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "proctor/internal/app/service/infra/logger" + "proctor/pkg/auth" +) + +type GateClient interface { + GetUserProfile(email string, token string) (*auth.UserDetail, error) +} + +type gateClient struct { + host string + profilePath string + protocol string + restClient *resty.Client +} + +func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetail, error) { + path := fmt.Sprintf("%s://%s/%s", g.protocol, g.host, g.profilePath) + formData := map[string]string{ + "email": email, + "access_token": token, + } + + profile := &Profile{} + response, err := g.restClient. + R(). + SetHeader("Accept", "application/json"). + SetFormData(formData). + SetResult(profile). + Get(path) + + logger.LogErrors(err, "get request to %s, with email %s", path, email) + if err != nil { + return nil, err + } + + if response.IsSuccess() { + userDetail := auth.UserDetail{ + Name: profile.Name, + Email: profile.Email, + Active: profile.Active, + Groups: profile.getGroups(), + } + + return &userDetail, nil + + } else { + return nil, fmt.Errorf("request to %s with email %s failed, returned body %s", path, email, response.String()) + } +} + +func NewGateClient() GateClient { + return &gateClient{ + protocol: Protocol(), + host: Host(), + profilePath: ProfilePath(), + restClient: resty.New(), + } +} + +type Profile struct { + Email string `json:"email"` + Name string `json:"name"` + Active bool `json:"active"` + Groups []Group `json:"groups"` +} + +type Group struct { + ID uint64 `json:"id"` + Name string `json:"name"` +} + +func (profile *Profile) getGroups() []string { + var groups []string + for _, group := range profile.Groups { + groups = append(groups, group.Name) + } + + return groups +} diff --git a/plugins/gate-auth-plugin/gate/config.go b/plugins/gate-auth-plugin/gate/config.go new file mode 100644 index 00000000..579ebe1a --- /dev/null +++ b/plugins/gate-auth-plugin/gate/config.go @@ -0,0 +1,23 @@ +package gate + +import "github.com/spf13/viper" + +func init() { + viper.AutomaticEnv() + viper.SetEnvPrefix("GATE_PLUGIN") +} + +func Protocol() string { + viper.SetDefault("PROTOCOL", "https") + return viper.GetString("PROTOCOL") +} + +func Host() string { + viper.SetDefault("HOST", "gate.gojek.co.id") + return viper.GetString("HOST") +} + +func ProfilePath() string { + viper.SetDefault("PROFILE_PATH", "api/v1/users/profile") + return viper.GetString("PROFILE_PATH") +} From 19830736437bcdaf8d8487439f46c9e82ef44488 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 15 Aug 2019 13:34:39 +0700 Subject: [PATCH 116/268] [Jasoet] Change Config Mechanism --- .env.test | 1 + .../app/service/execution/handler/http.go | 6 +- .../service/execution/service/execution.go | 2 +- .../service/execution_integration_test.go | 2 +- internal/app/service/infra/config/config.go | 255 ++++++++---------- .../app/service/infra/config/config_test.go | 177 +++++------- .../service/infra/db/migration/migrations.go | 2 +- .../app/service/infra/db/postgresql/client.go | 8 +- .../infra/db/postgresql/client_test.go | 6 +- internal/app/service/infra/db/redis/client.go | 6 +- .../app/service/infra/kubernetes/client.go | 14 +- .../kubernetes/client_integration_test.go | 6 +- .../service/infra/kubernetes/client_test.go | 9 +- .../service/infra/kubernetes/http/client.go | 21 +- .../infra/kubernetes/http/client_test.go | 18 +- internal/app/service/infra/logger/logrus.go | 2 +- internal/app/service/infra/mail/mailer.go | 4 +- .../app/service/infra/mail/mailer_test.go | 2 +- .../infra/plugin/plugin_integration_test.go | 50 ++-- .../app/service/schedule/worker/worker.go | 4 +- internal/app/service/server/api.go | 2 +- .../app/service/server/middleware/newrelic.go | 4 +- .../middleware/validate_client_version.go | 2 +- internal/app/service/server/router.go | 4 +- 24 files changed, 247 insertions(+), 360 deletions(-) diff --git a/.env.test b/.env.test index 7c07a062..234112d0 100644 --- a/.env.test +++ b/.env.test @@ -33,4 +33,5 @@ export PROCTOR_MAIL_SERVER_PORT=123 export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir +export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=Auth diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 67422a46..485867f8 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -34,8 +34,8 @@ type executionHTTPHandler struct { } var upgrader = websocket.Upgrader{ - ReadBufferSize: config.LogsStreamReadBufferSize(), - WriteBufferSize: config.LogsStreamWriteBufferSize(), + ReadBufferSize: config.Config().LogsStreamReadBufferSize, + WriteBufferSize: config.Config().LogsStreamWriteBufferSize, } func NewExecutionHTTPHandler( @@ -91,7 +91,7 @@ func (httpHandler *executionHTTPHandler) GetLogs() http.HandlerFunc { if context.Status == executionStatus.Created { logger.Debug("Execution is Created, Stream output from pod") - waitTime := config.KubeLogProcessWaitTime() * time.Second + waitTime := config.Config().KubeLogProcessWaitTime * time.Second podLog, _err := httpHandler.service.StreamJobLogs(context.Name, waitTime) logger.LogErrors(_err, "stream job log by execution name", context.Name) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 6841ec78..18626ae3 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -134,7 +134,7 @@ func (service *executionService) update(executionContext model.ExecutionContext) func (service *executionService) watchProcess(context model.ExecutionContext) { logger.Info("Start Watch Process for Job With Context ID: ", context.ExecutionID, ", name: ", context.Name, ", and Status: "+context.Status) - waitTime := config.KubeLogProcessWaitTime() * time.Second + waitTime := config.Config().KubeLogProcessWaitTime * time.Second err := service.kubernetesClient.WaitForReadyJob(context.Name, waitTime) if err != nil { diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 99c07a42..2f6358e8 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -98,7 +98,7 @@ func (suite *TestExecutionIntegrationSuite) TestStreamLogsSuccess() { executedJobname, err := suite.kubernetesClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) assert.NoError(t, err) - waitTime := config.KubePodsListWaitTime() * time.Second + waitTime := config.Config().KubePodsListWaitTime * time.Second logStream, err := suite.service.StreamJobLogs(executedJobname, waitTime) assert.NoError(t, err) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index dee747c1..650bf0af 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -4,173 +4,136 @@ import ( "encoding/json" "fmt" "github.com/spf13/viper" + "sync" "time" ) -func init() { - viper.AutomaticEnv() - viper.SetEnvPrefix("PROCTOR") +func GetStringDefault(viper *viper.Viper, key string, defaultValue string) string { + viper.SetDefault(key, defaultValue) + return viper.GetString(key) } -func KubeConfig() string { - return viper.GetString("KUBE_CONFIG") +func GetInt64Ref(viper *viper.Viper, key string) *int64 { + value := viper.GetInt64(key) + return &value } -func KubeContext() string { - viper.SetDefault("KUBE_CONTEXT", "default") - return viper.GetString("KUBE_CONTEXT") +func GetInt32Ref(viper *viper.Viper, key string) *int32 { + value := viper.GetInt32(key) + return &value } -func LogLevel() string { - return viper.GetString("LOG_LEVEL") -} - -func AppPort() string { - return viper.GetString("APP_PORT") -} - -func DefaultNamespace() string { - return viper.GetString("DEFAULT_NAMESPACE") -} - -func RedisAddress() string { - return viper.GetString("REDIS_ADDRESS") -} - -//TODO remove -func KubeClusterHostName() string { - return viper.GetString("KUBE_CLUSTER_HOST_NAME") -} - -//TODO Should Not be Used since we use ~/.kube/config -func KubeCACertEncoded() string { - return viper.GetString("KUBE_CA_CERT_ENCODED") -} - -//TODO remove -func KubeBasicAuthEncoded() string { - return viper.GetString("KUBE_BASIC_AUTH_ENCODED") -} - -func RedisMaxActiveConnections() int { - return viper.GetInt("REDIS_MAX_ACTIVE_CONNECTIONS") -} - -func LogsStreamReadBufferSize() int { - return viper.GetInt("LOGS_STREAM_READ_BUFFER_SIZE") -} - -func LogsStreamWriteBufferSize() int { - return viper.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE") -} - -//TODO test -func KubePodsListWaitTime() time.Duration { - return time.Duration(viper.GetInt("KUBE_POD_LIST_WAIT_TIME")) -} - -//TODO test -func KubeLogProcessWaitTime() time.Duration { - return time.Duration(viper.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")) -} - -func KubeJobActiveDeadlineSeconds() *int64 { - kubeJobActiveDeadlineSeconds := viper.GetInt64("KUBE_JOB_ACTIVE_DEADLINE_SECONDS") - return &kubeJobActiveDeadlineSeconds -} - -func KubeJobRetries() *int32 { - kubeJobRetries := int32(viper.GetInt("KUBE_JOB_RETRIES")) - return &kubeJobRetries -} - -func PostgresUser() string { - return viper.GetString("POSTGRES_USER") -} - -func PostgresPassword() string { - return viper.GetString("POSTGRES_PASSWORD") -} - -func PostgresHost() string { - return viper.GetString("POSTGRES_HOST") -} - -func PostgresPort() int { - return viper.GetInt("POSTGRES_PORT") -} - -func PostgresDatabase() string { - return viper.GetString("POSTGRES_DATABASE") -} - -func PostgresMaxConnections() int { - return viper.GetInt("POSTGRES_MAX_CONNECTIONS") -} - -func PostgresConnectionMaxLifetime() int { - return viper.GetInt("POSTGRES_CONNECTIONS_MAX_LIFETIME") -} - -func NewRelicAppName() string { - return viper.GetString("NEW_RELIC_APP_NAME") -} - -func NewRelicLicenceKey() string { - return viper.GetString("NEW_RELIC_LICENCE_KEY") -} - -func MinClientVersion() string { - return viper.GetString("MIN_CLIENT_VERSION") -} - -func ScheduledJobsFetchIntervalInMins() int { - return viper.GetInt("SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS") -} - -func MailUsername() string { - return viper.GetString("MAIL_USERNAME") -} - -func MailPassword() string { - return viper.GetString("MAIL_PASSWORD") -} - -func MailServerHost() string { - return viper.GetString("MAIL_SERVER_HOST") -} - -func MailServerPort() string { - return viper.GetString("MAIL_SERVER_PORT") -} - -func JobPodAnnotations() map[string]string { - var jsonStr = []byte(viper.GetString("JOB_POD_ANNOTATIONS")) +func GetMapFromJson(viper *viper.Viper, key string) map[string]string { + var jsonStr = []byte(viper.GetString(key)) var annotations map[string]string err := json.Unmarshal(jsonStr, &annotations) if err != nil { - _ = fmt.Errorf(err.Error(), "Invalid value for key PROCTOR_JOB_POD_ANNOTATIONS") + _ = fmt.Errorf("invalid Value for key %s, errors %v", key, err.Error()) } return annotations } -func SentryDSN() string { - return viper.GetString("SENTRY_DSN") -} +var once sync.Once +var config ProctorConfig + +type ProctorConfig struct { + viper *viper.Viper + KubeConfig string + KubeContext string + LogLevel string + AppPort string + DefaultNamespace string + RedisAddress string + LogsStreamReadBufferSize int + RedisMaxActiveConnections int + LogsStreamWriteBufferSize int + KubePodsListWaitTime time.Duration + KubeLogProcessWaitTime time.Duration + KubeJobActiveDeadlineSeconds *int64 + KubeJobRetries *int32 + PostgresUser string + PostgresPassword string + PostgresHost string + PostgresPort int + AuthPluginExported string + PostgresDatabase string + PostgresMaxConnections int + PostgresConnectionMaxLifetime int + NewRelicAppName string + NewRelicLicenceKey string + MinClientVersion string + ScheduledJobsFetchIntervalInMins int + MailUsername string + MailServerHost string + MailPassword string + MailServerPort string + JobPodAnnotations map[string]string + SentryDSN string + DocsPath string + AuthPluginBinary string +} + +func Load() ProctorConfig { + fang := viper.New() + fang.AutomaticEnv() + fang.SetEnvPrefix("PROCTOR") + proctorConfig := ProctorConfig{ + viper: fang, + KubeConfig: fang.GetString("KUBE_CONFIG"), + KubeContext: GetStringDefault(fang, "KUBE_CONTEXT", "default"), + LogLevel: GetStringDefault(fang, "LOG_LEVEL", "DEBUG"), + AppPort: GetStringDefault(fang, "APP_PORT", "5001"), + DefaultNamespace: fang.GetString("DEFAULT_NAMESPACE"), + RedisAddress: fang.GetString("REDIS_ADDRESS"), + RedisMaxActiveConnections: fang.GetInt("REDIS_MAX_ACTIVE_CONNECTIONS"), + LogsStreamReadBufferSize: fang.GetInt("LOGS_STREAM_READ_BUFFER_SIZE"), + LogsStreamWriteBufferSize: fang.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE"), + KubePodsListWaitTime: time.Duration(fang.GetInt("KUBE_POD_LIST_WAIT_TIME")), + KubeLogProcessWaitTime: time.Duration(fang.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")), + KubeJobActiveDeadlineSeconds: GetInt64Ref(fang, "KUBE_JOB_ACTIVE_DEADLINE_SECONDS"), + KubeJobRetries: GetInt32Ref(fang, "KUBE_JOB_RETRIES"), + PostgresUser: fang.GetString("POSTGRES_USER"), + PostgresPassword: fang.GetString("POSTGRES_PASSWORD"), + PostgresHost: fang.GetString("POSTGRES_HOST"), + PostgresPort: fang.GetInt("POSTGRES_PORT"), + PostgresDatabase: fang.GetString("POSTGRES_DATABASE"), + PostgresMaxConnections: fang.GetInt("POSTGRES_MAX_CONNECTIONS"), + PostgresConnectionMaxLifetime: fang.GetInt("POSTGRES_CONNECTIONS_MAX_LIFETIME"), + NewRelicAppName: fang.GetString("NEW_RELIC_APP_NAME"), + NewRelicLicenceKey: fang.GetString("NEW_RELIC_LICENCE_KEY"), + MinClientVersion: fang.GetString("MIN_CLIENT_VERSION"), + ScheduledJobsFetchIntervalInMins: fang.GetInt("SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS"), + MailUsername: fang.GetString("MAIL_USERNAME"), + MailServerHost: fang.GetString("MAIL_SERVER_HOST"), + MailPassword: fang.GetString("MAIL_PASSWORD"), + MailServerPort: fang.GetString("MAIL_SERVER_PORT"), + JobPodAnnotations: GetMapFromJson(fang, "JOB_POD_ANNOTATIONS"), + SentryDSN: fang.GetString("SENTRY_DSN"), + DocsPath: fang.GetString("DOCS_PATH"), + AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), + AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), + } -func DocsPath() string { - return viper.GetString("DOCS_PATH") + return proctorConfig } -//TODO test -func AuthPluginBinary() string { - return viper.GetString("AUTH_PLUGIN_BINARY") +var reset = false + +func Reset() { + reset = true } -//TODO test -func AuthPluginExported() string { - viper.SetDefault("AUTH_PLUGIN_EXPORTED", "Auth") - return viper.GetString("AUTH_PLUGIN_EXPORTED") +func Config() ProctorConfig { + once.Do(func() { + config = Load() + }) + + if reset { + config = Load() + reset = false + } + + return config } diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 52542b81..fc5b6e12 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -1,259 +1,204 @@ package config import ( + fake "github.com/brianvoe/gofakeit" + "github.com/stretchr/testify/assert" "os" + "strconv" "testing" - - "github.com/spf13/viper" - "github.com/stretchr/testify/assert" ) func TestEnvironment(t *testing.T) { - _ = os.Setenv("PROCTOR_KUBE_CONFIG", "in-cluster") + fake.Seed(0) + value := fake.FirstName() + _ = os.Setenv("PROCTOR_KUBE_CONFIG", value) - viper.AutomaticEnv() - - assert.Equal(t, "in-cluster", KubeConfig()) + assert.Equal(t, value, Load().KubeConfig) } func TestLogLevel(t *testing.T) { - _ = os.Setenv("PROCTOR_LOG_LEVEL", "debug") - - viper.AutomaticEnv() + fake.Seed(0) + value := fake.FirstName() + _ = os.Setenv("PROCTOR_LOG_LEVEL", value) - assert.Equal(t, "debug", LogLevel()) + assert.Equal(t, value, Load().LogLevel) } func TestAppPort(t *testing.T) { - _ = os.Setenv("PROCTOR_APP_PORT", "3000") - - viper.AutomaticEnv() + fake.Seed(0) + value := strconv.FormatInt(int64(fake.Number(1000, 4000)), 10) + _ = os.Setenv("PROCTOR_APP_PORT", value) - assert.Equal(t, "3000", AppPort()) + assert.Equal(t, value, Load().AppPort) } func TestDefaultNamespace(t *testing.T) { - _ = os.Setenv("PROCTOR_DEFAULT_NAMESPACE", "default") + fake.Seed(0) + value := fake.FirstName() + _ = os.Setenv("PROCTOR_DEFAULT_NAMESPACE", value) - viper.AutomaticEnv() - - assert.Equal(t, "default", DefaultNamespace()) + assert.Equal(t, value, Load().DefaultNamespace) } func TestRedisAddress(t *testing.T) { - _ = os.Setenv("PROCTOR_REDIS_ADDRESS", "localhost:6379") - - viper.AutomaticEnv() - - assert.Equal(t, "localhost:6379", RedisAddress()) -} - -func TestKubeClusterHostName(t *testing.T) { - _ = os.Setenv("PROCTOR_KUBE_CLUSTER_HOST_NAME", "somekube.io") - - viper.AutomaticEnv() + fake.Seed(0) + value := fake.FirstName() + _ = os.Setenv("PROCTOR_REDIS_ADDRESS", value) - assert.Equal(t, "somekube.io", KubeClusterHostName()) -} - -func TestKubeCACertEncoded(t *testing.T) { - _ = os.Setenv("PROCTOR_KUBE_CA_CERT_ENCODED", "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K") - - viper.AutomaticEnv() - - assert.Equal(t, "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K", KubeCACertEncoded()) -} - -func TestKubeBasicAuthEncoded(t *testing.T) { - _ = os.Setenv("PROCTOR_KUBE_BASIC_AUTH_ENCODED", "YWRtaW46cGFzc3dvcmQK") - - viper.AutomaticEnv() - - assert.Equal(t, "YWRtaW46cGFzc3dvcmQK", KubeBasicAuthEncoded()) + assert.Equal(t, value, Load().RedisAddress) } func TestRedisMaxActiveConnections(t *testing.T) { - _ = os.Setenv("PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS", "50") - - viper.AutomaticEnv() + fake.Seed(0) + number := fake.Number(10, 90) + value := strconv.FormatInt(int64(number), 10) + _ = os.Setenv("PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS", value) - assert.Equal(t, 50, RedisMaxActiveConnections()) + assert.Equal(t, number, Load().RedisMaxActiveConnections) } func TestLogsStreamReadBufferSize(t *testing.T) { _ = os.Setenv("PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE", "140") - viper.AutomaticEnv() - - assert.Equal(t, 140, LogsStreamReadBufferSize()) + assert.Equal(t, 140, Load().LogsStreamReadBufferSize) } func TestLogsStreamWriteBufferSize(t *testing.T) { _ = os.Setenv("PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE", "4096") - viper.AutomaticEnv() - - assert.Equal(t, 4096, LogsStreamWriteBufferSize()) + assert.Equal(t, 4096, Load().LogsStreamWriteBufferSize) } func TestKubeJobActiveDeadlineSeconds(t *testing.T) { _ = os.Setenv("PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS", "900") - viper.AutomaticEnv() - expectedValue := int64(900) - assert.Equal(t, &expectedValue, KubeJobActiveDeadlineSeconds()) + assert.Equal(t, &expectedValue, Load().KubeJobActiveDeadlineSeconds) } func TestKubeJobRetries(t *testing.T) { _ = os.Setenv("PROCTOR_KUBE_JOB_RETRIES", "0") - viper.AutomaticEnv() - expectedValue := int32(0) - assert.Equal(t, &expectedValue, KubeJobRetries()) + assert.Equal(t, &expectedValue, Load().KubeJobRetries) } func TestPostgresUser(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_USER", "postgres") - viper.AutomaticEnv() - - assert.Equal(t, "postgres", PostgresUser()) + assert.Equal(t, "postgres", Load().PostgresUser) } func TestPostgresPassword(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_PASSWORD", "ipsum-lorem") - viper.AutomaticEnv() - - assert.Equal(t, "ipsum-lorem", PostgresPassword()) + assert.Equal(t, "ipsum-lorem", Load().PostgresPassword) } func TestPostgresHost(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_HOST", "localhost") - viper.AutomaticEnv() - - assert.Equal(t, "localhost", PostgresHost()) + assert.Equal(t, "localhost", Load().PostgresHost) } func TestPostgresPort(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_PORT", "5432") - viper.AutomaticEnv() - - assert.Equal(t, 5432, PostgresPort()) + assert.Equal(t, 5432, Load().PostgresPort) } func TestPostgresDatabase(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_DATABASE", "proctord_development") - viper.AutomaticEnv() - - assert.Equal(t, "proctord_development", PostgresDatabase()) + assert.Equal(t, "proctord_development", Load().PostgresDatabase) } func TestPostgresMaxConnections(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_MAX_CONNECTIONS", "50") - viper.AutomaticEnv() - - assert.Equal(t, 50, PostgresMaxConnections()) + assert.Equal(t, 50, Load().PostgresMaxConnections) } func TestPostgresConnectionMaxLifetime(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME", "30") - viper.AutomaticEnv() - - assert.Equal(t, 30, PostgresConnectionMaxLifetime()) + assert.Equal(t, 30, Load().PostgresConnectionMaxLifetime) } func TestNewRelicAppName(t *testing.T) { _ = os.Setenv("PROCTOR_NEW_RELIC_APP_NAME", "PROCTORD") - viper.AutomaticEnv() - - assert.Equal(t, "PROCTORD", NewRelicAppName()) + assert.Equal(t, "PROCTORD", Load().NewRelicAppName) } func TestNewRelicLicenceKey(t *testing.T) { _ = os.Setenv("PROCTOR_NEW_RELIC_LICENCE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") - viper.AutomaticEnv() - - assert.Equal(t, "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr", NewRelicLicenceKey()) + assert.Equal(t, "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr", Load().NewRelicLicenceKey) } func TestMinClientVersion(t *testing.T) { _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") - viper.AutomaticEnv() - - assert.Equal(t, "0.2.0", MinClientVersion()) + assert.Equal(t, "0.2.0", Load().MinClientVersion) } func TestScheduledJobsFetchIntervalInMins(t *testing.T) { _ = os.Setenv("PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS", "5") - viper.AutomaticEnv() - - assert.Equal(t, 5, ScheduledJobsFetchIntervalInMins()) + assert.Equal(t, 5, Load().ScheduledJobsFetchIntervalInMins) } func TestMailUsername(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_USERNAME", "foo@bar.com") - viper.AutomaticEnv() - - assert.Equal(t, "foo@bar.com", MailUsername()) + assert.Equal(t, "foo@bar.com", Load().MailUsername) } func TestMailPassword(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_PASSWORD", "password") - viper.AutomaticEnv() - - assert.Equal(t, "password", MailPassword()) + assert.Equal(t, "password", Load().MailPassword) } func TestMailServerHost(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_SERVER_HOST", "127.0.0.1") - viper.AutomaticEnv() - - assert.Equal(t, "127.0.0.1", MailServerHost()) + assert.Equal(t, "127.0.0.1", Load().MailServerHost) } func TestMailServerPort(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_SERVER_PORT", "123") - viper.AutomaticEnv() - - assert.Equal(t, "123", MailServerPort()) + assert.Equal(t, "123", Load().MailServerPort) } func TestJobPodAnnotations(t *testing.T) { _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - viper.AutomaticEnv() - - assert.Equal(t, map[string]string{"key.one": "true"}, JobPodAnnotations()) + assert.Equal(t, map[string]string{"key.one": "true"}, Load().JobPodAnnotations) } func TestSentryDSN(t *testing.T) { _ = os.Setenv("PROCTOR_SENTRY_DSN", "domain") - viper.AutomaticEnv() - - assert.Equal(t, "domain", SentryDSN()) + assert.Equal(t, "domain", Load().SentryDSN) } func TestDocsPath(t *testing.T) { _ = os.Setenv("PROCTOR_DOCS_PATH", "path1") - viper.AutomaticEnv() + assert.Equal(t, "path1", Load().DocsPath) +} + +func TestAuthPluginBinary(t *testing.T) { + _ = os.Setenv("PROCTOR_AUTH_PLUGIN_BINARY", "path1") + + assert.Equal(t, "path1", Load().AuthPluginBinary) +} + +func TestAuthPluginExported(t *testing.T) { + _ = os.Setenv("PROCTOR_AUTH_PLUGIN_EXPORTED", "path1") - assert.Equal(t, "path1", DocsPath()) + assert.Equal(t, "path1", Load().AuthPluginExported) } diff --git a/internal/app/service/infra/db/migration/migrations.go b/internal/app/service/infra/db/migration/migrations.go index fbedceda..2fc1758c 100644 --- a/internal/app/service/infra/db/migration/migrations.go +++ b/internal/app/service/infra/db/migration/migrations.go @@ -15,7 +15,7 @@ import ( var migrationsPath, postgresConnectionURL string func init() { - postgresConnectionURL = fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", config.PostgresUser(), config.PostgresPassword(), config.PostgresHost(), config.PostgresPort(), config.PostgresDatabase()) + postgresConnectionURL = fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=disable", config.Config().PostgresUser, config.Config().PostgresPassword, config.Config().PostgresHost, config.Config().PostgresPort, config.Config().PostgresDatabase) migrationsPath = "file://./migrations" } diff --git a/internal/app/service/infra/db/postgresql/client.go b/internal/app/service/infra/db/postgresql/client.go index 6d0e4424..3fecb24a 100644 --- a/internal/app/service/infra/db/postgresql/client.go +++ b/internal/app/service/infra/db/postgresql/client.go @@ -23,16 +23,16 @@ type client struct { } func NewClient() Client { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.Config().PostgresDatabase, config.Config().PostgresUser, config.Config().PostgresPassword, config.Config().PostgresHost) db, err := sqlx.Connect("postgres", dataSourceName) if err != nil { panic(err.Error()) } - db.SetMaxIdleConns(config.PostgresMaxConnections()) - db.SetMaxOpenConns(config.PostgresMaxConnections()) - db.SetConnMaxLifetime(time.Duration(config.PostgresConnectionMaxLifetime()) * time.Minute) + db.SetMaxIdleConns(config.Config().PostgresMaxConnections) + db.SetMaxOpenConns(config.Config().PostgresMaxConnections) + db.SetConnMaxLifetime(time.Duration(config.Config().PostgresConnectionMaxLifetime) * time.Minute) return &client{ db: db, diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index d530550b..197b7ada 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -14,7 +14,7 @@ import ( ) func TestNamedExec(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.Config().PostgresDatabase, config.Config().PostgresUser, config.Config().PostgresPassword, config.Config().PostgresHost) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) @@ -48,7 +48,7 @@ func TestNamedExec(t *testing.T) { } func TestSelect(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.Config().PostgresDatabase, config.Config().PostgresUser, config.Config().PostgresPassword, config.Config().PostgresHost) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) @@ -80,7 +80,7 @@ func TestSelect(t *testing.T) { } func TestSelectForNoRows(t *testing.T) { - dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.Config().PostgresDatabase, config.Config().PostgresUser, config.Config().PostgresPassword, config.Config().PostgresHost) db, err := sqlx.Connect("postgres", dataSourceName) assert.NoError(t, err) diff --git a/internal/app/service/infra/db/redis/client.go b/internal/app/service/infra/db/redis/client.go index 41b6201f..147fd9bc 100644 --- a/internal/app/service/infra/db/redis/client.go +++ b/internal/app/service/infra/db/redis/client.go @@ -28,10 +28,10 @@ func NewClient() Client { func newPool() (*redis.Pool, error) { pool := &redis.Pool{ - MaxIdle: config.RedisMaxActiveConnections() / 2, - MaxActive: config.RedisMaxActiveConnections(), + MaxIdle: config.Config().RedisMaxActiveConnections / 2, + MaxActive: config.Config().RedisMaxActiveConnections, IdleTimeout: 5 * time.Second, - Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.RedisAddress()) }, + Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.Config().RedisAddress) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Minute { return nil diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index f44403c3..084418f1 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -31,7 +31,7 @@ func init() { Kind: "Job", APIVersion: "batch/v1", } - namespace = config.DefaultNamespace() + namespace = config.Config().DefaultNamespace } type KubernetesClient interface { @@ -50,15 +50,15 @@ type kubernetesClient struct { func NewClientSet() (*kubernetes.Clientset, error) { var kubeConfig *kubeRestClient.Config - if config.KubeConfig() == "out-of-cluster" { + if config.Config().KubeConfig == "out-of-cluster" { logger.Info("service is running outside kube cluster") home := os.Getenv("HOME") kubeConfigPath := filepath.Join(home, ".kube", "config") configOverrides := &clientcmd.ConfigOverrides{} - if config.KubeContext() != "default" { - configOverrides.CurrentContext = config.KubeContext() + if config.Config().KubeContext != "default" { + configOverrides.CurrentContext = config.Config().KubeContext } var err error @@ -153,7 +153,7 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m objectMeta := meta.ObjectMeta{ Name: executionName, Labels: label, - Annotations: config.JobPodAnnotations(), + Annotations: config.Config().JobPodAnnotations, } template := v1.PodTemplateSpec{ @@ -163,8 +163,8 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m jobSpec := batch.JobSpec{ Template: template, - ActiveDeadlineSeconds: config.KubeJobActiveDeadlineSeconds(), - BackoffLimit: config.KubeJobRetries(), + ActiveDeadlineSeconds: config.Config().KubeJobActiveDeadlineSeconds, + BackoffLimit: config.Config().KubeJobRetries, } jobToRun := batch.Job{ diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 205c5bb8..fbc93d1b 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -47,7 +47,7 @@ func (suite *IntegrationTestSuite) TestJobExecution() { LabelSelector: jobLabelSelector(executedJobname), } - namespace := config.DefaultNamespace() + namespace := config.Config().DefaultNamespace listOfJobs, err := suite.clientSet.BatchV1().Jobs(namespace).List(listOptions) assert.NoError(t, err) executedJob := listOfJobs.Items[0] @@ -59,8 +59,8 @@ func (suite *IntegrationTestSuite) TestJobExecution() { assert.Equal(t, expectedLabel, executedJob.ObjectMeta.Labels) assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) - assert.Equal(t, config.KubeJobActiveDeadlineSeconds(), executedJob.Spec.ActiveDeadlineSeconds) - assert.Equal(t, config.KubeJobRetries(), executedJob.Spec.BackoffLimit) + assert.Equal(t, config.Config().KubeJobActiveDeadlineSeconds, executedJob.Spec.ActiveDeadlineSeconds) + assert.Equal(t, config.Config().KubeJobRetries, executedJob.Spec.BackoffLimit) assert.Equal(t, v1.RestartPolicyNever, executedJob.Spec.Template.Spec.RestartPolicy) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 5e47bec7..88f9e013 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -38,7 +38,7 @@ func (suite *ClientTestSuite) SetupTest() { } suite.jobName = "job1" suite.podName = "pod1" - namespace := config.DefaultNamespace() + namespace := config.Config().DefaultNamespace suite.fakeClientSetStreaming = fakeclientset.NewSimpleClientset(&v1.Pod{ TypeMeta: meta.TypeMeta{ Kind: "Pod", @@ -67,6 +67,7 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + config.Reset() envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} sampleImageName := "img1" @@ -82,7 +83,7 @@ func (suite *ClientTestSuite) TestJobExecution() { TypeMeta: typeMeta, LabelSelector: jobLabelSelector(executedJobname), } - namespace := config.DefaultNamespace() + namespace := config.Config().DefaultNamespace listOfJobs, err := suite.fakeClientSet.BatchV1().Jobs(namespace).List(listOptions) assert.NoError(t, err) executedJob := listOfJobs.Items[0] @@ -97,8 +98,8 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedLabel, executedJob.Spec.Template.ObjectMeta.Labels) assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) - assert.Equal(t, config.KubeJobActiveDeadlineSeconds(), executedJob.Spec.ActiveDeadlineSeconds) - assert.Equal(t, config.KubeJobRetries(), executedJob.Spec.BackoffLimit) + assert.Equal(t, config.Config().KubeJobActiveDeadlineSeconds, executedJob.Spec.ActiveDeadlineSeconds) + assert.Equal(t, config.Config().KubeJobRetries, executedJob.Spec.BackoffLimit) assert.Equal(t, v1.RestartPolicyNever, executedJob.Spec.Template.Spec.RestartPolicy) diff --git a/internal/app/service/infra/kubernetes/http/client.go b/internal/app/service/infra/kubernetes/http/client.go index 860a2f9b..1cf9e8a6 100644 --- a/internal/app/service/infra/kubernetes/http/client.go +++ b/internal/app/service/infra/kubernetes/http/client.go @@ -1,27 +1,10 @@ package http import ( - "crypto/tls" - "crypto/x509" - "encoding/base64" "net/http" - "proctor/internal/app/service/infra/config" ) func NewClient() (*http.Client, error) { - caCert, err := base64.StdEncoding.DecodeString(config.KubeCACertEncoded()) - if err != nil { - return &http.Client{}, err - } - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: caCertPool, - }, - }, - } - return httpClient, err + httpClient := &http.Client{} + return httpClient, nil } diff --git a/internal/app/service/infra/kubernetes/http/client_test.go b/internal/app/service/infra/kubernetes/http/client_test.go index b5c295f2..0e2e1c52 100644 --- a/internal/app/service/infra/kubernetes/http/client_test.go +++ b/internal/app/service/infra/kubernetes/http/client_test.go @@ -1,29 +1,13 @@ package http import ( - "crypto/tls" - "crypto/x509" - "encoding/base64" - "net/http" - "proctor/internal/app/service/infra/config" "testing" "github.com/stretchr/testify/assert" ) func TestNewClient(t *testing.T) { - caCert, _ := base64.StdEncoding.DecodeString(config.KubeCACertEncoded()) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - expectedTransport := &http.Transport{ - TLSClientConfig: &tls.Config{ - RootCAs: caCertPool, - }, - } - httpClient, err := NewClient() - + assert.NotNil(t, httpClient) assert.NoError(t, err) - assert.Equal(t, expectedTransport, httpClient.Transport) } diff --git a/internal/app/service/infra/logger/logrus.go b/internal/app/service/infra/logger/logrus.go index 6adb0b39..d1e4e352 100644 --- a/internal/app/service/infra/logger/logrus.go +++ b/internal/app/service/infra/logger/logrus.go @@ -17,7 +17,7 @@ func Setup() { log.SetFormatter(&log.JSONFormatter{}) log.SetOutput(os.Stdout) - logLevel, err := log.ParseLevel(config.LogLevel()) + logLevel, err := log.ParseLevel(config.Config().LogLevel) if err != nil { log.Panic(err) } diff --git a/internal/app/service/infra/mail/mailer.go b/internal/app/service/infra/mail/mailer.go index c75426a9..0bee1fb5 100644 --- a/internal/app/service/infra/mail/mailer.go +++ b/internal/app/service/infra/mail/mailer.go @@ -22,11 +22,11 @@ type mailer struct { } func New(mailServerHost, mailServerPort string) Mailer { - auth := smtp.PlainAuth("", config.MailUsername(), config.MailPassword(), mailServerHost) + auth := smtp.PlainAuth("", config.Config().MailUsername, config.Config().MailPassword, mailServerHost) addr := mailServerHost + ":" + mailServerPort return &mailer{ - from: config.MailUsername(), + from: config.Config().MailUsername, addr: addr, auth: auth, } diff --git a/internal/app/service/infra/mail/mailer_test.go b/internal/app/service/infra/mail/mailer_test.go index 6a66b492..618a5d5d 100644 --- a/internal/app/service/infra/mail/mailer_test.go +++ b/internal/app/service/infra/mail/mailer_test.go @@ -100,7 +100,7 @@ func TestSendMail(t *testing.T) { stringifiedJobArgs := MapToString(executionContext.Args) var sendMailClient = `EHLO localhost HELO localhost -MAIL FROM:<` + config.MailUsername() + `> +MAIL FROM:<` + config.Config().MailUsername + `> RCPT TO:<` + recipients[0] + `> RCPT TO:<` + recipients[1] + `> DATA diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index 12e968e3..34faa580 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -15,9 +15,7 @@ type context interface { } type testContext struct { - pluginBinary string - exportedName string - goPlugin GoPlugin + goPlugin GoPlugin } func (context *testContext) setUp(t *testing.T) { @@ -25,9 +23,16 @@ func (context *testContext) setUp(t *testing.T) { if available != true || value != "true" { t.SkipNow() } - context.pluginBinary = config.AuthPluginBinary() - assert.NotEmpty(t, context.pluginBinary) - context.exportedName = config.AuthPluginExported() + + println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) + println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) + println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) + println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) + println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) + println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) + println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) + println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) + context.goPlugin = NewGoPlugin() assert.NotNil(t, context.goPlugin) } @@ -44,33 +49,38 @@ func newContext() context { return ctx } -func TestGoPlugin_LoadSuccessfully(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - raw, err := ctx.instance().goPlugin.Load(ctx.instance().pluginBinary, ctx.instance().exportedName) - assert.NoError(t, err) - assert.NotNil(t, raw) -} - func TestGoPlugin_LoadPluginFailed(t *testing.T) { - t.SkipNow() ctx := newContext() ctx.setUp(t) binary := "non-existing-binary" - raw, err := ctx.instance().goPlugin.Load(binary, ctx.instance().exportedName) + raw, err := ctx.instance().goPlugin.Load(binary, config.Config().AuthPluginExported) assert.EqualError(t, err, fmt.Sprintf("failed to load plugin binary from location: %s", binary)) assert.Nil(t, raw) } func TestGoPlugin_LoadExportedFailed(t *testing.T) { - t.SkipNow() ctx := newContext() ctx.setUp(t) exportedName := "non-existing-exported" - raw, err := ctx.instance().goPlugin.Load(ctx.instance().pluginBinary, exportedName) - assert.EqualError(t, err, fmt.Sprintf("failed to Lookup plugin binary from location: %s with Exported Name: %s", ctx.instance().pluginBinary, exportedName)) + raw, err := ctx.instance().goPlugin.Load(config.Config().AuthPluginBinary, exportedName) + assert.EqualError(t, err, fmt.Sprintf("failed to Lookup plugin binary from location: %s with Exported Name: %s", config.Config().AuthPluginBinary, exportedName)) assert.Nil(t, raw) } + +func TestGoPlugin_LoadSuccessfully(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + raw, err := ctx.instance().goPlugin.Load(config.Config().AuthPluginBinary, config.Config().AuthPluginExported) + assert.NoError(t, err) + assert.NotNil(t, raw) +} + +func TestGoPlugin_ShitShit(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + assert.True(t, true) +} diff --git a/internal/app/service/schedule/worker/worker.go b/internal/app/service/schedule/worker/worker.go index 44363ab1..4e39aa63 100644 --- a/internal/app/service/schedule/worker/worker.go +++ b/internal/app/service/schedule/worker/worker.go @@ -140,10 +140,10 @@ func Start() error { return err } kubeClient := kubernetes.NewKubernetesClient(httpClient) - mailer := mail.New(config.MailServerHost(), config.MailServerPort()) + mailer := mail.New(config.Config().MailServerHost, config.Config().AuthPluginExported) executionSvc := executionService.NewExecutionService(kubeClient, executionContextStore, metadataStore, secretStore) worker := NewWorker(executionSvc, executionContextStore, scheduleStore, scheduleContextStore, mailer) - ticker := time.NewTicker(time.Duration(config.ScheduledJobsFetchIntervalInMins()) * time.Minute) + ticker := time.NewTicker(time.Duration(config.Config().ScheduledJobsFetchIntervalInMins) * time.Minute) signalsChan := make(chan os.Signal, 1) worker.Run(ticker.C, signalsChan) diff --git a/internal/app/service/server/api.go b/internal/app/service/server/api.go index 08f7720d..a7aa5155 100644 --- a/internal/app/service/server/api.go +++ b/internal/app/service/server/api.go @@ -17,7 +17,7 @@ func Start() error { if err != nil { logger.Fatal(err) } - appPort := ":" + config.AppPort() + appPort := ":" + config.Config().AppPort server := negroni.New(negroni.NewRecovery()) router, err := NewRouter() diff --git a/internal/app/service/server/middleware/newrelic.go b/internal/app/service/server/middleware/newrelic.go index fe11b423..dd111bc7 100644 --- a/internal/app/service/server/middleware/newrelic.go +++ b/internal/app/service/server/middleware/newrelic.go @@ -10,8 +10,8 @@ import ( var newRelicApp newrelic.Application func InitNewRelic() error { - appName := config.NewRelicAppName() - licenceKey := config.NewRelicLicenceKey() + appName := config.Config().NewRelicAppName + licenceKey := config.Config().NewRelicLicenceKey newRelicConfig := newrelic.NewConfig(appName, licenceKey) newRelicConfig.Enabled = true app, err := newrelic.NewApplication(newRelicConfig) diff --git a/internal/app/service/server/middleware/validate_client_version.go b/internal/app/service/server/middleware/validate_client_version.go index 9fc9d7b0..f55bf9a2 100644 --- a/internal/app/service/server/middleware/validate_client_version.go +++ b/internal/app/service/server/middleware/validate_client_version.go @@ -24,7 +24,7 @@ func ValidateClientVersion(next http.Handler) http.Handler { return } - minClientVersion, err := version.NewVersion(config.MinClientVersion()) + minClientVersion, err := version.NewVersion(config.Config().MinClientVersion) if err != nil { logger.Error("Error while creating minClientVersion ", err.Error()) w.WriteHeader(http.StatusInternalServerError) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 32d7541e..99e8c332 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -55,9 +55,9 @@ func NewRouter() (*mux.Router, error) { }) router.HandleFunc("/docs", docs.APIDocHandler) - router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.DocsPath())))) + router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.Config().DocsPath)))) router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, path.Join(config.DocsPath(), "swagger.yml")) + http.ServeFile(w, r, path.Join(config.Config().DocsPath, "swagger.yml")) }) router = middleware.InstrumentNewRelic(router) From 4ee383a64d25c2b9a00d160c484cac78a0cd916a Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 15 Aug 2019 13:49:09 +0700 Subject: [PATCH 117/268] [Jasoet] Add Reset Mechanism for Config() using atomic boolean --- internal/app/service/infra/config/config.go | 30 ++++++++++++++++--- .../kubernetes/client_integration_test.go | 1 + 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 650bf0af..ac93682e 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/spf13/viper" "sync" + "sync/atomic" "time" ) @@ -119,10 +120,31 @@ func Load() ProctorConfig { return proctorConfig } -var reset = false +type AtomBool struct{ flag int32 } + +func (b *AtomBool) Set(value bool) { + var i int32 = 0 + if value { + i = 1 + } + atomic.StoreInt32(&(b.flag), int32(i)) +} + +func (b *AtomBool) Get() bool { + if atomic.LoadInt32(&(b.flag)) != 0 { + return true + } + return false +} + +var reset = new(AtomBool) + +func init() { + reset.Set(false) +} func Reset() { - reset = true + reset.Set(true) } func Config() ProctorConfig { @@ -130,9 +152,9 @@ func Config() ProctorConfig { config = Load() }) - if reset { + if reset.Get() { config = Load() - reset = false + reset.Set(false) } return config diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index fbc93d1b..56bfc804 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -31,6 +31,7 @@ func (suite *IntegrationTestSuite) SetupTest() { func (suite *IntegrationTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + config.Reset() envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} sampleImageName := "busybox" From 76bbaad2710855302481f13abd46de876292ca3e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Thu, 15 Aug 2019 13:52:10 +0700 Subject: [PATCH 118/268] [Jasoet] Remove unused log --- .../app/service/infra/plugin/plugin_integration_test.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index 34faa580..f0334796 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -24,15 +24,6 @@ func (context *testContext) setUp(t *testing.T) { t.SkipNow() } - println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) - println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) - println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) - println("==================== " + os.Getenv("PROCTOR_AUTH_PLUGIN_BINARY")) - println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) - println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) - println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) - println("xxxxxxxxxxxxxxxxxxxx " + config.Config().AuthPluginBinary) - context.goPlugin = NewGoPlugin() assert.NotNil(t, context.goPlugin) } From fab4852e808cabcd776ebc180d32610d6245b1ac Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 20 Aug 2019 11:42:31 +0700 Subject: [PATCH 119/268] [Jasoet] Change config mechanism for gate --- plugins/gate-auth-plugin/gate/client.go | 9 ++++-- plugins/gate-auth-plugin/gate/client_mock.go | 15 +++++++++ plugins/gate-auth-plugin/gate/config.go | 33 +++++++++++--------- 3 files changed, 40 insertions(+), 17 deletions(-) create mode 100644 plugins/gate-auth-plugin/gate/client_mock.go diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index d2c05411..2f96dfe2 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -12,6 +12,7 @@ type GateClient interface { } type gateClient struct { + config GateConfig host string profilePath string protocol string @@ -54,10 +55,12 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai } func NewGateClient() GateClient { + config := NewGateConfig() return &gateClient{ - protocol: Protocol(), - host: Host(), - profilePath: ProfilePath(), + config: config, + protocol: config.Protocol, + host: config.Host, + profilePath: config.ProfilePath, restClient: resty.New(), } } diff --git a/plugins/gate-auth-plugin/gate/client_mock.go b/plugins/gate-auth-plugin/gate/client_mock.go new file mode 100644 index 00000000..9be1406e --- /dev/null +++ b/plugins/gate-auth-plugin/gate/client_mock.go @@ -0,0 +1,15 @@ +package gate + +import ( + "github.com/stretchr/testify/mock" + "proctor/pkg/auth" +) + +type GateClientMock struct { + mock.Mock +} + +func (g *GateClientMock) GetUserProfile(email string, token string) (*auth.UserDetail, error) { + args := g.Called(email, token) + return args.Get(0).(*auth.UserDetail), args.Error(1) +} diff --git a/plugins/gate-auth-plugin/gate/config.go b/plugins/gate-auth-plugin/gate/config.go index 579ebe1a..51d8783d 100644 --- a/plugins/gate-auth-plugin/gate/config.go +++ b/plugins/gate-auth-plugin/gate/config.go @@ -2,22 +2,27 @@ package gate import "github.com/spf13/viper" -func init() { - viper.AutomaticEnv() - viper.SetEnvPrefix("GATE_PLUGIN") +type GateConfig struct { + Protocol string + Host string + ProfilePath string + viper viper.Viper } -func Protocol() string { - viper.SetDefault("PROTOCOL", "https") - return viper.GetString("PROTOCOL") -} +func NewGateConfig() GateConfig { + fang := viper.New() + fang.AutomaticEnv() + fang.SetEnvPrefix("GATE_PLUGIN") + fang.SetDefault("PROTOCOL", "https") -func Host() string { - viper.SetDefault("HOST", "gate.gojek.co.id") - return viper.GetString("HOST") -} + fang.SetDefault("HOST", "gate.gojek.co.id") + fang.SetDefault("PROFILE_PATH", "api/v1/users/profile") + config := GateConfig{ + Protocol: fang.GetString("PROTOCOL"), + Host: fang.GetString("HOST"), + ProfilePath: fang.GetString("PROFILE_PATH"), + viper: viper.Viper{}, + } -func ProfilePath() string { - viper.SetDefault("PROFILE_PATH", "api/v1/users/profile") - return viper.GetString("PROFILE_PATH") + return config } From 1dc2c6e2746d0fd5cadb0367f017253be2346af6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Fri, 23 Aug 2019 13:59:34 +0700 Subject: [PATCH 120/268] [GOJ-100075] Add template command --- internal/app/cli/command/template/template.go | 63 +++++++++ .../app/cli/command/template/template_test.go | 127 ++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 internal/app/cli/command/template/template.go create mode 100644 internal/app/cli/command/template/template_test.go diff --git a/internal/app/cli/command/template/template.go b/internal/app/cli/command/template/template.go new file mode 100644 index 00000000..80113753 --- /dev/null +++ b/internal/app/cli/command/template/template.go @@ -0,0 +1,63 @@ +package template + +import ( + "fmt" + + "github.com/fatih/color" + "github.com/spf13/cobra" + + "proctor/internal/app/cli/daemon" + utilFile "proctor/internal/app/cli/utility/file" + utilIO "proctor/internal/app/cli/utility/io" + modelMetadata "proctor/internal/pkg/model/metadata" +) + +func NewCmd(printer utilIO.Printer, proctorDClient daemon.Client) *cobra.Command { + return &cobra.Command{ + Use: "template", + Short: "Get input template of a procs", + Long: "To get input template of a procs, this command retrieve an example template derived from stored metadata", + Example: "proctor template say-hello-world say-hello-world.yaml", + Args: cobra.MinimumNArgs(2), + + Run: func(cmd *cobra.Command, args []string) { + if len(args) < 2 { + printer.Println("Incorrect command. See `proctor template --help` for usage", color.FgRed) + return + } + + userProvidedProcName := args[0] + filename := args[1] + + procList, err := proctorDClient.ListProcs() + if err != nil { + printer.Println(err.Error(), color.FgRed) + return + } + + desiredProc := modelMetadata.Metadata{} + for _, proc := range procList { + if userProvidedProcName == proc.Name { + desiredProc = proc + } + } + if len(desiredProc.Name) == 0 { + printer.Println(fmt.Sprintf("Proctor doesn't support Proc `%s`\nRun `proctor list` to view supported Procs", userProvidedProcName), color.FgRed) + return + } + + printer.Println("\nArgs", color.FgMagenta) + for _, arg := range desiredProc.EnvVars.Args { + printer.Println(fmt.Sprintf("%-40s %-100s", arg.Name, arg.Description), color.Reset) + } + + err = utilFile.WriteYAML(filename, desiredProc.EnvVars.Args) + if err != nil { + printer.Println(fmt.Sprintf("Error writing template file: %s", err.Error()), color.FgRed) + return + } + + printer.Println(fmt.Sprintf("\nTo %s, run:\nproctor execute %s -f %s ARG_ONE=foo ARG_TWO=bar", userProvidedProcName, userProvidedProcName, filename), color.FgGreen) + }, + } +} diff --git a/internal/app/cli/command/template/template_test.go b/internal/app/cli/command/template/template_test.go new file mode 100644 index 00000000..ea6e1dbb --- /dev/null +++ b/internal/app/cli/command/template/template_test.go @@ -0,0 +1,127 @@ +package template + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + + daemon2 "proctor/internal/app/cli/daemon" + "proctor/internal/app/cli/utility/io" + procMetadata "proctor/internal/pkg/model/metadata" + "proctor/internal/pkg/model/metadata/env" +) + +type TemplateCmdTestSuite struct { + suite.Suite + mockPrinter *io.MockPrinter + mockProctorDClient *daemon2.MockClient + testTemplateCmd *cobra.Command +} + +func (s *TemplateCmdTestSuite) SetupTest() { + s.mockPrinter = &io.MockPrinter{} + s.mockProctorDClient = &daemon2.MockClient{} + s.testTemplateCmd = NewCmd(s.mockPrinter, s.mockProctorDClient) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdUsage() { + assert.Equal(s.T(), "template", s.testTemplateCmd.Use) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdHelp() { + assert.Equal(s.T(), "Get input template of a procs", s.testTemplateCmd.Short) + assert.Equal(s.T(), "To get input template of a procs, this command retrieve an example template derived from stored metadata", s.testTemplateCmd.Long) + assert.Equal(s.T(), "proctor template say-hello-world say-hello-world.yaml", s.testTemplateCmd.Example) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdRun() { + t := s.T() + + filename := "/tmp/yaml-test-template" + defer os.Remove(filename) + + arg := env.VarMetadata{ + Name: "arg-one", + Description: "arg one description", + } + + secret := env.VarMetadata{ + Name: "secret-one", + Description: "secret one description", + } + + anyProc := procMetadata.Metadata{ + Name: "do-something", + Description: "does something", + Contributors: "user@example.com", + Organization: "org", + AuthorizedGroups: []string{"group_one", "group_two"}, + EnvVars: env.Vars{ + Args: []env.VarMetadata{arg}, + Secrets: []env.VarMetadata{secret}, + }, + } + procList := []procMetadata.Metadata{anyProc} + + s.mockProctorDClient.On("ListProcs").Return(procList, nil).Once() + + s.mockPrinter.On("Println", "\nArgs", color.FgMagenta).Once() + s.mockPrinter.On("Println", fmt.Sprintf("%-40s %-100s", arg.Name, arg.Description), color.Reset).Once() + s.mockPrinter.On("Println", fmt.Sprintf("\nTo %s, run:\nproctor execute %s -f %s ARG_ONE=foo ARG_TWO=bar", anyProc.Name, anyProc.Name, filename), color.FgGreen).Once() + + s.testTemplateCmd.Run(&cobra.Command{}, []string{anyProc.Name, filename}) + + templateFile, err := os.Open(filename) + assert.NoError(t, err) + defer templateFile.Close() + + templateBuffer, err := ioutil.ReadAll(templateFile) + assert.Equal(t, templateBuffer, []byte("# arg one description\narg-one:\n")) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdForIncorrectUsage() { + s.mockPrinter.On("Println", "Incorrect command. See `proctor template --help` for usage", color.FgRed).Once() + + s.testTemplateCmd.Run(&cobra.Command{}, []string{}) + + s.mockPrinter.AssertExpectations(s.T()) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdRunProctorDClientFailure() { + filename := "/tmp/yaml-test-template" + + s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, errors.New("test error")).Once() + s.mockPrinter.On("Println", "test error", color.FgRed).Once() + + s.testTemplateCmd.Run(&cobra.Command{}, []string{"do-something", filename}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + +func (s *TemplateCmdTestSuite) TestTemplateCmdRunProcNotSupported() { + filename := "/tmp/yaml-test-template" + + s.mockProctorDClient.On("ListProcs").Return([]procMetadata.Metadata{}, nil).Once() + testProcName := "do-something" + s.mockPrinter.On("Println", fmt.Sprintf("Proctor doesn't support Proc `%s`\nRun `proctor list` to view supported Procs", testProcName), color.FgRed).Once() + + s.testTemplateCmd.Run(&cobra.Command{}, []string{testProcName, filename}) + + s.mockProctorDClient.AssertExpectations(s.T()) + s.mockPrinter.AssertExpectations(s.T()) +} + +func TestTemplateCmdTestSuite(t *testing.T) { + suite.Run(t, new(TemplateCmdTestSuite)) +} From 12536e1503f65332bd8cd09b663a4757d1bf9615 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Fri, 23 Aug 2019 14:02:07 +0700 Subject: [PATCH 121/268] Add template to root command --- Makefile | 4 ++++ internal/app/cli/command/root.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/Makefile b/Makefile index c8d8986a..bf58467c 100644 --- a/Makefile +++ b/Makefile @@ -90,6 +90,10 @@ ftest.proctor.list: ftest.proctor.describe: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli describe say-hello-world +.PHONY: ftest.proctor.template +ftest.proctor.template: + LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli template say-hello-world say-hello-world.yaml + .PHONY: ftest.proctor.execute ftest.proctor.execute: LOCAL_CONFIG_DIR=$(CONFIG_DIR) $(BIN_DIR)/cli execute say-hello-world SAMPLE_ARG_ONE=foo SAMPLE_ARG_TWO=bar diff --git a/internal/app/cli/command/root.go b/internal/app/cli/command/root.go index cb54e3dc..fe2ddfe8 100644 --- a/internal/app/cli/command/root.go +++ b/internal/app/cli/command/root.go @@ -17,6 +17,7 @@ import ( scheduleList "proctor/internal/app/cli/command/schedule/list" "proctor/internal/app/cli/command/schedule/remove" "proctor/internal/app/cli/command/status" + "proctor/internal/app/cli/command/template" "proctor/internal/app/cli/command/version" "proctor/internal/app/cli/command/version/github" "proctor/internal/app/cli/daemon" @@ -51,6 +52,9 @@ func Execute(printer io.Printer, proctorDClient daemon.Client, githubClient gith listCmd := list.NewCmd(printer, proctorDClient) rootCmd.AddCommand(listCmd) + templateCmd := template.NewCmd(printer, proctorDClient) + rootCmd.AddCommand(templateCmd) + configCmd := config.NewCmd(printer) configShowCmd := view.NewCmd(printer) rootCmd.AddCommand(configCmd) From ba970670884fb6b0501649aee71a3d109297e033 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Fri, 23 Aug 2019 14:03:28 +0700 Subject: [PATCH 122/268] Add WriteYAML to utility --- internal/app/cli/utility/file/file.go | 16 ++++++++++++++++ internal/app/cli/utility/file/file_test.go | 21 +++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/internal/app/cli/utility/file/file.go b/internal/app/cli/utility/file/file.go index 2386583d..8a5cd343 100644 --- a/internal/app/cli/utility/file/file.go +++ b/internal/app/cli/utility/file/file.go @@ -5,6 +5,8 @@ import ( "os" "gopkg.in/yaml.v2" + + "proctor/internal/pkg/model/metadata/env" ) func ParseYAML(filename string, procArgs map[string]string) error { @@ -23,3 +25,17 @@ func ParseYAML(filename string, procArgs map[string]string) error { return nil } + +func WriteYAML(filename string, procArgs []env.VarMetadata) error { + var content string + + for _, procArg := range procArgs { + content += "# " + procArg.Description + content += "\n" + content += procArg.Name + ":" + content += "\n" + } + + err := ioutil.WriteFile(filename, []byte(content), 0644) + return err +} diff --git a/internal/app/cli/utility/file/file_test.go b/internal/app/cli/utility/file/file_test.go index 240b0bad..5b310111 100644 --- a/internal/app/cli/utility/file/file_test.go +++ b/internal/app/cli/utility/file/file_test.go @@ -3,6 +3,7 @@ package file import ( "io/ioutil" "os" + "proctor/internal/pkg/model/metadata/env" "testing" "github.com/stretchr/testify/assert" @@ -44,3 +45,23 @@ func TestParseYAMLError(t *testing.T) { assert.Contains(t, err.Error(), errorTest.ErrorMessage) } } + +func TestWriteYAML(t *testing.T) { + filename := "/tmp/yaml-test-write" + procArgs := []env.VarMetadata{ + {"foo", "bar"}, + {"moo", "zoo"}, + } + + err := WriteYAML(filename, procArgs) + assert.NoError(t, err) + defer os.Remove(filename) + + file, err := os.Open(filename) + assert.NoError(t, err) + defer file.Close() + + buffer, err := ioutil.ReadAll(file) + assert.NoError(t, err) + assert.Equal(t, buffer, []byte("# bar\nfoo:\n# zoo\nmoo:\n")) +} From 6e5950e8f877ce316ff90edcbb8f8a289167aed6 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 17:41:44 +0700 Subject: [PATCH 123/268] [Dembo] Refactor cli daemon client test to extract method for mock list porcs request --- internal/app/cli/daemon/client_test.go | 126 +++++++------------------ 1 file changed, 36 insertions(+), 90 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 70139bcb..cb280f95 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -49,6 +49,24 @@ func (s *ClientTestSuite) SetupTest() { s.testClient = NewClient(s.mockPrinter, s.mockConfigLoader) } +func mockListProcsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { + httpmock.RegisterStubRequest( + httpmock.NewStubRequest( + "GET", + "http://"+proctorConfig.Host+MetadataRoute, + func(req *http.Request) (*http.Response, error) { + return mockResponse, mockError + }, + ).WithHeader( + &http.Header{ + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, + }, + ), + ) +} + func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { t := s.T() @@ -70,21 +88,9 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { }, } - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, body) + mockError := error(nil) + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -103,21 +109,9 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "list proc error"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(500, "list proc error") + mockError := error(nil) + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -137,21 +131,9 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + var mockResponse *http.Response + mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -170,21 +152,9 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return nil, TestConnectionError{message: "Unknown Error", timeout: false} - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + var mockResponse *http.Response + mockError := TestConnectionError{message: "Unknown Error", timeout: false} + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -203,21 +173,9 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedUser() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(401, `{}`), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(401, `{}`) + mockError := error(nil) + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -235,21 +193,9 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(401, `{}`), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{""}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(401, `{}`) + mockError := error(nil) + mockListProcsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() procList, err := s.testClient.ListProcs() From b75c8068c58c95ac8353953b1f9a4ffdfef5e624 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 17:58:52 +0700 Subject: [PATCH 124/268] [Dembo] Refactor cli daemon client test to extract method for mock execute request --- internal/app/cli/daemon/client_test.go | 100 +++++++------------------ 1 file changed, 29 insertions(+), 71 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index cb280f95..43f6981c 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -226,29 +226,35 @@ func (s *ClientTestSuite) TestExecuteProc() { httpmock.Activate() defer httpmock.DeactivateAndReset() + mockResponse := httpmock.NewStringResponse(201, body) + mockError := error(nil) + mockExecuteRequest(proctorConfig, mockResponse, mockError) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + + executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) + + assert.NoError(t, err) + assert.Equal(t, expectedProcResponse, executeProcResponse) + s.mockConfigLoader.AssertExpectations(t) +} + +func mockExecuteRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", "http://"+proctorConfig.Host+ExecutionRoute, func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(201, body), nil + return mockResponse, mockError }, ).WithHeader( &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) - - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - - executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) - - assert.NoError(t, err) - assert.Equal(t, expectedProcResponse, executeProcResponse) - s.mockConfigLoader.AssertExpectations(t) } func (s *ClientTestSuite) TestSuccessScheduledJob() { @@ -339,21 +345,9 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ExecutionRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "Execute Error"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(500, "Execute Error") + mockError := error(nil) + mockExecuteRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) @@ -371,21 +365,9 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ExecutionRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(401, ""), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(401, "") + mockError := error(nil) + mockExecuteRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -404,21 +386,9 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ExecutionRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(401, ""), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{""}, - constant.AccessTokenHeaderKey: []string{""}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(401, "") + mockError := error(nil) + mockExecuteRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -437,21 +407,9 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ExecutionRoute, - func(req *http.Request) (*http.Response, error) { - return nil, TestConnectionError{message: "Unknown Error", timeout: false} - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + var mockResponse *http.Response = nil + mockError := TestConnectionError{message: "Unknown Error", timeout: false} + mockExecuteRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() From 4c7c393ce1bcfbf3ec05f95162946e792889e2a1 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 18:03:34 +0700 Subject: [PATCH 125/268] [Dembo] Refactor cli daemon client test to extract method for mock schedule request --- internal/app/cli/daemon/client_test.go | 42 +++++++++++--------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 43f6981c..041e1f40 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -274,21 +274,9 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ScheduleRoute, - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(201, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(201, body) + mockError := error(nil) + mockScheduleRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -313,27 +301,33 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { httpmock.Activate() defer httpmock.DeactivateAndReset() + mockResponse := httpmock.NewStringResponse(409, "Server Error!!!\nStatus Code: 409, Conflict") + mockError := error(nil) + mockScheduleRequest(proctorConfig, mockResponse, mockError) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + + _, err := s.testClient.ScheduleJob(procName, tags, time, notificationEmails, group, procArgs) + assert.Equal(t, "Server Error!!!\nStatus Code: 409, Conflict", err.Error()) + s.mockConfigLoader.AssertExpectations(t) +} + +func mockScheduleRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "POST", "http://"+proctorConfig.Host+ScheduleRoute, func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(409, "Server Error!!!\nStatus Code: 409, Conflict"), nil + return mockResponse, mockError }, ).WithHeader( &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) - - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - - _, err := s.testClient.ScheduleJob(procName, tags, time, notificationEmails, group, procArgs) - assert.Equal(t, "Server Error!!!\nStatus Code: 409, Conflict", err.Error()) - s.mockConfigLoader.AssertExpectations(t) } func (s *ClientTestSuite) TestExecuteProcInternalServerError() { From 115cef3f930da687c104645dad354cd310c48160 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 18:27:28 +0700 Subject: [PATCH 126/268] [Dembo] Refactor cli daemon client test to extract method for mock execution context request --- internal/app/cli/daemon/client_test.go | 126 +++++++------------------ 1 file changed, 36 insertions(+), 90 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 041e1f40..2c0ae92b 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -496,21 +496,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForSucceededProcs() { } responseBody := fmt.Sprintf(`{ "status": "%s" }`, constant.JobSucceeded) - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, responseBody), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, responseBody) + mockError := error(nil) + mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -540,21 +528,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForFailedProcs() { } responseBody := fmt.Sprintf(`{ "status": "%s" }`, constant.JobFailed) - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, responseBody), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, responseBody) + mockError := error(nil) + mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -573,21 +549,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForHTTPRequestFailure() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", - func(req *http.Request) (*http.Response, error) { - return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + var mockResponse *http.Response = nil + mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} + mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -607,21 +571,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForNonOKResponse() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "execute Error"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(500, "execute Error") + mockError := error(nil) + mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -661,21 +613,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForCompletedPr } responseBody := fmt.Sprintf(`{ "id": %v, "status": "%s" }`, fmt.Sprint(proc.executionID), proc.expectedExecutionContextStatus) - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(proc.executionID)+"/status", - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, responseBody), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, responseBody) + mockError := error(nil) + mockExecutionContextRequest(proctorConfig, proc.executionID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() @@ -695,21 +635,9 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForGetError() httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", - func(req *http.Request) (*http.Response, error) { - return nil, TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + var mockResponse *http.Response = nil + mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} + mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() @@ -762,6 +690,24 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingWhenPollCountR assert.Equal(t, executionResult, executionContextStatus) } +func mockExecutionContextRequest(proctorConfig config.ProctorConfig, executionID uint64, mockResponse *http.Response, mockError error) { + httpmock.RegisterStubRequest( + httpmock.NewStubRequest( + "GET", + "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(executionID)+"/status", + func(req *http.Request) (*http.Response, error) { + return mockResponse, mockError + }, + ).WithHeader( + &http.Header{ + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, + constant.ClientVersionHeaderKey: []string{version.ClientVersion}, + }, + ), + ) +} + func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { t := s.T() From 6aa5496cd95451c7ac8525f81467d79bc8c66539 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 18:57:10 +0700 Subject: [PATCH 127/268] [Dembo] Refactor cli daemon client test to extract method for mock describe schedule job request --- internal/app/cli/daemon/client_test.go | 80 ++++++++------------------ 1 file changed, 25 insertions(+), 55 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 2c0ae92b..e250cde2 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -718,21 +718,9 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, body) + mockError := error(nil) + mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -754,21 +742,9 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(400, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(400, body) + mockError := error(nil) + mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -787,21 +763,9 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWhenJobIDNotFound() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(404, "Job not found"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(404, "Job not found") + mockError := error(nil) + mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -820,28 +784,34 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { httpmock.Activate() defer httpmock.DeactivateAndReset() + mockResponse := httpmock.NewStringResponse(500, "Schedule Failed") + mockError := error(nil) + mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + + _, err := s.testClient.DescribeScheduledProc(jobID) + + assert.Equal(t, "Schedule Failed", err.Error()) + s.mockConfigLoader.AssertExpectations(t) +} + +func mockDescribeScheduledJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "Schedule Failed"), nil + return mockResponse, mockError }, ).WithHeader( &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) - - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - - _, err := s.testClient.DescribeScheduledProc(jobID) - - assert.Equal(t, "Schedule Failed", err.Error()) - s.mockConfigLoader.AssertExpectations(t) } func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { From 4e458a45836429956393db00f823973a6b24cda7 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 19:22:43 +0700 Subject: [PATCH 128/268] [Dembo] Refactor cli daemon client test to extract method for mock list scheduled jobs request --- internal/app/cli/daemon/client_test.go | 62 +++++++++----------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index e250cde2..5fbf7386 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -824,21 +824,9 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, body) + mockError := error(nil) + mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -859,21 +847,9 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenNoJobsScheduled() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(204, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(204, body) + mockError := error(nil) + mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -891,28 +867,34 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal httpmock.Activate() defer httpmock.DeactivateAndReset() + mockResponse := httpmock.NewStringResponse(500, "Schedule Error") + mockError := error(nil) + mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + + _, err := s.testClient.ListScheduledProcs() + + assert.Equal(t, "Schedule Error", err.Error()) + s.mockConfigLoader.AssertExpectations(t) +} + +func mockListScheduledJobsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "Schedule Error"), nil + return mockResponse, mockError }, ).WithHeader( &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) - - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - - _, err := s.testClient.ListScheduledProcs() - - assert.Equal(t, "Schedule Error", err.Error()) - s.mockConfigLoader.AssertExpectations(t) } func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { From 23ab01033bf34574e7dabd23d01a2d410933c7c1 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 19:33:02 +0700 Subject: [PATCH 129/268] [Dembo] Refactor cli daemon client test to extract method for mock remove scheduled job request --- internal/app/cli/daemon/client_test.go | 82 +++++++++----------------- 1 file changed, 27 insertions(+), 55 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 5fbf7386..603c0116 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -907,21 +907,9 @@ func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(200, body), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + mockResponse := httpmock.NewStringResponse(200, body) + mockError := error(nil) + mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -940,21 +928,10 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWithInvalidJobID() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(400, "Invalid Job ID"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + + mockResponse := httpmock.NewStringResponse(400, "Invalid Job ID") + mockError := error(nil) + mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -973,21 +950,10 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWhenJobIDNotFound() { httpmock.Activate() defer httpmock.DeactivateAndReset() - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(404, "Job not found"), nil - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) + + mockResponse := httpmock.NewStringResponse(404, "Job not found") + mockError := error(nil) + mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -1006,26 +972,32 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { httpmock.Activate() defer httpmock.DeactivateAndReset() + mockResponse := httpmock.NewStringResponse(500, "Schedule Error") + mockError := error(nil) + mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) + + s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() + + err := s.testClient.RemoveScheduledProc(jobID) + + assert.Equal(t, "Schedule Error", err.Error()) + s.mockConfigLoader.AssertExpectations(t) +} + +func mockRemoveScheduleJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), func(req *http.Request) (*http.Response, error) { - return httpmock.NewStringResponse(500, "Schedule Error"), nil + return mockResponse, mockError }, ).WithHeader( &http.Header{ - constant.UserEmailHeaderKey: []string{"proctor@example.com"}, - constant.AccessTokenHeaderKey: []string{"access-token"}, + constant.UserEmailHeaderKey: []string{proctorConfig.Email}, + constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, constant.ClientVersionHeaderKey: []string{version.ClientVersion}, }, ), ) - - s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - - err := s.testClient.RemoveScheduledProc(jobID) - - assert.Equal(t, "Schedule Error", err.Error()) - s.mockConfigLoader.AssertExpectations(t) } From d3ca03c27c818618e34542d467d9b998232e8a7d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 26 Aug 2019 19:45:30 +0700 Subject: [PATCH 130/268] [Dembo] Refactor cli daemon client test to delete mock requests to generic one --- internal/app/cli/daemon/client_test.go | 140 ++++++------------------- 1 file changed, 30 insertions(+), 110 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 603c0116..5c481f27 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -49,24 +49,6 @@ func (s *ClientTestSuite) SetupTest() { s.testClient = NewClient(s.mockPrinter, s.mockConfigLoader) } -func mockListProcsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+MetadataRoute, - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { t := s.T() @@ -239,24 +221,6 @@ func (s *ClientTestSuite) TestExecuteProc() { s.mockConfigLoader.AssertExpectations(t) } -func mockExecuteRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ExecutionRoute, - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestSuccessScheduledJob() { t := s.T() @@ -312,24 +276,6 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { s.mockConfigLoader.AssertExpectations(t) } -func mockScheduleRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "POST", - "http://"+proctorConfig.Host+ScheduleRoute, - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestExecuteProcInternalServerError() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} @@ -690,24 +636,6 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingWhenPollCountR assert.Equal(t, executionResult, executionContextStatus) } -func mockExecutionContextRequest(proctorConfig config.ProctorConfig, executionID uint64, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(executionID)+"/status", - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { t := s.T() @@ -796,24 +724,6 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { s.mockConfigLoader.AssertExpectations(t) } -func mockDescribeScheduledJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { t := s.T() @@ -879,24 +789,6 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal s.mockConfigLoader.AssertExpectations(t) } -func mockListScheduledJobsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), - func(req *http.Request) (*http.Response, error) { - return mockResponse, mockError - }, - ).WithHeader( - &http.Header{ - constant.UserEmailHeaderKey: []string{proctorConfig.Email}, - constant.AccessTokenHeaderKey: []string{proctorConfig.AccessToken}, - constant.ClientVersionHeaderKey: []string{version.ClientVersion}, - }, - ), - ) -} - func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { t := s.T() @@ -984,11 +876,39 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { s.mockConfigLoader.AssertExpectations(t) } +func mockListProcsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) +} + +func mockExecuteRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) +} + +func mockScheduleRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ScheduleRoute, mockResponse, mockError) +} + +func mockExecutionContextRequest(proctorConfig config.ProctorConfig, executionID uint64, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(executionID)+"/status", mockResponse, mockError) +} + func mockRemoveScheduleJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) +} + +func mockDescribeScheduledJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) +} + +func mockListScheduledJobsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), mockResponse, mockError) +} + +func mockRequest(proctorConfig config.ProctorConfig, method string, url string, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( - "DELETE", - fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), + method, + url, func(req *http.Request) (*http.Response, error) { return mockResponse, mockError }, From 46f7ad94d3514482fb59fbdb6ad0b29cfeaa538d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 27 Aug 2019 13:57:52 +0700 Subject: [PATCH 131/268] [Dembo] Refactor cli daemon client test to inline specific mock request --- internal/app/cli/daemon/client_test.go | 88 +++++++++----------------- 1 file changed, 30 insertions(+), 58 deletions(-) diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 5c481f27..de2837b6 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -72,7 +72,7 @@ func (s *ClientTestSuite) TestListProcsReturnsListOfProcsWithDetails() { mockResponse := httpmock.NewStringResponse(200, body) mockError := error(nil) - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -93,7 +93,7 @@ func (s *ClientTestSuite) TestListProcsReturnErrorFromResponseBody() { mockResponse := httpmock.NewStringResponse(500, "list proc error") mockError := error(nil) - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -115,7 +115,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideTimeoutError() { var mockResponse *http.Response mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -136,7 +136,7 @@ func (s *ClientTestSuite) TestListProcsReturnClientSideConnectionError() { var mockResponse *http.Response mockError := TestConnectionError{message: "Unknown Error", timeout: false} - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -157,7 +157,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedUser() { mockResponse := httpmock.NewStringResponse(401, `{}`) mockError := error(nil) - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -177,7 +177,7 @@ func (s *ClientTestSuite) TestListProcsForUnauthorizedErrorWithConfigMissing() { mockResponse := httpmock.NewStringResponse(401, `{}`) mockError := error(nil) - mockListProcsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() procList, err := s.testClient.ListProcs() @@ -210,7 +210,7 @@ func (s *ClientTestSuite) TestExecuteProc() { mockResponse := httpmock.NewStringResponse(201, body) mockError := error(nil) - mockExecuteRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -240,7 +240,7 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { mockResponse := httpmock.NewStringResponse(201, body) mockError := error(nil) - mockScheduleRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ScheduleRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -267,7 +267,7 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { mockResponse := httpmock.NewStringResponse(409, "Server Error!!!\nStatus Code: 409, Conflict") mockError := error(nil) - mockScheduleRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ScheduleRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -287,7 +287,7 @@ func (s *ClientTestSuite) TestExecuteProcInternalServerError() { mockResponse := httpmock.NewStringResponse(500, "Execute Error") mockError := error(nil) - mockExecuteRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() executeProcResponse, err := s.testClient.ExecuteProc(procName, procArgs) @@ -307,7 +307,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorized() { mockResponse := httpmock.NewStringResponse(401, "") mockError := error(nil) - mockExecuteRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -328,7 +328,7 @@ func (s *ClientTestSuite) TestExecuteProcUnAuthorizedWhenEmailAndAccessTokenNotS mockResponse := httpmock.NewStringResponse(401, "") mockError := error(nil) - mockExecuteRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -349,7 +349,7 @@ func (s *ClientTestSuite) TestExecuteProcsReturnClientSideConnectionError() { var mockResponse *http.Response = nil mockError := TestConnectionError{message: "Unknown Error", timeout: false} - mockExecuteRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -444,7 +444,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForSucceededProcs() { mockResponse := httpmock.NewStringResponse(200, responseBody) mockError := error(nil) - mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -476,7 +476,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForFailedProcs() { mockResponse := httpmock.NewStringResponse(200, responseBody) mockError := error(nil) - mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -497,7 +497,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForHTTPRequestFailure() { var mockResponse *http.Response = nil mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -519,7 +519,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusForNonOKResponse() { mockResponse := httpmock.NewStringResponse(500, "execute Error") mockError := error(nil) - mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -561,7 +561,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForCompletedPr mockResponse := httpmock.NewStringResponse(200, responseBody) mockError := error(nil) - mockExecutionContextRequest(proctorConfig, proc.executionID, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(proc.executionID)+"/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() @@ -583,7 +583,7 @@ func (s *ClientTestSuite) TestGetExecutionContextStatusWithPollingForGetError() var mockResponse *http.Response = nil mockError := TestConnectionError{message: "Unable to reach http://proctor.example.com/", timeout: true} - mockExecutionContextRequest(proctorConfig, 42, mockResponse, mockError) + mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/42/status", mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Twice() @@ -648,7 +648,7 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { mockResponse := httpmock.NewStringResponse(200, body) mockError := error(nil) - mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -672,7 +672,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { mockResponse := httpmock.NewStringResponse(400, body) mockError := error(nil) - mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -693,7 +693,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWhenJobIDNotFound() { mockResponse := httpmock.NewStringResponse(404, "Job not found") mockError := error(nil) - mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -714,7 +714,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { mockResponse := httpmock.NewStringResponse(500, "Schedule Failed") mockError := error(nil) - mockDescribeScheduledJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -736,7 +736,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { mockResponse := httpmock.NewStringResponse(200, body) mockError := error(nil) - mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -759,7 +759,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenNoJobsScheduled() { mockResponse := httpmock.NewStringResponse(204, body) mockError := error(nil) - mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -779,7 +779,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobsWhenServerReturnInternal mockResponse := httpmock.NewStringResponse(500, "Schedule Error") mockError := error(nil) - mockListScheduledJobsRequest(proctorConfig, mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -801,7 +801,7 @@ func (s *ClientTestSuite) TestSuccessRemoveScheduledJob() { mockResponse := httpmock.NewStringResponse(200, body) mockError := error(nil) - mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -823,7 +823,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWithInvalidJobID() { mockResponse := httpmock.NewStringResponse(400, "Invalid Job ID") mockError := error(nil) - mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -845,7 +845,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWhenJobIDNotFound() { mockResponse := httpmock.NewStringResponse(404, "Job not found") mockError := error(nil) - mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -866,7 +866,7 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { mockResponse := httpmock.NewStringResponse(500, "Schedule Error") mockError := error(nil) - mockRemoveScheduleJobRequest(proctorConfig, jobID, mockResponse, mockError) + mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -876,34 +876,6 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWitInternalServerError() { s.mockConfigLoader.AssertExpectations(t) } -func mockListProcsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+MetadataRoute, mockResponse, mockError) -} - -func mockExecuteRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ExecutionRoute, mockResponse, mockError) -} - -func mockScheduleRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "POST", "http://"+proctorConfig.Host+ScheduleRoute, mockResponse, mockError) -} - -func mockExecutionContextRequest(proctorConfig config.ProctorConfig, executionID uint64, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "GET", "http://"+proctorConfig.Host+ExecutionRoute+"/"+fmt.Sprint(executionID)+"/status", mockResponse, mockError) -} - -func mockRemoveScheduleJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) -} - -func mockDescribeScheduledJobRequest(proctorConfig config.ProctorConfig, jobID string, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) -} - -func mockListScheduledJobsRequest(proctorConfig config.ProctorConfig, mockResponse *http.Response, mockError error) { - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute), mockResponse, mockError) -} - func mockRequest(proctorConfig config.ProctorConfig, method string, url string, mockResponse *http.Response, mockError error) { httpmock.RegisterStubRequest( httpmock.NewStubRequest( From 80cb6213e8ffc09eb2fed69652f3e29edffad8b7 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 27 Aug 2019 18:12:10 +0700 Subject: [PATCH 132/268] [Dembo] Add test gate client get user profile success --- plugins/gate-auth-plugin/auth.go | 3 +- plugins/gate-auth-plugin/gate/client.go | 4 +- plugins/gate-auth-plugin/gate/client_test.go | 77 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 plugins/gate-auth-plugin/gate/client_test.go diff --git a/plugins/gate-auth-plugin/auth.go b/plugins/gate-auth-plugin/auth.go index 65e53113..af9cbdc0 100644 --- a/plugins/gate-auth-plugin/auth.go +++ b/plugins/gate-auth-plugin/auth.go @@ -1,6 +1,7 @@ package main import ( + "github.com/go-resty/resty/v2" "proctor/internal/app/service/infra/logger" "proctor/pkg/auth" "proctor/plugins/gate-auth-plugin/gate" @@ -46,7 +47,7 @@ func contains(groups []string, value string) bool { func newGateAuth() auth.Auth { return &gateAuth{ - gateClient: gate.NewGateClient(), + gateClient: gate.NewGateClient(resty.New()), } } diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index 2f96dfe2..646a2737 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -54,14 +54,14 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai } } -func NewGateClient() GateClient { +func NewGateClient(client *resty.Client) GateClient { config := NewGateConfig() return &gateClient{ config: config, protocol: config.Protocol, host: config.Host, profilePath: config.ProfilePath, - restClient: resty.New(), + restClient: client, } } diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go new file mode 100644 index 00000000..685c3785 --- /dev/null +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -0,0 +1,77 @@ +package gate + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "github.com/stretchr/testify/assert" + "github.com/thingful/httpmock" + "net/http" + "proctor/pkg/auth" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + gateClient GateClient +} + +func (context *testContext) setUp(t *testing.T) { + client := resty.New() + httpmock.ActivateNonDefault(client.GetClient()) + context.gateClient = NewGateClient(client) + assert.NotNil(t, context.gateClient) +} + +func (context *testContext) tearDown() { + httpmock.DeactivateAndReset() +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + return &testContext{} +} + +func TestGateClient_GetUserProfileSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + config := NewGateConfig() + body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` + + httpmock.RegisterStubRequest( + httpmock.NewStubRequest( + "GET", + fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), + func(req *http.Request) (*http.Response, error) { + response := httpmock.NewStringResponse(200, body) + response.Header.Set("Content-Type", "application/json") + return response, nil + }, + ), + ) + + expectedUserDetail := &auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: true, + Groups: []string{"system", "proctor_executor"}, + } + + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + actualUserDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) + + assert.NoError(t, err) + assert.NotNil(t, actualUserDetail) + assert.Equal(t, expectedUserDetail, actualUserDetail) + assert.NoError(t, httpmock.AllStubsCalled()) + ctx.tearDown() +} From 43b971a1dbe22bb75fe33832755e86500ff81277 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:24:57 +0700 Subject: [PATCH 133/268] Add poll count config --- .env.test | 1 + 1 file changed, 1 insertion(+) diff --git a/.env.test b/.env.test index ad63ee91..c446f64b 100644 --- a/.env.test +++ b/.env.test @@ -10,6 +10,7 @@ export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 export PROCTOR_KUBE_JOB_RETRIES=0 export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 +export PROCTOR_KUBE_WAIT_FOR_RESOURCE_POLL_COUNT=5 export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60 From 461b00e6c126904906abb04fae969ce71d9032ff Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:25:23 +0700 Subject: [PATCH 134/268] Tidy up long line --- internal/app/cli/config/config.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/app/cli/config/config.go b/internal/app/cli/config/config.go index e9b4a306..fc1f91d9 100644 --- a/internal/app/cli/config/config.go +++ b/internal/app/cli/config/config.go @@ -80,7 +80,13 @@ func (loader *loader) Load() (ProctorConfig, ConfigError) { connectionTimeout := time.Duration(viper.GetInt(ConnectionTimeoutSecs)) * time.Second procExecutionStatusPollCount := viper.GetInt(ProcExecutionStatusPollCount) - return ProctorConfig{Host: proctorHost, Email: emailId, AccessToken: accessToken, ConnectionTimeoutSecs: connectionTimeout, ProcExecutionStatusPollCount: procExecutionStatusPollCount}, ConfigError{} + return ProctorConfig{ + Host: proctorHost, + Email: emailId, + AccessToken: accessToken, + ConnectionTimeoutSecs: connectionTimeout, + ProcExecutionStatusPollCount: procExecutionStatusPollCount, + }, ConfigError{} } // Returns Config file directory From f1a6c0708ffd09d7c96f5b4017ab4774bf888848 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:26:28 +0700 Subject: [PATCH 135/268] Tidy import entries --- internal/app/service/execution/service/execution.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 6841ec78..04fb8519 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -5,9 +5,12 @@ import ( "bytes" "errors" "fmt" - "github.com/jmoiron/sqlx/types" "io" + "time" + + "github.com/jmoiron/sqlx/types" v1 "k8s.io/api/core/v1" + "proctor/internal/app/service/execution/model" "proctor/internal/app/service/execution/repository" "proctor/internal/app/service/execution/status" @@ -17,7 +20,6 @@ import ( "proctor/internal/app/service/infra/logger" svcMetadataRepository "proctor/internal/app/service/metadata/repository" svcSecretRepository "proctor/internal/app/service/secret/repository" - "time" ) type ExecutionService interface { From 9209f1dcea5a4dab4105b5d47110df9f13c1a27c Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:26:37 +0700 Subject: [PATCH 136/268] Fix type --- .../app/service/execution/service/execution_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 159ae7f1..4fd6c269 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -92,7 +92,7 @@ func (suite *TestExecutionIntegrationSuite) TestStreamLogsSuccess() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "busybox" executedJobname, err := suite.kubernetesClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) From 26dee2f0f6c06ba55412612a2a3fd5bcd52ebb0d Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:26:52 +0700 Subject: [PATCH 137/268] Add poll count to config entry --- internal/app/service/infra/config/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 71c00f9d..efcb7cab 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -61,6 +61,10 @@ func LogsStreamWriteBufferSize() int { return viper.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE") } +func KubeWaitForResourcePollCount() int { + return viper.GetInt("KUBE_WAIT_FOR_RESOURCE_POLL_COUNT") +} + func KubePodsListWaitTime() time.Duration { return time.Duration(viper.GetInt("KUBE_POD_LIST_WAIT_TIME")) } From c89b1af410e22022a16f0ddc04c54a6bc44e6848 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:27:23 +0700 Subject: [PATCH 138/268] Fix coverage not detected --- internal/app/service/infra/db/postgresql/client_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index d530550b..78db5e8f 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -70,7 +70,7 @@ func TestSelect(t *testing.T) { assert.NoError(t, err) executionContextResult := []executionContextModel.ExecutionContext{} - err = postgresClient.db.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) + err = postgresClient.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) assert.NoError(t, err) assert.Equal(t, executionContext.Status, executionContextResult[0].Status) From 70abd8336bcb8f65f415b3ff60388c2bb65fe60a Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:28:15 +0700 Subject: [PATCH 139/268] Add test for #Close and #GetDB method --- .../infra/db/postgresql/client_test.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/internal/app/service/infra/db/postgresql/client_test.go b/internal/app/service/infra/db/postgresql/client_test.go index 78db5e8f..45fd7dff 100644 --- a/internal/app/service/infra/db/postgresql/client_test.go +++ b/internal/app/service/infra/db/postgresql/client_test.go @@ -90,10 +90,36 @@ func TestSelectForNoRows(t *testing.T) { jobName := "test-job-name" executionContextResult := []executionContextModel.ExecutionContext{} - err = postgresClient.db.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) + err = postgresClient.Select(&executionContextResult, "SELECT status from execution_context where job_name = $1", jobName) assert.NoError(t, err) assert.Equal(t, 0, len(executionContextResult)) assert.NoError(t, err) } + +func TestClose(t *testing.T) { + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + + db, err := sqlx.Connect("postgres", dataSourceName) + assert.NoError(t, err) + + postgresClient := &client{db: db} + err = postgresClient.Close() + defer postgresClient.db.Close() + + assert.NoError(t, err) +} + +func TestGetDB(t *testing.T) { + dataSourceName := fmt.Sprintf("dbname=%s user=%s password=%s host=%s sslmode=disable", config.PostgresDatabase(), config.PostgresUser(), config.PostgresPassword(), config.PostgresHost()) + + db, err := sqlx.Connect("postgres", dataSourceName) + assert.NoError(t, err) + + postgresClient := &client{db: db} + defer postgresClient.db.Close() + + assert.Equal(t, db, postgresClient.GetDB()) + assert.NoError(t, err) +} From 9ad7a478c454dc7cc02d0eb811ed5880d3ecf1cf Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:28:45 +0700 Subject: [PATCH 140/268] Add generic timeout error --- internal/app/service/infra/kubernetes/client.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index f44403c3..b4c2ce4a 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -1,13 +1,12 @@ package kubernetes import ( + "errors" "fmt" "io" "net/http" "os" "path/filepath" - "proctor/internal/app/service/infra/config" - "proctor/internal/app/service/infra/logger" "time" uuid "github.com/satori/go.uuid" @@ -21,10 +20,14 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" kubeRestClient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" ) var typeMeta meta.TypeMeta var namespace string +var timeoutError = errors.New("timeout when waiting job to be available") func init() { typeMeta = meta.TypeMeta{ From ce469c230dff788f34827d362cb7763224c9b3a6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:29:43 +0700 Subject: [PATCH 141/268] Add kubernetes resource poll retries --- .../app/service/infra/kubernetes/client.go | 28 ++- .../service/infra/kubernetes/client_test.go | 194 +++++++++++++++++- 2 files changed, 209 insertions(+), 13 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index b4c2ce4a..2b7c134a 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -200,11 +200,13 @@ func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime t resultChan := watchJob.ResultChan() defer watchJob.Stop() - for { + var count int + for count < config.KubeWaitForResourcePollCount() { select { case watchEvent := <-resultChan: if watchEvent.Type == watch.Error { - return fmt.Errorf("watch error when waiting for job with list option %v", listOptions) + err = watcherError("job", listOptions) + count += 1 } job := watchEvent.Object.(*batch.Job) @@ -212,9 +214,13 @@ func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime t return nil } case <-timeoutChan: - return fmt.Errorf("timeout when waiting job to be available") + err = timeoutError + timeoutChan = time.After(waitTime) + count += 1 } } + + return err } func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) { @@ -234,21 +240,27 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t defer watchJob.Stop() var pod *v1.Pod - for { + var count int + for count < config.KubeWaitForResourcePollCount() { select { case event := <-resultChan: if event.Type == watch.Error { - return nil, fmt.Errorf("watch error when waiting for pod with list option %v", listOptions) + err = watcherError("pod", listOptions) + count += 1 } + pod = event.Object.(*v1.Pod) if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { return pod, nil } case <-timeoutChan: - return nil, fmt.Errorf("timeout when waiting pod to be available") + err = timeoutError + timeoutChan = time.After(waitTime) + count += 1 } } + return nil, err } func (client *kubernetesClient) JobExecutionStatus(executionName string) (string, error) { @@ -298,3 +310,7 @@ func (client *kubernetesClient) GetPodLogs(pod *v1.Pod) (io.ReadCloser, error) { } return response, nil } + +func watcherError(resource string, listOptions meta.ListOptions) error { + return fmt.Errorf("watch error when waiting for %s with list option %v", resource, listOptions) +} diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 5e47bec7..06d4ed4f 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -1,22 +1,24 @@ package kubernetes import ( + "fmt" "net/http" "os" - "proctor/internal/app/service/infra/config" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" batchV1 "k8s.io/api/batch/v1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" fakeclientset "k8s.io/client-go/kubernetes/fake" batch "k8s.io/client-go/kubernetes/typed/batch/v1" testing_kubernetes "k8s.io/client-go/testing" - utility "proctor/internal/pkg/constant" + + "proctor/internal/app/service/infra/config" + "proctor/internal/pkg/constant" ) type ClientTestSuite struct { @@ -111,6 +113,184 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedEnvVars, container.Env) } +func (suite *ClientTestSuite) TestWaitForReadyJob() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + + var testJob batchV1.Job + uniqueJobName := "proctor-job-1" + label := jobLabel(uniqueJobName) + objectMeta := meta.ObjectMeta{ + Name: uniqueJobName, + Labels: label, + } + testJob.ObjectMeta = objectMeta + waitTime := config.KubeLogProcessWaitTime() * time.Second + + go func() { + testJob.Status.Succeeded = 1 + watcher.Modify(&testJob) + + time.Sleep(time.Second * 1) + watcher.Stop() + }() + + err := suite.testClient.WaitForReadyJob(uniqueJobName, waitTime) + assert.NoError(t, err) +} + +func (suite *ClientTestSuite) TestWaitForReadyJobWatcherError() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + + var testJob batchV1.Job + uniqueJobName := "proctor-job-1" + label := jobLabel(uniqueJobName) + objectMeta := meta.ObjectMeta{ + Name: uniqueJobName, + Labels: label, + } + testJob.ObjectMeta = objectMeta + listOptions := meta.ListOptions{ + TypeMeta: typeMeta, + LabelSelector: jobLabelSelector(uniqueJobName), + } + waitTime := config.KubeLogProcessWaitTime() * time.Second + go func() { + watcher.Error(&testJob) + watcher.Error(&testJob) + watcher.Error(&testJob) + watcher.Error(&testJob) + watcher.Error(&testJob) + + time.Sleep(time.Second * 1) + watcher.Stop() + }() + + err := suite.testClient.WaitForReadyJob(uniqueJobName, waitTime) + assert.EqualError(t, err, fmt.Sprintf("watch error when waiting for job with list option %v", listOptions)) +} + +func (suite *ClientTestSuite) TestWaitForReadyJobTimeoutError() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + defer watcher.Stop() + + var testJob batchV1.Job + uniqueJobName := "proctor-job-1" + label := jobLabel(uniqueJobName) + objectMeta := meta.ObjectMeta{ + Name: uniqueJobName, + Labels: label, + } + testJob.ObjectMeta = objectMeta + waitTime := time.Millisecond * 100 + + go func() { + time.Sleep(time.Millisecond * 550) + watcher.Stop() + }() + + err := suite.testClient.WaitForReadyJob(uniqueJobName, waitTime) + assert.EqualError(t, err, "timeout when waiting job to be available") +} + +func (suite *ClientTestSuite) TestWaitForReadyPod() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + + var testPod v1.Pod + uniquePodName := "proctor-pod-1" + label := jobLabel(uniquePodName) + objectMeta := meta.ObjectMeta{ + Name: uniquePodName, + Labels: label, + } + testPod.ObjectMeta = objectMeta + waitTime := config.KubeLogProcessWaitTime() * time.Second + + go func() { + testPod.Status.Phase = v1.PodSucceeded + watcher.Modify(&testPod) + + time.Sleep(time.Second * 1) + watcher.Stop() + }() + + pod, err := suite.testClient.WaitForReadyPod(uniquePodName, waitTime) + assert.NoError(t, err) + assert.NotNil(t, pod) + assert.Equal(t, pod.Name, uniquePodName) +} + +func (suite *ClientTestSuite) TestWaitForReadyPodWatcherError() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + + var testPod v1.Pod + uniquePodName := "proctor-pod-1" + label := jobLabel(uniquePodName) + objectMeta := meta.ObjectMeta{ + Name: uniquePodName, + Labels: label, + } + testPod.ObjectMeta = objectMeta + listOptions := meta.ListOptions{ + LabelSelector: jobLabelSelector(uniquePodName), + } + waitTime := config.KubeLogProcessWaitTime() * time.Second + + go func() { + watcher.Error(&testPod) + watcher.Error(&testPod) + watcher.Error(&testPod) + watcher.Error(&testPod) + watcher.Error(&testPod) + + time.Sleep(time.Second * 1) + watcher.Stop() + }() + + _, err := suite.testClient.WaitForReadyPod(uniquePodName, waitTime) + assert.EqualError(t, err, fmt.Sprintf("watch error when waiting for pod with list option %v", listOptions)) +} + +func (suite *ClientTestSuite) TestWaitForReadyPodTimeoutError() { + t := suite.T() + + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + defer watcher.Stop() + + var testPod v1.Pod + uniquePodName := "proctor-pod-1" + label := jobLabel(uniquePodName) + objectMeta := meta.ObjectMeta{ + Name: uniquePodName, + Labels: label, + } + testPod.ObjectMeta = objectMeta + waitTime := time.Millisecond * 100 + + go func() { + time.Sleep(time.Millisecond * 550) + watcher.Stop() + }() + + _, err := suite.testClient.WaitForReadyPod(uniquePodName, waitTime) + assert.EqualError(t, err, "timeout when waiting job to be available") +} + func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { t := suite.T() @@ -143,7 +323,7 @@ func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { jobExecutionStatus, err := suite.testClient.JobExecutionStatus(uniqueJobName) assert.NoError(t, err) - assert.Equal(t, utility.JobSucceeded, jobExecutionStatus, "Should return SUCCEEDED") + assert.Equal(t, constant.JobSucceeded, jobExecutionStatus, "Should return SUCCEEDED") } func (suite *ClientTestSuite) TestShouldReturnFailedJobExecutionStatus() { @@ -177,7 +357,7 @@ func (suite *ClientTestSuite) TestShouldReturnFailedJobExecutionStatus() { jobExecutionStatus, err := suite.testClient.JobExecutionStatus(uniqueJobName) assert.NoError(t, err) - assert.Equal(t, utility.JobFailed, jobExecutionStatus, "Should return FAILED") + assert.Equal(t, constant.JobFailed, jobExecutionStatus, "Should return FAILED") } func (suite *ClientTestSuite) TestJobExecutionStatusForNonDefinitiveStatus() { @@ -206,7 +386,7 @@ func (suite *ClientTestSuite) TestJobExecutionStatusForNonDefinitiveStatus() { jobExecutionStatus, err := suite.testClient.JobExecutionStatus(uniqueJobName) assert.NoError(t, err) - assert.Equal(t, utility.NoDefinitiveJobExecutionStatusFound, jobExecutionStatus, "Should return NO_DEFINITIVE_JOB_EXECUTION_STATUS_FOUND") + assert.Equal(t, constant.NoDefinitiveJobExecutionStatusFound, jobExecutionStatus, "Should return NO_DEFINITIVE_JOB_EXECUTION_STATUS_FOUND") } func (suite *ClientTestSuite) TestShouldReturnJobExecutionStatusFetchError() { @@ -234,7 +414,7 @@ func (suite *ClientTestSuite) TestShouldReturnJobExecutionStatusFetchError() { jobExecutionStatus, err := suite.testClient.JobExecutionStatus(uniqueJobName) assert.NoError(t, err) - assert.Equal(t, utility.JobExecutionStatusFetchError, jobExecutionStatus, "Should return JOB_EXECUTION_STATUS_FETCH_ERROR") + assert.Equal(t, constant.JobExecutionStatusFetchError, jobExecutionStatus, "Should return JOB_EXECUTION_STATUS_FETCH_ERROR") } func TestClientTestSuite(t *testing.T) { From c6d95f670931a2e1ced07894927632a704b1191f Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Wed, 28 Aug 2019 11:29:58 +0700 Subject: [PATCH 142/268] Fix test result typo --- .../app/service/infra/kubernetes/client_integration_test.go | 4 ++-- internal/app/service/infra/kubernetes/client_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 205c5bb8..73f99514 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -31,7 +31,7 @@ func (suite *IntegrationTestSuite) SetupTest() { func (suite *IntegrationTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "busybox" executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) @@ -76,7 +76,7 @@ func (suite *IntegrationTestSuite) TestJobExecution() { func (suite *IntegrationTestSuite) TestJobExecutionStatus() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "busybox" executedJobname, err := suite.testClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 06d4ed4f..e3f3f523 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -69,7 +69,7 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - envVarsForContainer := map[string]string{"SAMPLE_ARG": "samle-value"} + envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "img1" executedJobname, err := suite.testClient.ExecuteJob(sampleImageName, envVarsForContainer) From e56fa8edf6f5620f268bcac4052d2900eefa4d97 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 11:41:15 +0700 Subject: [PATCH 143/268] [Dembo] Update test gate client get user profile success to make make better mock http request --- go.mod | 1 + go.sum | 2 + plugins/gate-auth-plugin/gate/client.go | 2 +- plugins/gate-auth-plugin/gate/client_test.go | 39 +++++++++++++------- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index c6a0468e..c0db3c57 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/goware/urlx v0.2.0 // indirect github.com/hashicorp/go-version v1.2.0 github.com/imdario/mergo v0.3.7 // indirect + github.com/jarcoal/httpmock v1.0.4 github.com/jmoiron/sqlx v1.2.0 github.com/json-iterator/go v1.1.6 // indirect github.com/lib/pq v1.1.1 diff --git a/go.sum b/go.sum index db7ac3a8..48bccebc 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jarcoal/httpmock v1.0.4 h1:jp+dy/+nonJE4g4xbVtl9QdrUNbn6/3hDT5R4nDIZnA= +github.com/jarcoal/httpmock v1.0.4/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index 646a2737..ef69de55 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -30,7 +30,7 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai response, err := g.restClient. R(). SetHeader("Accept", "application/json"). - SetFormData(formData). + SetQueryParams(formData). SetResult(profile). Get(path) diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index 685c3785..c2b89785 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -3,8 +3,8 @@ package gate import ( "fmt" "github.com/go-resty/resty/v2" + "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" - "github.com/thingful/httpmock" "net/http" "proctor/pkg/auth" "testing" @@ -43,19 +43,33 @@ func TestGateClient_GetUserProfileSuccess(t *testing.T) { ctx := newContext() ctx.setUp(t) + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + config := NewGateConfig() body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` - httpmock.RegisterStubRequest( - httpmock.NewStubRequest( - "GET", - fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), - func(req *http.Request) (*http.Response, error) { - response := httpmock.NewStringResponse(200, body) - response.Header.Set("Content-Type", "application/json") - return response, nil - }, - ), + httpmock.RegisterResponder( + "GET", + fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), + func(req *http.Request) (*http.Response, error) { + tokenParam := req.URL.Query()["access_token"][0] + if tokenParam != token { + return &http.Response{ + StatusCode: 401, + }, nil + } + emailParam := req.URL.Query()["email"][0] + if emailParam != email { + return &http.Response{ + StatusCode: 404, + Body: httpmock.NewRespBodyFromString(body), + }, nil + } + response := httpmock.NewStringResponse(200, body) + response.Header.Set("Content-Type", "application/json") + return response, nil + }, ) expectedUserDetail := &auth.UserDetail{ @@ -65,13 +79,10 @@ func TestGateClient_GetUserProfileSuccess(t *testing.T) { Groups: []string{"system", "proctor_executor"}, } - email := "w.albertusd@gmail.com" - token := "someunreadabletoken" actualUserDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) assert.NoError(t, err) assert.NotNil(t, actualUserDetail) assert.Equal(t, expectedUserDetail, actualUserDetail) - assert.NoError(t, httpmock.AllStubsCalled()) ctx.tearDown() } From a78047c64f7f78562aa688dfc8c1a54d943b3d58 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 13:09:03 +0700 Subject: [PATCH 144/268] [Dembo] Add test gate client get user profile unauthenticated --- plugins/gate-auth-plugin/gate/client.go | 4 ++ plugins/gate-auth-plugin/gate/client_test.go | 61 +++++++++++++------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index ef69de55..387f4edb 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -39,6 +39,10 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai return nil, err } + if response.StatusCode() == 401 { + return nil, fmt.Errorf("authentication failed, please check your access token") + } + if response.IsSuccess() { userDetail := auth.UserDetail{ Name: profile.Name, diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index c2b89785..c617f3d2 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -49,40 +49,59 @@ func TestGateClient_GetUserProfileSuccess(t *testing.T) { config := NewGateConfig() body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` + mockGetUserProfileAPI(config, token, email, body) + + expectedUserDetail := &auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: true, + Groups: []string{"system", "proctor_executor"}, + } + + actualUserDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) + + assert.NoError(t, err) + assert.NotNil(t, actualUserDetail) + assert.Equal(t, expectedUserDetail, actualUserDetail) + ctx.tearDown() +} + +func TestGateClient_GetUserProfileUnauthenticated(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + + config := NewGateConfig() + body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` + + mockGetUserProfileAPI(config, token, email, body) + + userDetail, err := ctx.instance().gateClient.GetUserProfile(email, "wrong-token") + + assert.Nil(t, userDetail) + assert.NotNil(t, err) + assert.Equal(t, "authentication failed, please check your access token", err.Error()) + ctx.tearDown() +} + +func mockGetUserProfileAPI(config GateConfig, token string, email string, body string) { httpmock.RegisterResponder( "GET", fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), func(req *http.Request) (*http.Response, error) { tokenParam := req.URL.Query()["access_token"][0] if tokenParam != token { - return &http.Response{ - StatusCode: 401, - }, nil + return httpmock.NewStringResponse(401, ""), nil } emailParam := req.URL.Query()["email"][0] if emailParam != email { - return &http.Response{ - StatusCode: 404, - Body: httpmock.NewRespBodyFromString(body), - }, nil + return httpmock.NewStringResponse(404, ""), nil } response := httpmock.NewStringResponse(200, body) response.Header.Set("Content-Type", "application/json") return response, nil }, ) - - expectedUserDetail := &auth.UserDetail{ - Name: "William Albertus Dembo", - Email: "w.albertusd@gmail.com", - Active: true, - Groups: []string{"system", "proctor_executor"}, - } - - actualUserDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) - - assert.NoError(t, err) - assert.NotNil(t, actualUserDetail) - assert.Equal(t, expectedUserDetail, actualUserDetail) - ctx.tearDown() } From 723a1e2ab8fd128462ff86819af7f4a2055512e0 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 13:14:38 +0700 Subject: [PATCH 145/268] [Dembo] Add test gate client get user profile not found --- plugins/gate-auth-plugin/gate/client.go | 3 +++ plugins/gate-auth-plugin/gate/client_test.go | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index 387f4edb..47149ebc 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -42,6 +42,9 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai if response.StatusCode() == 401 { return nil, fmt.Errorf("authentication failed, please check your access token") } + if response.StatusCode() == 404 { + return nil, fmt.Errorf("user not found for email %s", email) + } if response.IsSuccess() { userDetail := auth.UserDetail{ diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index c617f3d2..f5db001a 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -86,6 +86,26 @@ func TestGateClient_GetUserProfileUnauthenticated(t *testing.T) { ctx.tearDown() } +func TestGateClient_GetUserProfileNotFound(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + + config := NewGateConfig() + body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` + + mockGetUserProfileAPI(config, token, email, body) + + userDetail, err := ctx.instance().gateClient.GetUserProfile("random.email@gmail.com", token) + + assert.Nil(t, userDetail) + assert.NotNil(t, err) + assert.Equal(t, "user not found for email random.email@gmail.com", err.Error()) + ctx.tearDown() +} + func mockGetUserProfileAPI(config GateConfig, token string, email string, body string) { httpmock.RegisterResponder( "GET", From e8dfa3a619c6b52860f31c7be0414f89b8d41450 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 13:21:37 +0700 Subject: [PATCH 146/268] [Dembo] Refactor test gate client get user profile to move variable body --- plugins/gate-auth-plugin/gate/client_test.go | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index f5db001a..5413bf8d 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -45,11 +45,9 @@ func TestGateClient_GetUserProfileSuccess(t *testing.T) { email := "w.albertusd@gmail.com" token := "someunreadabletoken" - config := NewGateConfig() - body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` - mockGetUserProfileAPI(config, token, email, body) + mockGetUserProfileAPI(config, token, email) expectedUserDetail := &auth.UserDetail{ Name: "William Albertus Dembo", @@ -72,11 +70,9 @@ func TestGateClient_GetUserProfileUnauthenticated(t *testing.T) { email := "w.albertusd@gmail.com" token := "someunreadabletoken" - config := NewGateConfig() - body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` - mockGetUserProfileAPI(config, token, email, body) + mockGetUserProfileAPI(config, token, email) userDetail, err := ctx.instance().gateClient.GetUserProfile(email, "wrong-token") @@ -92,11 +88,9 @@ func TestGateClient_GetUserProfileNotFound(t *testing.T) { email := "w.albertusd@gmail.com" token := "someunreadabletoken" - config := NewGateConfig() - body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` - mockGetUserProfileAPI(config, token, email, body) + mockGetUserProfileAPI(config, token, email) userDetail, err := ctx.instance().gateClient.GetUserProfile("random.email@gmail.com", token) @@ -106,7 +100,7 @@ func TestGateClient_GetUserProfileNotFound(t *testing.T) { ctx.tearDown() } -func mockGetUserProfileAPI(config GateConfig, token string, email string, body string) { +func mockGetUserProfileAPI(config GateConfig, token string, email string) { httpmock.RegisterResponder( "GET", fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), @@ -119,6 +113,7 @@ func mockGetUserProfileAPI(config GateConfig, token string, email string, body s if emailParam != email { return httpmock.NewStringResponse(404, ""), nil } + body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` response := httpmock.NewStringResponse(200, body) response.Header.Set("Content-Type", "application/json") return response, nil From accf808f4aeb0e6162a85054dbde30421acdb0ae Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 13:27:38 +0700 Subject: [PATCH 147/268] [Dembo] Add test gate client get user profile server failure --- plugins/gate-auth-plugin/gate/client_test.go | 23 ++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index 5413bf8d..f484ba7c 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -120,3 +120,26 @@ func mockGetUserProfileAPI(config GateConfig, token string, email string) { }, ) } + +func TestGateClient_GetUserProfileServerFailure(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + config := NewGateConfig() + + httpmock.RegisterResponder( + "GET", + fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), + func(req *http.Request) (*http.Response, error) { + return httpmock.NewStringResponse(503, "Internal server error"), nil + }, + ) + + userDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) + + assert.Nil(t, userDetail) + assert.NotNil(t, err) + ctx.tearDown() +} From f70e3f257b2811e3bc3fb233c59e09334479db83 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 13:35:14 +0700 Subject: [PATCH 148/268] [Dembo] Add test gate client get user profile connection failure --- plugins/gate-auth-plugin/gate/client_test.go | 21 ++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index f484ba7c..8cb11bf7 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -143,3 +143,24 @@ func TestGateClient_GetUserProfileServerFailure(t *testing.T) { assert.NotNil(t, err) ctx.tearDown() } + +func TestGateClient_GetUserProfileConnectionFailure(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := "w.albertusd@gmail.com" + token := "someunreadabletoken" + config := NewGateConfig() + + httpmock.RegisterResponder( + "GET", + fmt.Sprintf("%s://%s/%s", config.Protocol, config.Host, config.ProfilePath), + httpmock.ConnectionFailure, + ) + + userDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) + + assert.Nil(t, userDetail) + assert.NotNil(t, err) + ctx.tearDown() +} From 9a9e3be4bc86641dfd0c30b76862fa14b34c9c4c Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 14:32:05 +0700 Subject: [PATCH 149/268] [Dembo] Add test plugin gate auth success --- plugins/gate-auth-plugin/auth_test.go | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 plugins/gate-auth-plugin/auth_test.go diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go new file mode 100644 index 00000000..66e577cd --- /dev/null +++ b/plugins/gate-auth-plugin/auth_test.go @@ -0,0 +1,58 @@ +package main + +import ( + "github.com/stretchr/testify/assert" + "proctor/pkg/auth" + "proctor/plugins/gate-auth-plugin/gate" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + gateAuth gateAuth + gateClient *gate.GateClientMock +} + +func (context *testContext) setUp(t *testing.T) { + context.gateAuth = gateAuth{} + context.gateClient = &gate.GateClientMock{} + context.gateAuth.gateClient = context.gateClient +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + return &testContext{} +} + +func TestGateAuth_AuthSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + email := "w.albertusd@gmail.com" + token := "unreadabletoken" + expectedUserDetail := &auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: true, + Groups: []string{"system", "proctor_executor"}, + } + ctx.instance().gateClient.On("GetUserProfile", email, token).Return(expectedUserDetail, nil) + + actualUserDetail, err := ctx.instance().gateAuth.Auth(email, token) + + assert.NoError(t, err) + assert.NotNil(t, actualUserDetail) + assert.Equal(t, expectedUserDetail, actualUserDetail) +} From f6aba751267b8715e9c7b8a738652dd43f414f47 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 14:49:18 +0700 Subject: [PATCH 150/268] [Dembo] Add test plugin gate auth wrong token --- plugins/gate-auth-plugin/auth_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go index 66e577cd..64ef3d11 100644 --- a/plugins/gate-auth-plugin/auth_test.go +++ b/plugins/gate-auth-plugin/auth_test.go @@ -1,6 +1,7 @@ package main import ( + "errors" "github.com/stretchr/testify/assert" "proctor/pkg/auth" "proctor/plugins/gate-auth-plugin/gate" @@ -56,3 +57,21 @@ func TestGateAuth_AuthSuccess(t *testing.T) { assert.NotNil(t, actualUserDetail) assert.Equal(t, expectedUserDetail, actualUserDetail) } + +func TestGateAuth_AuthWrongToken(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + email := "w.albertusd@gmail.com" + token := "unreadabletoken" + expectedError := errors.New("authentication failed, please check your access token") + var userDetail *auth.UserDetail + ctx.instance().gateClient.On("GetUserProfile", email, token).Return(userDetail, expectedError) + + actualUserDetail, actualError := ctx.instance().gateAuth.Auth(email, token) + + assert.Nil(t, actualUserDetail) + assert.Error(t, actualError) + assert.Equal(t, expectedError.Error(), actualError.Error()) +} From 439717a155b5bfdd6a22e6d5aff2a94696160076 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 14:51:56 +0700 Subject: [PATCH 151/268] [Dembo] Add test plugin gate auth wrong email --- plugins/gate-auth-plugin/auth_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go index 64ef3d11..30699f0f 100644 --- a/plugins/gate-auth-plugin/auth_test.go +++ b/plugins/gate-auth-plugin/auth_test.go @@ -75,3 +75,21 @@ func TestGateAuth_AuthWrongToken(t *testing.T) { assert.Error(t, actualError) assert.Equal(t, expectedError.Error(), actualError.Error()) } + +func TestGateAuth_AuthWrongEmail(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + email := "w.albertusd@gmail.com" + token := "unreadabletoken" + expectedError := errors.New("user not found for email w.albertusd@gmail.com") + var userDetail *auth.UserDetail + ctx.instance().gateClient.On("GetUserProfile", email, token).Return(userDetail, expectedError) + + actualUserDetail, actualError := ctx.instance().gateAuth.Auth(email, token) + + assert.Nil(t, actualUserDetail) + assert.Error(t, actualError) + assert.Equal(t, expectedError.Error(), actualError.Error()) +} From 6ce3a76be2375d7bfd285c875cbf5cad677a5f09 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 15:07:26 +0700 Subject: [PATCH 152/268] [Dembo] Add test plugin gate auth verify success --- plugins/gate-auth-plugin/auth_test.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go index 30699f0f..645d5fa0 100644 --- a/plugins/gate-auth-plugin/auth_test.go +++ b/plugins/gate-auth-plugin/auth_test.go @@ -2,10 +2,12 @@ package main import ( "errors" + "testing" + "github.com/stretchr/testify/assert" + "proctor/pkg/auth" "proctor/plugins/gate-auth-plugin/gate" - "testing" ) type context interface { @@ -93,3 +95,22 @@ func TestGateAuth_AuthWrongEmail(t *testing.T) { assert.Error(t, actualError) assert.Equal(t, expectedError.Error(), actualError.Error()) } + +func TestGateAuth_VerifySuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userDetail := auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: true, + Groups: []string{"system", "proctor_executor"}, + } + requiredGroups := []string{"system"} + + result, err := ctx.instance().gateAuth.Verify(userDetail, requiredGroups) + + assert.Equal(t, true, result) + assert.Nil(t, err) +} From 1d59b36dbe99b3480e7030047cd380191cb8abe2 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 15:18:49 +0700 Subject: [PATCH 153/268] [Dembo] Add test plugin gate auth verify inactive user --- plugins/gate-auth-plugin/auth_test.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go index 645d5fa0..010de9fc 100644 --- a/plugins/gate-auth-plugin/auth_test.go +++ b/plugins/gate-auth-plugin/auth_test.go @@ -112,5 +112,24 @@ func TestGateAuth_VerifySuccess(t *testing.T) { result, err := ctx.instance().gateAuth.Verify(userDetail, requiredGroups) assert.Equal(t, true, result) - assert.Nil(t, err) + assert.NoError(t, err) +} + +func TestGateAuth_VerifyInactiveUser(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userDetail := auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: false, + Groups: []string{"system", "proctor_executor"}, + } + requiredGroups := []string{"system"} + + result, err := ctx.instance().gateAuth.Verify(userDetail, requiredGroups) + + assert.Equal(t, false, result) + assert.NoError(t, err) } From 542b0a245fef2b0f96037dfead3354cac9e5d843 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 15:21:51 +0700 Subject: [PATCH 154/268] [Dembo] Add test plugin gate auth verify insufficient group --- plugins/gate-auth-plugin/auth_test.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/plugins/gate-auth-plugin/auth_test.go b/plugins/gate-auth-plugin/auth_test.go index 010de9fc..6b88212e 100644 --- a/plugins/gate-auth-plugin/auth_test.go +++ b/plugins/gate-auth-plugin/auth_test.go @@ -133,3 +133,22 @@ func TestGateAuth_VerifyInactiveUser(t *testing.T) { assert.Equal(t, false, result) assert.NoError(t, err) } + +func TestGateAuth_VerifyInsufficientGroup(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userDetail := auth.UserDetail{ + Name: "William Albertus Dembo", + Email: "w.albertusd@gmail.com", + Active: true, + Groups: []string{"system", "proctor_executor"}, + } + requiredGroups := []string{"system", "proctor_maintainer"} + + result, err := ctx.instance().gateAuth.Verify(userDetail, requiredGroups) + + assert.Equal(t, false, result) + assert.NoError(t, err) +} From 61b0ae0b9b7645ff9597fe76d120c76a118fb9ef Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 17:14:28 +0700 Subject: [PATCH 155/268] [Dembo] Add integration test plugin gate client get user profile success --- .../gate/client_integration_test.go | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 plugins/gate-auth-plugin/gate/client_integration_test.go diff --git a/plugins/gate-auth-plugin/gate/client_integration_test.go b/plugins/gate-auth-plugin/gate/client_integration_test.go new file mode 100644 index 00000000..c6e7cbe5 --- /dev/null +++ b/plugins/gate-auth-plugin/gate/client_integration_test.go @@ -0,0 +1,64 @@ +package gate + +import ( + "os" + "testing" + + "github.com/go-resty/resty/v2" + "github.com/stretchr/testify/assert" +) + +type integrationContext interface { + setUp(t *testing.T) + tearDown() + instance() *integrationTestContext +} + +type integrationTestContext struct { + gateClient GateClient + email string + token string +} + +func (context *integrationTestContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_PLUGIN_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + client := resty.New() + context.gateClient = NewGateClient(client) + email, _ := os.LookupEnv("GATE_PLUGIN_EMAIL") + assert.NotEmpty(t, email) + context.email = email + token, _ := os.LookupEnv("GATE_PLUGIN_TOKEN") + assert.NotEmpty(t, token) + context.token = token +} + +func (context *integrationTestContext) tearDown() { +} + +func (context *integrationTestContext) instance() *integrationTestContext { + return context +} + +func newIntegrationContext() integrationContext { + return &integrationTestContext{} +} + +func TestIntegrationGateClient_GetUserProfileSuccess(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + email := ctx.instance().email + token := ctx.instance().token + userDetail, err := ctx.instance().gateClient.GetUserProfile(email, token) + + assert.NoError(t, err) + assert.NotNil(t, userDetail) + assert.NotEmpty(t, userDetail.Email) + assert.NotEmpty(t, userDetail.Name) + assert.NotEmpty(t, userDetail.Active) + assert.NotNil(t, userDetail.Groups) +} From 70f25ee282829256c790bffcc5c85ab4fed0dfa0 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 17:18:28 +0700 Subject: [PATCH 156/268] [Dembo] Add makefile command to run plugin integration test --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index c9cedb56..58645d39 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,11 @@ itest: plugin.auth ENABLE_INTEGRATION_TEST=true \ go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./... +.PHONY: plugin.itest +plugin.itest: + ENABLE_PLUGIN_INTEGRATION_TEST=true \ + go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./plugins/... + .PHONY: server server: go build -race -o $(BIN_DIR)/server ./cmd/server/main.go From 8b1ab47a401b2e1343f19043313ec9aacb29ac15 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 17:23:57 +0700 Subject: [PATCH 157/268] [Dembo] Add integration test plugin gate client get user profile unauthenticated --- .../gate/client_integration_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/gate-auth-plugin/gate/client_integration_test.go b/plugins/gate-auth-plugin/gate/client_integration_test.go index c6e7cbe5..fcabb3d9 100644 --- a/plugins/gate-auth-plugin/gate/client_integration_test.go +++ b/plugins/gate-auth-plugin/gate/client_integration_test.go @@ -62,3 +62,16 @@ func TestIntegrationGateClient_GetUserProfileSuccess(t *testing.T) { assert.NotEmpty(t, userDetail.Active) assert.NotNil(t, userDetail.Groups) } + +func TestIntegrationGateClient_GetUserProfileUnauthenticated(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + email := ctx.instance().email + userDetail, err := ctx.instance().gateClient.GetUserProfile(email, "some-random-token") + + assert.Nil(t, userDetail) + assert.Error(t, err) + assert.Equal(t, "authentication failed, please check your access token", err.Error()) +} From 756c986f56e2f3c995f282ff610a3fc3d7d8d237 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 17:29:28 +0700 Subject: [PATCH 158/268] [Dembo] Add integration test plugin gate client get user profile not found --- .../gate/client_integration_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/plugins/gate-auth-plugin/gate/client_integration_test.go b/plugins/gate-auth-plugin/gate/client_integration_test.go index fcabb3d9..ab76abd0 100644 --- a/plugins/gate-auth-plugin/gate/client_integration_test.go +++ b/plugins/gate-auth-plugin/gate/client_integration_test.go @@ -75,3 +75,16 @@ func TestIntegrationGateClient_GetUserProfileUnauthenticated(t *testing.T) { assert.Error(t, err) assert.Equal(t, "authentication failed, please check your access token", err.Error()) } + +func TestIntegrationGateClient_GetUserProfileNotFound(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + token := ctx.instance().token + userDetail, err := ctx.instance().gateClient.GetUserProfile("random.email@gmail.com", token) + + assert.Nil(t, userDetail) + assert.Error(t, err) + assert.Equal(t, "user not found for email random.email@gmail.com", err.Error()) +} From fe876992daf53b4d0c038beee14b43f8249be1af Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 28 Aug 2019 17:56:53 +0700 Subject: [PATCH 159/268] [Dembo] Refactor test plugin gate client to extract constant message --- plugins/gate-auth-plugin/gate/client.go | 7 +++++-- plugins/gate-auth-plugin/gate/status/status.go | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 plugins/gate-auth-plugin/gate/status/status.go diff --git a/plugins/gate-auth-plugin/gate/client.go b/plugins/gate-auth-plugin/gate/client.go index 47149ebc..b163748f 100644 --- a/plugins/gate-auth-plugin/gate/client.go +++ b/plugins/gate-auth-plugin/gate/client.go @@ -2,9 +2,12 @@ package gate import ( "fmt" + "github.com/go-resty/resty/v2" + "proctor/internal/app/service/infra/logger" "proctor/pkg/auth" + "proctor/plugins/gate-auth-plugin/gate/status" ) type GateClient interface { @@ -40,10 +43,10 @@ func (g *gateClient) GetUserProfile(email string, token string) (*auth.UserDetai } if response.StatusCode() == 401 { - return nil, fmt.Errorf("authentication failed, please check your access token") + return nil, fmt.Errorf(status.TokenAuthenticationFailedMessage) } if response.StatusCode() == 404 { - return nil, fmt.Errorf("user not found for email %s", email) + return nil, fmt.Errorf(status.UserNotFoundMessage, email) } if response.IsSuccess() { diff --git a/plugins/gate-auth-plugin/gate/status/status.go b/plugins/gate-auth-plugin/gate/status/status.go new file mode 100644 index 00000000..6b8b9312 --- /dev/null +++ b/plugins/gate-auth-plugin/gate/status/status.go @@ -0,0 +1,6 @@ +package status + +const ( + TokenAuthenticationFailedMessage = "authentication failed, please check your access token" + UserNotFoundMessage = "user not found for email %s" +) From d9b16ff697cedf697a602d4800a39ca9daa25700 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 29 Aug 2019 13:44:15 +0700 Subject: [PATCH 160/268] [Dembo] Update response api mock for gate get user profile --- plugins/gate-auth-plugin/gate/client_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/gate-auth-plugin/gate/client_test.go b/plugins/gate-auth-plugin/gate/client_test.go index 8cb11bf7..f350f53e 100644 --- a/plugins/gate-auth-plugin/gate/client_test.go +++ b/plugins/gate-auth-plugin/gate/client_test.go @@ -113,7 +113,19 @@ func mockGetUserProfileAPI(config GateConfig, token string, email string) { if emailParam != email { return httpmock.NewStringResponse(404, ""), nil } - body := `{"email":"w.albertusd@gmail.com","name":"William Albertus Dembo","active":true,"groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}]}` + body := `{ + "email":"w.albertusd@gmail.com", + "uid": "7", + "name":"William Albertus Dembo", + "active":true, + "admin": true, + "home_dir": null, + "shell": null, + "public_key": "ssh-rsa some-public-key", + "user_login_id": "william.dembo", + "product_name": null, + "groups":[{"id":1,"name":"system"},{"id":2,"name":"proctor_executor"}] + }` response := httpmock.NewStringResponse(200, body) response.Header.Set("Content-Type", "application/json") return response, nil From 2b8d849866cc04e9bc8e6cd2b440dd1a6184606b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 10:39:31 +0700 Subject: [PATCH 161/268] [Dembo] Security middleware authentication success case --- .../security/middleware/authentication.go | 17 +++++ .../middleware/authentication_test.go | 72 +++++++++++++++++++ .../service/security/middleware/middleware.go | 7 ++ .../service/security/service/security_mock.go | 4 +- 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 internal/app/service/security/middleware/authentication.go create mode 100644 internal/app/service/security/middleware/authentication_test.go create mode 100644 internal/app/service/security/middleware/middleware.go diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go new file mode 100644 index 00000000..9c059cfa --- /dev/null +++ b/internal/app/service/security/middleware/authentication.go @@ -0,0 +1,17 @@ +package middleware + +import ( + "net/http" + + "proctor/internal/app/service/security/service" +) + +type authenticationMiddleware struct { + service service.SecurityService +} + +func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) +} diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go new file mode 100644 index 00000000..613201cc --- /dev/null +++ b/internal/app/service/security/middleware/authentication_test.go @@ -0,0 +1,72 @@ +package middleware + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/security/service" + "proctor/internal/app/service/server/middleware/parameter" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + authenticationMiddleware authenticationMiddleware + securityService service.SecurityService + testHandler http.HandlerFunc +} + +func (context *testContext) setUp(t *testing.T) { + context.authenticationMiddleware = authenticationMiddleware{} + context.securityService = &service.SecurityServiceMock{} + context.authenticationMiddleware.service = context.securityService + fn := func(rw http.ResponseWriter, req *http.Request) { + } + context.testHandler = http.HandlerFunc(fn) +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + return &testContext{} +} + +func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + authenticationMiddleware := ctx.instance().authenticationMiddleware + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + + body := map[string]string{} + body["name"] = "a-job" + requestBody, _ := json.Marshal(body) + + req, _ := http.NewRequest("GET", ts.URL, bytes.NewReader(requestBody)) + req.Header.Add("Content-Type", "application/json") + req.Header.Add(parameter.AccessTokenHeader, "a-token") + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go new file mode 100644 index 00000000..1f2ebd5d --- /dev/null +++ b/internal/app/service/security/middleware/middleware.go @@ -0,0 +1,7 @@ +package middleware + +import "net/http" + +type Middleware interface { + MiddlewareFunc(http.Handler) http.Handler +} diff --git a/internal/app/service/security/service/security_mock.go b/internal/app/service/security/service/security_mock.go index 656f1aa3..b820b729 100644 --- a/internal/app/service/security/service/security_mock.go +++ b/internal/app/service/security/service/security_mock.go @@ -14,9 +14,9 @@ func (m *SecurityServiceMock) initializePlugin() error { return args.Error(0) } -func (m *SecurityServiceMock) Auth(email string, token string) (auth.UserDetail, error) { +func (m *SecurityServiceMock) Auth(email string, token string) (*auth.UserDetail, error) { args := m.Called(email, token) - return args.Get(0).(auth.UserDetail), args.Error(1) + return args.Get(0).(*auth.UserDetail), args.Error(1) } func (m *SecurityServiceMock) Verify(userDetail auth.UserDetail, requiredGroups []string) (bool, error) { From b7e32fcf3b787fd64c2be05a2c9004578ee99d19 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 10:46:53 +0700 Subject: [PATCH 162/268] [Dembo] Security middleware authentication without token case --- .../security/middleware/authentication.go | 6 ++++ .../middleware/authentication_test.go | 29 +++++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 9c059cfa..30fca2fe 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -2,6 +2,7 @@ package middleware import ( "net/http" + "proctor/internal/app/service/server/middleware/parameter" "proctor/internal/app/service/security/service" ) @@ -12,6 +13,11 @@ type authenticationMiddleware struct { func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get(parameter.AccessTokenHeader) + if token == "" { + w.WriteHeader(http.StatusUnauthorized) + return + } next.ServeHTTP(w, r) }) } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 613201cc..35fdac3e 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -21,8 +21,8 @@ type context interface { type testContext struct { authenticationMiddleware authenticationMiddleware - securityService service.SecurityService - testHandler http.HandlerFunc + securityService service.SecurityService + testHandler http.HandlerFunc } func (context *testContext) setUp(t *testing.T) { @@ -70,3 +70,28 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } + +func TestAuthenticationMiddleware_MiddlewareFuncWithoutToken(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + authenticationMiddleware := ctx.instance().authenticationMiddleware + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + + body := map[string]string{} + body["name"] = "a-job" + requestBody, _ := json.Marshal(body) + + req, _ := http.NewRequest("GET", ts.URL, bytes.NewReader(requestBody)) + req.Header.Add("Content-Type", "application/json") + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} From 14c41b492f6859be0df47b53fb842c1b8e30cf58 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 10:58:39 +0700 Subject: [PATCH 163/268] [Dembo] Security middleware authentication without email case --- .../security/middleware/authentication.go | 7 ++-- .../middleware/authentication_test.go | 41 ++++++++++++------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 30fca2fe..f185b132 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -2,9 +2,9 @@ package middleware import ( "net/http" - "proctor/internal/app/service/server/middleware/parameter" "proctor/internal/app/service/security/service" + "proctor/internal/pkg/constant" ) type authenticationMiddleware struct { @@ -13,8 +13,9 @@ type authenticationMiddleware struct { func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - token := r.Header.Get(parameter.AccessTokenHeader) - if token == "" { + token := r.Header.Get(constant.AccessTokenHeaderKey) + userEmail := r.Header.Get(constant.UserEmailHeaderKey) + if token == "" || userEmail == "" { w.WriteHeader(http.StatusUnauthorized) return } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 35fdac3e..f86a9040 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -1,8 +1,6 @@ package middleware import ( - "bytes" - "encoding/json" "net/http" "net/http/httptest" "testing" @@ -10,7 +8,7 @@ import ( "github.com/stretchr/testify/assert" "proctor/internal/app/service/security/service" - "proctor/internal/app/service/server/middleware/parameter" + "proctor/internal/pkg/constant" ) type context interface { @@ -57,13 +55,9 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { client := &http.Client{} - body := map[string]string{} - body["name"] = "a-job" - requestBody, _ := json.Marshal(body) - - req, _ := http.NewRequest("GET", ts.URL, bytes.NewReader(requestBody)) - req.Header.Add("Content-Type", "application/json") - req.Header.Add(parameter.AccessTokenHeader, "a-token") + req, _ := http.NewRequest("GET", ts.URL, nil) + req.Header.Add(constant.AccessTokenHeaderKey, "a-token") + req.Header.Add(constant.UserEmailHeaderKey, "email@gmail.com") resp, _ := client.Do(req) defer resp.Body.Close() @@ -83,12 +77,29 @@ func TestAuthenticationMiddleware_MiddlewareFuncWithoutToken(t *testing.T) { client := &http.Client{} - body := map[string]string{} - body["name"] = "a-job" - requestBody, _ := json.Marshal(body) + req, _ := http.NewRequest("GET", ts.URL, nil) + req.Header.Add(constant.UserEmailHeaderKey, "email@gmail.com") + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} + +func TestAuthenticationMiddleware_MiddlewareFuncWithoutEmail(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + authenticationMiddleware := ctx.instance().authenticationMiddleware + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} - req, _ := http.NewRequest("GET", ts.URL, bytes.NewReader(requestBody)) - req.Header.Add("Content-Type", "application/json") + req, _ := http.NewRequest("GET", ts.URL, nil) + req.Header.Add(constant.AccessTokenHeaderKey, "a-token") resp, _ := client.Do(req) defer resp.Body.Close() From 2eb2faec87805bb291283daa3f5776ef25d3283e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 11:19:35 +0700 Subject: [PATCH 164/268] [Dembo] Security middleware authentication auth failed case --- .../security/middleware/authentication.go | 7 +++ .../middleware/authentication_test.go | 43 ++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index f185b132..4f97cdc0 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -3,6 +3,7 @@ package middleware import ( "net/http" + "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/security/service" "proctor/internal/pkg/constant" ) @@ -19,6 +20,12 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht w.WriteHeader(http.StatusUnauthorized) return } + userDetail, err := middleware.service.Auth(userEmail, token) + logger.LogErrors(err, "authentication user", userEmail) + if userDetail == nil { + w.WriteHeader(http.StatusUnauthorized) + return + } next.ServeHTTP(w, r) }) } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index f86a9040..ecc61e2a 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -1,8 +1,10 @@ package middleware import ( + "github.com/pkg/errors" "net/http" "net/http/httptest" + "proctor/pkg/auth" "testing" "github.com/stretchr/testify/assert" @@ -19,7 +21,7 @@ type context interface { type testContext struct { authenticationMiddleware authenticationMiddleware - securityService service.SecurityService + securityService *service.SecurityServiceMock testHandler http.HandlerFunc } @@ -48,6 +50,17 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active:true, + Groups: []string{"system", "proctor_maintainer"}, + } + securityService := ctx.instance().securityService + securityService. + On("Auth", "email@gmail.com", "a-token"). + Return(userDetail, nil) + authenticationMiddleware := ctx.instance().authenticationMiddleware testHandler := ctx.instance().testHandler ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) @@ -106,3 +119,31 @@ func TestAuthenticationMiddleware_MiddlewareFuncWithoutEmail(t *testing.T) { assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) } + +func TestAuthenticationMiddleware_MiddlewareFuncAuthFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + var userDetail *auth.UserDetail + securityService := ctx.instance().securityService + securityService. + On("Auth", "email@gmail.com", "a-token"). + Return(userDetail, errors.New("authentication failed, please check your access token")) + + authenticationMiddleware := ctx.instance().authenticationMiddleware + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + + req, _ := http.NewRequest("GET", ts.URL, nil) + req.Header.Add(constant.AccessTokenHeaderKey, "a-token") + req.Header.Add(constant.UserEmailHeaderKey, "email@gmail.com") + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) +} From 8d6ca7bc51a98365a6e4b4b9b8a0653ab7a2f12d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 11:39:10 +0700 Subject: [PATCH 165/268] [Dembo] Update security middleware authentication success case to give user detail to next request --- .../service/security/middleware/authentication.go | 6 +++++- .../security/middleware/authentication_test.go | 15 ++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 4f97cdc0..87ab51b8 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -1,6 +1,7 @@ package middleware import ( + "context" "net/http" "proctor/internal/app/service/infra/logger" @@ -8,6 +9,8 @@ import ( "proctor/internal/pkg/constant" ) +const ContextUserDetailKey string = "USER_DETAIL" + type authenticationMiddleware struct { service service.SecurityService } @@ -26,6 +29,7 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht w.WriteHeader(http.StatusUnauthorized) return } - next.ServeHTTP(w, r) + ctx := context.WithValue(r.Context(), ContextUserDetailKey, userDetail) + next.ServeHTTP(w, r.WithContext(ctx)) }) } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index ecc61e2a..109c20d5 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -13,12 +13,6 @@ import ( "proctor/internal/pkg/constant" ) -type context interface { - setUp(t *testing.T) - tearDown() - instance() *testContext -} - type testContext struct { authenticationMiddleware authenticationMiddleware securityService *service.SecurityServiceMock @@ -29,7 +23,7 @@ func (context *testContext) setUp(t *testing.T) { context.authenticationMiddleware = authenticationMiddleware{} context.securityService = &service.SecurityServiceMock{} context.authenticationMiddleware.service = context.securityService - fn := func(rw http.ResponseWriter, req *http.Request) { + fn := func(w http.ResponseWriter, r *http.Request) { } context.testHandler = http.HandlerFunc(fn) } @@ -41,7 +35,7 @@ func (context *testContext) instance() *testContext { return context } -func newContext() context { +func newContext() *testContext { return &testContext{} } @@ -62,7 +56,10 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { Return(userDetail, nil) authenticationMiddleware := ctx.instance().authenticationMiddleware - testHandler := ctx.instance().testHandler + fn := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, userDetail, r.Context().Value("USER_DETAIL")) + } + testHandler := http.HandlerFunc(fn) ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() From 5ce979a9cae53ed5e07a5e569e3b9e7b8cf44359 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 14:39:13 +0700 Subject: [PATCH 166/268] [Dembo] Security middleware authorization success case --- .../security/middleware/authorization.go | 17 ++++++ .../security/middleware/authorization_test.go | 54 +++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 internal/app/service/security/middleware/authorization.go create mode 100644 internal/app/service/security/middleware/authorization_test.go diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go new file mode 100644 index 00000000..ebf30826 --- /dev/null +++ b/internal/app/service/security/middleware/authorization.go @@ -0,0 +1,17 @@ +package middleware + +import ( + "net/http" + + "proctor/internal/app/service/security/service" +) + +type authorizationMiddleware struct { + service service.SecurityService +} + +func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) +} diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go new file mode 100644 index 00000000..b93f4407 --- /dev/null +++ b/internal/app/service/security/middleware/authorization_test.go @@ -0,0 +1,54 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +type authorizationContext struct { + authorizationMiddleware authorizationMiddleware + requestHandler func(http.Handler) http.Handler +} + +func (context *authorizationContext) setUp(t *testing.T) { + context.authorizationMiddleware = authorizationMiddleware{} + context.requestHandler = func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) + } +} + +func (context *authorizationContext) tearDown() { +} + +func (context *authorizationContext) instance() *authorizationContext { + return context +} + +func newAuthorizationContext() *authorizationContext { + return &authorizationContext{} +} + +func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", nil) + requestHandler := ctx.instance().requestHandler + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusOK, responseResult.StatusCode) +} From 4b557c7a7233c12a9e1469af25174d44455d5777 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 14:55:17 +0700 Subject: [PATCH 167/268] [Dembo] Security middleware authorization case without name --- .../security/middleware/authorization.go | 9 +++++- .../security/middleware/authorization_test.go | 28 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index ebf30826..2b37e2c1 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -1,8 +1,9 @@ package middleware import ( + "encoding/json" "net/http" - + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/security/service" ) @@ -12,6 +13,12 @@ type authorizationMiddleware struct { func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var job parameter.Job + _ = json.NewDecoder(r.Body).Decode(&job) + if job.Name == "" { + w.WriteHeader(http.StatusBadRequest) + return + } next.ServeHTTP(w, r) }) } diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index b93f4407..f14c2500 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -1,6 +1,8 @@ package middleware import ( + "bytes" + "encoding/json" "net/http" "net/http/httptest" "testing" @@ -38,8 +40,12 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + response := httptest.NewRecorder() - request := httptest.NewRequest(http.MethodPost, "/", nil) + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) requestHandler := ctx.instance().requestHandler testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -52,3 +58,23 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { responseResult := response.Result() assert.Equal(t, http.StatusOK, responseResult.StatusCode) } + +func TestAuthorizationMiddleware_MiddlewareFuncWithoutName(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", nil) + requestHandler := ctx.instance().requestHandler + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusBadRequest, responseResult.StatusCode) +} From 4b2b0991c1dbfb8bb8254ceb9a79b6a818683f54 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 15:31:08 +0700 Subject: [PATCH 168/268] [Dembo] Security middleware authorization case metadata error --- .../security/middleware/authorization.go | 12 ++++- .../security/middleware/authorization_test.go | 46 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index 2b37e2c1..4915d303 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -3,12 +3,16 @@ package middleware import ( "encoding/json" "net/http" + "proctor/internal/app/service/execution/handler/parameter" + "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/metadata/repository" "proctor/internal/app/service/security/service" ) type authorizationMiddleware struct { - service service.SecurityService + service service.SecurityService + metadataRepository repository.MetadataRepository } func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { @@ -19,6 +23,12 @@ func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) htt w.WriteHeader(http.StatusBadRequest) return } + _, err := middleware.metadataRepository.GetByName(job.Name) + logger.LogErrors(err, "get metadata", job.Name) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + next.ServeHTTP(w, r) }) } diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index f14c2500..4ea0663e 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -7,12 +7,17 @@ import ( "net/http/httptest" "testing" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/metadata/repository" + "proctor/internal/pkg/model/metadata" ) type authorizationContext struct { authorizationMiddleware authorizationMiddleware requestHandler func(http.Handler) http.Handler + metadataRepository *repository.MockMetadataRepository } func (context *authorizationContext) setUp(t *testing.T) { @@ -22,6 +27,8 @@ func (context *authorizationContext) setUp(t *testing.T) { next.ServeHTTP(w, r) }) } + context.metadataRepository = &repository.MockMetadataRepository{} + context.authorizationMiddleware.metadataRepository = context.metadataRepository } func (context *authorizationContext) tearDown() { @@ -43,11 +50,22 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { requestBody := map[string]string{} requestBody["name"] = "a-job" body, _ := json.Marshal(requestBody) + jobMetadata := &metadata.Metadata{ + Name: "a-job", + Description: "jobMetadata of a job", + ImageName: "ubuntu-18.04", + AuthorizedGroups: []string{"system", "proctor_maintainer"}, + Author: "systeam team", + Contributors: "proctor team", + Organization: "GoJek", + } response := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) requestHandler := ctx.instance().requestHandler + metadataRepository := ctx.metadataRepository + metadataRepository.On("GetByName", "a-job").Return(jobMetadata, nil) testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) @@ -78,3 +96,31 @@ func TestAuthorizationMiddleware_MiddlewareFuncWithoutName(t *testing.T) { responseResult := response.Result() assert.Equal(t, http.StatusBadRequest, responseResult.StatusCode) } + +func TestAuthorizationMiddleware_MiddlewareFuncMetadataError(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + var jobMetadata *metadata.Metadata + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestHandler := ctx.instance().requestHandler + + metadataRepository := ctx.metadataRepository + err := errors.New("metadata not found") + metadataRepository.On("GetByName", "a-job").Return(jobMetadata, err) + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusInternalServerError, responseResult.StatusCode) +} From 481635cd05aa3a128acf445fa0697963f73011c7 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 15:39:51 +0700 Subject: [PATCH 169/268] [Dembo] Security middleware authorization case without user detail --- .../security/middleware/authorization.go | 7 +++ .../security/middleware/authorization_test.go | 45 +++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index 4915d303..2ecf8f39 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -27,6 +27,13 @@ func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) htt logger.LogErrors(err, "get metadata", job.Name) if err != nil { w.WriteHeader(http.StatusInternalServerError) + return + } + + userDetail := r.Context().Value(ContextUserDetailKey) + if userDetail == nil { + w.WriteHeader(http.StatusUnauthorized) + return } next.ServeHTTP(w, r) diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 4ea0663e..8c0b3135 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -2,9 +2,11 @@ package middleware import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" + "proctor/pkg/auth" "testing" "github.com/pkg/errors" @@ -59,9 +61,17 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { Contributors: "proctor team", Organization: "GoJek", } + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active:true, + Groups: []string{"system", "proctor_maintainer"}, + } response := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) requestHandler := ctx.instance().requestHandler metadataRepository := ctx.metadataRepository @@ -124,3 +134,38 @@ func TestAuthorizationMiddleware_MiddlewareFuncMetadataError(t *testing.T) { responseResult := response.Result() assert.Equal(t, http.StatusInternalServerError, responseResult.StatusCode) } + +func TestAuthorizationMiddleware_MiddlewareFuncWithoutUserDetail(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + jobMetadata := &metadata.Metadata{ + Name: "a-job", + Description: "jobMetadata of a job", + ImageName: "ubuntu-18.04", + AuthorizedGroups: []string{"system", "proctor_maintainer"}, + Author: "systeam team", + Contributors: "proctor team", + Organization: "GoJek", + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestHandler := ctx.instance().requestHandler + metadataRepository := ctx.metadataRepository + + metadataRepository.On("GetByName", "a-job").Return(jobMetadata, nil) + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusUnauthorized, responseResult.StatusCode) +} From 41538e01666ccd75c701821b89cca56997d7eabc Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 16:12:10 +0700 Subject: [PATCH 170/268] [Dembo] Security middleware authorization case authorization failed --- .../security/middleware/authorization.go | 13 +++- .../security/middleware/authorization_test.go | 65 +++++++++++++++++-- 2 files changed, 70 insertions(+), 8 deletions(-) diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index 2ecf8f39..b4cd2303 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -8,6 +8,7 @@ import ( "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" "proctor/internal/app/service/security/service" + "proctor/pkg/auth" ) type authorizationMiddleware struct { @@ -18,12 +19,13 @@ type authorizationMiddleware struct { func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var job parameter.Job - _ = json.NewDecoder(r.Body).Decode(&job) + err := json.NewDecoder(r.Body).Decode(&job) + logger.LogErrors(err, "decode json", r.Body) if job.Name == "" { w.WriteHeader(http.StatusBadRequest) return } - _, err := middleware.metadataRepository.GetByName(job.Name) + jobMetadata, err := middleware.metadataRepository.GetByName(job.Name) logger.LogErrors(err, "get metadata", job.Name) if err != nil { w.WriteHeader(http.StatusInternalServerError) @@ -35,7 +37,12 @@ func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) htt w.WriteHeader(http.StatusUnauthorized) return } - + authorized, err := middleware.service.Verify(*(userDetail.(*auth.UserDetail)), jobMetadata.AuthorizedGroups) + logger.LogErrors(err, "authorization", jobMetadata, userDetail) + if !authorized { + w.WriteHeader(http.StatusForbidden) + return + } next.ServeHTTP(w, r) }) } diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 8c0b3135..12619d2d 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -6,20 +6,22 @@ import ( "encoding/json" "net/http" "net/http/httptest" - "proctor/pkg/auth" "testing" "github.com/pkg/errors" "github.com/stretchr/testify/assert" "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/security/service" "proctor/internal/pkg/model/metadata" + "proctor/pkg/auth" ) type authorizationContext struct { authorizationMiddleware authorizationMiddleware requestHandler func(http.Handler) http.Handler metadataRepository *repository.MockMetadataRepository + securityService *service.SecurityServiceMock } func (context *authorizationContext) setUp(t *testing.T) { @@ -31,6 +33,8 @@ func (context *authorizationContext) setUp(t *testing.T) { } context.metadataRepository = &repository.MockMetadataRepository{} context.authorizationMiddleware.metadataRepository = context.metadataRepository + context.securityService = &service.SecurityServiceMock{} + context.authorizationMiddleware.service = context.securityService } func (context *authorizationContext) tearDown() { @@ -62,9 +66,9 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { Organization: "GoJek", } userDetail := &auth.UserDetail{ - Name: "William Dembo", - Email: "email@gmail.com", - Active:true, + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, Groups: []string{"system", "proctor_maintainer"}, } @@ -73,9 +77,13 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) request = request.WithContext(requestContext) requestHandler := ctx.instance().requestHandler - metadataRepository := ctx.metadataRepository + metadataRepository := ctx.metadataRepository metadataRepository.On("GetByName", "a-job").Return(jobMetadata, nil) + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, jobMetadata.AuthorizedGroups).Return(true, nil) + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) @@ -169,3 +177,50 @@ func TestAuthorizationMiddleware_MiddlewareFuncWithoutUserDetail(t *testing.T) { responseResult := response.Result() assert.Equal(t, http.StatusUnauthorized, responseResult.StatusCode) } + +func TestAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + jobMetadata := &metadata.Metadata{ + Name: "a-job", + Description: "jobMetadata of a job", + ImageName: "ubuntu-18.04", + AuthorizedGroups: []string{"system", "proctor_maintainer"}, + Author: "systeam team", + Contributors: "proctor team", + Organization: "GoJek", + } + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, + Groups: []string{"system", "not_proctor_maintainer"}, + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) + requestHandler := ctx.instance().requestHandler + + metadataRepository := ctx.metadataRepository + metadataRepository.On("GetByName", "a-job").Return(jobMetadata, nil) + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, jobMetadata.AuthorizedGroups).Return(false, nil) + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusForbidden, responseResult.StatusCode) +} From ecaac869ba9e35af287f3c9ed4d83825ffc4d95e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 16:36:50 +0700 Subject: [PATCH 171/268] [Dembo] Refactor security middleware authorization to extract job metadata --- .../security/middleware/authorization_test.go | 40 ++++++------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 12619d2d..5b336abd 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -22,6 +22,7 @@ type authorizationContext struct { requestHandler func(http.Handler) http.Handler metadataRepository *repository.MockMetadataRepository securityService *service.SecurityServiceMock + jobMetadata *metadata.Metadata } func (context *authorizationContext) setUp(t *testing.T) { @@ -35,6 +36,15 @@ func (context *authorizationContext) setUp(t *testing.T) { context.authorizationMiddleware.metadataRepository = context.metadataRepository context.securityService = &service.SecurityServiceMock{} context.authorizationMiddleware.service = context.securityService + context.jobMetadata = &metadata.Metadata{ + Name: "a-job", + Description: "jobMetadata of a job", + ImageName: "ubuntu-18.04", + AuthorizedGroups: []string{"system", "proctor_maintainer"}, + Author: "systeam team", + Contributors: "proctor team", + Organization: "GoJek", + } } func (context *authorizationContext) tearDown() { @@ -56,15 +66,7 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { requestBody := map[string]string{} requestBody["name"] = "a-job" body, _ := json.Marshal(requestBody) - jobMetadata := &metadata.Metadata{ - Name: "a-job", - Description: "jobMetadata of a job", - ImageName: "ubuntu-18.04", - AuthorizedGroups: []string{"system", "proctor_maintainer"}, - Author: "systeam team", - Contributors: "proctor team", - Organization: "GoJek", - } + jobMetadata := ctx.jobMetadata userDetail := &auth.UserDetail{ Name: "William Dembo", Email: "email@gmail.com", @@ -151,15 +153,7 @@ func TestAuthorizationMiddleware_MiddlewareFuncWithoutUserDetail(t *testing.T) { requestBody := map[string]string{} requestBody["name"] = "a-job" body, _ := json.Marshal(requestBody) - jobMetadata := &metadata.Metadata{ - Name: "a-job", - Description: "jobMetadata of a job", - ImageName: "ubuntu-18.04", - AuthorizedGroups: []string{"system", "proctor_maintainer"}, - Author: "systeam team", - Contributors: "proctor team", - Organization: "GoJek", - } + jobMetadata := ctx.jobMetadata response := httptest.NewRecorder() request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) @@ -186,15 +180,7 @@ func TestAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { requestBody := map[string]string{} requestBody["name"] = "a-job" body, _ := json.Marshal(requestBody) - jobMetadata := &metadata.Metadata{ - Name: "a-job", - Description: "jobMetadata of a job", - ImageName: "ubuntu-18.04", - AuthorizedGroups: []string{"system", "proctor_maintainer"}, - Author: "systeam team", - Contributors: "proctor team", - Organization: "GoJek", - } + jobMetadata := ctx.jobMetadata userDetail := &auth.UserDetail{ Name: "William Dembo", Email: "email@gmail.com", From 5a19ea89f83e02604c0e2d9f8ec09f52e2f1f446 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 30 Aug 2019 17:15:13 +0700 Subject: [PATCH 172/268] [Dembo] Security middleware authorization success case for schedule --- .../security/middleware/authorization.go | 38 ++++++++++++++--- .../security/middleware/authorization_test.go | 41 ++++++++++++++++++- 2 files changed, 73 insertions(+), 6 deletions(-) diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index b4cd2303..7a854be0 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -1,12 +1,15 @@ package middleware import ( + "bytes" "encoding/json" + "io/ioutil" "net/http" "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/schedule/model" "proctor/internal/app/service/security/service" "proctor/pkg/auth" ) @@ -18,15 +21,14 @@ type authorizationMiddleware struct { func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var job parameter.Job - err := json.NewDecoder(r.Body).Decode(&job) + jobName, err := extractName(r) logger.LogErrors(err, "decode json", r.Body) - if job.Name == "" { + if jobName == "" { w.WriteHeader(http.StatusBadRequest) return } - jobMetadata, err := middleware.metadataRepository.GetByName(job.Name) - logger.LogErrors(err, "get metadata", job.Name) + jobMetadata, err := middleware.metadataRepository.GetByName(jobName) + logger.LogErrors(err, "get metadata", jobName) if err != nil { w.WriteHeader(http.StatusInternalServerError) return @@ -46,3 +48,29 @@ func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) htt next.ServeHTTP(w, r) }) } + +func extractName(r *http.Request) (string, error) { + bodyBytes, _ := ioutil.ReadAll(r.Body) + _ = r.Body.Close() + r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + + var job parameter.Job + err := json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&job) + if err != nil { + return "", err + } + if job.Name != "" { + return job.Name, nil + } + + var schedule model.Schedule + err = json.NewDecoder(bytes.NewReader(bodyBytes)).Decode(&schedule) + if err != nil { + return "", err + } + if schedule.JobName != "" { + return schedule.JobName, nil + } + + return "", nil +} diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 5b336abd..88d8b5d8 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -58,7 +58,7 @@ func newAuthorizationContext() *authorizationContext { return &authorizationContext{} } -func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { +func TestAuthorizationMiddleware_MiddlewareFuncExecutionSuccess(t *testing.T) { ctx := newAuthorizationContext() ctx.setUp(t) defer ctx.tearDown() @@ -97,6 +97,45 @@ func TestAuthorizationMiddleware_MiddlewareFuncSuccess(t *testing.T) { assert.Equal(t, http.StatusOK, responseResult.StatusCode) } +func TestAuthorizationMiddleware_MiddlewareFuncScheduleSuccess(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["jobName"] = "a-job" + body, _ := json.Marshal(requestBody) + jobMetadata := ctx.jobMetadata + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, + Groups: []string{"system", "proctor_maintainer"}, + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) + requestHandler := ctx.instance().requestHandler + + metadataRepository := ctx.metadataRepository + metadataRepository.On("GetByName", "a-job").Return(jobMetadata, nil) + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, jobMetadata.AuthorizedGroups).Return(true, nil) + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authorizationMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusOK, responseResult.StatusCode) +} + func TestAuthorizationMiddleware_MiddlewareFuncWithoutName(t *testing.T) { ctx := newAuthorizationContext() ctx.setUp(t) From 34a11ec6d560044d4f9070a76b6ff7a2cc1956f6 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Mon, 2 Sep 2019 10:02:27 +0700 Subject: [PATCH 173/268] Change retry mechanism of wait for ready job and pod --- .../app/service/infra/kubernetes/client.go | 118 +++++++++++------- .../service/infra/kubernetes/client_mock.go | 4 +- .../service/infra/kubernetes/client_test.go | 69 ++++------ 3 files changed, 99 insertions(+), 92 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 2b7c134a..abe3e9c9 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -191,32 +191,43 @@ func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime t LabelSelector: jobLabelSelector(executionName), } - watchJob, err := jobs.Watch(listOptions) - if err != nil { - return err - } - - timeoutChan := time.After(waitTime) - resultChan := watchJob.ResultChan() - defer watchJob.Stop() + var err error + for i := 0; i < config.KubeWaitForResourcePollCount(); i += 1 { + watchJob, watchErr := jobs.Watch(listOptions) + if watchErr != nil { + continue + } - var count int - for count < config.KubeWaitForResourcePollCount() { - select { - case watchEvent := <-resultChan: - if watchEvent.Type == watch.Error { - err = watcherError("job", listOptions) - count += 1 + timeoutChan := time.After(waitTime) + resultChan := watchJob.ResultChan() + + var job *batch.Job + for { + select { + case event := <-resultChan: + if event.Type == watch.Error { + err = watcherError("job", listOptions) + break + } + + // Ignore empty events + if event.Object == nil { + continue + } + + job = event.Object.(*batch.Job) + if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { + watchJob.Stop() + return nil + } + case <-timeoutChan: + err = timeoutError + break } - - job := watchEvent.Object.(*batch.Job) - if job.Status.Active >= 1 || job.Status.Succeeded >= 1 || job.Status.Failed >= 1 { - return nil + if err != nil { + watchJob.Stop() + break } - case <-timeoutChan: - err = timeoutError - timeoutChan = time.After(waitTime) - count += 1 } } @@ -230,33 +241,46 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t LabelSelector: jobLabelSelector(executionName), } - watchJob, err := kubernetesPods.Watch(listOptions) - if err != nil { - return nil, err - } + var err error + for i := 0; i < config.KubeWaitForResourcePollCount(); i += 1 { + watchJob, watchErr := kubernetesPods.Watch(listOptions) + if watchErr != nil { + continue + } - timeoutChan := time.After(waitTime) - resultChan := watchJob.ResultChan() - defer watchJob.Stop() - var pod *v1.Pod - - var count int - for count < config.KubeWaitForResourcePollCount() { - select { - case event := <-resultChan: - if event.Type == watch.Error { - err = watcherError("pod", listOptions) - count += 1 + timeoutChan := time.After(waitTime) + resultChan := watchJob.ResultChan() + defer watchJob.Stop() + + var pod *v1.Pod + for { + select { + case event := <-resultChan: + if event.Type == watch.Error { + err = watcherError("pod", listOptions) + watchJob.Stop() + break + } + + // Ignore empty events + if event.Object == nil { + continue + } + + pod = event.Object.(*v1.Pod) + if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { + watchJob.Stop() + return pod, nil + } + case <-timeoutChan: + err = timeoutError + watchJob.Stop() + break } - - pod = event.Object.(*v1.Pod) - if pod.Status.Phase == v1.PodRunning || pod.Status.Phase == v1.PodSucceeded || pod.Status.Phase == v1.PodFailed { - return pod, nil + if err != nil { + watchJob.Stop() + break } - case <-timeoutChan: - err = timeoutError - timeoutChan = time.After(waitTime) - count += 1 } } diff --git a/internal/app/service/infra/kubernetes/client_mock.go b/internal/app/service/infra/kubernetes/client_mock.go index 555ba50d..e8c55879 100644 --- a/internal/app/service/infra/kubernetes/client_mock.go +++ b/internal/app/service/infra/kubernetes/client_mock.go @@ -31,10 +31,12 @@ func (m *MockKubernetesClient) WaitForReadyJob(executionName string, waitTime ti args := m.Called(executionName, waitTime) return args.Error(0) } + func (m *MockKubernetesClient) WaitForReadyPod(executionName string, waitTime time.Duration) (*v1.Pod, error) { - args := m.Called(executionName) + args := m.Called(executionName, waitTime) return args.Get(0).(*v1.Pod), args.Error(1) } + func (m *MockKubernetesClient) GetPodLogs(pod *v1.Pod) (io.ReadCloser, error) { args := m.Called(pod) return args.Get(0).(io.ReadCloser), args.Error(1) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index e3f3f523..62ea12d3 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -116,9 +116,6 @@ func (suite *ClientTestSuite) TestJobExecution() { func (suite *ClientTestSuite) TestWaitForReadyJob() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - var testJob batchV1.Job uniqueJobName := "proctor-job-1" label := jobLabel(uniqueJobName) @@ -129,11 +126,13 @@ func (suite *ClientTestSuite) TestWaitForReadyJob() { testJob.ObjectMeta = objectMeta waitTime := config.KubeLogProcessWaitTime() * time.Second + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + go func() { testJob.Status.Succeeded = 1 watcher.Modify(&testJob) - time.Sleep(time.Second * 1) watcher.Stop() }() @@ -144,11 +143,8 @@ func (suite *ClientTestSuite) TestWaitForReadyJob() { func (suite *ClientTestSuite) TestWaitForReadyJobWatcherError() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - var testJob batchV1.Job - uniqueJobName := "proctor-job-1" + uniqueJobName := "proctor-job-2" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, @@ -160,15 +156,16 @@ func (suite *ClientTestSuite) TestWaitForReadyJobWatcherError() { LabelSelector: jobLabelSelector(uniqueJobName), } waitTime := config.KubeLogProcessWaitTime() * time.Second + + watcher := watch.NewRaceFreeFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + go func() { watcher.Error(&testJob) watcher.Error(&testJob) watcher.Error(&testJob) watcher.Error(&testJob) watcher.Error(&testJob) - - time.Sleep(time.Second * 1) - watcher.Stop() }() err := suite.testClient.WaitForReadyJob(uniqueJobName, waitTime) @@ -178,12 +175,8 @@ func (suite *ClientTestSuite) TestWaitForReadyJobWatcherError() { func (suite *ClientTestSuite) TestWaitForReadyJobTimeoutError() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - defer watcher.Stop() - var testJob batchV1.Job - uniqueJobName := "proctor-job-1" + uniqueJobName := "proctor-job-3" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, @@ -192,10 +185,8 @@ func (suite *ClientTestSuite) TestWaitForReadyJobTimeoutError() { testJob.ObjectMeta = objectMeta waitTime := time.Millisecond * 100 - go func() { - time.Sleep(time.Millisecond * 550) - watcher.Stop() - }() + watcher := watch.NewRaceFreeFake() + suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) err := suite.testClient.WaitForReadyJob(uniqueJobName, waitTime) assert.EqualError(t, err, "timeout when waiting job to be available") @@ -204,9 +195,6 @@ func (suite *ClientTestSuite) TestWaitForReadyJobTimeoutError() { func (suite *ClientTestSuite) TestWaitForReadyPod() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - var testPod v1.Pod uniquePodName := "proctor-pod-1" label := jobLabel(uniquePodName) @@ -217,11 +205,13 @@ func (suite *ClientTestSuite) TestWaitForReadyPod() { testPod.ObjectMeta = objectMeta waitTime := config.KubeLogProcessWaitTime() * time.Second + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + go func() { testPod.Status.Phase = v1.PodSucceeded watcher.Modify(&testPod) - time.Sleep(time.Second * 1) watcher.Stop() }() @@ -234,11 +224,8 @@ func (suite *ClientTestSuite) TestWaitForReadyPod() { func (suite *ClientTestSuite) TestWaitForReadyPodWatcherError() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - var testPod v1.Pod - uniquePodName := "proctor-pod-1" + uniquePodName := "proctor-pod-2" label := jobLabel(uniquePodName) objectMeta := meta.ObjectMeta{ Name: uniquePodName, @@ -250,15 +237,15 @@ func (suite *ClientTestSuite) TestWaitForReadyPodWatcherError() { } waitTime := config.KubeLogProcessWaitTime() * time.Second + watcher := watch.NewRaceFreeFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) + go func() { watcher.Error(&testPod) watcher.Error(&testPod) watcher.Error(&testPod) watcher.Error(&testPod) watcher.Error(&testPod) - - time.Sleep(time.Second * 1) - watcher.Stop() }() _, err := suite.testClient.WaitForReadyPod(uniquePodName, waitTime) @@ -268,12 +255,8 @@ func (suite *ClientTestSuite) TestWaitForReadyPodWatcherError() { func (suite *ClientTestSuite) TestWaitForReadyPodTimeoutError() { t := suite.T() - watcher := watch.NewFake() - suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) - defer watcher.Stop() - var testPod v1.Pod - uniquePodName := "proctor-pod-1" + uniquePodName := "proctor-pod-3" label := jobLabel(uniquePodName) objectMeta := meta.ObjectMeta{ Name: uniquePodName, @@ -282,10 +265,8 @@ func (suite *ClientTestSuite) TestWaitForReadyPodTimeoutError() { testPod.ObjectMeta = objectMeta waitTime := time.Millisecond * 100 - go func() { - time.Sleep(time.Millisecond * 550) - watcher.Stop() - }() + watcher := watch.NewFake() + suite.fakeClientSet.PrependWatchReactor("pods", testing_kubernetes.DefaultWatchReactor(watcher, nil)) _, err := suite.testClient.WaitForReadyPod(uniquePodName, waitTime) assert.EqualError(t, err, "timeout when waiting job to be available") @@ -299,7 +280,7 @@ func (suite *ClientTestSuite) TestShouldReturnSuccessJobExecutionStatus() { var activeJob batchV1.Job var succeededJob batchV1.Job - uniqueJobName := "proctor-job-2" + uniqueJobName := "proctor-job-4" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, @@ -334,7 +315,7 @@ func (suite *ClientTestSuite) TestShouldReturnFailedJobExecutionStatus() { var activeJob batchV1.Job var failedJob batchV1.Job - uniqueJobName := "proctor-job-1" + uniqueJobName := "proctor-job-5" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, @@ -367,7 +348,7 @@ func (suite *ClientTestSuite) TestJobExecutionStatusForNonDefinitiveStatus() { suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) var testJob batchV1.Job - uniqueJobName := "proctor-job-1" + uniqueJobName := "proctor-job-6" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, @@ -396,7 +377,7 @@ func (suite *ClientTestSuite) TestShouldReturnJobExecutionStatusFetchError() { suite.fakeClientSet.PrependWatchReactor("jobs", testing_kubernetes.DefaultWatchReactor(watcher, nil)) var testJob batchV1.Job - uniqueJobName := "proctor-job-3" + uniqueJobName := "proctor-job-7" label := jobLabel(uniqueJobName) objectMeta := meta.ObjectMeta{ Name: uniqueJobName, From 495af042fb1235d29a3449a3da83d7f7d85aa9b0 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 2 Sep 2019 11:20:46 +0700 Subject: [PATCH 174/268] [Jasoet|Dembo] Set machine ID for snowflakeId --- internal/app/service/infra/id/snowflake.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/app/service/infra/id/snowflake.go b/internal/app/service/infra/id/snowflake.go index eb844ee0..e522859f 100644 --- a/internal/app/service/infra/id/snowflake.go +++ b/internal/app/service/infra/id/snowflake.go @@ -1,15 +1,23 @@ package id -import "github.com/sony/sonyflake" +import ( + "github.com/sony/sonyflake" + "math/rand" + "time" +) var snowflake *sonyflake.Sonyflake +var snowflakeSetting sonyflake.Settings func init() { - var snowflakeSetting sonyflake.Settings - snowflake = sonyflake.NewSonyflake(snowflakeSetting) + rand.Seed(time.Now().UnixNano()) + snowflakeSetting.MachineID = func() (machineId uint16, e error) { + return uint16(rand.Uint64()), nil + } } func NextID() (uint64, error) { + snowflake = sonyflake.NewSonyflake(snowflakeSetting) return snowflake.NextID() } From 40f6ccef1049c8b297cbb0d91878e65ed859d2b0 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 2 Sep 2019 11:32:56 +0700 Subject: [PATCH 175/268] [Jasoet|Dembo] Tidyup security middleware --- .../security/middleware/authentication.go | 1 - .../middleware/authentication_test.go | 28 +++++++++---------- .../security/middleware/authorization_test.go | 24 ++++++++-------- .../service/security/middleware/middleware.go | 2 ++ 4 files changed, 28 insertions(+), 27 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 87ab51b8..60e02f91 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -9,7 +9,6 @@ import ( "proctor/internal/pkg/constant" ) -const ContextUserDetailKey string = "USER_DETAIL" type authenticationMiddleware struct { service service.SecurityService diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 109c20d5..7ad37a39 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -14,18 +14,18 @@ import ( ) type testContext struct { - authenticationMiddleware authenticationMiddleware - securityService *service.SecurityServiceMock - testHandler http.HandlerFunc + authMiddleware authenticationMiddleware + securityService *service.SecurityServiceMock + testHandler http.HandlerFunc } func (context *testContext) setUp(t *testing.T) { - context.authenticationMiddleware = authenticationMiddleware{} + context.authMiddleware = authenticationMiddleware{} context.securityService = &service.SecurityServiceMock{} - context.authenticationMiddleware.service = context.securityService + context.authMiddleware.service = context.securityService fn := func(w http.ResponseWriter, r *http.Request) { } - context.testHandler = http.HandlerFunc(fn) + context.testHandler = fn } func (context *testContext) tearDown() { @@ -55,12 +55,12 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { On("Auth", "email@gmail.com", "a-token"). Return(userDetail, nil) - authenticationMiddleware := ctx.instance().authenticationMiddleware + authMiddleware := ctx.instance().authMiddleware fn := func(w http.ResponseWriter, r *http.Request) { assert.Equal(t, userDetail, r.Context().Value("USER_DETAIL")) } testHandler := http.HandlerFunc(fn) - ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() client := &http.Client{} @@ -80,9 +80,9 @@ func TestAuthenticationMiddleware_MiddlewareFuncWithoutToken(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() - authenticationMiddleware := ctx.instance().authenticationMiddleware + authMiddleware := ctx.instance().authMiddleware testHandler := ctx.instance().testHandler - ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() client := &http.Client{} @@ -101,9 +101,9 @@ func TestAuthenticationMiddleware_MiddlewareFuncWithoutEmail(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() - authenticationMiddleware := ctx.instance().authenticationMiddleware + authMiddleware := ctx.instance().authMiddleware testHandler := ctx.instance().testHandler - ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() client := &http.Client{} @@ -128,9 +128,9 @@ func TestAuthenticationMiddleware_MiddlewareFuncAuthFailed(t *testing.T) { On("Auth", "email@gmail.com", "a-token"). Return(userDetail, errors.New("authentication failed, please check your access token")) - authenticationMiddleware := ctx.instance().authenticationMiddleware + authMiddleware := ctx.instance().authMiddleware testHandler := ctx.instance().testHandler - ts := httptest.NewServer(authenticationMiddleware.MiddlewareFunc(testHandler)) + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() client := &http.Client{} diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 88d8b5d8..42698a24 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -90,8 +90,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncExecutionSuccess(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusOK, responseResult.StatusCode) @@ -129,8 +129,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncScheduleSuccess(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusOK, responseResult.StatusCode) @@ -149,8 +149,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncWithoutName(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusBadRequest, responseResult.StatusCode) @@ -177,8 +177,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncMetadataError(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusInternalServerError, responseResult.StatusCode) @@ -204,8 +204,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncWithoutUserDetail(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusUnauthorized, responseResult.StatusCode) @@ -243,8 +243,8 @@ func TestAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { w.WriteHeader(http.StatusOK) }) - authorizationMiddleware := ctx.instance().authorizationMiddleware - requestHandler(authorizationMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) responseResult := response.Result() assert.Equal(t, http.StatusForbidden, responseResult.StatusCode) diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go index 1f2ebd5d..4580afcb 100644 --- a/internal/app/service/security/middleware/middleware.go +++ b/internal/app/service/security/middleware/middleware.go @@ -2,6 +2,8 @@ package middleware import "net/http" +const ContextUserDetailKey string = "USER_DETAIL" + type Middleware interface { MiddlewareFunc(http.Handler) http.Handler } From b8f014520d1cb9bd715404c1070cdb0929a6da5b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 2 Sep 2019 15:36:06 +0700 Subject: [PATCH 176/268] [Jasoet|Dembo] Add authentication and authorization to router --- .../security/middleware/authentication.go | 7 ++++++- .../security/middleware/authorization.go | 13 +++++++++++++ .../security/middleware/authorization_test.go | 18 ++++++++++++++++++ .../service/security/middleware/middleware.go | 9 ++++++++- internal/app/service/server/router.go | 13 +++++++++++-- 5 files changed, 56 insertions(+), 4 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 60e02f91..8473d562 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -9,7 +9,6 @@ import ( "proctor/internal/pkg/constant" ) - type authenticationMiddleware struct { service service.SecurityService } @@ -32,3 +31,9 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht next.ServeHTTP(w, r.WithContext(ctx)) }) } + +func NewAuthenticationMiddleware(securityService service.SecurityService) Middleware { + return &authenticationMiddleware{ + service: securityService, + } +} diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index 7a854be0..ab9cd739 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -6,6 +6,8 @@ import ( "io/ioutil" "net/http" + "github.com/gorilla/mux" + "proctor/internal/app/service/execution/handler/parameter" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" @@ -19,6 +21,10 @@ type authorizationMiddleware struct { metadataRepository repository.MetadataRepository } +func (middleware *authorizationMiddleware) Secure(router *mux.Router, path string, handler http.Handler) *mux.Route { + return router.NewRoute().Path(path).Handler(middleware.MiddlewareFunc(handler)) +} + func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { jobName, err := extractName(r) @@ -74,3 +80,10 @@ func extractName(r *http.Request) (string, error) { return "", nil } + +func NewAuthorizationMiddleware(securityService service.SecurityService, metadataRepository repository.MetadataRepository) AuthorizationMiddleware { + return &authorizationMiddleware{ + service: securityService, + metadataRepository: metadataRepository, + } +} diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index 42698a24..b2bebe9d 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -8,6 +8,7 @@ import ( "net/http/httptest" "testing" + "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -249,3 +250,20 @@ func TestAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { responseResult := response.Result() assert.Equal(t, http.StatusForbidden, responseResult.StatusCode) } + +func TestAuthorizationMiddleware_Secure(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + router := mux.NewRouter() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + authzMiddleware := ctx.instance().authorizationMiddleware + securedRouter := authzMiddleware.Secure(router, "/secure/path", handler) + + handledPath, err := securedRouter.GetPathTemplate() + assert.NoError(t, err) + assert.Equal(t, "/secure/path", handledPath) +} diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go index 4580afcb..7008475b 100644 --- a/internal/app/service/security/middleware/middleware.go +++ b/internal/app/service/security/middleware/middleware.go @@ -1,9 +1,16 @@ package middleware -import "net/http" +import ( + "github.com/gorilla/mux" + "net/http" +) const ContextUserDetailKey string = "USER_DETAIL" type Middleware interface { MiddlewareFunc(http.Handler) http.Handler } + +type AuthorizationMiddleware interface { + Secure(router *mux.Router, path string, handler http.Handler) *mux.Route +} diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 99e8c332..22e40ada 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -16,12 +16,15 @@ import ( "proctor/internal/app/service/infra/db/redis" "proctor/internal/app/service/infra/kubernetes" kubernetesHTTPClient "proctor/internal/app/service/infra/kubernetes/http" + "proctor/internal/app/service/infra/plugin" metadataHandler "proctor/internal/app/service/metadata/handler" metadataRepository "proctor/internal/app/service/metadata/repository" scheduleHTTPHandler "proctor/internal/app/service/schedule/handler" scheduleRepository "proctor/internal/app/service/schedule/repository" secretHTTPHandler "proctor/internal/app/service/secret/handler" secretRepository "proctor/internal/app/service/secret/repository" + securityMiddleware "proctor/internal/app/service/security/middleware" + "proctor/internal/app/service/security/service" "proctor/internal/app/service/server/middleware" ) @@ -37,6 +40,8 @@ func NewRouter() (*mux.Router, error) { return router, err } kubeClient := kubernetes.NewKubernetesClient(httpClient) + goPlugin := plugin.NewGoPlugin() + proctorConfig := config.Config() executionStore := executionContextRepository.NewExecutionContextRepository(postgresClient) scheduleStore := scheduleRepository.NewScheduleRepository(postgresClient) @@ -44,6 +49,7 @@ func NewRouter() (*mux.Router, error) { secretsStore := secretRepository.NewSecretRepository(redisClient) _executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) + _securityService := service.NewSecurityService(proctorConfig.AuthPluginBinary, proctorConfig.AuthPluginExported, goPlugin) executionHandler := executionHTTPHandler.NewExecutionHTTPHandler(_executionService, executionStore) jobMetadataHandler := metadataHandler.NewMetadataHTTPHandler(metadataStore) @@ -62,8 +68,11 @@ func NewRouter() (*mux.Router, error) { router = middleware.InstrumentNewRelic(router) router.Use(middleware.ValidateClientVersion) + authenticationMiddleware := securityMiddleware.NewAuthenticationMiddleware(_securityService) + router.Use(authenticationMiddleware.MiddlewareFunc) + authorizationMiddleware := securityMiddleware.NewAuthorizationMiddleware(_securityService, metadataStore) - router.HandleFunc("/execution", executionHandler.Post()).Methods("POST") + authorizationMiddleware.Secure(router, "/execution", executionHandler.Post()).Methods("POST") router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") @@ -72,7 +81,7 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") router.HandleFunc("/secret", jobSecretsHandler.Post()).Methods("POST") - router.HandleFunc("/schedule", scheduleHandler.Post()).Methods("POST") + authorizationMiddleware.Secure(router, "/schedule", scheduleHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.GetAll()).Methods("GET") router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Get()).Methods("GET") router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Delete()).Methods("DELETE") From aef8bcf15e619485ee71bf601d2324d1a7fd0e3a Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 2 Sep 2019 17:14:12 +0700 Subject: [PATCH 177/268] [Jasoet|Dembo] Integrate security service with plugin --- .env.test | 2 +- Makefile | 7 +- .../app/service/security/service/security.go | 6 +- .../service/security_integration_test.go | 133 ++++++++++++++ .../service/security/service/security_test.go | 166 ------------------ plugins/gate-auth-plugin/auth.go | 2 +- 6 files changed, 143 insertions(+), 173 deletions(-) create mode 100644 internal/app/service/security/service/security_integration_test.go delete mode 100644 internal/app/service/security/service/security_test.go diff --git a/.env.test b/.env.test index 234112d0..dbef83bf 100644 --- a/.env.test +++ b/.env.test @@ -34,4 +34,4 @@ export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir export PROCTOR_AUTH_PLUGIN_BINARY= -export PROCTOR_AUTH_PLUGIN_EXPORTED=Auth +export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth diff --git a/Makefile b/Makefile index 58645d39..c053cc35 100644 --- a/Makefile +++ b/Makefile @@ -36,12 +36,14 @@ itest: plugin.auth go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: plugin.itest -plugin.itest: +plugin.itest: plugin.auth + PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ ENABLE_PLUGIN_INTEGRATION_TEST=true \ - go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./plugins/... + go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: server server: + PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ go build -race -o $(BIN_DIR)/server ./cmd/server/main.go .PHONY: plugin.auth @@ -56,6 +58,7 @@ build-all: server cli plugin.auth .PHONY: start-server start-server: + PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ $(BIN_DIR)/server s generate: diff --git a/internal/app/service/security/service/security.go b/internal/app/service/security/service/security.go index 25f851d8..21fb892b 100644 --- a/internal/app/service/security/service/security.go +++ b/internal/app/service/security/service/security.go @@ -46,9 +46,9 @@ func (s *securityService) initializePlugin() error { if err != nil { return } - authInstance, ok := raw.(auth.Auth) - if !ok { - logger.Error("Failed to convert exported plugin to *auth.Auth type") + authInstance := *raw.(*auth.Auth) + if authInstance == nil { + logger.Error("Failed to convert exported plugin to auth.Auth type") return } s.authInstance = authInstance diff --git a/internal/app/service/security/service/security_integration_test.go b/internal/app/service/security/service/security_integration_test.go new file mode 100644 index 00000000..3168ca36 --- /dev/null +++ b/internal/app/service/security/service/security_integration_test.go @@ -0,0 +1,133 @@ +package service + +import ( + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/plugin" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + securityService SecurityService + email string + token string + proctorConfig config.ProctorConfig +} + +func (context *testContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_PLUGIN_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + context.proctorConfig = config.Config() + authPluginBinary := context.proctorConfig.AuthPluginBinary + authPluginExported := context.proctorConfig.AuthPluginExported + context.securityService = NewSecurityService(authPluginBinary, authPluginExported, plugin.NewGoPlugin()) + assert.NotNil(t, context.securityService) + email, _ := os.LookupEnv("GATE_PLUGIN_EMAIL") + assert.NotEmpty(t, email) + context.email = email + token, _ := os.LookupEnv("GATE_PLUGIN_TOKEN") + assert.NotEmpty(t, token) + context.token = token +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + ctx := &testContext{} + return ctx +} + +func TestSecurityService_AuthSuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := ctx.instance().email + token := ctx.instance().token + + userDetail, err := ctx.instance().securityService.Auth(email, token) + + assert.NoError(t, err) + assert.NotNil(t, userDetail) + assert.NotEmpty(t, userDetail.Email) + assert.NotEmpty(t, userDetail.Name) + assert.NotEmpty(t, userDetail.Active) + assert.NotNil(t, userDetail.Groups) +} + +func TestSecurityService_AuthPluginLoadFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := ctx.instance().email + token := ctx.instance().token + authPluginExported := ctx.instance().proctorConfig.AuthPluginExported + service := NewSecurityService("non-existent-plugin", authPluginExported, plugin.NewGoPlugin()) + + userDetail, err := service.Auth(email, token) + assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", "non-existent-plugin", authPluginExported)) + assert.Nil(t, userDetail) +} + +func TestSecurityService_AuthPluginFailedToCast(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := ctx.instance().email + token := ctx.instance().token + + authPluginBinary := ctx.instance().proctorConfig.AuthPluginBinary + service := NewSecurityService(authPluginBinary, "Verify", plugin.NewGoPlugin()) + + userDetail, err := service.Auth(email, token) + assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", authPluginBinary, "Verify")) + assert.Nil(t, userDetail) +} + +func TestSecurityService_VerifySuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := ctx.instance().email + token := ctx.instance().token + + userDetail, err := ctx.instance().securityService.Auth(email, token) + assert.NotNil(t, userDetail) + + group := userDetail.Groups + verified, err := ctx.instance().securityService.Verify(*userDetail, group) + assert.True(t, verified) + assert.NoError(t, err) +} + +func TestSecurityService_VerifyFailed(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + email := ctx.instance().email + token := ctx.instance().token + + userDetail, err := ctx.instance().securityService.Auth(email, token) + assert.NotNil(t, userDetail) + + group := []string{"non-existent-group", "proctor_executor"} + verified, err := ctx.instance().securityService.Verify(*userDetail, group) + assert.False(t, verified) + assert.NoError(t, err) +} diff --git a/internal/app/service/security/service/security_test.go b/internal/app/service/security/service/security_test.go deleted file mode 100644 index 52db5ff5..00000000 --- a/internal/app/service/security/service/security_test.go +++ /dev/null @@ -1,166 +0,0 @@ -package service - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "proctor/internal/app/service/infra/plugin" - "proctor/pkg/auth" - "testing" -) - -type context interface { - setUp(t *testing.T) - tearDown() - instance() *testContext -} - -type testContext struct { - pluginBinary string - exportedName string - goPlugin *plugin.GoPluginMock - securityService SecurityService - auth *auth.AuthMock -} - -func (context *testContext) setUp(t *testing.T) { - context.pluginBinary = "plugin-binary" - assert.NotEmpty(t, context.pluginBinary) - context.exportedName = "exported-name" - assert.NotEmpty(t, context.exportedName) - context.goPlugin = &plugin.GoPluginMock{} - assert.NotNil(t, context.goPlugin) - context.securityService = NewSecurityService(context.pluginBinary, context.exportedName, context.goPlugin) - assert.NotNil(t, context.securityService) - context.auth = &auth.AuthMock{} - assert.NotNil(t, context.auth) -} - -func (context *testContext) tearDown() { -} - -func (context *testContext) instance() *testContext { - return context -} - -func newContext() context { - ctx := &testContext{} - return ctx -} - -func TestSecurityService_AuthSuccess(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - email := "jasoet@go-jek.com" - token := "randomtokenforthesakeofopensource" - - userDetail := &auth.UserDetail{ - Name: "Deny Prasetyo", - Email: "jasoet87@gmail.com", - Active: true, - Groups: []string{"system", "proctor_executor"}, - } - ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) - - ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) - - expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) - assert.NoError(t, err) - assert.NotNil(t, expectedUserDetail) - assert.Equal(t, userDetail, expectedUserDetail) -} - -func TestSecurityService_AuthPluginLoadFailed(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - email := "jasoet@go-jek.com" - token := "randomtokenforthesakeofopensource" - - ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, fmt.Errorf("load error bro")) - - expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) - assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", ctx.instance().pluginBinary, ctx.instance().exportedName)) - assert.Nil(t, expectedUserDetail) -} - -func TestSecurityService_AuthPluginFailedToCast(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - email := "jasoet@go-jek.com" - token := "randomtokenforthesakeofopensource" - - userDetail := &auth.UserDetail{ - Name: "Deny Prasetyo", - Email: "jasoet87@gmail.com", - Active: true, - Groups: []string{"system", "proctor_executor"}, - } - - ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(userDetail, nil) - - expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) - assert.EqualError(t, err, fmt.Sprintf("failed to load and instantiate *auth.Auth from plugin location %s and exported name %s", ctx.instance().pluginBinary, ctx.instance().exportedName)) - assert.Nil(t, expectedUserDetail) -} - -func TestSecurityService_VerifySuccess(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - email := "jasoet@go-jek.com" - token := "randomtokenforthesakeofopensource" - - userDetail := &auth.UserDetail{ - Name: "Deny Prasetyo", - Email: "jasoet87@gmail.com", - Active: true, - Groups: []string{"system", "proctor_executor"}, - } - - ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) - - ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) - - expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) - assert.NoError(t, err) - assert.NotNil(t, expectedUserDetail) - assert.Equal(t, userDetail, expectedUserDetail) - - group := []string{"system", "proctor_executor"} - ctx.instance().auth.On("Verify", *expectedUserDetail, group).Return(true, nil) - verified, err := ctx.instance().securityService.Verify(*expectedUserDetail, group) - assert.True(t, verified) - assert.NoError(t, err) -} - -func TestSecurityService_VerifyFailed(t *testing.T) { - ctx := newContext() - ctx.setUp(t) - - email := "jasoet@go-jek.com" - token := "randomtokenforthesakeofopensource" - - userDetail := &auth.UserDetail{ - Name: "Deny Prasetyo", - Email: "jasoet87@gmail.com", - Active: true, - Groups: []string{"system", "proctor_executor"}, - } - - ctx.instance().auth.On("Auth", email, token).Return(userDetail, nil) - - ctx.instance().goPlugin.On("Load", ctx.instance().pluginBinary, ctx.instance().exportedName).Return(ctx.instance().auth, nil) - - expectedUserDetail, err := ctx.instance().securityService.Auth(email, token) - assert.NoError(t, err) - assert.NotNil(t, expectedUserDetail) - assert.Equal(t, userDetail, expectedUserDetail) - - group := []string{"system", "proctor_executor"} - ctx.instance().auth.On("Verify", *expectedUserDetail, group).Return(false, fmt.Errorf("verify error")) - verified, err := ctx.instance().securityService.Verify(*expectedUserDetail, group) - assert.False(t, verified) - assert.EqualError(t, err, "verify error") -} diff --git a/plugins/gate-auth-plugin/auth.go b/plugins/gate-auth-plugin/auth.go index af9cbdc0..8b8c2995 100644 --- a/plugins/gate-auth-plugin/auth.go +++ b/plugins/gate-auth-plugin/auth.go @@ -51,4 +51,4 @@ func newGateAuth() auth.Auth { } } -var Auth = newGateAuth() +var GateAuth = newGateAuth() From 97948fdaffdc7d3ed51e20bfe949d6590fc8a90e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 3 Sep 2019 11:46:51 +0700 Subject: [PATCH 178/268] [Dembo] Add env flag for auth --- .env.sample | 3 ++- .env.test | 1 + internal/app/service/infra/config/config.go | 7 ++++++ .../app/service/infra/config/config_test.go | 6 +++++ .../security/middleware/authentication.go | 8 +++++++ .../middleware/authentication_test.go | 21 +++++++++++++++++ .../security/middleware/authorization.go | 8 +++++++ .../security/middleware/authorization_test.go | 23 +++++++++++++++++++ 8 files changed, 76 insertions(+), 1 deletion(-) diff --git a/.env.sample b/.env.sample index e0426dc8..cb19df93 100644 --- a/.env.sample +++ b/.env.sample @@ -28,4 +28,5 @@ export PROCTOR_MAIL_SERVER_HOST="smtp.mail.com" export PROCTOR_MAIL_SERVER_PORT="123" export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" export PROCTOR_SENTRY_DSN="foo" -export PROCTOR_DOCS_PATH="/path/to/docs/dir" \ No newline at end of file +export PROCTOR_DOCS_PATH="/path/to/docs/dir" +export PROCTOR_AUTH_ENABLED=true diff --git a/.env.test b/.env.test index dbef83bf..798a2e65 100644 --- a/.env.test +++ b/.env.test @@ -35,3 +35,4 @@ export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth +export PROCTOR_AUTH_ENABLED=true diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index ac93682e..fec37943 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -14,6 +14,11 @@ func GetStringDefault(viper *viper.Viper, key string, defaultValue string) strin return viper.GetString(key) } +func GetBoolDefault(viper *viper.Viper, key string, defaultValue bool) bool { + viper.SetDefault(key, defaultValue) + return viper.GetBool(key) +} + func GetInt64Ref(viper *viper.Viper, key string) *int64 { value := viper.GetInt64(key) return &value @@ -74,6 +79,7 @@ type ProctorConfig struct { SentryDSN string DocsPath string AuthPluginBinary string + AuthEnabled bool } func Load() ProctorConfig { @@ -115,6 +121,7 @@ func Load() ProctorConfig { DocsPath: fang.GetString("DOCS_PATH"), AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), + AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", true), } return proctorConfig diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index fc5b6e12..642be974 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -202,3 +202,9 @@ func TestAuthPluginExported(t *testing.T) { assert.Equal(t, "path1", Load().AuthPluginExported) } + +func TestAuthEnabled(t *testing.T) { + _ = os.Setenv("PROCTOR_AUTH_ENABLED", "false") + + assert.Equal(t, false, Load().AuthEnabled) +} diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 8473d562..23c2e31d 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -4,6 +4,7 @@ import ( "context" "net/http" + "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/security/service" "proctor/internal/pkg/constant" @@ -11,10 +12,15 @@ import ( type authenticationMiddleware struct { service service.SecurityService + enabled bool } func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !middleware.enabled { + next.ServeHTTP(w, r) + return + } token := r.Header.Get(constant.AccessTokenHeaderKey) userEmail := r.Header.Get(constant.UserEmailHeaderKey) if token == "" || userEmail == "" { @@ -33,7 +39,9 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht } func NewAuthenticationMiddleware(securityService service.SecurityService) Middleware { + proctorConfig := config.Load() return &authenticationMiddleware{ service: securityService, + enabled: proctorConfig.AuthEnabled, } } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 7ad37a39..6bca9e5f 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -23,6 +23,7 @@ func (context *testContext) setUp(t *testing.T) { context.authMiddleware = authenticationMiddleware{} context.securityService = &service.SecurityServiceMock{} context.authMiddleware.service = context.securityService + context.authMiddleware.enabled = true fn := func(w http.ResponseWriter, r *http.Request) { } context.testHandler = fn @@ -144,3 +145,23 @@ func TestAuthenticationMiddleware_MiddlewareFuncAuthFailed(t *testing.T) { assert.Equal(t, http.StatusUnauthorized, resp.StatusCode) } + +func TestAuthenticationMiddleware_MiddlewareFuncDisabled(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + authMiddleware := ctx.instance().authMiddleware + authMiddleware.enabled = false + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + req, _ := http.NewRequest("GET", ts.URL, nil) + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index ab9cd739..2d12bbc4 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -9,6 +9,7 @@ import ( "github.com/gorilla/mux" "proctor/internal/app/service/execution/handler/parameter" + "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" "proctor/internal/app/service/schedule/model" @@ -19,6 +20,7 @@ import ( type authorizationMiddleware struct { service service.SecurityService metadataRepository repository.MetadataRepository + enabled bool } func (middleware *authorizationMiddleware) Secure(router *mux.Router, path string, handler http.Handler) *mux.Route { @@ -27,6 +29,10 @@ func (middleware *authorizationMiddleware) Secure(router *mux.Router, path strin func (middleware *authorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !middleware.enabled { + next.ServeHTTP(w, r) + return + } jobName, err := extractName(r) logger.LogErrors(err, "decode json", r.Body) if jobName == "" { @@ -82,8 +88,10 @@ func extractName(r *http.Request) (string, error) { } func NewAuthorizationMiddleware(securityService service.SecurityService, metadataRepository repository.MetadataRepository) AuthorizationMiddleware { + proctorConfig := config.Load() return &authorizationMiddleware{ service: securityService, metadataRepository: metadataRepository, + enabled: proctorConfig.AuthEnabled, } } diff --git a/internal/app/service/security/middleware/authorization_test.go b/internal/app/service/security/middleware/authorization_test.go index b2bebe9d..cfd8b7c4 100644 --- a/internal/app/service/security/middleware/authorization_test.go +++ b/internal/app/service/security/middleware/authorization_test.go @@ -37,6 +37,7 @@ func (context *authorizationContext) setUp(t *testing.T) { context.authorizationMiddleware.metadataRepository = context.metadataRepository context.securityService = &service.SecurityServiceMock{} context.authorizationMiddleware.service = context.securityService + context.authorizationMiddleware.enabled = true context.jobMetadata = &metadata.Metadata{ Name: "a-job", Description: "jobMetadata of a job", @@ -251,6 +252,28 @@ func TestAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { assert.Equal(t, http.StatusForbidden, responseResult.StatusCode) } +func TestAuthorizationMiddleware_MiddlewareFuncDisabled(t *testing.T) { + ctx := newAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + authzMiddleware := ctx.instance().authorizationMiddleware + authzMiddleware.enabled = false + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + ts := httptest.NewServer(authzMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + req, _ := http.NewRequest("GET", ts.URL, nil) + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + func TestAuthorizationMiddleware_Secure(t *testing.T) { ctx := newAuthorizationContext() ctx.setUp(t) From 8dbf85b3fb82bdaeb9d52d324ff624518e8025bb Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 3 Sep 2019 13:57:09 +0700 Subject: [PATCH 179/268] [Jasoet|Dembo] Rename wrong config name --- internal/app/service/infra/config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 07a13ad7..f26270ca 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -98,7 +98,7 @@ func Load() ProctorConfig { RedisMaxActiveConnections: fang.GetInt("REDIS_MAX_ACTIVE_CONNECTIONS"), LogsStreamReadBufferSize: fang.GetInt("LOGS_STREAM_READ_BUFFER_SIZE"), LogsStreamWriteBufferSize: fang.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE"), - KubeWaitForResourcePollCount: fang.GetInt("PROCTOR_KUBE_WAIT_FOR_RESOURCE_POLL_COUNT"), + KubeWaitForResourcePollCount: fang.GetInt("KUBE_WAIT_FOR_RESOURCE_POLL_COUNT"), KubePodsListWaitTime: time.Duration(fang.GetInt("KUBE_POD_LIST_WAIT_TIME")), KubeLogProcessWaitTime: time.Duration(fang.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")), KubeJobActiveDeadlineSeconds: GetInt64Ref(fang, "KUBE_JOB_ACTIVE_DEADLINE_SECONDS"), From 01eaf97665daed365d6a9134be46788b1d4d09b4 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 3 Sep 2019 14:07:00 +0700 Subject: [PATCH 180/268] [Jasoet|Dembo] Set auth default to false --- .env.sample | 2 +- .env.test | 2 +- internal/app/service/infra/config/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env.sample b/.env.sample index cb19df93..02913b56 100644 --- a/.env.sample +++ b/.env.sample @@ -29,4 +29,4 @@ export PROCTOR_MAIL_SERVER_PORT="123" export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" export PROCTOR_SENTRY_DSN="foo" export PROCTOR_DOCS_PATH="/path/to/docs/dir" -export PROCTOR_AUTH_ENABLED=true +export PROCTOR_AUTH_ENABLED=false diff --git a/.env.test b/.env.test index 4d1e472d..e8ae14cd 100644 --- a/.env.test +++ b/.env.test @@ -36,4 +36,4 @@ export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth -export PROCTOR_AUTH_ENABLED=true +export PROCTOR_AUTH_ENABLED=false diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index f26270ca..70980e45 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -123,7 +123,7 @@ func Load() ProctorConfig { DocsPath: fang.GetString("DOCS_PATH"), AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), - AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", true), + AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), } return proctorConfig From 1bef2fc4ea9d6285154d52ce85b07c7fd10f8d69 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 4 Sep 2019 14:46:46 +0700 Subject: [PATCH 181/268] [Dembo] Add docker capability --- .dockerignore | 10 ++++++++++ Dockerfile | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..00505ea8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.env +scripts/proctor.rb +_output/* + +*.swp +*.swo +.idea +.DS_Store +.vscode/ +dist/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fe61467e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.12 + +WORKDIR /go/src/app +COPY . . + +RUN make plugin.auth +RUN make server + +ENTRYPOINT ["./_output/bin/server"] +CMD ["s"] From b81d8e811a3291b97b1a0f2debd89885df6c9e3e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 5 Sep 2019 12:04:42 +0700 Subject: [PATCH 182/268] [Dembo] Use multistage build for docker --- Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe61467e..dab5919d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,14 @@ -FROM golang:1.12 - +FROM golang:1.12 AS builder WORKDIR /go/src/app COPY . . - RUN make plugin.auth RUN make server -ENTRYPOINT ["./_output/bin/server"] +FROM ubuntu:latest +WORKDIR /app/ +COPY --from=builder /go/src/app/_output/bin/server . +COPY --from=builder /go/src/app/_output/bin/plugin/auth.so . +ENV PROCTOR_AUTH_PLUGIN_BINARY=/app/auth.so + +ENTRYPOINT ["./server"] CMD ["s"] From 0100b148bed2faaa97310021e959c53809321f48 Mon Sep 17 00:00:00 2001 From: Bimo Horizon Date: Thu, 5 Sep 2019 12:43:57 +0700 Subject: [PATCH 183/268] Add error message when watch is throwing an error --- internal/app/service/infra/kubernetes/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index abe3e9c9..d83a9b37 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -195,6 +195,7 @@ func (client *kubernetesClient) WaitForReadyJob(executionName string, waitTime t for i := 0; i < config.KubeWaitForResourcePollCount(); i += 1 { watchJob, watchErr := jobs.Watch(listOptions) if watchErr != nil { + err = watchErr continue } @@ -245,12 +246,12 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t for i := 0; i < config.KubeWaitForResourcePollCount(); i += 1 { watchJob, watchErr := kubernetesPods.Watch(listOptions) if watchErr != nil { + err = watchErr continue } timeoutChan := time.After(waitTime) resultChan := watchJob.ResultChan() - defer watchJob.Stop() var pod *v1.Pod for { From 830ccb8b5cb719a699e63dbeb05136c484160ddc Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 5 Sep 2019 13:09:22 +0700 Subject: [PATCH 184/268] [Dembo] Copy migrations to docker image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index dab5919d..2de59d78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ FROM ubuntu:latest WORKDIR /app/ COPY --from=builder /go/src/app/_output/bin/server . COPY --from=builder /go/src/app/_output/bin/plugin/auth.so . +COPY --from=builder /go/src/app/migrations ./migrations ENV PROCTOR_AUTH_PLUGIN_BINARY=/app/auth.so ENTRYPOINT ["./server"] From 2d9d809b24c812f582b25a39488dcff47a105730 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 6 Sep 2019 14:52:55 +0700 Subject: [PATCH 185/268] [Dembo] Install ca-certificates in docker image --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index 2de59d78..fc2b4eeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,8 @@ RUN make plugin.auth RUN make server FROM ubuntu:latest +RUN apt-get update +RUN apt-get install -y ca-certificates WORKDIR /app/ COPY --from=builder /go/src/app/_output/bin/server . COPY --from=builder /go/src/app/_output/bin/plugin/auth.so . From 98d633adb120442178eaffd8414ef414866da7f6 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 6 Sep 2019 18:37:55 +0700 Subject: [PATCH 186/268] [Dembo] Change cli schedule id payload --- internal/app/cli/command/schedule/schedule.go | 2 +- internal/app/cli/daemon/client.go | 14 +++++++------- internal/app/cli/daemon/client_mock.go | 4 ++-- internal/app/cli/daemon/client_test.go | 4 ++-- internal/app/service/schedule/handler/http.go | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/app/cli/command/schedule/schedule.go b/internal/app/cli/command/schedule/schedule.go index 049630fe..0ec419ea 100644 --- a/internal/app/cli/command/schedule/schedule.go +++ b/internal/app/cli/command/schedule/schedule.go @@ -66,7 +66,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { print() return } - printer.Println(fmt.Sprintf("Scheduled Job UUID : %s", scheduledJobID), color.FgGreen) + printer.Println(fmt.Sprintf("Scheduled Job UUID : %d", scheduledJobID), color.FgGreen) }, } diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index e4c74844..87ab1361 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -40,7 +40,7 @@ type Client interface { StreamProcLogs(executionId uint64) error GetExecutionContextStatusWithPolling(executionId uint64) (*modelExecution.ExecutionResult, error) GetExecutionContextStatus(executionId uint64) (*modelExecution.ExecutionResult, error) - ScheduleJob(string, string, string, string, string, map[string]string) (string, error) + ScheduleJob(string, string, string, string, string, map[string]string) (uint64, error) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) DescribeScheduledProc(string) (modelSchedule.ScheduledJob, error) RemoveScheduledProc(string) error @@ -63,7 +63,7 @@ type ProcToExecute struct { } type ScheduleJobPayload struct { - ID string `json:"id"` + ID uint64 `json:"id"` Name string `json:"name"` Tags string `json:"tags"` Time string `json:"time"` @@ -80,10 +80,10 @@ func NewClient(printer io.Printer, proctorConfigLoader config.Loader) Client { } } -func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, jobArgs map[string]string) (string, error) { +func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, jobArgs map[string]string) (uint64, error) { err := c.loadProctorConfig() if err != nil { - return "", err + return 0, err } jobPayload := ScheduleJobPayload{ Name: name, @@ -96,7 +96,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, requestBody, err := json.Marshal(jobPayload) if err != nil { - return "", err + return 0, err } client := &http.Client{} @@ -108,12 +108,12 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, resp, err := client.Do(req) if err != nil { - return "", buildNetworkError(err) + return 0, buildNetworkError(err) } defer resp.Body.Close() if resp.StatusCode != http.StatusCreated { - return "", buildHTTPError(c, resp) + return 0, buildHTTPError(c, resp) } var scheduledJob ScheduleJobPayload diff --git a/internal/app/cli/daemon/client_mock.go b/internal/app/cli/daemon/client_mock.go index 9e33f79b..6fd259e0 100644 --- a/internal/app/cli/daemon/client_mock.go +++ b/internal/app/cli/daemon/client_mock.go @@ -42,9 +42,9 @@ func (m *MockClient) GetExecutionContextStatus(executionId uint64) (*modelExecut return args.Get(0).(*modelExecution.ExecutionResult), args.Error(1) } -func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, group string, jobArgs map[string]string) (string, error) { +func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, group string, jobArgs map[string]string) (uint64, error) { args := m.Called(name, tags, time, notificationEmails, group, jobArgs) - return args.Get(0).(string), args.Error(1) + return args.Get(0).(uint64), args.Error(1) } func (m *MockClient) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob, error) { diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index de2837b6..a4cd1939 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -225,7 +225,7 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - expectedProcResponse := "8965fce9-5025-43b3-b21c-920c5ff41cd9" + expectedProcResponse := uint64(7) procName := "run-sample" time := "*/1 * * * *" notificationEmails := "user@mail.com" @@ -233,7 +233,7 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { group := "test" procArgs := map[string]string{"ARG_ONE": "sample-value"} - body := `{"id":"8965fce9-5025-43b3-b21c-920c5ff41cd9","name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup", "group":"test"}` + body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup", "group":"test"}` httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 669b0ca8..686a4810 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -60,7 +60,7 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { _, err = cron.Parse(schedule.Cron) if err != nil { - logger.Error(fmt.Sprintf("Cron format is invalid: %s ", schedule.Tags), schedule.JobName, schedule.Cron) + logger.LogErrors(err, fmt.Sprintf("Cron format is invalid: %s ", schedule.Tags), schedule.JobName, schedule.Cron) response.WriteHeader(http.StatusBadRequest) _, _ = response.Write([]byte(status.ScheduleCronFormatInvalidError)) From a6f6f2e814c2df0bbecb5df615b1fef1bb969a2f Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 6 Sep 2019 19:03:01 +0700 Subject: [PATCH 187/268] [Dembo] Remove schedule job payload model from cli --- .../cli/command/schedule/describe/describe.go | 14 +++++++++--- .../schedule/describe/describe_test.go | 2 +- .../app/cli/command/schedule/list/list.go | 2 +- internal/app/cli/daemon/client.go | 20 +++++------------ internal/app/cli/daemon/client_mock.go | 2 +- internal/app/cli/daemon/client_test.go | 22 +++++++++---------- .../service/execution/service/execution.go | 1 + .../app/service/infra/kubernetes/client.go | 1 + internal/pkg/model/schedule/schedule_job.go | 2 +- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/internal/app/cli/command/schedule/describe/describe.go b/internal/app/cli/command/schedule/describe/describe.go index 1616d559..83d80d73 100644 --- a/internal/app/cli/command/schedule/describe/describe.go +++ b/internal/app/cli/command/schedule/describe/describe.go @@ -2,8 +2,11 @@ package describe import ( "fmt" + "strconv" + "github.com/fatih/color" "github.com/spf13/cobra" + "proctor/internal/app/cli/daemon" "proctor/internal/app/cli/utility/io" ) @@ -13,17 +16,22 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { Use: "describe", Short: "Describe scheduled job", Long: "This command helps to describe scheduled job", - Example: fmt.Sprintf("proctor schedule describe D958FCCC-F2B3-49D1-B83A-4E70A2A775A0"), + Example: fmt.Sprintf("proctor schedule describe 502376124721"), Run: func(cmd *cobra.Command, args []string) { - jobID := args[0] + jobID, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + printer.Println(err.Error(), color.FgRed) + return + } + scheduledProc, err := proctorDClient.DescribeScheduledProc(jobID) if err != nil { printer.Println(err.Error(), color.FgRed) return } - printer.Println(fmt.Sprintf("%-40s %-100s", "ID", scheduledProc.ID), color.Reset) + printer.Println(fmt.Sprintf("%-40s %-100d", "ID", scheduledProc.ID), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "PROC NAME", scheduledProc.Name), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "GROUP NAME", scheduledProc.Group), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "TAGS", scheduledProc.Tags), color.Reset) diff --git a/internal/app/cli/command/schedule/describe/describe_test.go b/internal/app/cli/command/schedule/describe/describe_test.go index aa7324bd..9d5b8a92 100644 --- a/internal/app/cli/command/schedule/describe/describe_test.go +++ b/internal/app/cli/command/schedule/describe/describe_test.go @@ -25,7 +25,7 @@ func (s *ScheduleCreateCmdTestSuite) SetupTest() { func (s *ScheduleCreateCmdTestSuite) TestScheduleCreateCmdHelp() { assert.Equal(s.T(), "Describe scheduled job", s.testScheduleDescribeCmd.Short) assert.Equal(s.T(), "This command helps to describe scheduled job", s.testScheduleDescribeCmd.Long) - assert.Equal(s.T(), "proctor schedule describe D958FCCC-F2B3-49D1-B83A-4E70A2A775A0", s.testScheduleDescribeCmd.Example) + assert.Equal(s.T(), "proctor schedule describe 502376124721", s.testScheduleDescribeCmd.Example) } func TestScheduleCreateCmdTestSuite(t *testing.T) { diff --git a/internal/app/cli/command/schedule/list/list.go b/internal/app/cli/command/schedule/list/list.go index 8d6d9b8f..22703b4d 100644 --- a/internal/app/cli/command/schedule/list/list.go +++ b/internal/app/cli/command/schedule/list/list.go @@ -25,7 +25,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { printer.Println(fmt.Sprintf("%-40s %-30s %-20s %s", "ID", "PROC NAME", "GROUP NAME", "TAGS"), color.FgGreen) for _, scheduledProc := range scheduledProcs { - printer.Println(fmt.Sprintf("%-40s %-30s %-20s %s", scheduledProc.ID, scheduledProc.Name, scheduledProc.Group, scheduledProc.Tags), color.Reset) + printer.Println(fmt.Sprintf("%-40d %-30s %-20s %s", scheduledProc.ID, scheduledProc.Name, scheduledProc.Group, scheduledProc.Tags), color.Reset) } }, } diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 87ab1361..62dcbbe8 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -42,7 +42,7 @@ type Client interface { GetExecutionContextStatus(executionId uint64) (*modelExecution.ExecutionResult, error) ScheduleJob(string, string, string, string, string, map[string]string) (uint64, error) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) - DescribeScheduledProc(string) (modelSchedule.ScheduledJob, error) + DescribeScheduledProc(uint64) (modelSchedule.ScheduledJob, error) RemoveScheduledProc(string) error } @@ -62,16 +62,6 @@ type ProcToExecute struct { Args map[string]string `json:"args"` } -type ScheduleJobPayload struct { - ID uint64 `json:"id"` - Name string `json:"name"` - Tags string `json:"tags"` - Time string `json:"time"` - NotificationEmails string `json:"notification_emails"` - Group string `json:"group_name"` - Args map[string]string `json:"args"` -} - func NewClient(printer io.Printer, proctorConfigLoader config.Loader) Client { return &client{ clientVersion: version.ClientVersion, @@ -85,7 +75,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, if err != nil { return 0, err } - jobPayload := ScheduleJobPayload{ + jobPayload := modelSchedule.ScheduledJob{ Name: name, Tags: tags, Time: time, @@ -116,7 +106,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, return 0, buildHTTPError(c, resp) } - var scheduledJob ScheduleJobPayload + var scheduledJob modelSchedule.ScheduledJob err = json.NewDecoder(resp.Body).Decode(&scheduledJob) return scheduledJob.ID, err @@ -197,7 +187,7 @@ func (c *client) ListScheduledProcs() ([]modelSchedule.ScheduledJob, error) { return scheduledProcsList, err } -func (c *client) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob, error) { +func (c *client) DescribeScheduledProc(jobID uint64) (modelSchedule.ScheduledJob, error) { err := c.loadProctorConfig() if err != nil { return modelSchedule.ScheduledJob{}, err @@ -206,7 +196,7 @@ func (c *client) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob client := &http.Client{ Timeout: c.connectionTimeoutSecs, } - url := fmt.Sprintf("http://"+c.proctordHost+ScheduleRoute+"/%s", jobID) + url := fmt.Sprintf("http://"+c.proctordHost+ScheduleRoute+"/%d", jobID) req, err := http.NewRequest("GET", url, nil) req.Header.Add(constant.UserEmailHeaderKey, c.emailId) req.Header.Add(constant.AccessTokenHeaderKey, c.accessToken) diff --git a/internal/app/cli/daemon/client_mock.go b/internal/app/cli/daemon/client_mock.go index 6fd259e0..1a22cd50 100644 --- a/internal/app/cli/daemon/client_mock.go +++ b/internal/app/cli/daemon/client_mock.go @@ -47,7 +47,7 @@ func (m *MockClient) ScheduleJob(name, tags, time, notificationEmails string, gr return args.Get(0).(uint64), args.Error(1) } -func (m *MockClient) DescribeScheduledProc(jobID string) (modelSchedule.ScheduledJob, error) { +func (m *MockClient) DescribeScheduledProc(jobID uint64) (modelSchedule.ScheduledJob, error) { args := m.Called(jobID) return args.Get(0).(modelSchedule.ScheduledJob), args.Error(1) } diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index a4cd1939..509554fc 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -640,15 +640,15 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - jobID := "8965fce9-5025-43b3-b21c-920c5ff41cd9" - body := `{"id":"8965fce9-5025-43b3-b21c-920c5ff41cd9","name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup"}` + jobID := uint64(7) + body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup"}` httpmock.Activate() defer httpmock.DeactivateAndReset() mockResponse := httpmock.NewStringResponse(200, body) mockError := error(nil) - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%d", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -664,7 +664,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - jobID := "invalid-job-id" + jobID := uint64(0) body := "Invalid Job ID" httpmock.Activate() @@ -672,7 +672,7 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWithInvalidJobID() { mockResponse := httpmock.NewStringResponse(400, body) mockError := error(nil) - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%d", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -686,14 +686,14 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWhenJobIDNotFound() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - jobID := "invalid-job-id" + jobID := uint64(7) httpmock.Activate() defer httpmock.DeactivateAndReset() mockResponse := httpmock.NewStringResponse(404, "Job not found") mockError := error(nil) - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%d", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -707,14 +707,14 @@ func (s *ClientTestSuite) TestDescribeScheduledJobWitInternalServerError() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - jobID := "invalid-job-id" + jobID := uint64(0) httpmock.Activate() defer httpmock.DeactivateAndReset() mockResponse := httpmock.NewStringResponse(500, "Schedule Failed") mockError := error(nil) - mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) + mockRequest(proctorConfig, "GET", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%d", jobID), mockResponse, mockError) s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() @@ -728,8 +728,8 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { t := s.T() proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} - jobID := "c3e040b1-c2b8-4d23-bebd-246c8b7c6f87" - body := `[{"id":"c3e040b1-c2b8-4d23-bebd-246c8b7c6f87","name":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","time":"0 2 * * *","tags":"sample,proctor"}]` + jobID := uint64(7) + body := `[{"id":7,"name":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","time":"0 2 * * *","tags":"sample,proctor"}]` httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 509145c9..cb44fce2 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -148,6 +148,7 @@ func (service *executionService) watchProcess(context model.ExecutionContext) { logger.Info("Job Ready for ", context.ExecutionID) pod, err := service.kubernetesClient.WaitForReadyPod(context.Name, waitTime) + logger.LogErrors(err, "wait for ready pod", pod) if err != nil { context.Status = status.PodCreationFailed return diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index e3cc661b..0daec2a9 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -285,6 +285,7 @@ func (client *kubernetesClient) WaitForReadyPod(executionName string, waitTime t } } + logger.Info("Wait for ready pod return pod ", nil, " and error ", err) return nil, err } diff --git a/internal/pkg/model/schedule/schedule_job.go b/internal/pkg/model/schedule/schedule_job.go index 153e3769..9b46887a 100644 --- a/internal/pkg/model/schedule/schedule_job.go +++ b/internal/pkg/model/schedule/schedule_job.go @@ -1,7 +1,7 @@ package schedule type ScheduledJob struct { - ID string `json:"id"` + ID uint64 `json:"id"` Name string `json:"name"` Args map[string]string `json:"args"` NotificationEmails string `json:"notification_emails"` From c86016efabf94618407fd0f58e99e2da2edac6f4 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 6 Sep 2019 19:25:58 +0700 Subject: [PATCH 188/268] [Dembo] Change schedule time to cron --- .../app/cli/command/schedule/describe/describe.go | 2 +- internal/app/cli/command/schedule/schedule.go | 12 ++++++------ internal/app/cli/daemon/client.go | 4 ++-- internal/app/cli/daemon/client_test.go | 14 +++++++------- internal/pkg/model/schedule/schedule_job.go | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/app/cli/command/schedule/describe/describe.go b/internal/app/cli/command/schedule/describe/describe.go index 83d80d73..cfaf3322 100644 --- a/internal/app/cli/command/schedule/describe/describe.go +++ b/internal/app/cli/command/schedule/describe/describe.go @@ -35,7 +35,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { printer.Println(fmt.Sprintf("%-40s %-100s", "PROC NAME", scheduledProc.Name), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "GROUP NAME", scheduledProc.Group), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "TAGS", scheduledProc.Tags), color.Reset) - printer.Println(fmt.Sprintf("%-40s %-100s", "Time", scheduledProc.Time), color.Reset) + printer.Println(fmt.Sprintf("%-40s %-100s", "Cron", scheduledProc.Cron), color.Reset) printer.Println(fmt.Sprintf("%-40s %-100s", "Notifier", scheduledProc.NotificationEmails), color.Reset) printer.Println("\nArgs", color.FgMagenta) diff --git a/internal/app/cli/command/schedule/schedule.go b/internal/app/cli/command/schedule/schedule.go index 0ec419ea..cf13fc73 100644 --- a/internal/app/cli/command/schedule/schedule.go +++ b/internal/app/cli/command/schedule/schedule.go @@ -14,13 +14,13 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { Use: "schedule", Short: "Create scheduled jobs", Long: "This command helps to create scheduled jobs", - Example: fmt.Sprintf("proctor schedule run-sample -g my-group -t '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar"), + Example: fmt.Sprintf("proctor schedule run-sample -g my-group -c '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar"), Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { procName := args[0] printer.Println(fmt.Sprintf("%-40s %-100s", "Creating Scheduled Job", procName), color.Reset) - time, err := cmd.Flags().GetString("time") + cron, err := cmd.Flags().GetString("cron") if err != nil { printer.Println(err.Error(), color.FgRed) } @@ -60,7 +60,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { printer.Println("With No Variables", color.FgRed) } - scheduledJobID, err := proctorDClient.ScheduleJob(procName, tags, time, notificationEmails, group, jobArgs) + scheduledJobID, err := proctorDClient.ScheduleJob(procName, tags, cron, notificationEmails, group, jobArgs) if err != nil { printer.Println(err.Error(), color.FgRed) print() @@ -70,10 +70,10 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { }, } - var Time, NotifyEmails, Tags, Group string + var Cron, NotifyEmails, Tags, Group string - scheduleCmd.PersistentFlags().StringVarP(&Time, "time", "t", "", "Schedule time") - _ = scheduleCmd.MarkFlagRequired("time") + scheduleCmd.PersistentFlags().StringVarP(&Cron, "cron", "c", "", "Schedule cron") + _ = scheduleCmd.MarkFlagRequired("cron") scheduleCmd.PersistentFlags().StringVarP(&Group, "group", "g", "", "Group Name") _ = scheduleCmd.MarkFlagRequired("group") scheduleCmd.PersistentFlags().StringVarP(&NotifyEmails, "notify", "n", "", "Notifier Email ID's") diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 62dcbbe8..39d58ed7 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -70,7 +70,7 @@ func NewClient(printer io.Printer, proctorConfigLoader config.Loader) Client { } } -func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, jobArgs map[string]string) (uint64, error) { +func (c *client) ScheduleJob(name, tags, cron, notificationEmails, group string, jobArgs map[string]string) (uint64, error) { err := c.loadProctorConfig() if err != nil { return 0, err @@ -78,7 +78,7 @@ func (c *client) ScheduleJob(name, tags, time, notificationEmails, group string, jobPayload := modelSchedule.ScheduledJob{ Name: name, Tags: tags, - Time: time, + Cron: cron, NotificationEmails: notificationEmails, Args: jobArgs, Group: group, diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 509554fc..2dc0e2e9 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -227,13 +227,13 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} expectedProcResponse := uint64(7) procName := "run-sample" - time := "*/1 * * * *" + cron := "*/1 * * * *" notificationEmails := "user@mail.com" tags := "db,backup" group := "test" procArgs := map[string]string{"ARG_ONE": "sample-value"} - body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup", "group":"test"}` + body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","cron":"*/1 * * * *","tags":"db,backup", "group":"test"}` httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -244,7 +244,7 @@ func (s *ClientTestSuite) TestSuccessScheduledJob() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - executeProcResponse, err := s.testClient.ScheduleJob(procName, tags, time, notificationEmails, group, procArgs) + executeProcResponse, err := s.testClient.ScheduleJob(procName, tags, cron, notificationEmails, group, procArgs) assert.NoError(t, err) assert.Equal(t, expectedProcResponse, executeProcResponse) @@ -256,7 +256,7 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} procName := "run-sample" - time := "*/1 * * * *" + cron := "*/1 * * * *" notificationEmails := "user@mail.com" tags := "db,backup" procArgs := map[string]string{"ARG_ONE": "sample-value"} @@ -271,7 +271,7 @@ func (s *ClientTestSuite) TestSchedulingAlreadyExistedScheduledJob() { s.mockConfigLoader.On("Load").Return(proctorConfig, config.ConfigError{}).Once() - _, err := s.testClient.ScheduleJob(procName, tags, time, notificationEmails, group, procArgs) + _, err := s.testClient.ScheduleJob(procName, tags, cron, notificationEmails, group, procArgs) assert.Equal(t, "Server Error!!!\nStatus Code: 409, Conflict", err.Error()) s.mockConfigLoader.AssertExpectations(t) } @@ -641,7 +641,7 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} jobID := uint64(7) - body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","time":"*/1 * * * *","tags":"db,backup"}` + body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","cron":"*/1 * * * *","tags":"db,backup"}` httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -729,7 +729,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} jobID := uint64(7) - body := `[{"id":7,"name":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","time":"0 2 * * *","tags":"sample,proctor"}]` + body := `[{"id":7,"name":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","cron":"0 2 * * *","tags":"sample,proctor"}]` httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/internal/pkg/model/schedule/schedule_job.go b/internal/pkg/model/schedule/schedule_job.go index 9b46887a..ae8325a5 100644 --- a/internal/pkg/model/schedule/schedule_job.go +++ b/internal/pkg/model/schedule/schedule_job.go @@ -5,7 +5,7 @@ type ScheduledJob struct { Name string `json:"name"` Args map[string]string `json:"args"` NotificationEmails string `json:"notification_emails"` - Time string `json:"time"` + Cron string `json:"cron"` Tags string `json:"tags"` Group string `json:"group_name"` } From 2d149fdd9598ff73c5fd224ea0bf66bb81087899 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 6 Sep 2019 19:40:37 +0700 Subject: [PATCH 189/268] [Dembo] Sync schedule payload model between cli and service --- internal/app/cli/command/schedule/schedule_test.go | 2 +- internal/app/cli/daemon/client_test.go | 4 ++-- internal/pkg/model/schedule/schedule_job.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/app/cli/command/schedule/schedule_test.go b/internal/app/cli/command/schedule/schedule_test.go index 3f637aeb..1df34018 100644 --- a/internal/app/cli/command/schedule/schedule_test.go +++ b/internal/app/cli/command/schedule/schedule_test.go @@ -25,7 +25,7 @@ func (s *ScheduleCreateCmdTestSuite) SetupTest() { func (s *ScheduleCreateCmdTestSuite) TestScheduleCreateCmdHelp() { assert.Equal(s.T(), "Create scheduled jobs", s.testScheduleCreateCmd.Short) assert.Equal(s.T(), "This command helps to create scheduled jobs", s.testScheduleCreateCmd.Long) - assert.Equal(s.T(), "proctor schedule run-sample -g my-group -t '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar", s.testScheduleCreateCmd.Example) + assert.Equal(s.T(), "proctor schedule run-sample -g my-group -c '0 2 * * *' -n 'username@mail.com' -T 'sample,proctor' ARG_ONE1=foobar", s.testScheduleCreateCmd.Example) } func TestScheduleCreateCmdTestSuite(t *testing.T) { diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 2dc0e2e9..52809afb 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -641,7 +641,7 @@ func (s *ClientTestSuite) TestSuccessDescribeScheduledJob() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} jobID := uint64(7) - body := `{"id":7,"name":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","cron":"*/1 * * * *","tags":"db,backup"}` + body := `{"id":7,"jobName":"run-sample","args":{"ARG_ONE":"sample-value"},"notification_emails":"user@mail.com","cron":"*/1 * * * *","tags":"db,backup"}` httpmock.Activate() defer httpmock.DeactivateAndReset() @@ -729,7 +729,7 @@ func (s *ClientTestSuite) TestSuccessListOfScheduledJobs() { proctorConfig := config.ProctorConfig{Host: "proctor.example.com", Email: "proctor@example.com", AccessToken: "access-token"} jobID := uint64(7) - body := `[{"id":7,"name":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","cron":"0 2 * * *","tags":"sample,proctor"}]` + body := `[{"id":7,"jobName":"run-sample","args":{"ARG2":"bar","ARG3":"test","ARG_ONE1":"foobar"},"notification_emails":"username@mail.com","cron":"0 2 * * *","tags":"sample,proctor"}]` httpmock.Activate() defer httpmock.DeactivateAndReset() diff --git a/internal/pkg/model/schedule/schedule_job.go b/internal/pkg/model/schedule/schedule_job.go index ae8325a5..0740f316 100644 --- a/internal/pkg/model/schedule/schedule_job.go +++ b/internal/pkg/model/schedule/schedule_job.go @@ -2,10 +2,10 @@ package schedule type ScheduledJob struct { ID uint64 `json:"id"` - Name string `json:"name"` + Name string `json:"jobName"` Args map[string]string `json:"args"` - NotificationEmails string `json:"notification_emails"` + NotificationEmails string `json:"notificationEmails"` Cron string `json:"cron"` Tags string `json:"tags"` - Group string `json:"group_name"` + Group string `json:"group"` } From 390b6769e1e33823b7673f9cf1cbc2c41faf9713 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 9 Sep 2019 16:38:35 +0700 Subject: [PATCH 190/268] [Dembo] Add readme content --- README.md | 112 ++++++++++++++++++++++------- assets/img/proctor_components.jpg | Bin 0 -> 33758 bytes docs/creating_procs.md | 115 ++++++++++++++++++++++++++++++ docs/glossary.md | 30 ++++++++ 4 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 assets/img/proctor_components.jpg create mode 100644 docs/creating_procs.md create mode 100644 docs/glossary.md diff --git a/README.md b/README.md index 7c18a731..8c4974ed 100644 --- a/README.md +++ b/README.md @@ -8,41 +8,104 @@

## Description +Proctor is a set of components that allow user to do automated task with configurable access policy. +Bundle repetitive task as a automation and turn it into `procs` to make it easier for user to do it themself. -Proctor is a developer friendly automation orchestrator. It helps everyone use automation and contribute to it +Before we goes deep down into explanation about proctor, you may want to read about [Proctor Glossary](docs/glossary.md) -### Dev environment setup +## Installation +This section provide installation for unix environment. + +General step * Install and setup golang -* Clone the repository -* Run `make build` +* Clone this repository +* Run `make build`. This will generate binary for proctor cli and service + +For proctor service + +* Make sure you have running Redis server +* Make sure you have running Postgres server +* Make sure you have running Kubernetes Cluster, for setting up local cluster, please refer [here](https://kubernetes.io/docs/setup/learning-environment/minikube/) +* Copy `.env.sample` into `.env` file. Please refer [here](#proctor-service-configuration-explanation) for configuration explanation +* Make sure you set correct value in `.env` for Kubernetes, Postgresql, and Redis +* Export value of `.env` by running `source .env` +* Run `./_output/bin/server s` to start proctor service + +For proctor cli + +* Run `./_output/bin/cli config PROCTOR_HOST=` to point you proctor cli to local proctor service +* Run `./_output/bin/cli` to see complete usage of proctor cli + +## Proctor Components +Here's the overview of proctor components. +![Proctor component](./assets/img/proctor_components.jpg) + +#### Proctor CLI +Proctor cli is a command line interface that used by client to interact with Proctor service. +Proctor cli read user configuration such as Proctor service host, user email, and user token from `~/.proctor/proctor.yaml`. + +#### Proctor Service +Proctor service govern the main process of Proctor such as: + * Create execution context + * Create and read procs metadata + * Create and read procs secret + * Order the execution of procs + * Get execution status and log of running procs + +#### Context Store +Currently Proctor service use postgres to store execution context of procs. + +#### Metadata Store +Metadata store contain all procs metadata, procs that doesn't have metadata on store cannot be executed. + +#### Secret Store +Secret store contain secret value that needed by procs to executed. + +#### Executor +Executor is the one that executing the procs, we use Kubernetes Job as executor. +Proctor service will send the procs name, needed args then executor will pull necessary image to run the procs. +Proctor service will occasionally contact executor to get status of requested procs. + +## Procs Execution Flow +Here's what happen between Proctor components when client want to execute a procs. + 1. Cli send execution request to service. This request consist of procs name, procs args, and user credentials. + 2. Service get metadata and secret for requested procs. + 3. Service create execution context to store data related to procs execution. + 4. Service tell the executor to run the procs image along with user argument and procs secret. + 5. Service watch the process run by executor by getting the log and execution status then write it to execution context. + +## Security flow +Some route is protected by authentication, authorization or both. +Authenticated user means that user should have account related with proctor. +Authorized user means that user should be part of groups that defined on procs meatadata, for example when procs authorized groups is `proctor-user`, and `dev` then user need to be a member of both groups. + +Proctor doesn't come with built in auth implementation, it's using configurable [plugin](#plugin) mechanism -### proctord +## Plugin +Proctor service support plugin for authentication and authorization using go plugin. +It create limitation that proctor service can only be used on Linux and MacOS (Until the time go plugin support other OS). -* `proctord` is the heart of the automation orchestrator -* It is a web service that handles management and execution of procs +By using authentication and authorization as a plugin you can customize this process using your organization current user management. -### Dev environment setup +#### How to make custom plugin +Creating plugin for auth (authentication and authorization) means creating a struct that implement [Auth interface](./pkg/auth/auth.go) then initialize it. +Begin using your custom plugin by building it with below command: +```sh +go build -race -buildmode=plugin -o ./_output/bin/plugin/auth.so +``` +Plug your custom plugin to proctor service by pointing the `PROCTOR_AUTH_PLUGIN_EXPORTED` environment variable to variable exported by plugin and `PROCTOR_AUTH_PLUGIN_BINARY` to `auth.so` file generated by command above. +After you finish it then proctor service will use your plugin. -* Ensure local postgres server is up and running -* Ensure local redis server is up and running -* Install kubectl -* Configure kubectl to point to desired kubernetes cluster. For setting up kubernetes cluster locally, refer [here](https://kubernetes.io/docs/getting-started-guides/minikube/) -* Run a kubectl proxy server on your local machine -* [Configure proctord](#proctord-configuration) -* Setup & Run database migrations by running this command `make db.setup` from the repo directory -* Start service by `make start-server` -* Run `curl {host-address:port}/ping` for health-check of service +#### Examples +If you like to jump in directly to example implementation then you can explore our default auth plugin [here](./plugins/gate-auth-plugin/auth.go). +This implementation is using [Gate](https://github.com/gate-sso/gate) as its user management. -#### proctord configuration +## Procs Creation -* Copy `.env.sample` into `.env` file -* Please refer meaning of `proctord` configuration [here](#proctord-configuration-explanation) -* Modify configuration for dev setup in `.env` file -* Export environment variables configured in `.env` file by running `source .env` -* `proctor server` gets configuration from environment variables. +You can read [here](./docs/creating_procs.md) to learn more about creating procs. -#### proctord configuration explanation +## Proctor Service Configuration Explanation * `PROCTOR_APP_PORT` is port on which service will run * `PROCTOR_LOG_LEVEL` defines log levels of service. Available options are: `debug`,`info`,`warn`,`error`,`fatal`,`panic` @@ -55,6 +118,7 @@ Proctor is a developer friendly automation orchestrator. It helps everyone use a * If a job doesn't reach completion, it is terminated after `PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS` * `PROCTOR_KUBE_JOB_RETRIES` is the number of retries for a kubernetes job (on failure) * `PROCTOR_DEFAULT_NAMESPACE` is the namespace under which jobs will be run in kubernetes cluster. By default, K8s has namespace "default". If you set another value, please create namespace in K8s before deploying `proctord` +* `PROCTOR_KUBE_CONTEXT` is used the name of context you want to use when running out of cluster. * `PROCTOR_KUBE_CLUSTER_HOST_NAME` is address/ip address to api-server of kube cluster. It is used for fetching logs of a pod using https * `PROCTOR_KUBE_CA_CERT_ENCODED` is the CA cert file encoded in base64. This is used for establishing authority while talking to kubernetes api-server on a public https call * `PROCTOR_KUBE_BASIC_AUTH_ENCODED` is the base64 encoded authentication of kubernetes. Enocde `username:password` to base64 and set this config. diff --git a/assets/img/proctor_components.jpg b/assets/img/proctor_components.jpg new file mode 100644 index 0000000000000000000000000000000000000000..ab3f99456e9e4304447db2fd71de22905dd252b8 GIT binary patch literal 33758 zcmeFZcT|&2yDuC?ks>HXkS>tWi&W_-J)wu*6-XdJKzheU?}QKtRe{ipw9o_#oq$Lc z2%r=}x^xi4k7vJof9HAEI_G_!z1Cj)oImztt^3a0*UVg#duFaSzsc{(-(LYYAsS!} zz@{1;5QMV3Lv@sNBZMRa*?i(Uil-Dk&<2|B`2exASWXyr?`IOIt3*)B{?}2 z4Hfl`n>T50Qe3B{qrFLY@%`o>ja>SpCdn1Di;6cX$tf?M{wLG#cK{l)OHr2{NG@>! zF4J5hp}F+C6To)S)k}cOB!6`H-_teHt7Ifs$S?mv*1HJ+kX|CWL_&6zjO;22$<>RW zb(!SKRnluTv}APjoVOSlZ*z$mjz2Ipf0WM6Bd+3~h8YD)+>?QtHQ<>f?Y*CogH)wF zQ;cAK&+9MRqyH11e`oi{Q=0$9{Xd>g1Fn-?bc}|C2A~Z1S@a*B`&)Mf2_CiT=(Ceu zYyyLXOW;bb1t&fW?2hBt{EAMY!D5eTZth)AtbXD+Rs9>##G*s&bvQ>m44tGo#x0| zgbdXQfFv(MRD|e2dWT)_HS|ClVAIv3;yIy|B40$}T6^Yx=u?*jDluc{0~lUCROywp zdmSi7lV-&MHK-biAHh%b@6c_^pLV$@PSp`!}*J=KwdM$vTtAXr5Q%n9JCTfV(Eo$U??D z^RdeBGp^U4q&iLqR6-sTP|?wGNpG(C&@Vbe4H{|Fgbu+>8!o>An>!q@_Ku)@K?RKy z?io|#!uLyF_22IGq8Mwl(Cf_Dr5})`6a*H2-pG!A!z`_Dk5qFNj_(7NCX~*at@Ch} z%Ka*0E>(?Q41iU3DtO#VO{kM49ro_Dv*w!!KxIB~fH?UA8~irZ}Q(5RsU@ zk03k|TcJ4wandNiZ@R}iv&e>T zsQ@h3b}HSA36a?~C2q*Gj4P2imE|a|SbA!sJ9;X_<#3<7a>Ta!o_o6J4a&Tw1Iji! zRz?rSMb&ULb$sb+Nm~64km-9f*wIi<-qI$;&sY5-kZEN`@;rr;+#++JJy(8%uVaPU z+dFA~`bpfpqU}D?p~!f0V6FcBm;(8vk-aJS(L`UVO}W4B{0HfX~TtCDXk>-HO)q}q!++EVx=u?cd?f1(~B+Lb~6k1^!zREvZ(GU+5 zn}?;C27WMrd|C@8h8{Kh<>Uo!+*_IQ3wPd>UL_O=UGbeefjefCFK+Z^*NL@3knIX`( zhkNgy`DWfeXm`Kc)E_9KlB~T6Om$g^01b}i&k(+BM7dszbnH#r;mA`m63g#K>MQIK zfYPG~;V~TF87+!SLPGg67l+uwlQqgR{B20XPoCPvL@lo;F5StEQCgO1=Fo%!%`~k9 zo9|O+)m%zLPpFj?w7;8{5M06%V51q|ruw>hTN2f4c8pE1?sjbPq`?zCxTJb~+7l+T zCo<47;WaOc690PX6rplAhei?B-Xz>I@bv;%%d?Rdp7shXE^f$_l9G~qvN%eDE%iAl z?z;9HpxYnvQ90TEFpwnOd|kUv{{}f})KCMs{7dbx>gXCLSr3EHiu#88vjtci>deIm z+;DARnO8&{qf55V5x1Y1VV^KFp2v#IbQ8pHf%0Do9q}7P5)O4Kj%|QGLDkbT5n7<*_DPhce$2Q5f99F zx7k)|#QpQWdnq;{zdri%m`&D0W-Z<7XXNGrBWD{?+{_>!_r+zytxO3l4inQ&>_}5X3#hv7 zfp=#c(P-K}9$aDCMl$so;AG!2^)vVJ$3}&qy_g=Qyw76V+3`|1)e;(NY-xsYmGhZgEkWxL9X9Em)f}0 zZjo##;GO)Pu4lsp&#;yJ;^NX{VDWD?2(OLwQVQ!-(;%mh2W4-$!r-Y|9}&BzHRd~P zk>Oxtguyr)rKGT+-cnl0AQ+Ug1y7xkZ;yD@NlP^37ZC34J>us4dDAw$+>3b*wclCG zno(uK%Tk?6)=$gruN4*=Qp6$KWuLb zGjl59I{&r<_4YJT;!|@Ffy0p9EBJ?p_*v4-Z@{mPwxirHdA|XpbsOjBcK3^(QWp{T zT}uykya$y=wq7!KOAbMW;FCn7w+ThCaFVQfsSN1gQmV60f|_-fVI#q3D;T?%-o`n* zfYnpZ_8BeRL~OPxakE@QjVA~o7xXF;9X-d22D=BvS;S|uXHJ6S@n6CS0O}eJjn4-e z%UE{VN<78D{vKye%G;D#6ZCI@;9FTrwa&`K@hc<7QB-`ul4Dw}wt~Z`TOPwz#h<%| zBu}+lp1R&1Rn)4Ao6~qVYm%d=VaP-f|+|xy1PXC;Zju zphLmffbEXqn%JgPLeUTBaYhuXgN{N4n4K8LTMH&l9P0d4Wp{sK9h11oe2TLmMJ-eJ z!|}hu?<}d?F)k_)V|>%IBz`PKS*b)Py*u6rBX#ft$YE3WUyZ>l?FSJ`O$T6#BY;)P zo3cSut{F=0A|_92P;ucQLEEadJ`+8CPZ58_lpta>DpEA;l4w>sE4MCmoo41-Qrb)I zRKcgR;3&G1MQ9+_rY=a%MjTxtGGOIBM9018#0=tU9od3tRLaTodeN^D5vS)%00_Nf-t+ zVP-N*d4*{MbNfZC!EAZY3e$Rj1Jq!5mCy~QROhsF6$iBEkD?I-3OFhjMB%FE(n|8E zSjbkJAWKSwJA%2u{Av(oI&r0s0BYRgEVIi5e_Z!4t7~Z%a}Gz;!&1Zbmt$E>ly^Tq zX!rfzpuJSsIH-A|;+Fk2L|M(xTeye7-|1chO`nC#;dP&QbkuLOnxYd!{6(z1z90#q zZA-FGFO3Z(4Gq;Z)+>{@vBlCe3w3Vo#zES+>L+h}D@BRze$VwrW zdU8%zx3b-ar>-F^WUtQMdxX^i&n+cHQN+ZrZB?#?tS?P< zkHc57L8S(#;^EPE({F+A1*>j?KUkFS8}`g@26xI%hpGPt44-lc2Ets-egmqiI%>W( z{s!r#&>3SV>)@UVCEA)bStSKnn)dO~ORyQ02T`-}LU@PUh9-7Uia?=$elUG|{m@)Cq zM!t(6)jewN1QT>u#AiW98_TkyHbFNAwkgp1dq*kmi9>b20p;nRgzj8+9mQjorUeV@ zgRD?1G`nx5r*uPo4$ERMp-EBbyCXziR**2ie(aHm)#;miat8lff&4EX<~f)b>1=;nB`^>c=QLQ5^FdCCatfRQtc>M!li<}H1wta;0Gox%2-hTCvgs%R`(ycc`Y zV-AHj?!cs^ld0L$;`BEeL+D^C{6^~#`{&3x)<8Fs`y?Ecnv#+D%x%`HkvB@hdXV`x z-%GJFB}wqQqcUyBL{X+QQ;kRWF3R+0rn1&tlrhnhoLAJo^i!rh$-MSRKmwj9gbguJ zdngT%IR7htGRKy(g;MpO1HSpC&F{E)nNh8mTsQ1e3zo`jgIWPS1BMoAB`&?bl$2GF zsS48tCz`L?atIVNKhDKH{l#k1Q(+OD##E=tpwrIBCDt3H_%393KVex(dBw?pE7x|{ zLCJ7KnpnH75Nyj5;Q8$}72tY^FzX2OwCfwMR>+Uz23f(#gE3ORg?z=#p}A6Fl*}8a z9Sf%LS=JF20T&&w9>|Xq6PeNQ1EFi*7pfHP2{R>wnKHGood+~eCjmo0|BBz@yz_GM z#Hh_;+O%0zS81R1y{eaDnzv{+Cc+NDZYf=h%CWrzr0+O^Qd>;bM@>vJoGG zFeLk)X6!YvEo1l~tA+US%jh1???;#R2qX70j(-Dka=x?#2KU9EXuc-86W;7>1|a8q zFz+M*Vn6D=wOkQZJN=u29SdViRV<$kZWqKxAG#_B72>rPcJjI$;dDOUT5hGKVgxq9 zj&H_`dURv2ISOK%4&p?;3Z$JE`qI0uXplT4QT|JY9Z3-v<6@=jpJ;}#Kk!Js3s4a`}T=E**zL92>Afpd`7nUfnI3d*5A3s$P$74rNbDQev|E#|MS$+Sr`u=aNz8l$W z)`%osrW*!Wo9>ss^T~+Wv&6-@os!y#$d^0MKNxlagBhj!sPpFLpWZ0=`Y-hEzmX)2 z=B=AcB~683RASIBwmRBUUE7%{$`rsPT75s2++F4Rx%lV~vtcZL@7}0};Fs1fIk~Q1 zPDB%bMWwdR>W8lEdvAv1^=xe_{b2c3jD49z zq?U?tL1w~ut)H^Pr@8!J6~1Oi?t;6I2;cn#J>w8FjN(PmIsLHpDboAs@)WLq`@t0~ zD6E!gN$1X?XkTs83=gVk>hTE z=?`n@qGYeFkqO$st_D-ji;5{qb~U5@1#?T~5-mKuL;Fk6NhzViBJ}#^qkahm52mUp zQn0N`iQBOEomiDDqyeXf_(r~f_?Wg7%~2;4jpt7Nb?PYl#@qOu_uc|9;iaRkwq!?g zNHeWak?(L(CS*G;Tha=eqbQZQaw`*R* zfG_rX{voAH&5DC`kcwk&>gj9O&u7}9LbgBO1}X1d`zeZ2D4+7LK7qG}g}ixEy@~$~ zD1LZ}_~Wl;B2(#}yVY2xz_OTMUTIk3)QOo!o;!WG)>9pI?w{oQ&75$%+cC#fe1rNC zUdx5U<~3-^E^T_h0SHbi*xOM=jl|xNaYP5yOeS$m(C;Kg{}sP~qhBdEd!n!>#H(p} zxM;4smwy7<66o)-@WhY*2Kaf+^FBM483AIt)$(r{3JfO=QJxVjau26_D=0E!G=J7N z^m}5)>}Z3a#afU!cir}(WBtY0vXn0?0v3inupvbcH!;Jun7r;*uFVROXC0m@% zo)oGHE%ha7HS9}{7H#kz?=@ZXcL#*a9ZTQ7_qbN>oVLh79qObejS>Y*Q*JLc5e2N= zKX{<%7pTCG4;4o{cVv+wO|Zt>?$s;Iat8xK9zt0nJXhIGyz9PBv|4*+U>0UV#cVNA zr)Dys&E%OJ6po)I!)8m$L0jzm-Hq)7(OVT!_M;vhDdNi#!eB(lR0AGY5rZ&1rb)qa ze3JP&>pc>zUC|$ry@f56A&GBHEY)TV$G3`WwP`a8mv@u|1T5-0=?^~?< zS>Q$|ky)mdbdRm8fC5J|(Wkz&mF{*KgATCX5t}S({8XiU;|5<$u$!vrlhMS6wmAJN zSBeI;NaSlbjLN#>Xv?om!8P+3ac=dyGmpHOQA6(Pob6rka3-O#uuHSInPZ9%YZeLR zA8S_QwV!Sl`a4X$USJQOju8Prz-d4b0^%SU+difgrsRbmp|FdvNN=r>)iuJpyPKGA zw{9$FA9AcP4<;&I&#cquELjnk65s{mozsxBL+=+?G#8OmWaeGYqqVPP$X^V1-$_Bl5c(|D^kSh?VKyRLefTxzoQ>ge(Q^`=$fhcX_j zmR6FcVr<{FdDx&a6FIP8EO1)fkeQ)!=;3+SS}=QFP|5gD|13gxR&RC}D}nn%;x|`6 z=k!^I>E!g&X)B%Q8-sfewcI-^J=3hSH=uI2k{6PkXby~S$7pm7v|QE8b7hy{m6I<+ z>e5csf-{@EHIwku$@N=lG~g&_4to-a;k@);)g{}4DKU)L6Of}DGU!uz+RNUE>D(A!?}XFp0% z$J9$*i)Wxs1 z=kRsvRlzCo?xp>Ze-mc^;zBAw9QdMpyE<$;%rCI|$x^%gA-mqTui{bO@KD3<6OPvU z)9x{N@nEwD)VVyPYdQ?~Q{L&D4RdG{l!oh)7=6-r-jSsUY=}MYR}#p-;B!F&+8BK{ zn?Wd8S_O!$m`%YajhSkI_30L=OdFXsjTW91+z_HAeIdU9HwdJ0k?dPQHrh{|OgQSV z)BPGZy|k7y(lb>IdHcfkEsDK#Rwy*nr~FvD^@OVo0J8m4p7YNI1zwU=GOv%O>q!Ps zHr%e(Yq4PAA%JQU3|r!*TeE1tl|Sp_+6aNjE7s!}kGVyWL-tA^?2l)aSmMX*Z*NF; zmKD5U$6xn3#U>IE$a=;ZFqrZD%0bf5*2}zn$;BK4JhnkIY0}nB+#%4E7+ZYNe$G~W#txUKExboyRo3_(5 zFs3n67+ra^>Aa55alZOSHe(QW7NH)63p@u0n@o4}5?iY4?Y%OuWKtxWu!@S14S4G3_cdP$QWb1EVgjLu%v$ zH=K&cks^J&pn8J%VotDhankD){rM@>)6mq)M`>>~u!S~wwJI6l7f z`FcLNpO|NCk@rujI!Hfj&_`Roh?&kB;}tj|$4si2Xe|ZyJ=jBrwop1^Y4SuKly=M2 zsi#@5RMqE+2|iH}@r-S8>8*gIOO;Ry<)GB`X>X!AInYb_Q*bR!325LGmzJ5~bzk)$ zMY!dxcSb7~P2eq~qHz^4*qA`?a zsmYx)O0jt0_m)97#jLaW>wd}ZLiU6ii071VV-}&C1=m`R;7v*TnCcVda8F~iW)|9> z#qQ@@?72hf9NwYMLy~hiVwrc(ph2?Ozn)7Hq(H92-ft*qPqo(4`i|3hiqKobI>e5z z-ZiXKMOqI8lYg&EF8O5oo~64F_c^&f*!?9@GS6A}*ki-W*C=+(bgcb6$Atxt5pyf< zFM!9VJu;&uDRnW95zY$F9Tc=A=NXx~7noA7#~A(2A&1ezz}`a2IvB|x8qN{2P#=JRFkp z&6d#igdj!S`?mP&39|8*JRs@7vq(|WQBeiv&;0Cj5gLCji{Byd|4U|9%1b>q)(Q@vJ%7?t)@2N7!JL0A zQ+B@1dsr)%mk}8i0tHpy;7)OF{E?==L zr5w|+y%A3|fYEujvK&@a=M{EWi0F!aMS?Qc-6=(df%}DB+)F5xOx0Bh#lySf&VS#j zN(&+&K93KYobmRzTeH_r`yP0kJR#q$P6w1d{cooI9jA)(`2l9Mx;(S|SU)YK>9H*J zqZ_tWVihDzJrX>8nT___JZXVZM5wqYCtseu_$?&~;BF2ky0B=N;Ae2c*OV8H^pU^S z)K$Mli8bQ);C<`6@!}4|q&Ed72HzV%y|uW$%X2fBacN5PS#ad>KL+>ON9rmm`J(vX z5}m3%1GyW$QncsFp4)*!=5ba=%?}=FE#wO)y?{q1ZM(fQms&puwPQp>?bj7o{d`tu^H#}L`ST(1$>?jq=D)Z5f1Wzm$4(lB*K}`r-)_|(d#mYoR$idrc(J2* zpNA{u*Oi~v$#tUfYGV#Hc|Wr~E><26f@Zcg*D`Wls^N=U<)NJkXF7uw zFZ`7y@K1VOAm3!H@zB@s^!|J0S(AqXFaOWIBw4v;#_TYvp*qH6=2^lP;K=O%guChJ zjf+sjs}hBWuhE`h_O8KN)#9V3hkxHq{UaDc0xCOc&Gap9ly%brXFXyn)GQA#m@Jyo z@)S}{c>Bq{&lf}3Ht+2i9yB)AxP=&x#LD&!bR>q-pf5biI#Z1U%zLJ-TC zT|;8o`DRy`j=;8-OLdHkUJhdOqZM75uF=20hO#sBSR^`%wo`_9!t!nwwJw}K zSL4}ZZ}qNc{f;9;7Xrb|BJ?p^7gTk?$;ah6N6rP2pk zH$cZ|K!ozUaSv7RxLZkEw-tKz+CI+HA-9L%Rrd5BgMOy4CpK-h%ciO;oeG_3poOAD z1F?BcQOlA|b>?arUY4QhWP{X_wNR6qjbNq#j_4bF|5mZfDlluc;MVr+_QxTq?X&cr zu#Udjt>X5*pR$MT`in&Y$e`|^Z5cmJl6@XUN+@P&;I1|tR!g+9T9K-85bohvwuq0!1g5?qWA`O-oRT}nLvRQuj68}3KkCe%ik z5DbOJG{4+*RPX72^*8j+z~7F{0saU@0RZG?&@WTtnwun7egk|>VoTC|g@5H3hkd%M ze`I1#QSy(0oZq`%mUp#FL_#Oxo0Nt6)l(){l|a5s1Zjh(=!9vXIM+{o`%7}I!BS(t zbl-G(Qfaq+`2M1~X!ypAE}Ct~Cg+yJeUyu0o4r9doZttvO|nJYd{( zD0|!LX_rcE_qRvY<0+(#YE5*y$K;9M!dspjszGswHRE=(P1Su7#(@83+T)W*6aD84 z=}bi`yO|aF&ws@?yi!2xO1`jcN?#YeFv~-EYXIr1Rk^jY~>+`S3rd^Zx5?vLt$}*nE$lV;|M)4l<*} zmW4h(B)RY)MB)b*7q^52yP?qMBi3Gi{`xSwSbn)U$IO_>7gM^UuXqkJhdk&}`S>ND z@&g-m-|=a@vW-%A3Nm)mr6Jp&`Sn=&Rmbos1B7Snfpla!Rx&S^>TYHtuAsxoI$)rg z-)cmtSi;Ok;G>FZQNDQNgcg?+Gc?sO#iKE?Q(+TN6n=%Vg0QH7`61f&V8*8C_UG)8 znzU+XK_k~SPTum+ZA#|!HVbQsbIqJlcAh8GUcXU(-9cPIPIn?3K51k6Qgyp&yj>-B zA?o3a$kK4@g@I6=l11r8vRv(8lup}{-*E*xRx=C}|`=CBu#d}t= zZI6;~L|Ovf|E8zPC~qi-F&utp?0xhCgT|`1ELAhjRv($$w56ka0oOg+KsQ(Q167J{ zvHe+}MbW#Va-O09xsRt%=c;V1zF-@+6~w<)`)NI7;>A5^#&^)Y<_(e-A-GI@MZ^q5 z+Q{HwksLZ(;qDE=Okcc@0wZ9RqgwiuAvXP)+|(&LXF1Re`iWnpR&K@)vR5%p&9JK3 zhjixduy*+zQ}_lC7VSl>>b+`s<*EjsmD~lM=e?F;PnV<})vBs<)xqaW#>LEeOsMs; z8tL`dMqFd7h;Cr+9Yp8{*&FUy00)jtiOf33B^9o7v9oLLvd_PtRSahj-AO|&9JyIO zNTRqB6JM`aXP-W&x~i)G%H0cggPCXu0zyRY2n8@yEKH|(qedNcKN>DlzjINGn&O#G zHo8+MmDaH7O{`6Ru7r4iiczmgaSxlza{=)bsx;?mGI&!Bf0=TqrLer2YMcS3?~RuhT&Ji$7Gm0mh-rvhLX`0=ScOePQvnVzx#Ra$4nuA(=G*>T81LvAY$S- zK!{M>nH>pb%kNAQQp}B>aRX(2m8;yZzNh8{>X~qM za`xDBQsdZiKLVTc;1NQfQVj!u(aJPZ1-hGeSDO#zL(bLJNx^DfddgM;Q+jeC3e4`@ z?lL=JhNLb?0q_ORe?zme^{T%GGJ)F0C}ncoA%e+^(cqU@SK0;kweV#jy3lWc^NM3F zf)6o=(3kQANmd&4tXL*kJhNPUygB=Pz^aeV(sz}XRW;!rGv)3?QI@JUuQ52Was+p; zHjpntLoMQ3Zn^gU&s?but$^&Sb*W8_BB>y@m44DHtJLn*k{LaMA+5eo$^}F;I-s&X z3R3O4wgLH~yHhGzzkQTBNm=gn`uS1AiN;Ov6zGyE2w!0u#=iqA@-xs8LIGCGybM`X?pbWY*jvogFAxc3U#rnQ;;83Tke>$`*n?wKi^A zpCz|oZe0Fym4hCY02YpV8F_#h@SEmZwl~~Pxif5{P|z2|9q+}>%aJfhsdAi&a#ro< z76*B8DP2`|s?@_ej=pP;C@oDV9j8sd)Z0M!#kjs(l`>`W)ph3VjZ{#99jy}k7va`> zGJ8ihI^Xq_4p>Yx<1X>{y5`TQ1Y&YSpFaa0_h3#dUung6M1762h2AXS=8_Ose4#1V zYLd>vCNT4s9||=;&umr3F9RY;qHF%?T%m{dO}yM zs4I0agFEY9NDNEul8Lx3N@ozR2ijA<%>BTl>8sCMi=9f-mXCZTk1TqgTzPA9<(qc+ zI$z)jtWjPAU7x})UkRR!ADRM%#eQ=6@^rpN2%jfetUfN*uOeGoTrd#3gb=gUo5>7J zC&Nq=yp#mEX5!-;!6|vgO1B7^sp|IK2BWux42fa_=<&_L;#87N^1)5#PCo%6kQ-J*g`ySou6OR%)2)@l*G1bq>Ujf=CgD%*J`pzYI11 z|KW}Q^6fX-6JG~xr#sxjyRg3E5si1x54DeRn9_A84)yorxZP@FAP8(!s4YFrsSRYp z(&F#3rCs6YN?fK)y3z&lqpsqri3(bfTP?rzD1NkXTV>t|` z`5hc$$IP5+K!bx*o8KLJ9vzv9)jlR#Mx$PQ)w0wT@m`$5GKnU>ve=ZJl7^P&f|YeXSMgSv^b3a<7I@Rn#v8<~N<31iQ< zQjDIPGgYq-%l#lx9)xQgdLFzNk3*{KyY^KLkK-EnDS#!1cW)Wc!CMlCo$AI{Q)LO1 zeJphXW{--1FSLyk8k#cJ{p&W87m}yP(533VaUG$(L3Ayck!fNUrs$iA(8%JCZ?X;# zi|sZo)7ZTp$$Vhi@bh;XpQ3ry#jm$9ZBVdtrvi`$SxZuxcX%w0WvcB5WjEh+=+V^~ z*V|l75l~?y!^#RDf}y${RtO+1%4HI5HT->Rb=T6Fi9ykcxw~pa!iZBRc2#B|k`>I~ zc(e8g)Hs!!Ta<+R5^#+|NjI0c5F{KEkf7E#g_*Uyv!#*h4sDLy$oQ}TyJgFPBm9K=L*p@s@;I)_C?21GoH~*w zr($g7g*<%H(L8=1^5##KqOx*~E{@b>nj9k>_J&OWwAJ_Iihl!SCCh(|^sKyEUNQa+ zSm8+vV>UnAH@SbMeDGuGZ@@=gfiuaPlkbb7K6SPdJ|@Oana$6%3L--F13|Iz6ps8J z0xcgDmWirc#P-NOJq;CO_{gtR(@xErw3q*If&R-^)7{0Ls`B~{zV}k12KF_+%%o|S zmd3L^sZ6z~_19V_W#Y68F`Y7w2L3v+3V-99F^RWBTbnzDmy=TIc0WT6 zH5OSKq~%@MLQu=csSB86+y~+KST7skdWD;59bB`w51ly9>bV~@9J#I#$cBy}XcAfC`3su39+>Zt9mRG@H` zYAEg{1h>TG7%}GPm6~;xNx7Uyq{Hy8>)CXoq#!B7FPC)34+@1~O(ij1SFS*f96S)a z-5jE6UdP5e;qoR9(tqfx{Gd&>BjHz{cDTFiIKtF4pY6J!1|_CzOzAyI4EFe$tvYIW1(iQ zz(*d`l{@%oEm4X&B@XLDktIDGy~#Sc&5x6WSoX$`AwUHVho9<&<#B3GV=796 zR-g{vE~1?$<42$CAuF}K>DF(|Nc-)1wpxIQrmt>$VZ$7%WC>l6*!^?nM`~de-xHMi0Uk)E9VBL1iCmVowp7&2f&?&YKTD zXT>j7b^_aHrbs&|VaL^NY(q%iLEwt?hz1UbvPvYazB1yC^E4=-xEec61vBkhvf1|I! zqehBJH0F5TXk@wQYtWNdA_iX>kj7oPctqo-s>nvI_qTx$vX|-&TyQW3#h|dP_ zRaz86>du4)+Yz#R4k;?WU^+blfuMp>(Z9Mi_meR$+bg-S@bBxkGp^+AK6b#iE(`N@ z5$Wl-TOX}~Ka{UAtDz=)(W=-_Zd`s@&Um_~Q<8h)C439_ebbBb?K7l_TpO)E(j(Tu zuxI?jFXK^G!05-*Vkh%Yu&xqwM2lO`j12jH157c^dg9Y@3;Gb_RdPdQU*pto0QK^% ze?yCL$jCK~ZDopWy%KK;z5iYL&Y1S`ub#H08hc@+-nXj}+_BMkC8|tj<+i#@4?kBQ zV}6>q9fa*aS-!Dv2Mjc;-L~5u^gBBV_zkclA}>zdIqRLO5Knab%}a zs}AHx7=a_Czi>J?)qc6F#8 zSeK(Y2C@1+YTYU2L>6Hp?Px|WVru@m?M`y7gkqoGJ*ftKFf-U-oUo75V2=#U^ffCu zX16I}(w_E2zc1;is=ilQmTt3@bvt?DluB4c+tWnYh$Ib7EThPRZ}J_m$aGa# zBtMo1jY$}fabw&sfOc{-;l>&WGHu5~Sxyaj{gyLz?}hG-xL4eC!yq#+4JMcRV)4XX zHmY?wSDs<32D>xL&uUYL8DWQ@AV&QzoVlI(CczO~rYl4xC;UHF{4hZ58d`g_Ra2$ zua%p1w({jmsRw$i4jZF5yY~%W`feC+VUz~tOwL$r&Qks`KK`6vl`0AlHkrvbQ?;xf zC2n{Lnc1kD1WlK_OX7Nem;{SRFdgZ~M{v4ATBR~g9V-(kYN#m%ir&1gL~XHoQ=oo< zoikpiS{pG-zJli?mSd2W-ASn4k@TDSOYk({JFkY}Le=U7TALT6Uml^Qmx@v0O+QCa z+@5OF(C5(jMsWx9ls@9W<;t1$a18Epgl8mb%L02y;^%hro#)r`m9pfEcTk*YmipmjiLIJ7Zy=V?(a@ zJj<>Y@jOe;89DXKC`P(e8csftnk@xZqCS8(Grx5KQ$uybuwIKb*el%ft8oTZ&U^(9 zqcycy)n`Tym33T3o8tS&49V~mZ*2j|*M%LHiX#2~w&B@Q^1O1IhrHj5cEdY5P!4{iSDP9N-E9w!bqwi7L!Ja|P&79hYUtH)9 z6BX)Fe!R3rX`3`J$1Y?N4K`rI(W(FG`gxMi0UELT{h1XZ-&YP%j4{WRs(W_%jTYuu zq*iz#6|&A>aHG=S+fFN{&RRd(^x-X&YB~rLn{#&0XnxNpM!9F|8frpI_xiSHRQ6V~ zZsBx~XXA}H{nAvB*AB3$vh^!~rv+SmB7F(%0@>)VnRd$qx}U!~-p#f~LB6WTuZz|f z_3=pxa55XGGY(|$0GUI}4EEtR;|~HSs|O$Sy9g|2Yc%>nG9GN;aA@}hf;ZP8G#H!0 z7?W~+AUg_LB}TFSM*iwEiV4Tm&2_IVDK%r~tR}VjY||y0mV6$xu6s^Q2Z&!lpYD2? z??`0iGDA!sO<`t0T(L0q$8MbsIlM<&b7sO}Vd=8*3f@)fFxt{okJcDozlR(+fWPqR zNm-iQ&TJD}Rm5<5Usy$_^^abYVbPt@$nJ;HQ=;q8{jLDa#o4R&9p_Z5bY+ZY?wme( zyl2LYb-8=}N8^}l0|~BZcPyQw#uSbWApp8FFk>hf8_;x2L|fzvQZhF@(~lc<@G~jt zc96yx%%?VdyET!e*NIt_936C1ukkNrd`IdrE{{v9(@T3o`uPb^(Wu4zRC3kq2hx96 z{f|)opYp1IF8GIO|L45E@8@EXZ08@0E@HMhKDE-i7;OjU0^i$6|9?kAOdkIk4iTD8 zTcoVq@;prQAdT6|ZkE<|zwBpMp-i3f^U0@3gul^Fo6s#dmap<~| z*rQ2^wT8*q+N4=mT_G|qsAQ2YSbvt*3GeYl_NsbupK6T6`LrYqpdm_h9IHLnJ_{EHJ zk0F8*B0qGv+g1+b^}6KiH-u5ZRkI*nA$aSJg{F*Z>848IhL<^zPYs2AZ;RDm%__o} zky-Z#Yh~+EAbCx{S*(gx?8XLCWE{TBy^B;HPIles9g0nxDjdApyg7OC$046}dBzk@ zdC`Ox|02u02#bk2woL23#qe+ROZ*>4jQuksINfn$!tLN$N);@(N?gh&W6gI5V+U1T za>u(~$0yV*x^Z^8wFBE_EQw9XmeDE^dPZ{Tc;RBawtmZW^hj{E6TSQZK<>bQG}quf z+Xci%6~U^L`9*qfg+6mFN7_Lz(UUdY3$3-X&=gM020Kj&^(I(th^4c!JJFQg{pN6Q zj28K$U!WNE?L`%SyyyYLW}F3eyB5OQB6)#h)={ILf5h<4apDYsx^Rs{@8^0ZjNY=D zOXO~#klz-k&a0w95wY&uNfOOjl!BIrDO^Sg*}%i_MX+|M1&!y16}`NiYbfn%EAxmO z)L09vHu?H)zmwqLo{Pc>P`4SBJKMu1y?Ad9Z?T z6(!+3TkEQAKD|Y<5#vZ{7h|+_24V{pI=LNSnFCE0O&X~Z5ESd6hZdW_gqeD!YcZ90 zC=HV;xt|{Ir44iO{OU#GX1 zqRRZWIp8@0=TueaT7k{u$|rp|yGV?(;*CGM*f0MK9Q-GsAiKM-{<#;zY#x_1r20>O z_q9K_{7)?Z$(w(-%$TFGS|}Hv#XUxKuf6;UPtPjP{fHMs)yen@Wd1uSF}c-$B;pRv zFQQeOd%nm1ix+k9pQXTmLVR4al={y}jPS0VmXLc)C7oiEh!_)1Qbo7ht49)Sm=U70 zXcG!0dPcs^kvIE5nfqsdf^hX#jA)QasQn15Mc#Y2`_lO1qT%4}`GyPp5CGPotyn!W zDAn&PFd~7nU)bN>%ubw25#D`-6UqeVMS+;Xpd&1GGyIp13n@*hgyf3UL?!KLFVMkQ^Z8=H-nFo11k(KD{>Nbqay?wCPR1JU%Atc zr3VUlzU)PTMF?KdKHE#mT|b8l74LP|z9EwIDU=G*C1p4ce0pcxH2KS+Xr9aMg{Riz z%*R;YPEK;qEi%JQX>i8jfkb)i+V#X%x_3Kqaj@2`#ejI?w+a4!)124ojQB^Q7hZ6q z4w$FDJf>b-5~SgBNbSpe;bb&vMHw+L7=@e5an&w_$;kTQoSjySLoVYB>c^#HNGhnB&LBH3tHK*$+N(cf!>djMbK&vMwZ6} zdvQ+M=I^onAEjkYR;eR;(crWA6B(3=r};LT9aCJk!ll9*T}-}IU*Al;a|*0LS(WnI z4bMQ<>bS-#_Bfw5iF;d@Li07e-G|GLA1}U3beSA@%RkGStiBEG1vAk`_g<9*04|gr zu&LmkO-UDA;6F4SM^R_9%U3V0c2wdJl3)Ftj6n3BOtOnx^f`8yFpPG1JxW`acs*!R zY;+4QFO0)~vhxH1?T+at{!$IP`~wB>pE(@m>sdN0=`gDT^ZEJgQ`m!&^qteB;6D3K z%jukcN{WlarfwlDl9Eq?;Q;wG%z5Z~m}odRvHXL5?xV+cDn&Bx8s#wcE%BLREoGcw z9P6QyRhqGHBev6jeA?4rR|KFsMTAdtK9=G32a<3+j z4UBjnN|7<&%v_nq*hKKkxpR)L67rrraW^r%H%y`JV>Ih@qV5)bulZYRqZe17+oye* zaJ1NE@sTj$QW~a*6JH{Sgcrq^a2{I!2vq_AtRkurk51e&o12wHO7%?TSh@`1BDZv=GY?+424cWgH1#eSd-763;;$aP;+DL*TJaNeoa=jI9I8WZEnG z>0+tk9y}A>Oj?vR5;+%+&o|(a3zAQ5D*7zmF;27hWeiom(*msQy(=_95WXU!R)W zY505Ro&Y0H|9EqM2j#+)L*3;rhLnH4Y}{|vw>=wtrw(osxSV1X* zG!aPX0qIRZke1LSG=TsD3DT>8^sTai(33zQRDn=Juc0c6^dcHUZ%R|TfKmltJa^x7 z_1yEG?Y{Ty_r1@3&iuhMi&iPQup+9iw zB04ys$@M6oo;aRoSPBBdp_i;+Lybfs!$16=9vKEni&j zejCAW%kK#zr2~FnU>0_r^FYHQOY~!a?dW6Pl<489YDB;Rw}h34ee4N)!FDq@6JL9t z9Bw{#0Ggh}4#usOhZg?4dMV`37}@_~n?Ivoy1@8d+luCyu>~-8jU4J~>#NBn{{J7NfzYHRU?88#ab%oR7x|a zLb4+?wuGF~Y>H1`Qn0K4*6_bI|M&j>oGw>9`?B-AYi^WrlTVY}jiIIuOSb)sJU;d5 z`+XnYkM1hQPK*Y=m3;DJ2S9V4V6SMFgN&J}g`Oj->3-V5pt$Q+aBOhi^}iA8{@(Qc zvElzH{YBTC)~_air0dnqXFEO1j0|Q!()D5mRf`GZPs;G_dJOF^DsQQ86YmN#fBs5A zJEh6^taLK=%ZtOv*;xK9^QA+RlBo<~_N9I2riEb&?=Sz+_}^cnbO;#3e&&v-i!#q! zw9F<38NSII`;~+C8Xb+s;iEty4U@z+&Y%;JpTnS!<^0&oG5!U){UW=m<*yW1kM0Za z8mbiH*H+83gDb!IAILm3{Gs+S{zbEW+5CeicQaNC$_;5ta+)d&Vn->@9xfIh`>pD~ z(CB}FTmOuT|Njvh{j&i6&);MI1YYuIsGonEp^$o=!zE^^eZ}^s{-h(+`ERE6%^xQ6 z546eAl>ZJzdH!$k%|HJ|bU}D8kN)tMk{3-u?~UH{)@5Ofwk$;N%cwaUEC>QKB--72 z4s;h1n-Z4c=GEwWt!kyY$ToxO)WrP~c zrx<4X6d6&>m+7&(0;qZwm2B#fn|*F(W@CaDmEu`APpueIS6w6nEa9x;v@bq?a3pY% zlhr8@PGY(b>9Texx%ycYE<-KD4iA&D`?bYD0x;n5XCOspUYbivpn8$ zB|JCx*2?W6ZgV8ZeD5ols`OTfWKkhf4&xN`+>eyh!*p*kG*QNWBCLx9fk z-9T?@)9%)eN5@V-O_N!+)72-VvOF6ApPocE(yFhdqrN>5oc~^ad66W$>zu}^bG11DjKSw zf*H2N!|%0hZByNlK?!n-DV)rOx1mSP5%Wfx&Mmw(5Omf@(QnH~Q8sqsLad%h0E;Hb zeOzez#uwXtKxM~Lxkr}Yo9*}Y+uq3;{J;)Z!WrWevw>MG0iBdEZLUgl$-9!tR#|u2 z=sam*;5g>EnQ}x=EZ^`%RiXvA!K|&x-5omoBTem`Dm0jHf)98_|4%qwfB6l>+Nn^S--%?{Lg@&-)nrPZQA;f$Tp7qQ6r(mYi{J zt{0qDZo_LIf#qFEN`N?Q&3C}UDO<9H@kyErUsKG^aGX6<^s(RmW%QRHd-(C5D9e!Kvwi1H)5xLxVe0dLM_YdT#f+$y z4NDIJXd@FOZ{n*f13(&X>9Js#KB18+n)c1&`A9I=I@da<;uDX%#|^(HnZM6m{trc- zDJVoobJB+vY?J3?6YQBfMcn;-yWWzj7#=R zUNS=3kA13*2KI!B`qc@s;dj+RqA13%6drd;{py!lXn+GjcWT5way@e_N$VUaX6_Xq zy)fI1LqWS9Q$uQ|TM{G#BO}JzBVYg~RNzUQ(C7Ep_b)UrnRw*iW485-ynUpG;cih* zFkM#IskvQq)T3 z``u8IO0{@tZE3Cg=}RHC+b4N@SpYr$ez<{FFG_WDy2v88%y+a0y@K* zkz>~rNz;a7xd4o*Vaq5mnKoO(|FkUT$n69MxJ8IO&(|!-8XGGS*^NOheDe31;SiZHfLcr-3Fj?lZP9P7)@;~dHu|r!aEq+5oWw*nv7+?M98TK`79y=xCC9um zhbyz3MhR_n`Y;JRkI&@iqz8e>;JRN+Jdu;tObtaWTpsCq?GW_Xn~9j74i~h6E&Lv{ zKA&-HWOOZ}oaxdL5u=K@$HH3hDNK150ytC)NCorP7Ws5t)YKpFL~&57=!a~Wn!;A% z0?|fb5E)bYVHN0Qf^snOje8Dbk|{AD=yA!^*bM|oyT4VJaI&?GUyae=+(yFzQAic| z@ni}Lp86#;Oz*7_*}C%bzq9(W{noH*HyMB>Sp6O`KI;hJMy%8=tC#+>vf4*r?Ef<4 z^7xNI2x$zSny^v(N^x&zd(ZQD!H@ClTj@ktk7z*vBYkbJC}aEpf}U|1QG9kb%O1}> z(9&dL2Iz=fkPr7fE=>M)kxTe=n{&P>vUVbR-^jR+WMjn0o%B%>-Q!hrDiMqJgIc3D zz#Q5fpK$5>R~)K-o__tm=V<;>{^n2Frv5Xa^o#FFW@*;2>tUtFR|bzo&EPQf+GgcG z7)ayG6#|oVT)b{M)42>Lw69LL3RK^2u7Nio9;VwOSCVVUZa8H>fBNz_Bac6ifc?)i;)nl;7){#l*|qv_`)^kE`hTSW zu1}S1`1ljn8V_RvOEEhK6M2)8+2j;`LqvC%1PbUGv#jd9oa0}%c6A_oyh$m5rc+^N z+H9?yV(Yo}(=#8MaL-}_FTJy{-;UXyWxuAf;p6)3pwxSLU&w!FW2SbNdjXqb0wck_ zW&NWe#I_V}`+H6LY#suaEKVl9VwMqA{iuXGN`3Y!_k$#h>l!+is=4 zS%ArtFI1v1fX9Q_x0Rx-!I^A~n^JH3vRDOr71k%eJf1z1uCuZ|RlPcqf#?gq3iCNu zeJnX8*VGx|2}TVstV-~YF+1UM(hsz}I*ODY(Ar4b&2}=rpj5e3)vaCV;<6#TBHxxZ z1}{#L(jO+oAQ|iwPiQ5cWliVW5$yTJ{^tMoJ+#^T;jwV&@Q8!c6yB2~MVW4Ei#S6Y_|^b(K}xGRu>nXeRV=Fx4KPUExFLizQPvk z|5;>+v;5RsaMZhO48Zav(bQz_ypX#cz|PntbtjKd@Py zC;afd7)Sda&6{AcK-N8u=EnXtwzUq0o5rO8UngKt#w$rI&m}(g&37 zA%jy|3;)aZQ}L2?jF>whITQLZ=O38U^LtccC^wu@8>&L2pxSsz^;SXi8f2P4dR_DLfJ(Jdu;r%t8@mLm-?kC1t$q7P$6DAFk zCOkprA-0{m^2GM0lLF46R;CAczEafa<&0)pcdm#T*4KY*|NmM$RN~>?2_*+3lek4SMpdlM70g0w`Hrzy z&#MUND5po4jdNoHAl?;J8(NH1aWm)qX!Ks~|Bs3|?t8iM<=e;jzc2qI`SE}Ghp5EY z#e)6Jffjw0D-!*wmh=Th+-0>uRlTdnx$9qGwrDR8XUgZ2SY4#5rxRZIxSn$HqWWhG z{Qo*o-&U?lO<4<2L*h4`D!QoJS>09QGXZAFQAgtD$aZ%L5Tv&<#v=SWF}-NkUPm2O zyL?6136Cley#HhtqSKZQ>A@l3D!CJ;T2ZH@?|L-Ge5K&%wCHE0Y&ERJMIfaTI}y}r za?~5#am_GDvWH`;0Up>wpaLL=ZdXW-`bUD~2gkI>n-(wc%@sB3MNNOSCO)2q2l%gR z88>c&dJbB*qm44|Ub??m0#_>$EGcr%ley>YYU{M!z~rFJvSPk$=A0#7)ei!Kh#@&t z8~_W7D>}*Vk^{)Gk_6ItXV=QTBTF18PoJkXmxoH?g$ZAalV$|@D!l0>A~)n+sJd-!aN`H19;O%OB^B5_p)| zQ^^}^>Hx;h2XNF#*pI!)IQl)$Cfv}`-Vw>|W(r**sL+$gwv z;w3d)?uyeL<1SkR6YZRNc0{w83o&a#022A)==)0F^06ZGv{8|4u1C5_GDw!NT)w5< zxyvm=DU+frQIGvHESPQWqUVxl6av_9zf6-?4TbSX@s$f&m0MZz5kwH;iAlaW=+{dQ zoZURtO7?{6K`=w@c&1^k@qC-@s0rv2D)IS&r$)kw#Zi#{ls5XT@OL)f*>yxDulU!L?c!0MRIFGX#K zj1Uu?pS6^9Jd+EgkjH!Z;x%d;o|Kul28QzXwiKn`NU{gsZY=>M&uUIrPLFlE=Ulg& zcclxl2O$Fj>=vg^SQA^j$EBpQ8^dN_Mn(^lGa0bRa$lzx21caC6ibIC9Ug=<#_4ql zIfZi^0ooI^P!lrd!^Ov zb8P}-HIDcZZsdbQO>LLtNKIeE!Lr-kAKzH)3o7)d!l2p#_^NMw4j)fPB5~}kKs~t~ zZ*IYFn=yCxiFG6+F*B~kHa*efU`k17VMNEBwx4I?0DH@0 zR`yrbXps?#qM@p8Ub9s`?1;4-0ET^+Ps*0BdtmmQ;&g;o(#M$wQF!RERR~|ZxjX#G zg#kOC!?Z&~b7X_?OX6HphTl}=yE9vhACvNuV2rN`F)XNp5k3E?8a~p+1;})}d@Od( zkPZGu-miJAps}xXJxy02VD;TnjU1bBC!8Z+CBEE(#O8234j(VcmA}LL+C_RXybp(a z^SwUknBmc->pRys`R}wt|D(1o&-NDHt_Cx^%Md5pMD{{mN;&g8`fWHhOKPaakO#I< zJ$(K2#TO3CQd?lK;dQS)%0g>*3EMt>>%lq0;ocZUh8kd5)5{V*<{}vQq7WcKMnMcj zc&yb4_O%yVMqViPm+Ez%x)X>Ktl<{x@JfV!dleO*fbaJB7WYK=Zu(MqF+>(>L-Gq! z%s=e*Y)v3d=3bWW>d2dNlA#tQoTeY`Ivu%$?WImPc z-}{0QsI0FfY}{wfOfcH=S`lAHIt_c9-RNTyfj;IAC)beFj(m3*_?Ny?KrhIqC!fL> z$({;`zg@~zS5)6^Wz-wXitoIet?WFtyBjvardRP;S<}0_YPJd$=Kjmgz1)HGjrzx5 z2)p%k?LFew#u^QEm1p8zNY9t1V)jF|R6WFF9K~+kD;Ldqz#i5JUbiT-wcj+&P!#HM z*<`+8>n&rbwrXX<$WXnJhECUv?uHCUg>vNZ*^|)A-dkYB8pr3Sy(ql$E-ry&qmQ%c zq`a**QGkCYFNPHsdJOjsVBDt5fxAv49F&P8?}Yg2Dpg7bCeH)hp^*=fgH4CM?)exI z7CX1$w)NdsZ7Ye;ETgBAi?5!#g3)8=%+4saK+?U`X6%&dT2I2O1?B8c5E3WuI#9uq zo$89^vi#{mH+nqJOJ>gY^sWi%*RjUsd*Gb4>$^Hz+>I+7__!S)^e-}d4PuH)cpXY_ zY?IF3m$_hSnvyK5IxfDCJ%pyuUU{>xBGsCQoQ}}}G$^b4sz-#^Twhi~F*#XV_4B#C zE&_+v@Vd{)U9BZ@a+W%8(_BJ>KLX5i&m%!?V5vaU zeGy&7E_HSw@dP?cfo4>mJkteU7w34Xa~OlpW5SoV^WLpKS5Y8v+e*!jc6FNRJDI;ztZDS67;OA_gmd_Yy_js@ez z20qOCHk%ft%(AZ--u;;=Xu@0YaWiki8W6tt)Mp;Y&mX0iC93{ky|Slh?OT(kEybmFXx$T%(529}2kYME*Xrbo0h zU9r}SRyFj!yr!|0x^NOL?C6tGkOL4|3@WSrn&4AbA#>+xcK@{oec;J?ux)M{=8iT! zld!mlw8B0Zf!}(CPKD~XLXN&i8N#Ex7d5CY`jX&Dr4g$e_jX&VF5$inVnNul!mEpz z6vpcAsP7`~JS6)@G9Evwi>Qxn1@1f^461vI)TjiBmnI@?&V_p2LW?vkIt;J%)jdAu zwWg8Ubq>-=N16tREEiUqq`;uChEq}Z8?vN>R8}=LBQ)2jeLiywb`$!qRn)a*C(Y=w z`#Oo&)a7IG*P=SrEM`O%DdkDn&>208u=9*%XwneOX;*o#*&$2fxS1zMdrV#iG%030 z)YJ{fsx6Q98I{NyX2+ic1d#R`Ko>!l8>8|;nyZ^R!=Nf9beJ(6 zvf?FD-@*6nd?jRSxf&b0FiF!bb3^>7Nw5>!b5w`%!lo^*g#@wGLy% z3J{jq{RqEMsxGqJx;$OUgw{{>bHZlJJAxK>Tzi1@HQCB$c~|9P>0)RH`|TxYhh|m; z+?J?>P5g#5!)(f!+fdlC*LS54H3RZ%)5f>g@VZvI^tR?9V^KJMQo3JtBPZnOL7kty0cy_L zKGq;VeY5pYksY7(kF=y`S3^|9hthfDi%{=p+~0NKN$wD7HWQ!_Or?( zOJ86r=~WdMSNAaM1k~}Zk-R?BJfKZ=*EB&rpvm9*$aI5obIC;XK+UqoJo4Jd2%vwb zCL|?jV3z#_l#W?6@{3o&o}}2h6lY?$9Rf=_a~Gw)cH!`_RCv8x5CC|%0Rs2=0|x2%@0Z8UK^}1CtM|1pvvKVTmEYs zK{8@g{LPJZm#P%RNNw@rWf6Fl9H2MJA2ZQP0zkXI#yH2nyq@ zb#IIWjoik2yOa^fJ*$0Toh^iF8Xh>H+uezR&8*uG*0Fw?6FzE>nXjW%nfo$^G%B$e zQm%@sLor$FmANPx7_&T>x$bD+wx!mR`@juydP&z$aD)1MtWrV&Q>buIk#B(@;m!bt zJJOquW0%h!${v%4CJqTH(*ZGv8O?}wu9^0#nG)jduO6{FCI}%tel5*q;_y96P1BwV>+23 zax`vdf2F$s80NkN24KR!WHJ$QCa&$~SXlE=6kgamMJi ztSu0ebia_)u;d6(n}&(+2v&+Cvr(b)Wp{ISqq=f&Px>`5K%bd(_)1iwzo<3mOpY8M zG1ZE2gPb+emJGme>m{^M_+hIhjH@f`O9iZlhkEU`I+39G#_pATB9!4*njEm`HVWIu z^Qi*3prdX4P&a|wRv>74#-<<8Sd(579~vE`KEVVA4d1P@>#4|35=7>aHpjI_PqEMR5$%3Ajf zaYMI0&Tn?R4>*^5xP3W;@XBK#FGsvTUGbbC;-7zKqHu6?$yFF&n5z;}mOIUxYRg&z zc6O;mF(^7*CtW zc6_4|8fT8Igi=VQEe-6b)2V?+u0R++!<|H%jcb>0)0Rr_9k~K=e=1xTYM)hl&wk_z zG&`9P1RVYn{tc~orEug5)cKVEi)Q)mCEuUz)Iq{h%2TIm>pcj0R?LY@Jp8j;yg&UI zA2l%Cy=ISUM5tmUaOtYHa5GDPv_e)hMnXTMC_KI=`N+&6SFqOuA?!-+dZUKTXpE8P z;`2~Ys^gu{CxqH^7js2IXFgr28W^u(h~-XjSqb@6m79&he-2O~j+qxsa&eiYL|6DT zvbEm-&x3dJ#qEtiR`=#M_gxe23ro3+4vw!Mmt{)i_r6+wEFPcapTT*zNwANIZLlTx zU(kO}DSTxxkiQ3J;J~_(+1Z=HScq59%WF4k3Ah@u%{XGzrL$m4i>P~W(Uo^VUb18%UPutMte&Qd4_TvCvAG~G z9Hnxav%uTj!7x*I?fU-cBBZnISh$&^Gdv;F4r-u+$1(Er>$`~b7{?f54dfMNk<=vclnCCp|wW+{}$)pk%UPRN!GBc0`xU!jIM zr(dX;CEKgkLK4n#hxxMXc#KE=a{K1dk#+NR@eF-y*y@}-Ct=`DZyPVCV9LZpui8ZK zAu3V$Hh%Tqr%TjscQ*nZn#&!B-1E5LLkuGEArObhSITQt)agTtK^@*))RHelef}GeXR)RJsy6INdW?%?HuL;gT+JdId`~ zfh8WHiWoQwznz8DqD#u2o{8dX#XmI;u5${kmt11Ct7*XKil%jW=7hx+7RKc6? ze!2Wfnez}Svjq*~oqakuQ^+}8N6p0BUw<2muFgElBm5}~tF3C3jc=ePS;j@)#K+_s z(eHAU+HBdoVDeaqz2cWRinDur9xphU6T&9uA3P+$8u$gXJVl_vpIXc!qcVbDzj1x{ zZ2!`MA9F7YFiT3`G^>mLkrGe&6Q7ti3qT$}xwy|pm`>66V2tAQoGkTn<-ZT=TUkWJQRft1>JBWaCU2HlsrT6PL- b=4;4ixb(J4P*2~&bqW@@za9|wuOt5lsgukw literal 0 HcmV?d00001 diff --git a/docs/creating_procs.md b/docs/creating_procs.md new file mode 100644 index 00000000..064c559b --- /dev/null +++ b/docs/creating_procs.md @@ -0,0 +1,115 @@ +# Creating procs + +Main purpose of proctor is running procs so you need to know how to create procs. +Here are the recommended steps to create procs. + +#### 1. Define the task you want to automate. +Candidate task for a procs usually involving access to restricted resources or having complicated steps. + +#### 2. Define the interface of the task. +A task takes input, performs operation and provide output. + +#### 3. Research how to automate the task +Before start writing the script, please do research on how to do the automation by reading the documentation or stuff. + +#### 4. Write the script +Because procs will be run by executor(which is computer) you need to write down the script. +Write a script such that it's runnable on your local machine. For this step you can hardcode on input, we'll extract it out later. + +#### 5. Test the script +Don't skip this step, please test it on your local. + +#### 6. Package script in a docker image +Proctor leverage container to easily run task with all it's dependency, that's why you need to package your script into a docker image. +Install every dependency on your dockerfile. Provide an `ENTRYPOINT` to run the script by default when the container is spun up from the image. + +#### 7. Extract all hard-coded variables and secrets as configurable ENV vars +The image are meant to be reusable so user can use it according their use case, this is why every variables that define the behaviour of the automation should be extracted as args. + +#### 8. Perform validations on ENV vars +Some ENV vars are mandatory in order to use the automation, adding validation before running script helps failing fast and provide better error messages to user. + +#### 9. Test the image (Pass all variables as env) +Build the image then run it in local docker to make sure your image run as expected. +After you complete this step, your task can be automated using docker on any machine, post these steps you automation will evolve into a `proc` + +#### 10. Create metadata +Create metadata file to describe information about your procs to user. +Metadata look like this; +```json +{ + "name": "echo-worker", + "description": "This procs will echo your name", + "image_name": "walbertusd/echo-worker", + "env_vars": { + "secrets": [ + { + "name": "SECRET_NAME", + "description": "My other secret name" + } + ], + "args": [ + { + "name": "NAME", + "description": "Name to be echoed" + } + ] + }, + "authorized_groups": [ + "my-group" + ], + "author": "Dembo", + "contributors": "Dembo", + "organization": "GoJek" +} +``` + +#### 11. Upload metadata +Send `POST` request to your proctor service on `/metadata`, it receive array of metadata as json so your request body should look like this: +```json +[ + { + "name": "echo-worker", + "description": "This procs will echo your name", + "image_name": "walbertusd/echo-worker", + "env_vars": { + "secrets": [ + { + "name": "SECRET_NAME", + "description": "My other secret name" + } + ], + "args": [ + { + "name": "NAME", + "description": "Name to be echoed" + } + ] + }, + "authorized_groups": [ + "my-group" + ], + "author": "Dembo", + "contributors": "Dembo", + "organization": "GoJek" + } +] +``` + +#### 12. Store secret +User aren't supposed to know the secret value to run your jobs so make sure to keep it secret. +Send `POST` request to your proctor service on `/secret`, your request body will look like this: +```json +{ + "job_name": "echo-worker", + "secrets": { + "SECRET_NAME": "Iron Man" + } +} +``` + +#### 13. Test your proctor using CLI +Execute your procs using CLI, make sure it success and the resulting log is correct. + +#### 14. Complete +Congratulations! You've just automated one repetitive task! diff --git a/docs/glossary.md b/docs/glossary.md new file mode 100644 index 00000000..152c83f3 --- /dev/null +++ b/docs/glossary.md @@ -0,0 +1,30 @@ +# Proctor Glossary + +### Execution context +Execution context or context is a record of procs execution request by user to execute. +Context contain data such as: + * procs name + * context name + * user email + * procs tag + * args given + * procs output + * procs status + +### Procs +Procs is a job to execute using Proctor bundled as docker image and have set of metadata and secrets. + +### Procs metadata +Procs metadata or mostly addressed as metadata contain metadata of procs such as: + * name: Name of the procs + * description: Description of the procs + * author: Who create this procs + * contributors: People that contribute to the procs + * organization: Which org own this procs + * env_vars: + * secrets: Secret value that required by procs to run + * args: Arguments that can be passed to procs + +### Secret +Secret variable required to run procs such as credentials for cloud platform. +Proctor user shouldn't know the value of secret. From a95aaae7f364cf94c1493bbece6fbaed0110bfb3 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 10 Sep 2019 14:21:56 +0700 Subject: [PATCH 191/268] [Dembo] Add travis stage to publish docker image --- .travis.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.travis.yml b/.travis.yml index b6b21e34..f4bca488 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,18 +7,36 @@ go: services: - redis-server - postgresql + - docker + +env: + global: + - secure: qQqXZ/2rvDqyvlQOqjPzE7I7XKwug18Hxl65gGfL9G69HiY2BFY4rN/hBxXQXXWTJK4F3LYNgpGgSYHC8v6UdDBUoltxcSEgiYluDXkz31cej/O1pUXR+cH1H3y9AM+D8S89YmDlBuT3OthZRE1K77hwXgETOxAe59UzP81COK/vrZBW8n/cQjCRnLPoNzfI/jIhHAIWmcNRS5hxD9zPvN98P5n2H0tfzEMNhvwW3EUoziEk6srKUu0bhsH4Djl6MR8vGkqWn1TM3LzwaxADLw9qNlZnT6+EHJbScqcdm19KFM/13ADzSNOi7mnLaiiIXreffNZhbUETpyG508bMPS9G0FcKiz1b/8hWPGWVQQYDDVZ4F2Hw3i7sGC+7wnbUUYinjgpAfH6Omp1lfRzzSEFQMPlFYb1RPBDE5cgBJaiS6cmx98H/c3gbu7euEIVxggOItg1Msyek/FTraFxhHn3KGMcG7CvlqXfUfdoPBLHYQ+plTUTZ2IBfnnAvvrKZJ1Fqp7S3LZvZDvIm0ksg3iaM8TSylEViYu8+aB4Zi/EEwJTP7m8CU0I0LBno9HYkGEAfXtW9XeS5CtFJG5mBqw9cgXjWBEXdY376Ptp1/76//AgpRF7R9zXNBwcB5sOkM57Z5ZBgo7PE9duTZQ09hTsPlgR0f4q4rZreUe/Hx00= + - secure: GrY4QImh1SiTcQ59sVcJTnRpdc0yLQxUHBkhAZIdj5h+qRpDoWUS+QUoEj+F37GpDkSuKBI863mXOFWsszKq7bP5cu+S8xI45ZpEGSO0AJ4en/Mf6sFOEghx5Kn6EYKRrcZCOFruFztwQX1Men3WtcCgfz7ID1yc7d+NXl5Ai0OiiFax9DfNwDyD/PE3qk5tuiUH5Blo+603xqchCA0Qsr1v8NQPgdQJgsSVddpPrJAeDOHu4uEtSjtxHjxIJBuHMu5Z7lb6ksXSqW65ntpVWOpzhlyYQSIEsuCp7Y+o+R2X/P5ocqMGAwql7EgeVG/HRrpNETn66c0boXwIcX+pkZAcn0OnJOzpRLWvSEH/jJuadEZO1XtrI4xoBbU79XPlHlEjKnWkfhalUAYYu+mts0ijNa4xLrusTB5WSgpuLYjioOBpmC5WDdj4mb+WSQVk2z3Q/fiZ76mghzCM7Of0k6Agu7nInR/2W2+tcJnRbsm3gUhvCXxjebbAIVtclDg0E/pQhaYG/u7eDE6GJweZfeXXaoW6qiRD8697PMCKmGaxeuH9d0qxBvanW05Yi13/nX0iULb0e809uwOzuZIAHg4IxJV1KfMj6N/Ve8wCIQNySIsvtGDkA0SaqElNu+9AVavrASUmaF7yH8fMPqPZFQwiK2M84zCwUnA+a+DbaTE= before_script: - sudo service redis-server start stages: - test + - name: docker publish + if: (branch = master OR tag IS present) and type != pull_request jobs: include: - stage: test script: - make db.setup test + - stage: docker publish + install: skip + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - docker build -t proctorgojek/proctor-service:latest . + - if [ -n "$TRAVIS_TAG" ]; then + docker tag proctorgojek/proctor-service:latest proctorgojek/proctor-service:$TRAVIS_TAG; + fi + - docker images + - docker push proctorgojek/proctor-service #after_success: # - scripts/release.sh From 666cd62f82880eb30cb34bd80cb67207f668bcae Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 10 Sep 2019 14:40:55 +0700 Subject: [PATCH 192/268] [Dembo] Change docker login on travis --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index f4bca488..d1e3f4ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,8 +11,8 @@ services: env: global: - - secure: qQqXZ/2rvDqyvlQOqjPzE7I7XKwug18Hxl65gGfL9G69HiY2BFY4rN/hBxXQXXWTJK4F3LYNgpGgSYHC8v6UdDBUoltxcSEgiYluDXkz31cej/O1pUXR+cH1H3y9AM+D8S89YmDlBuT3OthZRE1K77hwXgETOxAe59UzP81COK/vrZBW8n/cQjCRnLPoNzfI/jIhHAIWmcNRS5hxD9zPvN98P5n2H0tfzEMNhvwW3EUoziEk6srKUu0bhsH4Djl6MR8vGkqWn1TM3LzwaxADLw9qNlZnT6+EHJbScqcdm19KFM/13ADzSNOi7mnLaiiIXreffNZhbUETpyG508bMPS9G0FcKiz1b/8hWPGWVQQYDDVZ4F2Hw3i7sGC+7wnbUUYinjgpAfH6Omp1lfRzzSEFQMPlFYb1RPBDE5cgBJaiS6cmx98H/c3gbu7euEIVxggOItg1Msyek/FTraFxhHn3KGMcG7CvlqXfUfdoPBLHYQ+plTUTZ2IBfnnAvvrKZJ1Fqp7S3LZvZDvIm0ksg3iaM8TSylEViYu8+aB4Zi/EEwJTP7m8CU0I0LBno9HYkGEAfXtW9XeS5CtFJG5mBqw9cgXjWBEXdY376Ptp1/76//AgpRF7R9zXNBwcB5sOkM57Z5ZBgo7PE9duTZQ09hTsPlgR0f4q4rZreUe/Hx00= - - secure: GrY4QImh1SiTcQ59sVcJTnRpdc0yLQxUHBkhAZIdj5h+qRpDoWUS+QUoEj+F37GpDkSuKBI863mXOFWsszKq7bP5cu+S8xI45ZpEGSO0AJ4en/Mf6sFOEghx5Kn6EYKRrcZCOFruFztwQX1Men3WtcCgfz7ID1yc7d+NXl5Ai0OiiFax9DfNwDyD/PE3qk5tuiUH5Blo+603xqchCA0Qsr1v8NQPgdQJgsSVddpPrJAeDOHu4uEtSjtxHjxIJBuHMu5Z7lb6ksXSqW65ntpVWOpzhlyYQSIEsuCp7Y+o+R2X/P5ocqMGAwql7EgeVG/HRrpNETn66c0boXwIcX+pkZAcn0OnJOzpRLWvSEH/jJuadEZO1XtrI4xoBbU79XPlHlEjKnWkfhalUAYYu+mts0ijNa4xLrusTB5WSgpuLYjioOBpmC5WDdj4mb+WSQVk2z3Q/fiZ76mghzCM7Of0k6Agu7nInR/2W2+tcJnRbsm3gUhvCXxjebbAIVtclDg0E/pQhaYG/u7eDE6GJweZfeXXaoW6qiRD8697PMCKmGaxeuH9d0qxBvanW05Yi13/nX0iULb0e809uwOzuZIAHg4IxJV1KfMj6N/Ve8wCIQNySIsvtGDkA0SaqElNu+9AVavrASUmaF7yH8fMPqPZFQwiK2M84zCwUnA+a+DbaTE= + - DOCKER_USERNAME=proctorbuilder + - secure: "ZCOM7TMiOsBO55V0VfAI1WQKODLZ4SUq50ngjZBJ2RfrOhHZ9zAqJgjw32bJtzUxuFX7Ez/44UEwd/Z28zbQjnzvIm8jwAwM+zxGH8zPlT9K9nuEN5edCO8oCYb5IdN4uPQkDw2w9FSNkRf8WfgS5Ru53SGiH+SVf2x/DT767HyP8EKhHWXJkKloxvR8wx7dSTr0vyQiNrslhh3EAVGQciZXLh2c6/pkZr0z/I5TpQnvWbvU0w9n9CqhhOaDiYI5zgKrpD/BdnYBmZtxfWO/RDyivacLDgozGcrV3iTJ6FkDbMgmhzrF0sAQL9LSrDOzUq6Yfe476c5BRHPgvh/gBHrpY6y7jmFGQ4vxER+vI9/rVILRLQfUSAYJBwuTAu7Wf/lZulw0GoQfIdaKa8RBCItaEN47HAPglxnPJBcl1ClYLBNc5XaHARdpKRurRz64AtBjCbTmoeUrdw9qktnKHSzxIPGQT1nKGvoizjpLUOURSwCp1T0GXxgcNX/65S/MruSFk5wf6jRz5x7ErVlv8MJLXWORV3EUR8sieXUTL5K0d2jq4NaGYTkbg9v4DwcgxMEm579gn+9MHPWGrrbC/k5xM03HjGYX44MnZb7jLuC0TRaYgnh4V8pe+ffxRY/+FLqDAnGtT7CxJsEONFy9jZrG9wVm2e0wVUxU9KxPvWE=" before_script: - sudo service redis-server start @@ -30,7 +30,7 @@ jobs: - stage: docker publish install: skip script: - - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - docker build -t proctorgojek/proctor-service:latest . - if [ -n "$TRAVIS_TAG" ]; then docker tag proctorgojek/proctor-service:latest proctorgojek/proctor-service:$TRAVIS_TAG; From d0737c0318376c51b41e7c8ff15d416c7ad32e30 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 11 Sep 2019 11:18:37 +0700 Subject: [PATCH 193/268] [Dembo] Add tags to schedule --- .../service/schedule/repository/schedule.go | 14 ++--- .../schedule/repository/schedule_test.go | 59 +++++++++++++++---- .../14_AddColumnTagsToSchedule.down.sql | 1 + migrations/14_AddColumnTagsToSchedule.up.sql | 1 + 4 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 migrations/14_AddColumnTagsToSchedule.down.sql create mode 100644 migrations/14_AddColumnTagsToSchedule.up.sql diff --git a/internal/app/service/schedule/repository/schedule.go b/internal/app/service/schedule/repository/schedule.go index f37b8f8a..83d8d76f 100644 --- a/internal/app/service/schedule/repository/schedule.go +++ b/internal/app/service/schedule/repository/schedule.go @@ -57,7 +57,7 @@ func (repository *scheduleRepository) Disable(id uint64) error { func (repository *scheduleRepository) Insert(context model.Schedule) (uint64, error) { snowflakeID, _ := id.NextID() context.ID = snowflakeID - sql := "INSERT INTO schedule (id, job_name, args,cron,notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :cron, :notification_emails, :user_email, :group, :enabled)" + sql := "INSERT INTO schedule (id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled) VALUES (:id, :job_name, :args, :tags, :cron, :notification_emails, :user_email, :group, :enabled)" _, err := repository.postgresqlClient.NamedExec(sql, &context) if err != nil { return 0, err @@ -75,7 +75,7 @@ func (repository *scheduleRepository) Delete(id uint64) error { } func (repository *scheduleRepository) GetByID(id uint64) (*model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email,\"group\", enabled, created_at, updated_at FROM schedule WHERE id=$1 " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email,\"group\", enabled, created_at, updated_at FROM schedule WHERE id=$1 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, id) if err != nil { @@ -90,7 +90,7 @@ func (repository *scheduleRepository) GetByID(id uint64) (*model.Schedule, error } func (repository *scheduleRepository) GetByUserEmail(userEmail string) ([]model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE user_email=$1 " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE user_email=$1 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, userEmail) if err != nil { @@ -101,7 +101,7 @@ func (repository *scheduleRepository) GetByUserEmail(userEmail string) ([]model. } func (repository *scheduleRepository) GetByJobName(jobName string) ([]model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE job_name=$1 " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE job_name=$1 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, jobName) if err != nil { @@ -112,7 +112,7 @@ func (repository *scheduleRepository) GetByJobName(jobName string) ([]model.Sche } func (repository *scheduleRepository) GetAllEnabled() ([]model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, true) if err != nil { @@ -123,7 +123,7 @@ func (repository *scheduleRepository) GetAllEnabled() ([]model.Schedule, error) } func (repository *scheduleRepository) GetAll() ([]model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql) if err != nil { @@ -134,7 +134,7 @@ func (repository *scheduleRepository) GetAll() ([]model.Schedule, error) { } func (repository *scheduleRepository) GetEnabledByID(id uint64) (*model.Schedule, error) { - sql := "SELECT id, job_name, args, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 AND id=$2 " + sql := "SELECT id, job_name, args, tags, cron, notification_emails, user_email, \"group\", enabled, created_at, updated_at FROM schedule WHERE enabled=$1 AND id=$2 " var schedules []model.Schedule err := repository.postgresqlClient.Select(&schedules, sql, true, id) if err != nil { diff --git a/internal/app/service/schedule/repository/schedule_test.go b/internal/app/service/schedule/repository/schedule_test.go index 09d3ae26..89661f1d 100644 --- a/internal/app/service/schedule/repository/schedule_test.go +++ b/internal/app/service/schedule/repository/schedule_test.go @@ -50,14 +50,21 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_Insert() { assert.NotNil(t, id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetByID(id) + actualSchedule, err := suite.repository.GetByID(id) assert.NoError(t, err) - assert.NotNil(t, expectedSchedule) - - assert.Equal(t, id, expectedSchedule.ID) - assert.NotNil(t, expectedSchedule.CreatedAt) - assert.NotNil(t, expectedSchedule.UpdatedAt) - assert.Equal(t, expectedSchedule.Args[mapKey], mapValue) + assert.NotNil(t, actualSchedule) + + assert.Equal(t, id, actualSchedule.ID) + assert.NotNil(t, actualSchedule.CreatedAt) + assert.NotNil(t, actualSchedule.UpdatedAt) + assert.Equal(t, mapValue, actualSchedule.Args[mapKey]) + assert.Equal(t, schedule.JobName, actualSchedule.JobName) + assert.Equal(t, schedule.UserEmail, actualSchedule.UserEmail) + assert.Equal(t, schedule.Cron, actualSchedule.Cron) + assert.Equal(t, schedule.Tags, actualSchedule.Tags) + assert.Equal(t, schedule.NotificationEmails, actualSchedule.NotificationEmails) + assert.Equal(t, schedule.Group, actualSchedule.Group) + assert.Equal(t, schedule.Enabled, actualSchedule.Enabled) } func (suite *ScheduleTestSuite) TestScheduleRepository_Delete() { @@ -100,6 +107,10 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetAll() { assert.NotNil(t, schedules) size := len(schedules) assert.Equal(t, recordCount, size) + for _, schedule := range schedules { + assert.NotNil(t, schedule) + assertScheduleCompleteParam(t, schedule) + } } func (suite *ScheduleTestSuite) TestScheduleRepository_GetByUserEmail() { @@ -120,6 +131,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetByUserEmail() { for _, schedule := range schedules { assert.Equal(t, suppliedEmail, schedule.UserEmail) + assertScheduleCompleteParam(t, schedule) } } @@ -141,6 +153,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetByJobName() { for _, schedule := range schedules { assert.Equal(t, suppliedJobName, schedule.JobName) + assertScheduleCompleteParam(t, schedule) } } @@ -161,6 +174,7 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetAllEnabled() { for _, schedule := range schedules { assert.True(t, schedule.Enabled) + assertScheduleCompleteParam(t, schedule) } } @@ -189,16 +203,27 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_GetEnabledByID() { assert.NotNil(t, id) assert.NoError(t, err) - expectedSchedule, err := suite.repository.GetEnabledByID(id) + actualSchedule, err := suite.repository.GetEnabledByID(id) assert.NoError(t, err) - assert.NotNil(t, expectedSchedule) - assert.True(t, expectedSchedule.Enabled) + assert.NotNil(t, actualSchedule) + assert.True(t, actualSchedule.Enabled) willNotExistsID := uint64(17777717) unexpectedSchedule, err := suite.repository.GetEnabledByID(willNotExistsID) assert.Error(t, err) assert.Nil(t, unexpectedSchedule) + assert.Equal(t, id, actualSchedule.ID) + assert.NotNil(t, actualSchedule.CreatedAt) + assert.NotNil(t, actualSchedule.UpdatedAt) + assert.Equal(t, mapValue, actualSchedule.Args[mapKey]) + assert.Equal(t, schedule.JobName, actualSchedule.JobName) + assert.Equal(t, schedule.UserEmail, actualSchedule.UserEmail) + assert.Equal(t, schedule.Cron, actualSchedule.Cron) + assert.Equal(t, schedule.Tags, actualSchedule.Tags) + assert.Equal(t, schedule.NotificationEmails, actualSchedule.NotificationEmails) + assert.Equal(t, schedule.Group, actualSchedule.Group) + assert.Equal(t, schedule.Enabled, actualSchedule.Enabled) } func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { @@ -245,6 +270,20 @@ func (suite *ScheduleTestSuite) TestScheduleRepository_EnableDisable() { } +func assertScheduleCompleteParam(t *testing.T, schedule model.Schedule) { + assert.NotNil(t, schedule) + assert.NotNil(t, schedule.CreatedAt) + assert.NotNil(t, schedule.UpdatedAt) + assert.NotEmpty(t, schedule.Args) + assert.NotEmpty(t, schedule.JobName) + assert.NotEmpty(t, schedule.UserEmail) + assert.NotEmpty(t, schedule.Cron) + assert.NotEmpty(t, schedule.Tags) + assert.NotEmpty(t, schedule.NotificationEmails) + assert.NotEmpty(t, schedule.Group) + assert.NotNil(t, schedule.Enabled) +} + func populateSeedDataForTest(repository ScheduleRepository, count int, seedField map[string]string) error { for i := 0; i < count; i++ { fake.Seed(0) diff --git a/migrations/14_AddColumnTagsToSchedule.down.sql b/migrations/14_AddColumnTagsToSchedule.down.sql new file mode 100644 index 00000000..80f6b053 --- /dev/null +++ b/migrations/14_AddColumnTagsToSchedule.down.sql @@ -0,0 +1 @@ +ALTER TABLE schedule DROP COLUMN if EXISTS tags; diff --git a/migrations/14_AddColumnTagsToSchedule.up.sql b/migrations/14_AddColumnTagsToSchedule.up.sql new file mode 100644 index 00000000..4b446e3d --- /dev/null +++ b/migrations/14_AddColumnTagsToSchedule.up.sql @@ -0,0 +1 @@ +ALTER TABLE schedule ADD COLUMN tags varchar(255); From cbcbfee6b39711e3b0f11ed04b35ef444181e832 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 11 Sep 2019 16:23:24 +0700 Subject: [PATCH 194/268] [Dembo] Add tags to schedule payload --- internal/app/cli/daemon/client.go | 1 + internal/pkg/model/schedule/schedule_job.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/app/cli/daemon/client.go b/internal/app/cli/daemon/client.go index 39d58ed7..b22bef70 100644 --- a/internal/app/cli/daemon/client.go +++ b/internal/app/cli/daemon/client.go @@ -82,6 +82,7 @@ func (c *client) ScheduleJob(name, tags, cron, notificationEmails, group string, NotificationEmails: notificationEmails, Args: jobArgs, Group: group, + Enabled: true, } requestBody, err := json.Marshal(jobPayload) diff --git a/internal/pkg/model/schedule/schedule_job.go b/internal/pkg/model/schedule/schedule_job.go index 0740f316..21d8d831 100644 --- a/internal/pkg/model/schedule/schedule_job.go +++ b/internal/pkg/model/schedule/schedule_job.go @@ -8,4 +8,5 @@ type ScheduledJob struct { Cron string `json:"cron"` Tags string `json:"tags"` Group string `json:"group"` + Enabled bool `json:"enabled"` } From 78927e7194172b4b4fe140380c9db9ec36b81801 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 13 Sep 2019 10:56:01 +0700 Subject: [PATCH 195/268] [Dembo] Add mechanism for middleware to ignore route --- .../security/middleware/authentication.go | 27 ++++++++++++++-- .../middleware/authentication_test.go | 31 +++++++++++++++++-- .../service/security/middleware/middleware.go | 1 + 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 23c2e31d..468ebfd3 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -4,6 +4,8 @@ import ( "context" "net/http" + "github.com/gorilla/mux" + "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/security/service" @@ -11,8 +13,9 @@ import ( ) type authenticationMiddleware struct { - service service.SecurityService - enabled bool + service service.SecurityService + enabled bool + excludedRoutes []*mux.Route } func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { @@ -21,6 +24,10 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht next.ServeHTTP(w, r) return } + if middleware.isRequestExcluded(r) { + next.ServeHTTP(w, r) + return + } token := r.Header.Get(constant.AccessTokenHeaderKey) userEmail := r.Header.Get(constant.UserEmailHeaderKey) if token == "" || userEmail == "" { @@ -38,6 +45,22 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht }) } +func (middleware *authenticationMiddleware) Exclude(routes ...*mux.Route) { + for _, route := range routes { + middleware.excludedRoutes = append(middleware.excludedRoutes, route) + } +} + +func (middleware *authenticationMiddleware) isRequestExcluded(r *http.Request) bool { + for _, route := range middleware.excludedRoutes { + match := mux.RouteMatch{} + if route.Match(r, &match) { + return true + } + } + return false +} + func NewAuthenticationMiddleware(securityService service.SecurityService) Middleware { proctorConfig := config.Load() return &authenticationMiddleware{ diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 6bca9e5f..2bfe019a 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -1,6 +1,7 @@ package middleware import ( + "github.com/gorilla/mux" "github.com/pkg/errors" "net/http" "net/http/httptest" @@ -46,9 +47,9 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { defer ctx.tearDown() userDetail := &auth.UserDetail{ - Name: "William Dembo", - Email: "email@gmail.com", - Active:true, + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, Groups: []string{"system", "proctor_maintainer"}, } securityService := ctx.instance().securityService @@ -165,3 +166,27 @@ func TestAuthenticationMiddleware_MiddlewareFuncDisabled(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) } + +func TestAuthenticationMiddleware_Exclude(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + excludedRoute := &mux.Route{} + excludedRoute.Path("/ping").Methods("GET") + + authMiddleware := ctx.instance().authMiddleware + authMiddleware.Exclude(excludedRoute) + + testHandler := ctx.instance().testHandler + ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + req, _ := http.NewRequest("GET", ts.URL+"/ping", nil) + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go index 7008475b..6b53efb4 100644 --- a/internal/app/service/security/middleware/middleware.go +++ b/internal/app/service/security/middleware/middleware.go @@ -9,6 +9,7 @@ const ContextUserDetailKey string = "USER_DETAIL" type Middleware interface { MiddlewareFunc(http.Handler) http.Handler + Exclude(...*mux.Route) } type AuthorizationMiddleware interface { From 47c30fac1ce3f2bf7a8c0dbf183e84c064a57a46 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 13 Sep 2019 11:21:04 +0700 Subject: [PATCH 196/268] [Dembo] Exclude common route from authentication --- internal/app/service/server/router.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 22e40ada..a9077e9a 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -56,21 +56,24 @@ func NewRouter() (*mux.Router, error) { jobSecretsHandler := secretHTTPHandler.NewSecretHTTPHandler(secretsStore) scheduleHandler := scheduleHTTPHandler.NewScheduleHTTPHandler(scheduleStore, metadataStore) - router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { + authenticationMiddleware := securityMiddleware.NewAuthenticationMiddleware(_securityService) + authorizationMiddleware := securityMiddleware.NewAuthorizationMiddleware(_securityService, metadataStore) + + pingRoute := router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { _, _ = fmt.Fprintf(w, "pong") }) - router.HandleFunc("/docs", docs.APIDocHandler) - router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.Config().DocsPath)))) - router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { + docsRoute := router.HandleFunc("/docs", docs.APIDocHandler) + docsSubRoute := router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.Config().DocsPath)))) + swaggerRoute := router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { http.ServeFile(w, r, path.Join(config.Config().DocsPath, "swagger.yml")) }) + authenticationMiddleware.Exclude(pingRoute, docsRoute, docsSubRoute, swaggerRoute) + router = middleware.InstrumentNewRelic(router) router.Use(middleware.ValidateClientVersion) - authenticationMiddleware := securityMiddleware.NewAuthenticationMiddleware(_securityService) router.Use(authenticationMiddleware.MiddlewareFunc) - authorizationMiddleware := securityMiddleware.NewAuthorizationMiddleware(_securityService, metadataStore) authorizationMiddleware.Secure(router, "/execution", executionHandler.Post()).Methods("POST") router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") From 7e90d70f7e7f28147d3ef254fecd009335c3d8f8 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 13 Sep 2019 13:05:05 +0700 Subject: [PATCH 197/268] [Dembo] Add content for security in README --- README.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8c4974ed..521165d5 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,29 @@ Some route is protected by authentication, authorization or both. Authenticated user means that user should have account related with proctor. Authorized user means that user should be part of groups that defined on procs meatadata, for example when procs authorized groups is `proctor-user`, and `dev` then user need to be a member of both groups. -Proctor doesn't come with built in auth implementation, it's using configurable [plugin](#plugin) mechanism +A request need head these headers to pass auth process: +``` +'Access-Token: ' +'Email-Id: ' +``` + +List of routes that require authentication: + - POST /execution + - GET /execution/{contextId}/status + - GET /execution/logs + - GET /metadata + - POST /metadata + - POST /secret + - POST /schedule + - GET /schedule + - GET /schedule/{scheduleID} + - DELETE /schedule/{scheduleID} + +List of routes that require authorization: + - POST /execution + - POST /schedule + +Proctor doesn't come with built in auth implementation, it's using configurable [plugin](#plugin) mechanism. ## Plugin Proctor service support plugin for authentication and authorization using go plugin. From b0a812eb02bc3317161e1c1fd1154ac9439fff01 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 13 Sep 2019 16:09:23 +0700 Subject: [PATCH 198/268] [Dembo] Add configuration for kubernetes service name --- .env.sample | 1 + .env.test | 1 + internal/app/service/infra/config/config.go | 2 ++ internal/app/service/infra/config/config_test.go | 7 +++++++ 4 files changed, 11 insertions(+) diff --git a/.env.sample b/.env.sample index 02913b56..7a02c8d3 100644 --- a/.env.sample +++ b/.env.sample @@ -5,6 +5,7 @@ export PROCTOR_DEFAULT_NAMESPACE="default" export PROCTOR_REDIS_ADDRESS="localhost:6379" export PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS="10" export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS="60" +export PROCTOR_KUBE_SERVICE_ACCOUNT_NAME="default" export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE="140" export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE="4096" export PROCTOR_KUBE_CLUSTER_HOST_NAME="localhost:8001" diff --git a/.env.test b/.env.test index e8ae14cd..fed3a3e2 100644 --- a/.env.test +++ b/.env.test @@ -8,6 +8,7 @@ export PROCTOR_REDIS_ADDRESS=localhost:6379 export PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS=10 export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 export PROCTOR_KUBE_JOB_RETRIES=0 +export PROCTOR_KUBE_SERVICE_ACCOUNT_NAME=default export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 export PROCTOR_KUBE_WAIT_FOR_RESOURCE_POLL_COUNT=5 diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 70980e45..2e6bd9b0 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -60,6 +60,7 @@ type ProctorConfig struct { KubeLogProcessWaitTime time.Duration KubeJobActiveDeadlineSeconds *int64 KubeJobRetries *int32 + KubeServiceAccountName string PostgresUser string PostgresPassword string PostgresHost string @@ -103,6 +104,7 @@ func Load() ProctorConfig { KubeLogProcessWaitTime: time.Duration(fang.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")), KubeJobActiveDeadlineSeconds: GetInt64Ref(fang, "KUBE_JOB_ACTIVE_DEADLINE_SECONDS"), KubeJobRetries: GetInt32Ref(fang, "KUBE_JOB_RETRIES"), + KubeServiceAccountName: fang.GetString("KUBE_SERVICE_ACCOUNT_NAME"), PostgresUser: fang.GetString("POSTGRES_USER"), PostgresPassword: fang.GetString("POSTGRES_PASSWORD"), PostgresHost: fang.GetString("POSTGRES_HOST"), diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 642be974..5b7dd151 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -83,6 +83,13 @@ func TestKubeJobRetries(t *testing.T) { assert.Equal(t, &expectedValue, Load().KubeJobRetries) } +func TestKubeServiceName(t *testing.T) { + _ = os.Setenv("PROCTOR_KUBE_SERVICE_ACCOUNT_NAME", "proctor") + + expectedValue := "proctor" + assert.Equal(t, expectedValue, Load().KubeServiceAccountName) +} + func TestPostgresUser(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_USER", "postgres") From c7663a53fb00cee8865735c35251243acf14c657 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Fri, 13 Sep 2019 16:54:11 +0700 Subject: [PATCH 199/268] [Dembo] Edit kubernetes execute job with custom service name --- internal/app/service/infra/kubernetes/client.go | 5 +++-- .../app/service/infra/kubernetes/client_integration_test.go | 2 ++ internal/app/service/infra/kubernetes/client_test.go | 2 ++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 0daec2a9..6c66787f 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -149,8 +149,9 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m } podSpec := v1.PodSpec{ - Containers: []v1.Container{container}, - RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{container}, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: config.Load().KubeServiceAccountName, } objectMeta := meta.ObjectMeta{ diff --git a/internal/app/service/infra/kubernetes/client_integration_test.go b/internal/app/service/infra/kubernetes/client_integration_test.go index 56f0c7f4..bb2c6084 100644 --- a/internal/app/service/infra/kubernetes/client_integration_test.go +++ b/internal/app/service/infra/kubernetes/client_integration_test.go @@ -31,6 +31,7 @@ func (suite *IntegrationTestSuite) SetupTest() { func (suite *IntegrationTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + _ = os.Setenv("PROCTOR_KUBE_SERVICE_ACCOUNT_NAME", "default") config.Reset() envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "busybox" @@ -59,6 +60,7 @@ func (suite *IntegrationTestSuite) TestJobExecution() { expectedLabel := jobLabel(executedJobname) assert.Equal(t, expectedLabel, executedJob.ObjectMeta.Labels) assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) + assert.Equal(t, "default", executedJob.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, config.Config().KubeJobActiveDeadlineSeconds, executedJob.Spec.ActiveDeadlineSeconds) assert.Equal(t, config.Config().KubeJobRetries, executedJob.Spec.BackoffLimit) diff --git a/internal/app/service/infra/kubernetes/client_test.go b/internal/app/service/infra/kubernetes/client_test.go index 961a705c..872f80b3 100644 --- a/internal/app/service/infra/kubernetes/client_test.go +++ b/internal/app/service/infra/kubernetes/client_test.go @@ -69,6 +69,7 @@ func (suite *ClientTestSuite) SetupTest() { func (suite *ClientTestSuite) TestJobExecution() { t := suite.T() _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") + _ = os.Setenv("PROCTOR_KUBE_SERVICE_ACCOUNT_NAME", "default") config.Reset() envVarsForContainer := map[string]string{"SAMPLE_ARG": "sample-value"} sampleImageName := "img1" @@ -99,6 +100,7 @@ func (suite *ClientTestSuite) TestJobExecution() { assert.Equal(t, expectedLabel, executedJob.ObjectMeta.Labels) assert.Equal(t, expectedLabel, executedJob.Spec.Template.ObjectMeta.Labels) assert.Equal(t, map[string]string{"key.one": "true"}, executedJob.Spec.Template.Annotations) + assert.Equal(t, "default", executedJob.Spec.Template.Spec.ServiceAccountName) assert.Equal(t, config.Config().KubeJobActiveDeadlineSeconds, executedJob.Spec.ActiveDeadlineSeconds) assert.Equal(t, config.Config().KubeJobRetries, executedJob.Spec.BackoffLimit) From 3be5e90a8693b899caeb91dd16efef1cdf4c1733 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 17 Sep 2019 11:38:56 +0700 Subject: [PATCH 200/268] [Dembo] Add notification mechanism interfaces --- .../app/service/notification/service/notification.go | 9 +++++++++ pkg/notification/event/event.go | 11 +++++++++++ pkg/notification/observer.go | 7 +++++++ 3 files changed, 27 insertions(+) create mode 100644 internal/app/service/notification/service/notification.go create mode 100644 pkg/notification/event/event.go create mode 100644 pkg/notification/observer.go diff --git a/internal/app/service/notification/service/notification.go b/internal/app/service/notification/service/notification.go new file mode 100644 index 00000000..a1cd98da --- /dev/null +++ b/internal/app/service/notification/service/notification.go @@ -0,0 +1,9 @@ +package service + +import ( + "proctor/pkg/notification/event" +) + +type NotificationService interface { + Notify(evt event.Event) +} diff --git a/pkg/notification/event/event.go b/pkg/notification/event/event.go new file mode 100644 index 00000000..28037b4e --- /dev/null +++ b/pkg/notification/event/event.go @@ -0,0 +1,11 @@ +package event + +type Event interface { + Type() string + User() UserData + Content() map[string]string +} + +type UserData struct { + Email string +} diff --git a/pkg/notification/observer.go b/pkg/notification/observer.go new file mode 100644 index 00000000..84d1e653 --- /dev/null +++ b/pkg/notification/observer.go @@ -0,0 +1,7 @@ +package notification + +import "proctor/pkg/notification/event" + +type Observer interface { + OnNotify(evt event.Event) +} From 4d099861db0e41c18148d5b9e24390e4c81ce7fe Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 17 Sep 2019 14:01:10 +0700 Subject: [PATCH 201/268] [Jasoet] Change binary name for proctor2 --- .goreleaser.yml | 2 +- docs/glossary.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 7b25cea4..64a57e9f 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -10,7 +10,7 @@ builds: - id: "proctor-cli" main: ./cmd/cli/main.go - binary: proctor + binary: proctor2 env: - CGO_ENABLED=0 archive: diff --git a/docs/glossary.md b/docs/glossary.md index 152c83f3..342c6f2d 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,7 +1,7 @@ # Proctor Glossary ### Execution context -Execution context or context is a record of procs execution request by user to execute. +Execution context (or simply context) is a record of an execution that happens on proctor, both from user execution request or invocation from the scheduler. Context contain data such as: * procs name * context name @@ -10,11 +10,11 @@ Context contain data such as: * args given * procs output * procs status - + ### Procs -Procs is a job to execute using Proctor bundled as docker image and have set of metadata and secrets. +Procs is a job to execute using Proctor, its bundled as Docker image and have a set of metadata and secrets. Both metadata and secret is a key-value pair -### Procs metadata +### Metadata Procs metadata or mostly addressed as metadata contain metadata of procs such as: * name: Name of the procs * description: Description of the procs From 81ffc87e4119ab79aab57530cd0667134c7d30fd Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 17 Sep 2019 14:34:52 +0700 Subject: [PATCH 202/268] [Jasoet] Enable prometheus metrics exporter --- go.mod | 1 + go.sum | 6 ++++++ internal/app/service/execution/service/execution.go | 3 +++ internal/app/service/infra/metrics/prometheus.go | 13 +++++++++++++ internal/app/service/server/router.go | 5 ++++- 5 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 internal/app/service/infra/metrics/prometheus.go diff --git a/go.mod b/go.mod index c0db3c57..0a434dc8 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/newrelic/go-agent v2.9.0+incompatible github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/pkg/errors v0.8.1 + github.com/prometheus/client_golang v0.9.3 github.com/robfig/cron v1.2.0 github.com/satori/go.uuid v1.2.0 github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 48bccebc..906326f9 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,7 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/badoux/checkmail v0.0.0-20181210160741-9661bd69e9ad h1:kXfVkP8xPSJXzicomzjECcw6tv1Wl9h1lNenWBfNKdg= github.com/badoux/checkmail v0.0.0-20181210160741-9661bd69e9ad/go.mod h1:r5ZalvRl3tXevRNJkwIB6DC4DD3DMjIlY9NEU1XGoaQ= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91 h1:GMmnK0dvr0Sf0gx3DvTbln0c8DE07B7sPVD9dgHOqo4= github.com/briandowns/spinner v0.0.0-20190319032542-ac46072a5a91/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU= @@ -143,6 +144,7 @@ github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= @@ -172,12 +174,16 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3 h1:9iH4JKXLzFbOAdtqv/a+j8aewx2Y8lAjAydhbaScPF8= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index cb44fce2..2bfb4ac6 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "proctor/internal/app/service/infra/metrics" "time" "github.com/jmoiron/sqlx/types" @@ -84,6 +85,8 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st Status: status.Created, } + metrics.ExecutionCounter.Inc() + metadata, err := service.metadataRepository.GetByName(jobName) if err != nil { context.Status = status.RequirementNotMet diff --git a/internal/app/service/infra/metrics/prometheus.go b/internal/app/service/infra/metrics/prometheus.go new file mode 100644 index 00000000..35988115 --- /dev/null +++ b/internal/app/service/infra/metrics/prometheus.go @@ -0,0 +1,13 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + ExecutionCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "execution_total", + Help: "The total number of executions", + }) +) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index a9077e9a..124c53ac 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -2,6 +2,7 @@ package server import ( "fmt" + "github.com/prometheus/client_golang/prometheus/promhttp" "net/http" "path" @@ -69,7 +70,9 @@ func NewRouter() (*mux.Router, error) { http.ServeFile(w, r, path.Join(config.Config().DocsPath, "swagger.yml")) }) - authenticationMiddleware.Exclude(pingRoute, docsRoute, docsSubRoute, swaggerRoute) + metricsRoute := router.Handle("/metrics", promhttp.Handler()) + + authenticationMiddleware.Exclude(pingRoute, docsRoute, docsSubRoute, swaggerRoute, metricsRoute) router = middleware.InstrumentNewRelic(router) router.Use(middleware.ValidateClientVersion) From f9687f4f50f5f9361451d3097c27f71b9b868172 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 17 Sep 2019 14:44:56 +0700 Subject: [PATCH 203/268] [Dembo] Add execution event --- pkg/notification/event/execution.go | 42 ++++++++++++ pkg/notification/event/execution_test.go | 81 ++++++++++++++++++++++++ pkg/notification/event/type.go | 5 ++ 3 files changed, 128 insertions(+) create mode 100644 pkg/notification/event/execution.go create mode 100644 pkg/notification/event/execution_test.go create mode 100644 pkg/notification/event/type.go diff --git a/pkg/notification/event/execution.go b/pkg/notification/event/execution.go new file mode 100644 index 00000000..6207e68f --- /dev/null +++ b/pkg/notification/event/execution.go @@ -0,0 +1,42 @@ +package event + +import ( + "encoding/json" + "strconv" + + "proctor/internal/app/service/execution/model" +) + +type executionEvent struct { + userEmail string + context model.ExecutionContext +} + +func (evt executionEvent) Type() string { + return string(executionEventType) +} + +func (evt executionEvent) User() UserData { + return UserData{ + Email: evt.userEmail, + } +} + +func (evt executionEvent) Content() map[string]string { + executionContext := evt.context + jobArgsByte, _ := json.Marshal(executionContext.Args) + return map[string]string{ + "ExecutionID": strconv.FormatUint(executionContext.ExecutionID, 10), + "JobName": executionContext.JobName, + "ImageTag": executionContext.ImageTag, + "Args": string(jobArgsByte), + "Status": string(executionContext.Status), + } +} + +func NewExecutionEvent(userEmail string, context model.ExecutionContext) Event { + return executionEvent{ + userEmail: userEmail, + context: context, + } +} diff --git a/pkg/notification/event/execution_test.go b/pkg/notification/event/execution_test.go new file mode 100644 index 00000000..18119baf --- /dev/null +++ b/pkg/notification/event/execution_test.go @@ -0,0 +1,81 @@ +package event + +import ( + "encoding/json" + "strconv" + "testing" + "time" + + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/execution/handler/parameter" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/status" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() context +} + +type executionTestContext struct { + executionEvent executionEvent +} + +func (context *executionTestContext) setUp(t *testing.T) { +} + +func (context *executionTestContext) tearDown() { +} + +func (context *executionTestContext) instance() context { + return context +} + +func newExecutionTestContext() context { + ctx := &executionTestContext{} + return ctx +} + +func TestExecutionEvent(t *testing.T) { + ctx := newExecutionTestContext() + ctx.setUp(t) + defer ctx.tearDown() + + userEmail := "mrproctor@example.com" + executionContextId := uint64(7) + job := parameter.Job{ + Name: "sample-job-name", + Args: map[string]string{"argOne": "sample-arg"}, + } + executionContext := model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.Created, + Output: types.GzippedText("test"), + } + actualEvent := NewExecutionEvent(userEmail, executionContext).(executionEvent) + + expectedEventType := "EXECUTION_EVENT" + expectedUserData := UserData{ + Email: userEmail, + } + jobArgsByte, _ := json.Marshal(job.Args) + expectedContent := map[string]string{ + "ExecutionID": strconv.FormatUint(executionContext.ExecutionID, 10), + "JobName": executionContext.JobName, + "ImageTag": executionContext.ImageTag, + "Args": string(jobArgsByte), + "Status": "CREATED", + } + + assert.Equal(t, expectedEventType, actualEvent.Type()) + assert.Equal(t, expectedUserData, actualEvent.User()) + assert.Equal(t, expectedContent, actualEvent.Content()) +} diff --git a/pkg/notification/event/type.go b/pkg/notification/event/type.go new file mode 100644 index 00000000..40bb823c --- /dev/null +++ b/pkg/notification/event/type.go @@ -0,0 +1,5 @@ +package event + +type Type string + +const executionEventType Type = "EXECUTION_EVENT" From c7767a88faffce008d6f5e885b3e3b56ea7945d4 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 17 Sep 2019 14:51:08 +0700 Subject: [PATCH 204/268] [Dembo] Add observer mock --- pkg/notification/observer_mock.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 pkg/notification/observer_mock.go diff --git a/pkg/notification/observer_mock.go b/pkg/notification/observer_mock.go new file mode 100644 index 00000000..6b8c91e4 --- /dev/null +++ b/pkg/notification/observer_mock.go @@ -0,0 +1,15 @@ +package notification + +import ( + "github.com/stretchr/testify/mock" + "proctor/pkg/notification/event" +) + +type ObserverMock struct { + mock.Mock +} + +func (m *ObserverMock) OnNotify(evt event.Event) { + m.Called(evt) + return +} From 885e52c5e80a37f2da7c9ff4e2003e0ec68df802 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 17 Sep 2019 16:34:27 +0700 Subject: [PATCH 205/268] [Dembo] Add slack notification client --- .../slack-notification-plugin/slack/client.go | 34 ++++++ .../slack/client_test.go | 102 ++++++++++++++++++ .../slack-notification-plugin/slack/config.go | 18 ++++ .../slack/message.go | 5 + .../slack/message_mock.go | 14 +++ 5 files changed, 173 insertions(+) create mode 100644 plugins/slack-notification-plugin/slack/client.go create mode 100644 plugins/slack-notification-plugin/slack/client_test.go create mode 100644 plugins/slack-notification-plugin/slack/config.go create mode 100644 plugins/slack-notification-plugin/slack/message.go create mode 100644 plugins/slack-notification-plugin/slack/message_mock.go diff --git a/plugins/slack-notification-plugin/slack/client.go b/plugins/slack-notification-plugin/slack/client.go new file mode 100644 index 00000000..3fca4173 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/client.go @@ -0,0 +1,34 @@ +package slack + +import ( + "github.com/go-resty/resty/v2" +) + +type SlackClient interface { + Publish(message Message) error +} + +type slackClient struct { + client *resty.Client + config SlackConfig +} + +func (s *slackClient) Publish(message Message) error { + messageJson, err := message.JSON() + if err != nil { + return err + } + path := s.config.url + _, err = s.client.R(). + SetBody(messageJson). + SetHeader("Content-Type", "application/json"). + Post(path) + return err +} + +func NewSlackClient(client *resty.Client) SlackClient { + return &slackClient{ + client: client, + config: NewSlackConfig(), + } +} diff --git a/plugins/slack-notification-plugin/slack/client_test.go b/plugins/slack-notification-plugin/slack/client_test.go new file mode 100644 index 00000000..d4877fe6 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/client_test.go @@ -0,0 +1,102 @@ +package slack + +import ( + "errors" + "io/ioutil" + "net/http" + "testing" + + "github.com/go-resty/resty/v2" + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" +) + +type slackClientTest struct { + client SlackClient +} + +func (context *slackClientTest) setUp(t *testing.T) { + httpClient := resty.New() + httpmock.ActivateNonDefault(httpClient.GetClient()) + context.client = NewSlackClient(httpClient) + assert.NotNil(t, context.client) +} + +func (context *slackClientTest) tearDown() { + httpmock.DeactivateAndReset() +} + +func newNotificationServiceTestContext() *slackClientTest { + return &slackClientTest{} +} + +func TestSlackClient_Publish(t *testing.T) { + ctx := newNotificationServiceTestContext() + ctx.setUp(t) + defer ctx.tearDown() + + config := NewSlackConfig() + message := MessageMock{} + message.On("JSON").Return("message sent", nil) + + httpmock.RegisterResponder( + "POST", + config.url, + func(req *http.Request) (*http.Response, error) { + body, err := ioutil.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, "message sent", string(body)) + + contentType := req.Header.Get("Content-type") + assert.Equal(t, "application/json", contentType) + + response := httpmock.NewStringResponse(200, "") + return response, nil + }, + ) + err := ctx.client.Publish(&message) + assert.NoError(t, err) + assert.Equal(t, 1, httpmock.GetTotalCallCount()) +} + +func TestSlackClient_PublishErrorJSON(t *testing.T) { + ctx := newNotificationServiceTestContext() + ctx.setUp(t) + defer ctx.tearDown() + + message := MessageMock{} + message.On("JSON").Return("", errors.New("JSON unmarshal error")).Once() + + err := ctx.client.Publish(&message) + assert.Error(t, err) + assert.Equal(t, 0, httpmock.GetTotalCallCount()) +} + +func TestSlackClient_PublishErrorRequest(t *testing.T) { + ctx := newNotificationServiceTestContext() + ctx.setUp(t) + defer ctx.tearDown() + + config := NewSlackConfig() + message := MessageMock{} + message.On("JSON").Return("message sent", nil) + + httpmock.RegisterResponder( + "POST", + config.url, + func(req *http.Request) (*http.Response, error) { + body, err := ioutil.ReadAll(req.Body) + assert.NoError(t, err) + assert.Equal(t, "message sent", string(body)) + + contentType := req.Header.Get("Content-type") + assert.Equal(t, "application/json", contentType) + + response := httpmock.NewStringResponse(503, "") + return response, errors.New("internal server error") + }, + ) + err := ctx.client.Publish(&message) + assert.Error(t, err) + assert.Equal(t, 1, httpmock.GetTotalCallCount()) +} diff --git a/plugins/slack-notification-plugin/slack/config.go b/plugins/slack-notification-plugin/slack/config.go new file mode 100644 index 00000000..da21e17f --- /dev/null +++ b/plugins/slack-notification-plugin/slack/config.go @@ -0,0 +1,18 @@ +package slack + +import "github.com/spf13/viper" + +type SlackConfig struct { + url string +} + +func NewSlackConfig() SlackConfig { + fang := viper.New() + fang.AutomaticEnv() + fang.SetEnvPrefix("SLACK_PLUGIN") + fang.SetDefault("URL", "https://hooks.slack.com/services") + config := SlackConfig{ + url: fang.GetString("URL"), + } + return config +} diff --git a/plugins/slack-notification-plugin/slack/message.go b/plugins/slack-notification-plugin/slack/message.go new file mode 100644 index 00000000..60f70b16 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message.go @@ -0,0 +1,5 @@ +package slack + +type Message interface { + JSON() (string, error) +} diff --git a/plugins/slack-notification-plugin/slack/message_mock.go b/plugins/slack-notification-plugin/slack/message_mock.go new file mode 100644 index 00000000..20b3267f --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message_mock.go @@ -0,0 +1,14 @@ +package slack + +import ( + "github.com/stretchr/testify/mock" +) + +type MessageMock struct { + mock.Mock +} + +func (m *MessageMock) JSON() (string, error) { + args := m.Called() + return args.Get(0).(string), args.Error(1) +} From b970e6e77492fb9e65d4210d5433952af4f3364d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 17 Sep 2019 17:00:57 +0700 Subject: [PATCH 206/268] [Dembo] Add standard slack notification message --- .../slack/message.go | 20 +++++++++++++++++++ .../slack/message_test.go | 13 ++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 plugins/slack-notification-plugin/slack/message_test.go diff --git a/plugins/slack-notification-plugin/slack/message.go b/plugins/slack-notification-plugin/slack/message.go index 60f70b16..42195167 100644 --- a/plugins/slack-notification-plugin/slack/message.go +++ b/plugins/slack-notification-plugin/slack/message.go @@ -1,5 +1,25 @@ package slack +import "encoding/json" + type Message interface { JSON() (string, error) } + +type standardMessage struct { + Text string `json:"text"` +} + +func (message *standardMessage) JSON() (string, error) { + byteMessage, err := json.Marshal(message) + if err != nil { + return "", err + } + return string(byteMessage), nil +} + +func NewStandardMessage(text string) Message { + return &standardMessage{ + Text: text, + } +} diff --git a/plugins/slack-notification-plugin/slack/message_test.go b/plugins/slack-notification-plugin/slack/message_test.go new file mode 100644 index 00000000..b948044a --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message_test.go @@ -0,0 +1,13 @@ +package slack + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestStandardMessage_JSON(t *testing.T) { + message := NewStandardMessage("content") + result, err := message.JSON() + assert.NoError(t, err) + assert.Equal(t, "{\"text\":\"content\"}", result) +} From 12d1f48059ca9635b9dc2ff59e7596decdebc634 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 11:13:18 +0700 Subject: [PATCH 207/268] [Dembo] Add slack notification plugin main struct --- pkg/notification/event/event_mock.go | 22 +++++++ pkg/notification/event/execution.go | 2 +- pkg/notification/event/type.go | 2 +- .../slack/client_mock.go | 12 ++++ .../slack_notification.go | 26 ++++++++ .../slack_notification_test.go | 65 +++++++++++++++++++ 6 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 pkg/notification/event/event_mock.go create mode 100644 plugins/slack-notification-plugin/slack/client_mock.go create mode 100644 plugins/slack-notification-plugin/slack_notification.go create mode 100644 plugins/slack-notification-plugin/slack_notification_test.go diff --git a/pkg/notification/event/event_mock.go b/pkg/notification/event/event_mock.go new file mode 100644 index 00000000..e8bc1667 --- /dev/null +++ b/pkg/notification/event/event_mock.go @@ -0,0 +1,22 @@ +package event + +import "github.com/stretchr/testify/mock" + +type EventMock struct { + mock.Mock +} + +func (m *EventMock) Type() string { + args := m.Called() + return args.Get(0).(string) +} + +func (m *EventMock) User() UserData { + args := m.Called() + return args.Get(0).(UserData) +} + +func (m *EventMock) Content() map[string]string { + args := m.Called() + return args.Get(0).(map[string]string) +} diff --git a/pkg/notification/event/execution.go b/pkg/notification/event/execution.go index 6207e68f..1f5d662f 100644 --- a/pkg/notification/event/execution.go +++ b/pkg/notification/event/execution.go @@ -13,7 +13,7 @@ type executionEvent struct { } func (evt executionEvent) Type() string { - return string(executionEventType) + return string(ExecutionEventType) } func (evt executionEvent) User() UserData { diff --git a/pkg/notification/event/type.go b/pkg/notification/event/type.go index 40bb823c..cd3b00af 100644 --- a/pkg/notification/event/type.go +++ b/pkg/notification/event/type.go @@ -2,4 +2,4 @@ package event type Type string -const executionEventType Type = "EXECUTION_EVENT" +const ExecutionEventType Type = "EXECUTION_EVENT" diff --git a/plugins/slack-notification-plugin/slack/client_mock.go b/plugins/slack-notification-plugin/slack/client_mock.go new file mode 100644 index 00000000..12edcc50 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/client_mock.go @@ -0,0 +1,12 @@ +package slack + +import "github.com/stretchr/testify/mock" + +type SlackClientMock struct { + mock.Mock +} + +func (m *SlackClientMock) Publish(message Message) error { + args := m.Called(message) + return args.Error(0) +} diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go new file mode 100644 index 00000000..1a298a7a --- /dev/null +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -0,0 +1,26 @@ +package main + +import ( + "encoding/json" + "proctor/pkg/notification" + "proctor/pkg/notification/event" + "proctor/plugins/slack-notification-plugin/slack" +) + +type slackNotification struct { + slackClient slack.SlackClient +} + +func (notification *slackNotification) OnNotify(evt event.Event) { + evtDataJSON, _ := json.Marshal(evt.Content()) + textMessage := "User: " + evt.User().Email + "\n" + textMessage += "Execute job with detail: " + message := slack.NewStandardMessage(string(evtDataJSON)) + _ = notification.slackClient.Publish(message) +} + +func NewSlackNotification(slackClient slack.SlackClient) notification.Observer { + return &slackNotification{ + slackClient: slackClient, + } +} diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go new file mode 100644 index 00000000..c80a5ee6 --- /dev/null +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -0,0 +1,65 @@ +package main + +import ( + "proctor/pkg/notification" + "proctor/pkg/notification/event" + "proctor/plugins/slack-notification-plugin/slack" + "testing" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + slackNotification notification.Observer + slackClient *slack.SlackClientMock + event *event.EventMock +} + +func (context *testContext) setUp(t *testing.T) { + context.slackClient = &slack.SlackClientMock{} + context.slackNotification = NewSlackNotification(context.slackClient) + context.event = &event.EventMock{} +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + return &testContext{} +} + +func TestSlackNotification_OnNotify(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userData := event.UserData{ + Email: "proctor@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := ctx.instance().event + evt.On("Type").Return(event.ExecutionEventType) + evt.On("User").Return(userData) + evt.On("Content").Return(content) + + message := slack.NewStandardMessage("{\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + ctx.instance().slackClient.On("Publish", message).Return(nil) + + ctx.instance().slackNotification.OnNotify(evt) + + ctx.instance().slackClient.AssertExpectations(t) +} From 0735f94a279f39f8dc7227c75b0f77918f1bff6d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 11:15:40 +0700 Subject: [PATCH 208/268] [Dembo] Change observer signature to return error --- pkg/notification/observer.go | 2 +- pkg/notification/observer_mock.go | 6 +++--- plugins/slack-notification-plugin/slack_notification.go | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/notification/observer.go b/pkg/notification/observer.go index 84d1e653..4988e3b0 100644 --- a/pkg/notification/observer.go +++ b/pkg/notification/observer.go @@ -3,5 +3,5 @@ package notification import "proctor/pkg/notification/event" type Observer interface { - OnNotify(evt event.Event) + OnNotify(evt event.Event) error } diff --git a/pkg/notification/observer_mock.go b/pkg/notification/observer_mock.go index 6b8c91e4..49a9cac4 100644 --- a/pkg/notification/observer_mock.go +++ b/pkg/notification/observer_mock.go @@ -9,7 +9,7 @@ type ObserverMock struct { mock.Mock } -func (m *ObserverMock) OnNotify(evt event.Event) { - m.Called(evt) - return +func (m *ObserverMock) OnNotify(evt event.Event) error { + args := m.Called(evt) + return args.Error(0) } diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 1a298a7a..9ca2e6fd 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -11,12 +11,13 @@ type slackNotification struct { slackClient slack.SlackClient } -func (notification *slackNotification) OnNotify(evt event.Event) { +func (notification *slackNotification) OnNotify(evt event.Event) error { evtDataJSON, _ := json.Marshal(evt.Content()) textMessage := "User: " + evt.User().Email + "\n" textMessage += "Execute job with detail: " message := slack.NewStandardMessage(string(evtDataJSON)) _ = notification.slackClient.Publish(message) + return nil } func NewSlackNotification(slackClient slack.SlackClient) notification.Observer { From c1ddb05eea85ee80ae171842460f0476067925ff Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 11:21:29 +0700 Subject: [PATCH 209/268] [Dembo] Add test for slack notification plugin on publish error --- .../slack_notification.go | 9 +++-- .../slack_notification_test.go | 38 ++++++++++++++++++- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 9ca2e6fd..cef4890e 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -12,12 +12,15 @@ type slackNotification struct { } func (notification *slackNotification) OnNotify(evt event.Event) error { - evtDataJSON, _ := json.Marshal(evt.Content()) + evtDataJSON, err := json.Marshal(evt.Content()) + if err != nil { + return err + } textMessage := "User: " + evt.User().Email + "\n" textMessage += "Execute job with detail: " message := slack.NewStandardMessage(string(evtDataJSON)) - _ = notification.slackClient.Publish(message) - return nil + err = notification.slackClient.Publish(message) + return err } func NewSlackNotification(slackClient slack.SlackClient) notification.Observer { diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index c80a5ee6..ef5998ca 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -1,10 +1,14 @@ package main import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "proctor/pkg/notification" "proctor/pkg/notification/event" "proctor/plugins/slack-notification-plugin/slack" - "testing" ) type context interface { @@ -59,7 +63,37 @@ func TestSlackNotification_OnNotify(t *testing.T) { message := slack.NewStandardMessage("{\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") ctx.instance().slackClient.On("Publish", message).Return(nil) - ctx.instance().slackNotification.OnNotify(evt) + err := ctx.instance().slackNotification.OnNotify(evt) + assert.NoError(t, err) + + ctx.instance().slackClient.AssertExpectations(t) +} + +func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userData := event.UserData{ + Email: "proctor@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := ctx.instance().event + evt.On("Type").Return(event.ExecutionEventType) + evt.On("User").Return(userData) + evt.On("Content").Return(content) + + message := slack.NewStandardMessage("{\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + ctx.instance().slackClient.On("Publish", message).Return(errors.New("publish error")) + + err := ctx.instance().slackNotification.OnNotify(evt) + assert.Error(t, err) ctx.instance().slackClient.AssertExpectations(t) } From c0ebbf301834a13c76b730504b213b23d9d47698 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 11:24:58 +0700 Subject: [PATCH 210/268] [Dembo] Change message structure to publish --- plugins/slack-notification-plugin/slack_notification.go | 3 ++- plugins/slack-notification-plugin/slack_notification_test.go | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index cef4890e..d1d04378 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -18,7 +18,8 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { } textMessage := "User: " + evt.User().Email + "\n" textMessage += "Execute job with detail: " - message := slack.NewStandardMessage(string(evtDataJSON)) + textMessage += string(evtDataJSON) + message := slack.NewStandardMessage(textMessage) err = notification.slackClient.Publish(message) return err } diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index ef5998ca..5b4398c7 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -60,7 +60,7 @@ func TestSlackNotification_OnNotify(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - message := slack.NewStandardMessage("{\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + message := slack.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") ctx.instance().slackClient.On("Publish", message).Return(nil) err := ctx.instance().slackNotification.OnNotify(evt) @@ -89,7 +89,7 @@ func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - message := slack.NewStandardMessage("{\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + message := slack.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") ctx.instance().slackClient.On("Publish", message).Return(errors.New("publish error")) err := ctx.instance().slackNotification.OnNotify(evt) From 332f2c7042c5e2daf4483838f84e5ba5bf294ebd Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 11:38:05 +0700 Subject: [PATCH 211/268] [Dembo] Add slack client integration test --- .../slack/client_integration_test.go | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 plugins/slack-notification-plugin/slack/client_integration_test.go diff --git a/plugins/slack-notification-plugin/slack/client_integration_test.go b/plugins/slack-notification-plugin/slack/client_integration_test.go new file mode 100644 index 00000000..1da7d5e3 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/client_integration_test.go @@ -0,0 +1,53 @@ +package slack + +import ( + "os" + "testing" + + "github.com/go-resty/resty/v2" + "github.com/stretchr/testify/assert" +) + +type integrationContext interface { + setUp(t *testing.T) + tearDown() + instance() *integrationTestContext +} + +type integrationTestContext struct { + slackClient SlackClient + slackUrl string +} + +func (context *integrationTestContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_PLUGIN_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + client := resty.New() + context.slackClient = NewSlackClient(client) + slackUrl, _ := os.LookupEnv("SLACK_PLUGIN_URL") + assert.NotEmpty(t, slackUrl) + context.slackUrl = slackUrl +} + +func (context *integrationTestContext) tearDown() { +} + +func (context *integrationTestContext) instance() *integrationTestContext { + return context +} + +func newIntegrationContext() integrationContext { + return &integrationTestContext{} +} + +func TestSlackClientIntegration_Publish(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + message := NewStandardMessage("Message from slack plugin integration test with standard message") + err := ctx.instance().slackClient.Publish(message) + assert.NoError(t, err) +} From 494a57ec4b3ade019683d9f803cd278dc56fa6fd Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 12:05:13 +0700 Subject: [PATCH 212/268] [Dembo] Add proctor config for notification plugin binary and exported --- internal/app/service/infra/config/config.go | 4 ++++ internal/app/service/infra/config/config_test.go | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 2e6bd9b0..1e5a48e3 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -82,6 +82,8 @@ type ProctorConfig struct { DocsPath string AuthPluginBinary string AuthEnabled bool + NotificationPluginBinary string + NotificationPluginExported string } func Load() ProctorConfig { @@ -126,6 +128,8 @@ func Load() ProctorConfig { AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), + NotificationPluginBinary: fang.GetString("NOTIFICATION_PLUGIN_BINARY"), + NotificationPluginExported: fang.GetString("NOTIFICATION_PLUGIN_EXPORTED"), } return proctorConfig diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 5b7dd151..6ae0e488 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -215,3 +215,15 @@ func TestAuthEnabled(t *testing.T) { assert.Equal(t, false, Load().AuthEnabled) } + +func TestNotificationPluginBinary(t *testing.T) { + _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_BINARY", "path-notification") + + assert.Equal(t, "path-notification", Load().NotificationPluginBinary) +} + +func TestNotificationPluginExported(t *testing.T) { + _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_EXPORTED", "plugin-notification") + + assert.Equal(t, "plugin-notification", Load().NotificationPluginExported) +} From fd8c80442432c09453629a60550ba198c4ffa80f Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 13:29:56 +0700 Subject: [PATCH 213/268] [Dembo] Add test integration plugin with slack notification --- .env.test | 2 ++ Makefile | 12 ++++++++--- .../infra/plugin/plugin_integration_test.go | 21 +++++++++++++++++-- .../slack_notification.go | 6 +++++- .../slack_notification_test.go | 6 +++--- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/.env.test b/.env.test index fed3a3e2..9cf693e7 100644 --- a/.env.test +++ b/.env.test @@ -38,3 +38,5 @@ export PROCTOR_DOCS_PATH=/path/to/docs/dir export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth export PROCTOR_AUTH_ENABLED=false +export PROCTOR_NOTIFICATION_PLUGIN_BINARY= +export PROCTOR_NOTIFICATION_PLUGIN_EXPORTED=SlackNotification diff --git a/Makefile b/Makefile index ad78c7bd..c9e1b915 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,16 @@ test: go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: itest -itest: plugin.auth +itest: plugin.auth plugin.slack PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ + PROCTOR_NOTIFICATION_PLUGIN_BINARY=$(PLUGIN_DIR)/slack.so \ ENABLE_INTEGRATION_TEST=true \ go test -p 1 -race -coverprofile=$(OUT_DIR)/coverage.out ./... .PHONY: plugin.itest -plugin.itest: plugin.auth +plugin.itest: plugin.auth plugin.slack PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ + PROCTOR_NOTIFICATION_PLUGIN_BINARY=$(PLUGIN_DIR)/slack.so \ ENABLE_PLUGIN_INTEGRATION_TEST=true \ go test -race -coverprofile=$(OUT_DIR)/coverage.out ./... @@ -50,11 +52,15 @@ server: plugin.auth: go build -race -buildmode=plugin -o $(PLUGIN_DIR)/auth.so ./plugins/gate-auth-plugin/auth.go +.PHONY: plugin.slack +plugin.slack: + go build -race -buildmode=plugin -o $(PLUGIN_DIR)/slack.so ./plugins/slack-notification-plugin/slack_notification.go + .PHONY: cli cli: go build -race -o $(BIN_DIR)/cli ./cmd/cli/main.go -build-all: server cli plugin.auth +build-all: server cli plugin.auth plugin.slack .PHONY: start-server start-server: diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index f0334796..48b5b5d8 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -2,10 +2,13 @@ package plugin import ( "fmt" - "github.com/stretchr/testify/assert" "os" - "proctor/internal/app/service/infra/config" + "strings" "testing" + + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/infra/config" ) type context interface { @@ -69,6 +72,20 @@ func TestGoPlugin_LoadSuccessfully(t *testing.T) { assert.NotNil(t, raw) } +func TestGoPlugin_LoadNotificationSuccessfully(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + + pluginsBinary := strings.Split(config.Config().NotificationPluginBinary, ",") + pluginsExported := strings.Split(config.Config().NotificationPluginExported, ",") + for idx, pluginBinary := range pluginsBinary { + pluginExported := pluginsExported[idx] + raw, err := ctx.instance().goPlugin.Load(pluginBinary, pluginExported) + assert.NoError(t, err) + assert.NotNil(t, raw) + } +} + func TestGoPlugin_ShitShit(t *testing.T) { ctx := newContext() ctx.setUp(t) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index d1d04378..477d0e4b 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "github.com/go-resty/resty/v2" "proctor/pkg/notification" "proctor/pkg/notification/event" "proctor/plugins/slack-notification-plugin/slack" @@ -24,8 +25,11 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { return err } -func NewSlackNotification(slackClient slack.SlackClient) notification.Observer { +func newSlackNotification() notification.Observer { + slackClient := slack.NewSlackClient(&resty.Client{}) return &slackNotification{ slackClient: slackClient, } } + +var SlackNotification = newSlackNotification() diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index 5b4398c7..94364880 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -6,7 +6,6 @@ import ( "github.com/stretchr/testify/assert" - "proctor/pkg/notification" "proctor/pkg/notification/event" "proctor/plugins/slack-notification-plugin/slack" ) @@ -18,14 +17,15 @@ type context interface { } type testContext struct { - slackNotification notification.Observer + slackNotification *slackNotification slackClient *slack.SlackClientMock event *event.EventMock } func (context *testContext) setUp(t *testing.T) { context.slackClient = &slack.SlackClientMock{} - context.slackNotification = NewSlackNotification(context.slackClient) + context.slackNotification = &slackNotification{} + context.slackNotification.slackClient = context.slackClient context.event = &event.EventMock{} } From 00255267845ae045997f7bc427377d2768796531 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 14:55:51 +0700 Subject: [PATCH 214/268] [Dembo] Add notification service --- .../notification/service/notification.go | 53 +++++++++++++ .../service/notification_integration_test.go | 78 +++++++++++++++++++ .../slack_notification.go | 2 +- 3 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 internal/app/service/notification/service/notification_integration_test.go diff --git a/internal/app/service/notification/service/notification.go b/internal/app/service/notification/service/notification.go index a1cd98da..49842fd6 100644 --- a/internal/app/service/notification/service/notification.go +++ b/internal/app/service/notification/service/notification.go @@ -1,9 +1,62 @@ package service import ( + "proctor/internal/app/service/infra/logger" + "proctor/internal/app/service/infra/plugin" + "proctor/pkg/notification" "proctor/pkg/notification/event" + "strings" + "sync" ) type NotificationService interface { Notify(evt event.Event) } + +type notificationService struct { + observers []notification.Observer + goPlugin plugin.GoPlugin + pluginsBinary string + pluginsExportedName string + once sync.Once +} + +func (s *notificationService) Notify(evt event.Event) { + s.initializePlugin() + for _, observer := range s.observers { + err := observer.OnNotify(evt) + logger.LogErrors(err, "notify event to observer", evt, observer) + } +} + +func (s *notificationService) initializePlugin() { + s.once.Do(func() { + s.observers = []notification.Observer{} + pluginsBinary := strings.Split(s.pluginsBinary, ",") + pluginsExported := strings.Split(s.pluginsExportedName, ",") + + for idx, pluginBinary := range pluginsBinary { + raw, err := s.goPlugin.Load(pluginBinary, pluginsExported[idx]) + logger.LogErrors(err, "Load GoPlugin binary") + if err != nil { + return + } + observer := *raw.(*notification.Observer) + if observer == nil { + logger.Error("Failed to convert exported plugin to notification.Observer type") + return + } + s.observers = append(s.observers, observer) + } + logger.Info("Number of notification plugin ", len(s.observers)) + }) +} + +func NewNotificationService(pluginsBinary string, pluginsExportedName string, goPlugin plugin.GoPlugin) NotificationService { + return ¬ificationService{ + goPlugin: goPlugin, + pluginsBinary: pluginsBinary, + pluginsExportedName: pluginsExportedName, + once: sync.Once{}, + } +} diff --git a/internal/app/service/notification/service/notification_integration_test.go b/internal/app/service/notification/service/notification_integration_test.go new file mode 100644 index 00000000..264c620b --- /dev/null +++ b/internal/app/service/notification/service/notification_integration_test.go @@ -0,0 +1,78 @@ +package service + +import ( + "os" + "testing" + "time" + + "github.com/jmoiron/sqlx/types" + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/execution/handler/parameter" + "proctor/internal/app/service/execution/model" + "proctor/internal/app/service/execution/status" + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/plugin" + "proctor/pkg/notification/event" +) + +type context interface { + setUp(t *testing.T) + tearDown() + instance() *testContext +} + +type testContext struct { + notificationService NotificationService + config config.ProctorConfig +} + +func (context *testContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_PLUGIN_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + context.config = config.Config() + pluginsBinary := context.config.NotificationPluginBinary + pluginsExportedName := context.config.NotificationPluginExported + assert.NotEmpty(t, pluginsBinary) + assert.NotEmpty(t, pluginsExportedName) + context.notificationService = NewNotificationService(pluginsBinary, pluginsExportedName, plugin.NewGoPlugin()) +} + +func (context *testContext) tearDown() { +} + +func (context *testContext) instance() *testContext { + return context +} + +func newContext() context { + return &testContext{} +} + +func TestNotificationService_NotifySuccess(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + executionContextId := uint64(1) + userEmail := "mrproctor@example.com" + job := parameter.Job{ + Name: "notification-service-integration-test", + Args: map[string]string{"data": "dummy"}, + } + executionContext := model.ExecutionContext{ + ExecutionID: executionContextId, + UserEmail: userEmail, + JobName: job.Name, + ImageTag: "test", + Args: job.Args, + CreatedAt: time.Now(), + Status: status.Finished, + Output: types.GzippedText("test"), + } + executionEvent := event.NewExecutionEvent(userEmail, executionContext) + + ctx.instance().notificationService.Notify(executionEvent) +} diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 477d0e4b..1b36205f 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -26,7 +26,7 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { } func newSlackNotification() notification.Observer { - slackClient := slack.NewSlackClient(&resty.Client{}) + slackClient := slack.NewSlackClient(resty.New()) return &slackNotification{ slackClient: slackClient, } From 5614c5854fcb4f440d1e4f11660179aac6e07866 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 14:59:11 +0700 Subject: [PATCH 215/268] [Dembo] Add notification service to execution handler --- Makefile | 1 + internal/app/service/execution/handler/http.go | 16 ++++++++++++---- .../app/service/execution/handler/http_test.go | 16 ++++++++++++++-- .../notification/service/notification_mock.go | 14 ++++++++++++++ internal/app/service/server/router.go | 18 ++++++++++++++---- 5 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 internal/app/service/notification/service/notification_mock.go diff --git a/Makefile b/Makefile index c9e1b915..07dd9ecf 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,7 @@ build-all: server cli plugin.auth plugin.slack .PHONY: start-server start-server: PROCTOR_AUTH_PLUGIN_BINARY=$(PLUGIN_DIR)/auth.so \ + PROCTOR_NOTIFICATION_PLUGIN_BINARY=$(PLUGIN_DIR)/slack.so \ $(BIN_DIR)/server s generate: diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 485867f8..1d267724 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -19,7 +19,9 @@ import ( executionStatus "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" + serviceNotification "proctor/internal/app/service/notification/service" "proctor/internal/pkg/model/execution" + "proctor/pkg/notification/event" ) type ExecutionHTTPHandler interface { @@ -29,8 +31,9 @@ type ExecutionHTTPHandler interface { } type executionHTTPHandler struct { - service service.ExecutionService - repository repository.ExecutionContextRepository + service service.ExecutionService + repository repository.ExecutionContextRepository + notificationService serviceNotification.NotificationService } var upgrader = websocket.Upgrader{ @@ -41,10 +44,12 @@ var upgrader = websocket.Upgrader{ func NewExecutionHTTPHandler( executionService service.ExecutionService, repository repository.ExecutionContextRepository, + notificationService serviceNotification.NotificationService, ) ExecutionHTTPHandler { return &executionHTTPHandler{ - service: executionService, - repository: repository, + service: executionService, + repository: repository, + notificationService: notificationService, } } @@ -185,6 +190,9 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { return } + evt := event.NewExecutionEvent(userEmail, *context) + httpHandler.notificationService.Notify(evt) + responseBody := &execution.ExecutionResult{ ExecutionId: context.ExecutionID, JobName: context.JobName, diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 1db4a685..9a7e0c3e 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -8,7 +8,6 @@ import ( "io/ioutil" "net/http" "net/http/httptest" - "proctor/internal/pkg/model/execution" "strings" "testing" "time" @@ -26,7 +25,10 @@ import ( "proctor/internal/app/service/execution/service" "proctor/internal/app/service/execution/status" "proctor/internal/app/service/infra/kubernetes" + serviceNotification "proctor/internal/app/service/notification/service" "proctor/internal/pkg/constant" + "proctor/internal/pkg/model/execution" + "proctor/pkg/notification/event" ) type ExecutionHTTPHandlerTestSuite struct { @@ -34,6 +36,7 @@ type ExecutionHTTPHandlerTestSuite struct { mockExecutionerService *service.MockExecutionService mockExecutionerContextRepository *repository.MockExecutionContextRepository mockKubernetesClient kubernetes.MockKubernetesClient + mockNotificationService *serviceNotification.NotificationServiceMock testExecutionHTTPHandler ExecutionHTTPHandler } @@ -41,7 +44,12 @@ func (suite *ExecutionHTTPHandlerTestSuite) SetupTest() { suite.mockExecutionerService = &service.MockExecutionService{} suite.mockExecutionerContextRepository = &repository.MockExecutionContextRepository{} suite.mockKubernetesClient = kubernetes.MockKubernetesClient{} - suite.testExecutionHTTPHandler = NewExecutionHTTPHandler(suite.mockExecutionerService, suite.mockExecutionerContextRepository) + suite.mockNotificationService = &serviceNotification.NotificationServiceMock{} + suite.testExecutionHTTPHandler = NewExecutionHTTPHandler( + suite.mockExecutionerService, + suite.mockExecutionerContextRepository, + suite.mockNotificationService, + ) } type logsHandlerServer struct { @@ -285,6 +293,10 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHa suite.mockExecutionerService.On("Execute", job.Name, userEmail, job.Args).Return(context, "test", nil).Once() defer suite.mockExecutionerService.AssertExpectations(t) + expectedEvent := event.NewExecutionEvent(userEmail, *context) + suite.mockNotificationService.On("Notify", expectedEvent) + defer suite.mockNotificationService.AssertExpectations(t) + suite.testExecutionHTTPHandler.Post()(responseRecorder, req) assert.Equal(t, http.StatusCreated, responseRecorder.Code) diff --git a/internal/app/service/notification/service/notification_mock.go b/internal/app/service/notification/service/notification_mock.go new file mode 100644 index 00000000..21d23011 --- /dev/null +++ b/internal/app/service/notification/service/notification_mock.go @@ -0,0 +1,14 @@ +package service + +import ( + "github.com/stretchr/testify/mock" + "proctor/pkg/notification/event" +) + +type NotificationServiceMock struct { + mock.Mock +} + +func (m NotificationServiceMock) Notify(evt event.Event) { + m.Called(evt) +} diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index a9077e9a..788b8319 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -19,12 +19,13 @@ import ( "proctor/internal/app/service/infra/plugin" metadataHandler "proctor/internal/app/service/metadata/handler" metadataRepository "proctor/internal/app/service/metadata/repository" + notificationService "proctor/internal/app/service/notification/service" scheduleHTTPHandler "proctor/internal/app/service/schedule/handler" scheduleRepository "proctor/internal/app/service/schedule/repository" secretHTTPHandler "proctor/internal/app/service/secret/handler" secretRepository "proctor/internal/app/service/secret/repository" securityMiddleware "proctor/internal/app/service/security/middleware" - "proctor/internal/app/service/security/service" + securityService "proctor/internal/app/service/security/service" "proctor/internal/app/service/server/middleware" ) @@ -49,9 +50,18 @@ func NewRouter() (*mux.Router, error) { secretsStore := secretRepository.NewSecretRepository(redisClient) _executionService := executionService.NewExecutionService(kubeClient, executionStore, metadataStore, secretsStore) - _securityService := service.NewSecurityService(proctorConfig.AuthPluginBinary, proctorConfig.AuthPluginExported, goPlugin) - - executionHandler := executionHTTPHandler.NewExecutionHTTPHandler(_executionService, executionStore) + _securityService := securityService.NewSecurityService( + proctorConfig.AuthPluginBinary, + proctorConfig.AuthPluginExported, + goPlugin, + ) + _notificationService := notificationService.NewNotificationService( + proctorConfig.NotificationPluginBinary, + proctorConfig.NotificationPluginExported, + goPlugin, + ) + + executionHandler := executionHTTPHandler.NewExecutionHTTPHandler(_executionService, executionStore, _notificationService) jobMetadataHandler := metadataHandler.NewMetadataHTTPHandler(metadataStore) jobSecretsHandler := secretHTTPHandler.NewSecretHTTPHandler(secretsStore) scheduleHandler := scheduleHTTPHandler.NewScheduleHTTPHandler(scheduleStore, metadataStore) From 91e4348ada2e842a47c478ad2b2e42089564ab3b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 15:24:17 +0700 Subject: [PATCH 216/268] [Dembo] Refactor move standard message to own package --- .../slack-notification-plugin/slack/client.go | 7 ++++--- .../slack/client_integration_test.go | 6 ++++-- .../slack/client_mock.go | 7 +++++-- .../slack/client_test.go | 20 ++++++++++--------- .../slack/{ => message}/message.go | 6 +++--- .../slack/{ => message}/message_mock.go | 2 +- .../slack/{ => message}/message_test.go | 6 +++--- .../slack_notification.go | 5 +++-- .../slack_notification_test.go | 9 +++++---- 9 files changed, 39 insertions(+), 29 deletions(-) rename plugins/slack-notification-plugin/slack/{ => message}/message.go (70%) rename plugins/slack-notification-plugin/slack/{ => message}/message_mock.go (92%) rename plugins/slack-notification-plugin/slack/{ => message}/message_test.go (64%) diff --git a/plugins/slack-notification-plugin/slack/client.go b/plugins/slack-notification-plugin/slack/client.go index 3fca4173..07c141c4 100644 --- a/plugins/slack-notification-plugin/slack/client.go +++ b/plugins/slack-notification-plugin/slack/client.go @@ -2,10 +2,11 @@ package slack import ( "github.com/go-resty/resty/v2" + "proctor/plugins/slack-notification-plugin/slack/message" ) type SlackClient interface { - Publish(message Message) error + Publish(messageObject message.Message) error } type slackClient struct { @@ -13,8 +14,8 @@ type slackClient struct { config SlackConfig } -func (s *slackClient) Publish(message Message) error { - messageJson, err := message.JSON() +func (s *slackClient) Publish(messageObject message.Message) error { + messageJson, err := messageObject.JSON() if err != nil { return err } diff --git a/plugins/slack-notification-plugin/slack/client_integration_test.go b/plugins/slack-notification-plugin/slack/client_integration_test.go index 1da7d5e3..f8185fbd 100644 --- a/plugins/slack-notification-plugin/slack/client_integration_test.go +++ b/plugins/slack-notification-plugin/slack/client_integration_test.go @@ -4,6 +4,8 @@ import ( "os" "testing" + "proctor/plugins/slack-notification-plugin/slack/message" + "github.com/go-resty/resty/v2" "github.com/stretchr/testify/assert" ) @@ -47,7 +49,7 @@ func TestSlackClientIntegration_Publish(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() - message := NewStandardMessage("Message from slack plugin integration test with standard message") - err := ctx.instance().slackClient.Publish(message) + messageObject := message.NewStandardMessage("Message from slack plugin integration test with standard message") + err := ctx.instance().slackClient.Publish(messageObject) assert.NoError(t, err) } diff --git a/plugins/slack-notification-plugin/slack/client_mock.go b/plugins/slack-notification-plugin/slack/client_mock.go index 12edcc50..c34ccd07 100644 --- a/plugins/slack-notification-plugin/slack/client_mock.go +++ b/plugins/slack-notification-plugin/slack/client_mock.go @@ -1,12 +1,15 @@ package slack -import "github.com/stretchr/testify/mock" +import ( + "github.com/stretchr/testify/mock" + "proctor/plugins/slack-notification-plugin/slack/message" +) type SlackClientMock struct { mock.Mock } -func (m *SlackClientMock) Publish(message Message) error { +func (m *SlackClientMock) Publish(message message.Message) error { args := m.Called(message) return args.Error(0) } diff --git a/plugins/slack-notification-plugin/slack/client_test.go b/plugins/slack-notification-plugin/slack/client_test.go index d4877fe6..1301871e 100644 --- a/plugins/slack-notification-plugin/slack/client_test.go +++ b/plugins/slack-notification-plugin/slack/client_test.go @@ -6,6 +6,8 @@ import ( "net/http" "testing" + "proctor/plugins/slack-notification-plugin/slack/message" + "github.com/go-resty/resty/v2" "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" @@ -36,8 +38,8 @@ func TestSlackClient_Publish(t *testing.T) { defer ctx.tearDown() config := NewSlackConfig() - message := MessageMock{} - message.On("JSON").Return("message sent", nil) + messageObject := message.MessageMock{} + messageObject.On("JSON").Return("message sent", nil) httpmock.RegisterResponder( "POST", @@ -54,7 +56,7 @@ func TestSlackClient_Publish(t *testing.T) { return response, nil }, ) - err := ctx.client.Publish(&message) + err := ctx.client.Publish(&messageObject) assert.NoError(t, err) assert.Equal(t, 1, httpmock.GetTotalCallCount()) } @@ -64,10 +66,10 @@ func TestSlackClient_PublishErrorJSON(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() - message := MessageMock{} - message.On("JSON").Return("", errors.New("JSON unmarshal error")).Once() + messageObject := message.MessageMock{} + messageObject.On("JSON").Return("", errors.New("JSON unmarshal error")).Once() - err := ctx.client.Publish(&message) + err := ctx.client.Publish(&messageObject) assert.Error(t, err) assert.Equal(t, 0, httpmock.GetTotalCallCount()) } @@ -78,8 +80,8 @@ func TestSlackClient_PublishErrorRequest(t *testing.T) { defer ctx.tearDown() config := NewSlackConfig() - message := MessageMock{} - message.On("JSON").Return("message sent", nil) + messageObject := message.MessageMock{} + messageObject.On("JSON").Return("message sent", nil) httpmock.RegisterResponder( "POST", @@ -96,7 +98,7 @@ func TestSlackClient_PublishErrorRequest(t *testing.T) { return response, errors.New("internal server error") }, ) - err := ctx.client.Publish(&message) + err := ctx.client.Publish(&messageObject) assert.Error(t, err) assert.Equal(t, 1, httpmock.GetTotalCallCount()) } diff --git a/plugins/slack-notification-plugin/slack/message.go b/plugins/slack-notification-plugin/slack/message/message.go similarity index 70% rename from plugins/slack-notification-plugin/slack/message.go rename to plugins/slack-notification-plugin/slack/message/message.go index 42195167..39a36f0b 100644 --- a/plugins/slack-notification-plugin/slack/message.go +++ b/plugins/slack-notification-plugin/slack/message/message.go @@ -1,4 +1,4 @@ -package slack +package message import "encoding/json" @@ -10,8 +10,8 @@ type standardMessage struct { Text string `json:"text"` } -func (message *standardMessage) JSON() (string, error) { - byteMessage, err := json.Marshal(message) +func (messageObject *standardMessage) JSON() (string, error) { + byteMessage, err := json.Marshal(messageObject) if err != nil { return "", err } diff --git a/plugins/slack-notification-plugin/slack/message_mock.go b/plugins/slack-notification-plugin/slack/message/message_mock.go similarity index 92% rename from plugins/slack-notification-plugin/slack/message_mock.go rename to plugins/slack-notification-plugin/slack/message/message_mock.go index 20b3267f..e2745eb3 100644 --- a/plugins/slack-notification-plugin/slack/message_mock.go +++ b/plugins/slack-notification-plugin/slack/message/message_mock.go @@ -1,4 +1,4 @@ -package slack +package message import ( "github.com/stretchr/testify/mock" diff --git a/plugins/slack-notification-plugin/slack/message_test.go b/plugins/slack-notification-plugin/slack/message/message_test.go similarity index 64% rename from plugins/slack-notification-plugin/slack/message_test.go rename to plugins/slack-notification-plugin/slack/message/message_test.go index b948044a..8d0b861b 100644 --- a/plugins/slack-notification-plugin/slack/message_test.go +++ b/plugins/slack-notification-plugin/slack/message/message_test.go @@ -1,4 +1,4 @@ -package slack +package message import ( "github.com/stretchr/testify/assert" @@ -6,8 +6,8 @@ import ( ) func TestStandardMessage_JSON(t *testing.T) { - message := NewStandardMessage("content") - result, err := message.JSON() + messageObject := NewStandardMessage("content") + result, err := messageObject.JSON() assert.NoError(t, err) assert.Equal(t, "{\"text\":\"content\"}", result) } diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 1b36205f..1901e550 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -6,6 +6,7 @@ import ( "proctor/pkg/notification" "proctor/pkg/notification/event" "proctor/plugins/slack-notification-plugin/slack" + "proctor/plugins/slack-notification-plugin/slack/message" ) type slackNotification struct { @@ -20,8 +21,8 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { textMessage := "User: " + evt.User().Email + "\n" textMessage += "Execute job with detail: " textMessage += string(evtDataJSON) - message := slack.NewStandardMessage(textMessage) - err = notification.slackClient.Publish(message) + messageObject := message.NewStandardMessage(textMessage) + err = notification.slackClient.Publish(messageObject) return err } diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index 94364880..8fc6bf4e 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -2,6 +2,7 @@ package main import ( "errors" + "proctor/plugins/slack-notification-plugin/slack/message" "testing" "github.com/stretchr/testify/assert" @@ -60,8 +61,8 @@ func TestSlackNotification_OnNotify(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - message := slack.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") - ctx.instance().slackClient.On("Publish", message).Return(nil) + messageObject := message.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + ctx.instance().slackClient.On("Publish", messageObject).Return(nil) err := ctx.instance().slackNotification.OnNotify(evt) assert.NoError(t, err) @@ -89,8 +90,8 @@ func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - message := slack.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") - ctx.instance().slackClient.On("Publish", message).Return(errors.New("publish error")) + messageObject := message.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + ctx.instance().slackClient.On("Publish", messageObject).Return(errors.New("publish error")) err := ctx.instance().slackNotification.OnNotify(evt) assert.Error(t, err) From 5a68f3e804372842a50021986ac75b45e1d0d79f Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 15:27:46 +0700 Subject: [PATCH 217/268] [Dembo] Refactor extract standard message to own file --- .../slack/message/message.go | 20 ------------------ .../slack/message/standard.go | 21 +++++++++++++++++++ .../{message_test.go => standard_test.go} | 0 3 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 plugins/slack-notification-plugin/slack/message/standard.go rename plugins/slack-notification-plugin/slack/message/{message_test.go => standard_test.go} (100%) diff --git a/plugins/slack-notification-plugin/slack/message/message.go b/plugins/slack-notification-plugin/slack/message/message.go index 39a36f0b..b9343dc4 100644 --- a/plugins/slack-notification-plugin/slack/message/message.go +++ b/plugins/slack-notification-plugin/slack/message/message.go @@ -1,25 +1,5 @@ package message -import "encoding/json" - type Message interface { JSON() (string, error) } - -type standardMessage struct { - Text string `json:"text"` -} - -func (messageObject *standardMessage) JSON() (string, error) { - byteMessage, err := json.Marshal(messageObject) - if err != nil { - return "", err - } - return string(byteMessage), nil -} - -func NewStandardMessage(text string) Message { - return &standardMessage{ - Text: text, - } -} diff --git a/plugins/slack-notification-plugin/slack/message/standard.go b/plugins/slack-notification-plugin/slack/message/standard.go new file mode 100644 index 00000000..bd748029 --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message/standard.go @@ -0,0 +1,21 @@ +package message + +import "encoding/json" + +type standardMessage struct { + Text string `json:"text"` +} + +func (messageObject *standardMessage) JSON() (string, error) { + byteMessage, err := json.Marshal(messageObject) + if err != nil { + return "", err + } + return string(byteMessage), nil +} + +func NewStandardMessage(text string) Message { + return &standardMessage{ + Text: text, + } +} diff --git a/plugins/slack-notification-plugin/slack/message/message_test.go b/plugins/slack-notification-plugin/slack/message/standard_test.go similarity index 100% rename from plugins/slack-notification-plugin/slack/message/message_test.go rename to plugins/slack-notification-plugin/slack/message/standard_test.go From e4e9dc28e42444e8cb2deba310177007c592a71a Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 16:10:50 +0700 Subject: [PATCH 218/268] [Dembo] Refactor move formatting to message object --- .../slack/client_integration_test.go | 5 +++-- .../slack/message/standard.go | 18 ++++++++++++++--- .../slack/message/standard_test.go | 20 +++++++++++++++++-- .../slack_notification.go | 12 ++--------- .../slack_notification_test.go | 4 ++-- 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/plugins/slack-notification-plugin/slack/client_integration_test.go b/plugins/slack-notification-plugin/slack/client_integration_test.go index f8185fbd..b19332db 100644 --- a/plugins/slack-notification-plugin/slack/client_integration_test.go +++ b/plugins/slack-notification-plugin/slack/client_integration_test.go @@ -49,7 +49,8 @@ func TestSlackClientIntegration_Publish(t *testing.T) { ctx.setUp(t) defer ctx.tearDown() - messageObject := message.NewStandardMessage("Message from slack plugin integration test with standard message") - err := ctx.instance().slackClient.Publish(messageObject) + messageObject := message.MessageMock{} + messageObject.On("JSON").Return("{\"text\":\"Message from slack plugin integration test with standard message\"}", nil) + err := ctx.instance().slackClient.Publish(&messageObject) assert.NoError(t, err) } diff --git a/plugins/slack-notification-plugin/slack/message/standard.go b/plugins/slack-notification-plugin/slack/message/standard.go index bd748029..5d49ff52 100644 --- a/plugins/slack-notification-plugin/slack/message/standard.go +++ b/plugins/slack-notification-plugin/slack/message/standard.go @@ -1,12 +1,24 @@ package message -import "encoding/json" +import ( + "encoding/json" + "proctor/pkg/notification/event" +) type standardMessage struct { Text string `json:"text"` + evt event.Event } func (messageObject *standardMessage) JSON() (string, error) { + evtDataJSON, err := json.Marshal(messageObject.evt.Content()) + if err != nil { + return "", err + } + textMessage := "User: " + messageObject.evt.User().Email + "\n" + textMessage += "Execute job with detail: " + textMessage += string(evtDataJSON) + messageObject.Text = textMessage byteMessage, err := json.Marshal(messageObject) if err != nil { return "", err @@ -14,8 +26,8 @@ func (messageObject *standardMessage) JSON() (string, error) { return string(byteMessage), nil } -func NewStandardMessage(text string) Message { +func NewStandardMessage(evt event.Event) Message { return &standardMessage{ - Text: text, + evt: evt, } } diff --git a/plugins/slack-notification-plugin/slack/message/standard_test.go b/plugins/slack-notification-plugin/slack/message/standard_test.go index 8d0b861b..4f723cd8 100644 --- a/plugins/slack-notification-plugin/slack/message/standard_test.go +++ b/plugins/slack-notification-plugin/slack/message/standard_test.go @@ -2,12 +2,28 @@ package message import ( "github.com/stretchr/testify/assert" + "proctor/pkg/notification/event" "testing" ) func TestStandardMessage_JSON(t *testing.T) { - messageObject := NewStandardMessage("content") + userData := event.UserData{ + Email: "proctor@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := event.EventMock{} + evt.On("User").Return(userData) + evt.On("Content").Return(content) + defer evt.AssertExpectations(t) + + messageObject := NewStandardMessage(&evt) result, err := messageObject.JSON() assert.NoError(t, err) - assert.Equal(t, "{\"text\":\"content\"}", result) + assert.NotEmpty(t, result) } diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 1901e550..782e49c3 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "github.com/go-resty/resty/v2" "proctor/pkg/notification" "proctor/pkg/notification/event" @@ -14,15 +13,8 @@ type slackNotification struct { } func (notification *slackNotification) OnNotify(evt event.Event) error { - evtDataJSON, err := json.Marshal(evt.Content()) - if err != nil { - return err - } - textMessage := "User: " + evt.User().Email + "\n" - textMessage += "Execute job with detail: " - textMessage += string(evtDataJSON) - messageObject := message.NewStandardMessage(textMessage) - err = notification.slackClient.Publish(messageObject) + messageObject := message.NewStandardMessage(evt) + err := notification.slackClient.Publish(messageObject) return err } diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index 8fc6bf4e..db1ff6e1 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -61,7 +61,7 @@ func TestSlackNotification_OnNotify(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - messageObject := message.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + messageObject := message.NewStandardMessage(evt) ctx.instance().slackClient.On("Publish", messageObject).Return(nil) err := ctx.instance().slackNotification.OnNotify(evt) @@ -90,7 +90,7 @@ func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - messageObject := message.NewStandardMessage("User: proctor@example.com\nExecute job with detail: {\"Args\":\"args\",\"ExecutionID\":\"7\",\"ImageTag\":\"test\",\"JobName\":\"test-job\",\"Status\":\"CREATED\"}") + messageObject := message.NewStandardMessage(evt) ctx.instance().slackClient.On("Publish", messageObject).Return(errors.New("publish error")) err := ctx.instance().slackNotification.OnNotify(evt) From cc82a272009ec79ea05afc0c50829f7f378df319 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 16:29:03 +0700 Subject: [PATCH 219/268] [Dembo] Add slack execution message --- .../slack/message/component.go | 12 +++++ .../slack/message/execution.go | 52 +++++++++++++++++++ .../slack/message/execution_test.go | 41 +++++++++++++++ 3 files changed, 105 insertions(+) create mode 100644 plugins/slack-notification-plugin/slack/message/component.go create mode 100644 plugins/slack-notification-plugin/slack/message/execution.go create mode 100644 plugins/slack-notification-plugin/slack/message/execution_test.go diff --git a/plugins/slack-notification-plugin/slack/message/component.go b/plugins/slack-notification-plugin/slack/message/component.go new file mode 100644 index 00000000..8c875f1d --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message/component.go @@ -0,0 +1,12 @@ +package message + +type textComponent struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type sectionComponent struct { + Type string `json:"type"` + Text textComponent `json:"text"` + Fields []textComponent `json:"fields"` +} diff --git a/plugins/slack-notification-plugin/slack/message/execution.go b/plugins/slack-notification-plugin/slack/message/execution.go new file mode 100644 index 00000000..81dd934d --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message/execution.go @@ -0,0 +1,52 @@ +package message + +import ( + "encoding/json" + "errors" + "proctor/pkg/notification/event" +) + +type executionMessage struct { + evt event.Event + Blocks []sectionComponent `json:"blocks"` +} + +func (messageObject *executionMessage) JSON() (string, error) { + evt := messageObject.evt + if evt.Type() != string(event.ExecutionEventType) { + return "", errors.New("event type mismatch") + } + userEmail := evt.User().Email + messageObject.Blocks = []sectionComponent{} + + section := sectionComponent{} + section.Type = "section" + section.Text = textComponent{ + Type: "text", + Text: userEmail + " execute job with details:", + } + section.Fields = []textComponent{} + for key, value := range evt.Content() { + keyComponent := textComponent{} + keyComponent.Type = "mrkdwn" + keyComponent.Text = "*" + key + "*" + valueComponent := textComponent{} + valueComponent.Type = "text" + valueComponent.Text = value + + section.Fields = append(section.Fields, keyComponent, valueComponent) + } + + messageObject.Blocks = append(messageObject.Blocks, section) + byteMessage, err := json.Marshal(messageObject) + if err != nil { + return "", err + } + return string(byteMessage), nil +} + +func NewExecutionMessage(evt event.Event) Message { + return &executionMessage{ + evt: evt, + } +} diff --git a/plugins/slack-notification-plugin/slack/message/execution_test.go b/plugins/slack-notification-plugin/slack/message/execution_test.go new file mode 100644 index 00000000..2548d0ae --- /dev/null +++ b/plugins/slack-notification-plugin/slack/message/execution_test.go @@ -0,0 +1,41 @@ +package message + +import ( + "github.com/stretchr/testify/assert" + "proctor/pkg/notification/event" + "testing" +) + +func TestExecutionMessage_JSON(t *testing.T) { + userData := event.UserData{ + Email: "proctor@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := event.EventMock{} + evt.On("Type").Return(string(event.ExecutionEventType)) + evt.On("User").Return(userData) + evt.On("Content").Return(content) + defer evt.AssertExpectations(t) + + messageObject := NewExecutionMessage(&evt) + result, err := messageObject.JSON() + assert.NoError(t, err) + assert.NotEmpty(t, result) +} + +func TestExecutionMessage_JSONMismatch(t *testing.T) { + evt := event.EventMock{} + evt.On("Type").Return("not-execution-event") + defer evt.AssertExpectations(t) + + messageObject := NewExecutionMessage(&evt) + result, err := messageObject.JSON() + assert.Error(t, err) + assert.Empty(t, result) +} From c17f231ca75933841fb6ee73eac7615c66a46cb3 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 16:58:27 +0700 Subject: [PATCH 220/268] [Dembo] Change slack notification to use execution message --- .../slack-notification-plugin/slack/message/execution.go | 4 ++-- plugins/slack-notification-plugin/slack_notification.go | 2 +- .../slack-notification-plugin/slack_notification_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/slack-notification-plugin/slack/message/execution.go b/plugins/slack-notification-plugin/slack/message/execution.go index 81dd934d..1ee28822 100644 --- a/plugins/slack-notification-plugin/slack/message/execution.go +++ b/plugins/slack-notification-plugin/slack/message/execution.go @@ -22,7 +22,7 @@ func (messageObject *executionMessage) JSON() (string, error) { section := sectionComponent{} section.Type = "section" section.Text = textComponent{ - Type: "text", + Type: "plain_text", Text: userEmail + " execute job with details:", } section.Fields = []textComponent{} @@ -31,7 +31,7 @@ func (messageObject *executionMessage) JSON() (string, error) { keyComponent.Type = "mrkdwn" keyComponent.Text = "*" + key + "*" valueComponent := textComponent{} - valueComponent.Type = "text" + valueComponent.Type = "plain_text" valueComponent.Text = value section.Fields = append(section.Fields, keyComponent, valueComponent) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 782e49c3..0053bd40 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -13,7 +13,7 @@ type slackNotification struct { } func (notification *slackNotification) OnNotify(evt event.Event) error { - messageObject := message.NewStandardMessage(evt) + messageObject := message.NewExecutionMessage(evt) err := notification.slackClient.Publish(messageObject) return err } diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index db1ff6e1..447672f7 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -41,7 +41,7 @@ func newContext() context { return &testContext{} } -func TestSlackNotification_OnNotify(t *testing.T) { +func TestSlackNotification_OnNotifyExecution(t *testing.T) { ctx := newContext() ctx.setUp(t) defer ctx.tearDown() @@ -61,7 +61,7 @@ func TestSlackNotification_OnNotify(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - messageObject := message.NewStandardMessage(evt) + messageObject := message.NewExecutionMessage(evt) ctx.instance().slackClient.On("Publish", messageObject).Return(nil) err := ctx.instance().slackNotification.OnNotify(evt) @@ -70,7 +70,7 @@ func TestSlackNotification_OnNotify(t *testing.T) { ctx.instance().slackClient.AssertExpectations(t) } -func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { +func TestSlackNotification_OnNotifyExecutionErrorPublish(t *testing.T) { ctx := newContext() ctx.setUp(t) defer ctx.tearDown() @@ -90,7 +90,7 @@ func TestSlackNotification_OnNotifyErrorPublish(t *testing.T) { evt.On("User").Return(userData) evt.On("Content").Return(content) - messageObject := message.NewStandardMessage(evt) + messageObject := message.NewExecutionMessage(evt) ctx.instance().slackClient.On("Publish", messageObject).Return(errors.New("publish error")) err := ctx.instance().slackNotification.OnNotify(evt) From 5d6a78fde8253936c434d1c477c8356490a9242e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 17:05:33 +0700 Subject: [PATCH 221/268] [Dembo] Add error when slack request fail --- plugins/slack-notification-plugin/slack/client.go | 9 ++++++++- .../slack/client_integration_test.go | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/plugins/slack-notification-plugin/slack/client.go b/plugins/slack-notification-plugin/slack/client.go index 07c141c4..f555cc16 100644 --- a/plugins/slack-notification-plugin/slack/client.go +++ b/plugins/slack-notification-plugin/slack/client.go @@ -1,6 +1,7 @@ package slack import ( + "errors" "github.com/go-resty/resty/v2" "proctor/plugins/slack-notification-plugin/slack/message" ) @@ -20,10 +21,16 @@ func (s *slackClient) Publish(messageObject message.Message) error { return err } path := s.config.url - _, err = s.client.R(). + response, err := s.client.R(). SetBody(messageJson). SetHeader("Content-Type", "application/json"). Post(path) + if err != nil { + return err + } + if response.IsError() { + return errors.New(response.String()) + } return err } diff --git a/plugins/slack-notification-plugin/slack/client_integration_test.go b/plugins/slack-notification-plugin/slack/client_integration_test.go index b19332db..df0cff83 100644 --- a/plugins/slack-notification-plugin/slack/client_integration_test.go +++ b/plugins/slack-notification-plugin/slack/client_integration_test.go @@ -54,3 +54,14 @@ func TestSlackClientIntegration_Publish(t *testing.T) { err := ctx.instance().slackClient.Publish(&messageObject) assert.NoError(t, err) } + +func TestSlackClientIntegration_PublishFormatError(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + messageObject := message.MessageMock{} + messageObject.On("JSON").Return("{\"nothingness\":\"Message from slack plugin integration test with standard message\"}", nil) + err := ctx.instance().slackClient.Publish(&messageObject) + assert.Error(t, err) +} From b4e257027a8fbd40c0541f5b1880031498b07cdc Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 17:14:14 +0700 Subject: [PATCH 222/268] [Dembo] Slack notification will send standard message for unsupported event --- .../slack_notification.go | 11 ++++++- .../slack_notification_test.go | 33 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 0053bd40..20dd0563 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -13,7 +13,16 @@ type slackNotification struct { } func (notification *slackNotification) OnNotify(evt event.Event) error { - messageObject := message.NewExecutionMessage(evt) + var messageObject message.Message + switch evt.Type() { + case string(event.ExecutionEventType): + messageObject = message.NewExecutionMessage(evt) + default: + messageObject = message.NewStandardMessage(evt) + } + if evt.Type() == string(event.ExecutionEventType) { + + } err := notification.slackClient.Publish(messageObject) return err } diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index 447672f7..1a60f169 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -57,7 +57,7 @@ func TestSlackNotification_OnNotifyExecution(t *testing.T) { "Status": "CREATED", } evt := ctx.instance().event - evt.On("Type").Return(event.ExecutionEventType) + evt.On("Type").Return(string(event.ExecutionEventType)) evt.On("User").Return(userData) evt.On("Content").Return(content) @@ -86,7 +86,7 @@ func TestSlackNotification_OnNotifyExecutionErrorPublish(t *testing.T) { "Status": "CREATED", } evt := ctx.instance().event - evt.On("Type").Return(event.ExecutionEventType) + evt.On("Type").Return(string(event.ExecutionEventType)) evt.On("User").Return(userData) evt.On("Content").Return(content) @@ -98,3 +98,32 @@ func TestSlackNotification_OnNotifyExecutionErrorPublish(t *testing.T) { ctx.instance().slackClient.AssertExpectations(t) } + +func TestSlackNotification_OnNotifyUnsupportedEvent(t *testing.T) { + ctx := newContext() + ctx.setUp(t) + defer ctx.tearDown() + + userData := event.UserData{ + Email: "proctor@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := ctx.instance().event + evt.On("Type").Return("Unsupported event") + evt.On("User").Return(userData) + evt.On("Content").Return(content) + + messageObject := message.NewStandardMessage(evt) + ctx.instance().slackClient.On("Publish", messageObject).Return(nil) + + err := ctx.instance().slackNotification.OnNotify(evt) + assert.NoError(t, err) + + ctx.instance().slackClient.AssertExpectations(t) +} From 8e97e63f73e0cd77d22e0f51c17607d0856e222c Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 17:16:29 +0700 Subject: [PATCH 223/268] [Dembo] Change content of standard slack message --- plugins/slack-notification-plugin/slack/message/standard.go | 2 +- .../slack-notification-plugin/slack/message/standard_test.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/slack-notification-plugin/slack/message/standard.go b/plugins/slack-notification-plugin/slack/message/standard.go index 5d49ff52..21ce46e3 100644 --- a/plugins/slack-notification-plugin/slack/message/standard.go +++ b/plugins/slack-notification-plugin/slack/message/standard.go @@ -16,7 +16,7 @@ func (messageObject *standardMessage) JSON() (string, error) { return "", err } textMessage := "User: " + messageObject.evt.User().Email + "\n" - textMessage += "Execute job with detail: " + textMessage += "Emit event" + messageObject.evt.Type() + " with detail: " textMessage += string(evtDataJSON) messageObject.Text = textMessage byteMessage, err := json.Marshal(messageObject) diff --git a/plugins/slack-notification-plugin/slack/message/standard_test.go b/plugins/slack-notification-plugin/slack/message/standard_test.go index 4f723cd8..5c0c5fd7 100644 --- a/plugins/slack-notification-plugin/slack/message/standard_test.go +++ b/plugins/slack-notification-plugin/slack/message/standard_test.go @@ -18,6 +18,7 @@ func TestStandardMessage_JSON(t *testing.T) { "Status": "CREATED", } evt := event.EventMock{} + evt.On("Type").Return("unsupported-event") evt.On("User").Return(userData) evt.On("Content").Return(content) defer evt.AssertExpectations(t) From 480adc9b4b3b6a58cc81b0ea338e4e2f0427f27f Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 17:19:00 +0700 Subject: [PATCH 224/268] [Dembo] Remove unused code --- plugins/slack-notification-plugin/slack_notification.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 20dd0563..c3aa57a1 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -19,9 +19,6 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { messageObject = message.NewExecutionMessage(evt) default: messageObject = message.NewStandardMessage(evt) - } - if evt.Type() == string(event.ExecutionEventType) { - } err := notification.slackClient.Publish(messageObject) return err From 35f73b04e6e4fdcdb36a83d5ee5aa1510b367295 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 17:20:19 +0700 Subject: [PATCH 225/268] [Dembo] Refactor slack notification to extract method for generate message --- plugins/slack-notification-plugin/slack_notification.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index c3aa57a1..3d8063c9 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -13,6 +13,12 @@ type slackNotification struct { } func (notification *slackNotification) OnNotify(evt event.Event) error { + messageObject := notification.generateMessage(evt) + err := notification.slackClient.Publish(messageObject) + return err +} + +func (notification *slackNotification) generateMessage(evt event.Event) message.Message { var messageObject message.Message switch evt.Type() { case string(event.ExecutionEventType): @@ -20,8 +26,7 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { default: messageObject = message.NewStandardMessage(evt) } - err := notification.slackClient.Publish(messageObject) - return err + return messageObject } func newSlackNotification() notification.Observer { From 3f2b0d9a68d159e041edb3669e66b915a350405b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 18:09:22 +0700 Subject: [PATCH 226/268] [Dembo] Update readme about plugin --- README.md | 20 ++++++------------- docs/plugin.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 docs/plugin.md diff --git a/README.md b/README.md index 521165d5..a33f2e8c 100644 --- a/README.md +++ b/README.md @@ -105,23 +105,15 @@ List of routes that require authorization: Proctor doesn't come with built in auth implementation, it's using configurable [plugin](#plugin) mechanism. ## Plugin -Proctor service support plugin for authentication and authorization using go plugin. -It create limitation that proctor service can only be used on Linux and MacOS (Until the time go plugin support other OS). -By using authentication and authorization as a plugin you can customize this process using your organization current user management. +Proctor service use plugin for: + 1. Authentication + 2. Authorization + 3. Notification -#### How to make custom plugin -Creating plugin for auth (authentication and authorization) means creating a struct that implement [Auth interface](./pkg/auth/auth.go) then initialize it. -Begin using your custom plugin by building it with below command: -```sh -go build -race -buildmode=plugin -o ./_output/bin/plugin/auth.so -``` -Plug your custom plugin to proctor service by pointing the `PROCTOR_AUTH_PLUGIN_EXPORTED` environment variable to variable exported by plugin and `PROCTOR_AUTH_PLUGIN_BINARY` to `auth.so` file generated by command above. -After you finish it then proctor service will use your plugin. +It create limitation that proctor service can only be used on Linux and MacOS (Until the time go plugin support other OS). -#### Examples -If you like to jump in directly to example implementation then you can explore our default auth plugin [here](./plugins/gate-auth-plugin/auth.go). -This implementation is using [Gate](https://github.com/gate-sso/gate) as its user management. +For details about plugin please read [here](./docs/plugin.md) ## Procs Creation diff --git a/docs/plugin.md b/docs/plugin.md new file mode 100644 index 00000000..b5f0db63 --- /dev/null +++ b/docs/plugin.md @@ -0,0 +1,52 @@ +# Plugins + +Proctor using GO plugin, for official documentation about it please read [here](https://golang.org/pkg/plugin/). + +Proctor decide to use plugin for some feature to make it easily integrate into different component. + +### Notification Plugin + +Plugin can be made for publishing notification to external channel such as slack. +In order to use notification plugin you should compile the plugin and fill these environment variables: + 1. PROCTOR_NOTIFICATION_PLUGIN_BINARY + + Fill this variable with path to compiled plugin separated by comma since it's possible to use multiple notification channel. + + 2. PROCTOR_NOTIFICATION_PLUGIN_EXPORTED + + Fill this variable with name of variable exported from respective plugin binary separated by comma + +### Authentication and Authorization Plugin + +Authentication and authorization process is delegated to plugin so you can use any existing user management system with proctor. +In order to use auth plugin you should compile the plugin and fill these environment variables: + 1. PROCTOR_AUTH_PLUGIN_BINARY + + Fill this variable with path to compiled plugin, only single plugin is allowed + + 2. PROCTOR_AUTH_PLUGIN_EXPORTED + + Fill this variable with name of variable exported from plugin binary + + 3. PROCTOR_AUTH_ENABLED + + Fill this with `true` to activate auth using plugin + +### Provided plugin + +#### Gate Auth Plugin + +In order to use gate auth plugin, you need a running (Gate)[https://github.com/gate-sso/gate] server. +Authenticated user mean an user that registered to Gate server. +Autorized user mean an user need to be member of at least one group from groups list specified on authorized_groups metadata for procs. + +Compile gate auth plugin by running `make plugin.auth` and fill `PROCTOR_AUTH_PLUGIN_BINARY` with generated `auth.so` in `./_output/bin/plugin/auth.so`. + + + +#### Slack Notification Plugin + +Proctor will send notification to slack when some event happen, see below for a list of events and it's content. +Create a [slack app(https://api.slack.com/incoming-webhooks) then fill `SLACK_PLUGIN_URL` environment variable with incoming webhook url, it should look like `https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX`. + +Compile slack notification plugin by running `make plugin.slack` and fill `PROCTOR_NOTIFICATION_PLUGIN_BINARY` with generated `slack.so` in `./_output/bin/plugin/slack.so`. From 74a39d6294c8031be3c7a9b040b8dab0f37a7a80 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 18 Sep 2019 18:23:28 +0700 Subject: [PATCH 227/268] [Dembo] Change event type signature to return type --- pkg/notification/event/event.go | 2 +- pkg/notification/event/event_mock.go | 4 ++-- pkg/notification/event/execution.go | 4 ++-- pkg/notification/event/execution_test.go | 2 +- .../slack-notification-plugin/slack/message/execution.go | 2 +- .../slack/message/execution_test.go | 4 ++-- plugins/slack-notification-plugin/slack/message/standard.go | 2 +- .../slack/message/standard_test.go | 2 +- plugins/slack-notification-plugin/slack_notification.go | 2 +- .../slack-notification-plugin/slack_notification_test.go | 6 +++--- 10 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pkg/notification/event/event.go b/pkg/notification/event/event.go index 28037b4e..b250a4ba 100644 --- a/pkg/notification/event/event.go +++ b/pkg/notification/event/event.go @@ -1,7 +1,7 @@ package event type Event interface { - Type() string + Type() Type User() UserData Content() map[string]string } diff --git a/pkg/notification/event/event_mock.go b/pkg/notification/event/event_mock.go index e8bc1667..4ec91125 100644 --- a/pkg/notification/event/event_mock.go +++ b/pkg/notification/event/event_mock.go @@ -6,9 +6,9 @@ type EventMock struct { mock.Mock } -func (m *EventMock) Type() string { +func (m *EventMock) Type() Type { args := m.Called() - return args.Get(0).(string) + return args.Get(0).(Type) } func (m *EventMock) User() UserData { diff --git a/pkg/notification/event/execution.go b/pkg/notification/event/execution.go index 1f5d662f..55fedaec 100644 --- a/pkg/notification/event/execution.go +++ b/pkg/notification/event/execution.go @@ -12,8 +12,8 @@ type executionEvent struct { context model.ExecutionContext } -func (evt executionEvent) Type() string { - return string(ExecutionEventType) +func (evt executionEvent) Type() Type { + return ExecutionEventType } func (evt executionEvent) User() UserData { diff --git a/pkg/notification/event/execution_test.go b/pkg/notification/event/execution_test.go index 18119baf..02d175aa 100644 --- a/pkg/notification/event/execution_test.go +++ b/pkg/notification/event/execution_test.go @@ -75,7 +75,7 @@ func TestExecutionEvent(t *testing.T) { "Status": "CREATED", } - assert.Equal(t, expectedEventType, actualEvent.Type()) + assert.Equal(t, expectedEventType, string(actualEvent.Type())) assert.Equal(t, expectedUserData, actualEvent.User()) assert.Equal(t, expectedContent, actualEvent.Content()) } diff --git a/plugins/slack-notification-plugin/slack/message/execution.go b/plugins/slack-notification-plugin/slack/message/execution.go index 1ee28822..ef4fbd8a 100644 --- a/plugins/slack-notification-plugin/slack/message/execution.go +++ b/plugins/slack-notification-plugin/slack/message/execution.go @@ -13,7 +13,7 @@ type executionMessage struct { func (messageObject *executionMessage) JSON() (string, error) { evt := messageObject.evt - if evt.Type() != string(event.ExecutionEventType) { + if evt.Type() != event.ExecutionEventType { return "", errors.New("event type mismatch") } userEmail := evt.User().Email diff --git a/plugins/slack-notification-plugin/slack/message/execution_test.go b/plugins/slack-notification-plugin/slack/message/execution_test.go index 2548d0ae..b92cead4 100644 --- a/plugins/slack-notification-plugin/slack/message/execution_test.go +++ b/plugins/slack-notification-plugin/slack/message/execution_test.go @@ -18,7 +18,7 @@ func TestExecutionMessage_JSON(t *testing.T) { "Status": "CREATED", } evt := event.EventMock{} - evt.On("Type").Return(string(event.ExecutionEventType)) + evt.On("Type").Return(event.ExecutionEventType) evt.On("User").Return(userData) evt.On("Content").Return(content) defer evt.AssertExpectations(t) @@ -31,7 +31,7 @@ func TestExecutionMessage_JSON(t *testing.T) { func TestExecutionMessage_JSONMismatch(t *testing.T) { evt := event.EventMock{} - evt.On("Type").Return("not-execution-event") + evt.On("Type").Return(event.Type("not-execution-event")) defer evt.AssertExpectations(t) messageObject := NewExecutionMessage(&evt) diff --git a/plugins/slack-notification-plugin/slack/message/standard.go b/plugins/slack-notification-plugin/slack/message/standard.go index 21ce46e3..aca54b50 100644 --- a/plugins/slack-notification-plugin/slack/message/standard.go +++ b/plugins/slack-notification-plugin/slack/message/standard.go @@ -16,7 +16,7 @@ func (messageObject *standardMessage) JSON() (string, error) { return "", err } textMessage := "User: " + messageObject.evt.User().Email + "\n" - textMessage += "Emit event" + messageObject.evt.Type() + " with detail: " + textMessage += "Emit event" + string(messageObject.evt.Type()) + " with detail: " textMessage += string(evtDataJSON) messageObject.Text = textMessage byteMessage, err := json.Marshal(messageObject) diff --git a/plugins/slack-notification-plugin/slack/message/standard_test.go b/plugins/slack-notification-plugin/slack/message/standard_test.go index 5c0c5fd7..3d7c7575 100644 --- a/plugins/slack-notification-plugin/slack/message/standard_test.go +++ b/plugins/slack-notification-plugin/slack/message/standard_test.go @@ -18,7 +18,7 @@ func TestStandardMessage_JSON(t *testing.T) { "Status": "CREATED", } evt := event.EventMock{} - evt.On("Type").Return("unsupported-event") + evt.On("Type").Return(event.Type("unsupported-event")) evt.On("User").Return(userData) evt.On("Content").Return(content) defer evt.AssertExpectations(t) diff --git a/plugins/slack-notification-plugin/slack_notification.go b/plugins/slack-notification-plugin/slack_notification.go index 3d8063c9..66d9fdea 100644 --- a/plugins/slack-notification-plugin/slack_notification.go +++ b/plugins/slack-notification-plugin/slack_notification.go @@ -21,7 +21,7 @@ func (notification *slackNotification) OnNotify(evt event.Event) error { func (notification *slackNotification) generateMessage(evt event.Event) message.Message { var messageObject message.Message switch evt.Type() { - case string(event.ExecutionEventType): + case event.ExecutionEventType: messageObject = message.NewExecutionMessage(evt) default: messageObject = message.NewStandardMessage(evt) diff --git a/plugins/slack-notification-plugin/slack_notification_test.go b/plugins/slack-notification-plugin/slack_notification_test.go index 1a60f169..b43e67b1 100644 --- a/plugins/slack-notification-plugin/slack_notification_test.go +++ b/plugins/slack-notification-plugin/slack_notification_test.go @@ -57,7 +57,7 @@ func TestSlackNotification_OnNotifyExecution(t *testing.T) { "Status": "CREATED", } evt := ctx.instance().event - evt.On("Type").Return(string(event.ExecutionEventType)) + evt.On("Type").Return(event.ExecutionEventType) evt.On("User").Return(userData) evt.On("Content").Return(content) @@ -86,7 +86,7 @@ func TestSlackNotification_OnNotifyExecutionErrorPublish(t *testing.T) { "Status": "CREATED", } evt := ctx.instance().event - evt.On("Type").Return(string(event.ExecutionEventType)) + evt.On("Type").Return(event.ExecutionEventType) evt.On("User").Return(userData) evt.On("Content").Return(content) @@ -115,7 +115,7 @@ func TestSlackNotification_OnNotifyUnsupportedEvent(t *testing.T) { "Status": "CREATED", } evt := ctx.instance().event - evt.On("Type").Return("Unsupported event") + evt.On("Type").Return(event.Type("Unsupported event")) evt.On("User").Return(userData) evt.On("Content").Return(content) From 1f33a5828a3f3166882caa3c833bc758acdc3ed9 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 19 Sep 2019 10:11:48 +0700 Subject: [PATCH 228/268] [Dembo] Change config type for notification plugin binary --- internal/app/service/infra/config/config.go | 8 ++++++-- internal/app/service/infra/config/config_test.go | 5 +++-- .../app/service/infra/plugin/plugin_integration_test.go | 2 +- internal/app/service/notification/service/notification.go | 6 +++--- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 1e5a48e3..c278358d 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/spf13/viper" + "strings" "sync" "sync/atomic" "time" @@ -82,7 +83,7 @@ type ProctorConfig struct { DocsPath string AuthPluginBinary string AuthEnabled bool - NotificationPluginBinary string + NotificationPluginBinary []string NotificationPluginExported string } @@ -128,10 +129,13 @@ func Load() ProctorConfig { AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), - NotificationPluginBinary: fang.GetString("NOTIFICATION_PLUGIN_BINARY"), NotificationPluginExported: fang.GetString("NOTIFICATION_PLUGIN_EXPORTED"), } + notificationPluginsBinary := strings.Split(fang.GetString("NOTIFICATION_PLUGIN_BINARY"), ",") + proctorConfig.NotificationPluginBinary = []string{} + proctorConfig.NotificationPluginBinary = append(proctorConfig.NotificationPluginBinary, notificationPluginsBinary...) + return proctorConfig } diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 6ae0e488..19585628 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -217,9 +217,10 @@ func TestAuthEnabled(t *testing.T) { } func TestNotificationPluginBinary(t *testing.T) { - _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_BINARY", "path-notification") + _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_BINARY", "path-notification,second-path") - assert.Equal(t, "path-notification", Load().NotificationPluginBinary) + expected := []string{"path-notification", "second-path"} + assert.Equal(t, expected, Load().NotificationPluginBinary) } func TestNotificationPluginExported(t *testing.T) { diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index 48b5b5d8..d1b3d56d 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -76,7 +76,7 @@ func TestGoPlugin_LoadNotificationSuccessfully(t *testing.T) { ctx := newContext() ctx.setUp(t) - pluginsBinary := strings.Split(config.Config().NotificationPluginBinary, ",") + pluginsBinary := config.Config().NotificationPluginBinary pluginsExported := strings.Split(config.Config().NotificationPluginExported, ",") for idx, pluginBinary := range pluginsBinary { pluginExported := pluginsExported[idx] diff --git a/internal/app/service/notification/service/notification.go b/internal/app/service/notification/service/notification.go index 49842fd6..98a6cd69 100644 --- a/internal/app/service/notification/service/notification.go +++ b/internal/app/service/notification/service/notification.go @@ -16,7 +16,7 @@ type NotificationService interface { type notificationService struct { observers []notification.Observer goPlugin plugin.GoPlugin - pluginsBinary string + pluginsBinary []string pluginsExportedName string once sync.Once } @@ -32,7 +32,7 @@ func (s *notificationService) Notify(evt event.Event) { func (s *notificationService) initializePlugin() { s.once.Do(func() { s.observers = []notification.Observer{} - pluginsBinary := strings.Split(s.pluginsBinary, ",") + pluginsBinary := s.pluginsBinary pluginsExported := strings.Split(s.pluginsExportedName, ",") for idx, pluginBinary := range pluginsBinary { @@ -52,7 +52,7 @@ func (s *notificationService) initializePlugin() { }) } -func NewNotificationService(pluginsBinary string, pluginsExportedName string, goPlugin plugin.GoPlugin) NotificationService { +func NewNotificationService(pluginsBinary []string, pluginsExportedName string, goPlugin plugin.GoPlugin) NotificationService { return ¬ificationService{ goPlugin: goPlugin, pluginsBinary: pluginsBinary, From 456f62164278b852fd328edb4c0804a9c1d84e77 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 19 Sep 2019 10:22:23 +0700 Subject: [PATCH 229/268] [Dembo] Change config type for notification plugin exported name --- internal/app/service/infra/config/config.go | 7 ++++--- internal/app/service/infra/config/config_test.go | 5 +++-- .../service/infra/plugin/plugin_integration_test.go | 3 +-- .../app/service/notification/service/notification.go | 12 ++++-------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index c278358d..ad653e17 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -84,7 +84,7 @@ type ProctorConfig struct { AuthPluginBinary string AuthEnabled bool NotificationPluginBinary []string - NotificationPluginExported string + NotificationPluginExported []string } func Load() ProctorConfig { @@ -129,13 +129,14 @@ func Load() ProctorConfig { AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), - NotificationPluginExported: fang.GetString("NOTIFICATION_PLUGIN_EXPORTED"), } notificationPluginsBinary := strings.Split(fang.GetString("NOTIFICATION_PLUGIN_BINARY"), ",") - proctorConfig.NotificationPluginBinary = []string{} proctorConfig.NotificationPluginBinary = append(proctorConfig.NotificationPluginBinary, notificationPluginsBinary...) + notificationPluginsExported := strings.Split(fang.GetString("NOTIFICATION_PLUGIN_EXPORTED"), ",") + proctorConfig.NotificationPluginExported = append([]string{}, notificationPluginsExported...) + return proctorConfig } diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 19585628..3db88ae9 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -224,7 +224,8 @@ func TestNotificationPluginBinary(t *testing.T) { } func TestNotificationPluginExported(t *testing.T) { - _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_EXPORTED", "plugin-notification") + _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_EXPORTED", "plugin-notification,second-plugin") - assert.Equal(t, "plugin-notification", Load().NotificationPluginExported) + expected := []string{"plugin-notification", "second-plugin"} + assert.Equal(t, expected, Load().NotificationPluginExported) } diff --git a/internal/app/service/infra/plugin/plugin_integration_test.go b/internal/app/service/infra/plugin/plugin_integration_test.go index d1b3d56d..c2051a54 100644 --- a/internal/app/service/infra/plugin/plugin_integration_test.go +++ b/internal/app/service/infra/plugin/plugin_integration_test.go @@ -3,7 +3,6 @@ package plugin import ( "fmt" "os" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -77,7 +76,7 @@ func TestGoPlugin_LoadNotificationSuccessfully(t *testing.T) { ctx.setUp(t) pluginsBinary := config.Config().NotificationPluginBinary - pluginsExported := strings.Split(config.Config().NotificationPluginExported, ",") + pluginsExported := config.Config().NotificationPluginExported for idx, pluginBinary := range pluginsBinary { pluginExported := pluginsExported[idx] raw, err := ctx.instance().goPlugin.Load(pluginBinary, pluginExported) diff --git a/internal/app/service/notification/service/notification.go b/internal/app/service/notification/service/notification.go index 98a6cd69..5b745e93 100644 --- a/internal/app/service/notification/service/notification.go +++ b/internal/app/service/notification/service/notification.go @@ -5,7 +5,6 @@ import ( "proctor/internal/app/service/infra/plugin" "proctor/pkg/notification" "proctor/pkg/notification/event" - "strings" "sync" ) @@ -17,7 +16,7 @@ type notificationService struct { observers []notification.Observer goPlugin plugin.GoPlugin pluginsBinary []string - pluginsExportedName string + pluginsExportedName []string once sync.Once } @@ -32,11 +31,8 @@ func (s *notificationService) Notify(evt event.Event) { func (s *notificationService) initializePlugin() { s.once.Do(func() { s.observers = []notification.Observer{} - pluginsBinary := s.pluginsBinary - pluginsExported := strings.Split(s.pluginsExportedName, ",") - - for idx, pluginBinary := range pluginsBinary { - raw, err := s.goPlugin.Load(pluginBinary, pluginsExported[idx]) + for idx, pluginBinary := range s.pluginsBinary { + raw, err := s.goPlugin.Load(pluginBinary, s.pluginsExportedName[idx]) logger.LogErrors(err, "Load GoPlugin binary") if err != nil { return @@ -52,7 +48,7 @@ func (s *notificationService) initializePlugin() { }) } -func NewNotificationService(pluginsBinary []string, pluginsExportedName string, goPlugin plugin.GoPlugin) NotificationService { +func NewNotificationService(pluginsBinary []string, pluginsExportedName []string, goPlugin plugin.GoPlugin) NotificationService { return ¬ificationService{ goPlugin: goPlugin, pluginsBinary: pluginsBinary, From 258cd2adaea0174f0e01cd4bf1d7d14af12dbc9a Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 19 Sep 2019 10:51:36 +0700 Subject: [PATCH 230/268] [Dembo] Add integration test for slack notification plugin --- .../slack_notification_integration_test.go | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 plugins/slack-notification-plugin/slack_notification_integration_test.go diff --git a/plugins/slack-notification-plugin/slack_notification_integration_test.go b/plugins/slack-notification-plugin/slack_notification_integration_test.go new file mode 100644 index 00000000..4660b366 --- /dev/null +++ b/plugins/slack-notification-plugin/slack_notification_integration_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "github.com/go-resty/resty/v2" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "proctor/pkg/notification/event" + "proctor/plugins/slack-notification-plugin/slack" +) + +type integrationContext interface { + setUp(t *testing.T) + tearDown() + instance() *integrationTestContext +} + +type integrationTestContext struct { + slackNotification *slackNotification + event *event.EventMock +} + +func (context *integrationTestContext) setUp(t *testing.T) { + value, available := os.LookupEnv("ENABLE_PLUGIN_INTEGRATION_TEST") + if available != true || value != "true" { + t.SkipNow() + } + slackUrl, _ := os.LookupEnv("SLACK_PLUGIN_URL") + assert.NotEmpty(t, slackUrl) + + context.slackNotification = &slackNotification{} + context.slackNotification.slackClient = slack.NewSlackClient(resty.New()) + context.event = &event.EventMock{} +} + +func (context *integrationTestContext) tearDown() { +} + +func (context *integrationTestContext) instance() *integrationTestContext { + return context +} + +func newIntegrationContext() integrationContext { + return &integrationTestContext{} +} + +func TestSlackNotificationIntegration_OnNotifyExecution(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + userData := event.UserData{ + Email: "slack_notification_integration_execution_event@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := ctx.instance().event + evt.On("Type").Return(event.ExecutionEventType) + evt.On("User").Return(userData) + evt.On("Content").Return(content) + + err := ctx.instance().slackNotification.OnNotify(evt) + assert.NoError(t, err) +} + +func TestSlackNotificationIntegration_OnNotifyUnsupportedEvent(t *testing.T) { + ctx := newIntegrationContext() + ctx.setUp(t) + defer ctx.tearDown() + + userData := event.UserData{ + Email: "slack_notification_integration_unsupported_event@example.com", + } + content := map[string]string{ + "ExecutionID": "7", + "JobName": "test-job", + "ImageTag": "test", + "Args": "args", + "Status": "CREATED", + } + evt := ctx.instance().event + evt.On("Type").Return(event.Type("Unsupported event")) + evt.On("User").Return(userData) + evt.On("Content").Return(content) + + err := ctx.instance().slackNotification.OnNotify(evt) + assert.NoError(t, err) +} From 7a9027acf1f916e572fa18f7a7ce9489722e0503 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 20 Sep 2019 10:24:24 +0700 Subject: [PATCH 231/268] [Jasoet] Enable Admin Authorization --- internal/app/cli/daemon/client_test.go | 2 - .../app/service/execution/status/execution.go | 22 +- internal/app/service/infra/config/config.go | 19 +- internal/app/service/infra/plugin/plugin.go | 23 +- .../middleware/admin_authorization.go | 54 +++++ .../middleware/admin_authorization_test.go | 215 ++++++++++++++++++ .../security/middleware/authentication.go | 2 +- .../security/middleware/authorization.go | 2 +- .../service/security/middleware/middleware.go | 1 + .../service/security_integration_test.go | 6 +- internal/app/service/server/router.go | 5 +- pkg/auth/auth.go | 2 +- .../gate-auth-plugin/gate/status/status.go | 2 +- 13 files changed, 315 insertions(+), 40 deletions(-) create mode 100644 internal/app/service/security/middleware/admin_authorization.go create mode 100644 internal/app/service/security/middleware/admin_authorization_test.go diff --git a/internal/app/cli/daemon/client_test.go b/internal/app/cli/daemon/client_test.go index 52809afb..8dbdf92d 100644 --- a/internal/app/cli/daemon/client_test.go +++ b/internal/app/cli/daemon/client_test.go @@ -820,7 +820,6 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWithInvalidJobID() { httpmock.Activate() defer httpmock.DeactivateAndReset() - mockResponse := httpmock.NewStringResponse(400, "Invalid Job ID") mockError := error(nil) mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) @@ -842,7 +841,6 @@ func (s *ClientTestSuite) TestRemoveScheduledJobWhenJobIDNotFound() { httpmock.Activate() defer httpmock.DeactivateAndReset() - mockResponse := httpmock.NewStringResponse(404, "Job not found") mockError := error(nil) mockRequest(proctorConfig, "DELETE", fmt.Sprintf("http://"+proctorConfig.Host+ScheduleRoute+"/%s", jobID), mockResponse, mockError) diff --git a/internal/app/service/execution/status/execution.go b/internal/app/service/execution/status/execution.go index f50ec2d6..422556ed 100644 --- a/internal/app/service/execution/status/execution.go +++ b/internal/app/service/execution/status/execution.go @@ -3,15 +3,15 @@ package status type ExecutionStatus string const ( - Received ExecutionStatus = "RECEIVED" - RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" - Created ExecutionStatus = "CREATED" - CreationFailed ExecutionStatus = "CREATION_FAILED" - JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" - JobReady ExecutionStatus = "JOB_READY" - PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" - PodReady ExecutionStatus = "POD_READY" - PodFailed ExecutionStatus = "POD_FAILED" - FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" - Finished ExecutionStatus = "FINISHED" + Received ExecutionStatus = "RECEIVED" + RequirementNotMet ExecutionStatus = "REQUIREMENT_NOT_MET" + Created ExecutionStatus = "CREATED" + CreationFailed ExecutionStatus = "CREATION_FAILED" + JobCreationFailed ExecutionStatus = "JOB_CREATION_FAILED" + JobReady ExecutionStatus = "JOB_READY" + PodCreationFailed ExecutionStatus = "POD_CREATION_FAILED" + PodReady ExecutionStatus = "POD_READY" + PodFailed ExecutionStatus = "POD_FAILED" + FetchPodLogFailed ExecutionStatus = "FETCH_POD_LOG_FAILED" + Finished ExecutionStatus = "FINISHED" ) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index ad653e17..a050fb90 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -15,6 +15,15 @@ func GetStringDefault(viper *viper.Viper, key string, defaultValue string) strin return viper.GetString(key) } +func GetArrayString(viper *viper.Viper, key string) []string { + return strings.Split(viper.GetString(key), ",") +} + +func GetArrayStringDefault(viper *viper.Viper, key string, defaultValue []string) []string { + viper.SetDefault(key, strings.Join(defaultValue, ",")) + return strings.Split(viper.GetString(key), ",") +} + func GetBoolDefault(viper *viper.Viper, key string, defaultValue bool) bool { viper.SetDefault(key, defaultValue) return viper.GetBool(key) @@ -85,6 +94,7 @@ type ProctorConfig struct { AuthEnabled bool NotificationPluginBinary []string NotificationPluginExported []string + AuthRequiredAdminGroup []string } func Load() ProctorConfig { @@ -129,14 +139,11 @@ func Load() ProctorConfig { AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), + NotificationPluginBinary: GetArrayString(fang, "NOTIFICATION_PLUGIN_BINARY"), + NotificationPluginExported: GetArrayString(fang, "NOTIFICATION_PLUGIN_EXPORTED"), + AuthRequiredAdminGroup: GetArrayStringDefault(fang, "AUTH_REQUIRED_ADMIN_GROUP", []string{"proctor_admin"}), } - notificationPluginsBinary := strings.Split(fang.GetString("NOTIFICATION_PLUGIN_BINARY"), ",") - proctorConfig.NotificationPluginBinary = append(proctorConfig.NotificationPluginBinary, notificationPluginsBinary...) - - notificationPluginsExported := strings.Split(fang.GetString("NOTIFICATION_PLUGIN_EXPORTED"), ",") - proctorConfig.NotificationPluginExported = append([]string{}, notificationPluginsExported...) - return proctorConfig } diff --git a/internal/app/service/infra/plugin/plugin.go b/internal/app/service/infra/plugin/plugin.go index 4671344b..33b03603 100644 --- a/internal/app/service/infra/plugin/plugin.go +++ b/internal/app/service/infra/plugin/plugin.go @@ -13,21 +13,20 @@ type GoPlugin interface { type goPlugin struct{} func (g *goPlugin) Load(pluginBinary string, exportedName string) (plugin.Symbol, error) { - binary, err := plugin.Open(pluginBinary) - logger.LogErrors(err, "load auth plugin binary from location: ", pluginBinary) - if err != nil { - return nil, fmt.Errorf("failed to load plugin binary from location: %s", pluginBinary) - } + binary, err := plugin.Open(pluginBinary) + logger.LogErrors(err, "load auth plugin binary from location: ", pluginBinary) + if err != nil { + return nil, fmt.Errorf("failed to load plugin binary from location: %s", pluginBinary) + } - raw, err := binary.Lookup(exportedName) - logger.LogErrors(err, "Lookup ", pluginBinary, " for ", exportedName) - if err != nil { - return nil, fmt.Errorf("failed to Lookup plugin binary from location: %s with Exported Name: %s", pluginBinary, exportedName) - } - return raw, nil + raw, err := binary.Lookup(exportedName) + logger.LogErrors(err, "Lookup ", pluginBinary, " for ", exportedName) + if err != nil { + return nil, fmt.Errorf("failed to Lookup plugin binary from location: %s with Exported Name: %s", pluginBinary, exportedName) + } + return raw, nil } func NewGoPlugin() GoPlugin { return &goPlugin{} } - diff --git a/internal/app/service/security/middleware/admin_authorization.go b/internal/app/service/security/middleware/admin_authorization.go new file mode 100644 index 00000000..1c693222 --- /dev/null +++ b/internal/app/service/security/middleware/admin_authorization.go @@ -0,0 +1,54 @@ +package middleware + +import ( + "net/http" + "proctor/internal/app/service/security/service" + + "github.com/gorilla/mux" + + "proctor/internal/app/service/infra/config" + "proctor/internal/app/service/infra/logger" + "proctor/pkg/auth" +) + +type adminAuthorizationMiddleware struct { + service service.SecurityService + requiredGroup []string + enabled bool +} + +func (middleware *adminAuthorizationMiddleware) Secure(router *mux.Router, path string, handler http.Handler) *mux.Route { + return router.NewRoute().Path(path).Handler(middleware.MiddlewareFunc(handler)) +} + +func (middleware *adminAuthorizationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !middleware.enabled { + next.ServeHTTP(w, r) + return + } + + userDetail := r.Context().Value(ContextUserDetailKey) + if userDetail == nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + + authorized, err := middleware.service.Verify(*(userDetail.(*auth.UserDetail)), middleware.requiredGroup) + logger.LogErrors(err, "authorization", middleware.requiredGroup, userDetail) + if !authorized { + w.WriteHeader(http.StatusForbidden) + return + } + next.ServeHTTP(w, r) + }) +} + +func NewAdminAuthorizationMiddleware(securityService service.SecurityService) AuthorizationMiddleware { + proctorConfig := config.Config() + return &adminAuthorizationMiddleware{ + service: securityService, + requiredGroup: proctorConfig.AuthRequiredAdminGroup, + enabled: proctorConfig.AuthEnabled, + } +} diff --git a/internal/app/service/security/middleware/admin_authorization_test.go b/internal/app/service/security/middleware/admin_authorization_test.go new file mode 100644 index 00000000..a7d366d3 --- /dev/null +++ b/internal/app/service/security/middleware/admin_authorization_test.go @@ -0,0 +1,215 @@ +package middleware + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" + + "proctor/internal/app/service/security/service" + "proctor/pkg/auth" +) + +type adminAuthorizationContext struct { + authorizationMiddleware adminAuthorizationMiddleware + requestHandler func(http.Handler) http.Handler + securityService *service.SecurityServiceMock +} + +func (context *adminAuthorizationContext) setUp(t *testing.T) { + context.authorizationMiddleware = adminAuthorizationMiddleware{} + context.requestHandler = func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + next.ServeHTTP(w, r) + }) + } + context.securityService = &service.SecurityServiceMock{} + context.authorizationMiddleware.service = context.securityService + context.authorizationMiddleware.enabled = true + context.authorizationMiddleware.requiredGroup = []string{"proctor_admin", "system"} +} + +func (context *adminAuthorizationContext) tearDown() { +} + +func (context *adminAuthorizationContext) instance() *adminAuthorizationContext { + return context +} + +func newAdminAuthorizationContext() *adminAuthorizationContext { + return &adminAuthorizationContext{} +} + +func TestAdminAuthorizationMiddleware_MiddlewareFuncExecutionSuccess(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, + Groups: []string{"system", "proctor_admin"}, + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) + requestHandler := ctx.instance().requestHandler + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, ctx.authorizationMiddleware.requiredGroup).Return(true, nil) + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusOK, responseResult.StatusCode) +} + +func TestAdminAuthorizationMiddleware_MiddlewareFuncScheduleSuccess(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["jobName"] = "a-job" + body, _ := json.Marshal(requestBody) + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, + Groups: []string{"system", "proctor_maintainer"}, + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) + requestHandler := ctx.instance().requestHandler + + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, ctx.authorizationMiddleware.requiredGroup).Return(true, nil) + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusOK, responseResult.StatusCode) +} + +func TestAdminAuthorizationMiddleware_MiddlewareFuncWithoutUserDetail(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestHandler := ctx.instance().requestHandler + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusUnauthorized, responseResult.StatusCode) +} + +func TestAdminAuthorizationMiddleware_MiddlewareFuncFailed(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + requestBody := map[string]string{} + requestBody["name"] = "a-job" + body, _ := json.Marshal(requestBody) + userDetail := &auth.UserDetail{ + Name: "William Dembo", + Email: "email@gmail.com", + Active: true, + Groups: []string{"system", "not_proctor_maintainer"}, + } + + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body)) + requestContext := context.WithValue(request.Context(), ContextUserDetailKey, userDetail) + request = request.WithContext(requestContext) + requestHandler := ctx.instance().requestHandler + + securityService := ctx.securityService + securityService.On("Verify", *userDetail, ctx.authorizationMiddleware.requiredGroup).Return(false, nil) + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + + authzMiddleware := ctx.instance().authorizationMiddleware + requestHandler(authzMiddleware.MiddlewareFunc(testHandler)).ServeHTTP(response, request) + + responseResult := response.Result() + assert.Equal(t, http.StatusForbidden, responseResult.StatusCode) +} + +func TestAdminAuthorizationMiddleware_MiddlewareFuncDisabled(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + authzMiddleware := ctx.instance().authorizationMiddleware + authzMiddleware.enabled = false + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + ts := httptest.NewServer(authzMiddleware.MiddlewareFunc(testHandler)) + defer ts.Close() + + client := &http.Client{} + req, _ := http.NewRequest("GET", ts.URL, nil) + + resp, _ := client.Do(req) + defer resp.Body.Close() + + assert.Equal(t, http.StatusOK, resp.StatusCode) +} + +func TestAdminAuthorizationMiddleware_Secure(t *testing.T) { + ctx := newAdminAuthorizationContext() + ctx.setUp(t) + defer ctx.tearDown() + + router := mux.NewRouter() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }) + authzMiddleware := ctx.instance().authorizationMiddleware + securedRouter := authzMiddleware.Secure(router, "/secure/path", handler) + + handledPath, err := securedRouter.GetPathTemplate() + assert.NoError(t, err) + assert.Equal(t, "/secure/path", handledPath) +} diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 468ebfd3..20670a03 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -62,7 +62,7 @@ func (middleware *authenticationMiddleware) isRequestExcluded(r *http.Request) b } func NewAuthenticationMiddleware(securityService service.SecurityService) Middleware { - proctorConfig := config.Load() + proctorConfig := config.Config() return &authenticationMiddleware{ service: securityService, enabled: proctorConfig.AuthEnabled, diff --git a/internal/app/service/security/middleware/authorization.go b/internal/app/service/security/middleware/authorization.go index 2d12bbc4..6e12b917 100644 --- a/internal/app/service/security/middleware/authorization.go +++ b/internal/app/service/security/middleware/authorization.go @@ -88,7 +88,7 @@ func extractName(r *http.Request) (string, error) { } func NewAuthorizationMiddleware(securityService service.SecurityService, metadataRepository repository.MetadataRepository) AuthorizationMiddleware { - proctorConfig := config.Load() + proctorConfig := config.Config() return &authorizationMiddleware{ service: securityService, metadataRepository: metadataRepository, diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go index 6b53efb4..7df2d6b1 100644 --- a/internal/app/service/security/middleware/middleware.go +++ b/internal/app/service/security/middleware/middleware.go @@ -13,5 +13,6 @@ type Middleware interface { } type AuthorizationMiddleware interface { + MiddlewareFunc(http.Handler) http.Handler Secure(router *mux.Router, path string, handler http.Handler) *mux.Route } diff --git a/internal/app/service/security/service/security_integration_test.go b/internal/app/service/security/service/security_integration_test.go index 3168ca36..6b59219e 100644 --- a/internal/app/service/security/service/security_integration_test.go +++ b/internal/app/service/security/service/security_integration_test.go @@ -19,9 +19,9 @@ type context interface { type testContext struct { securityService SecurityService - email string - token string - proctorConfig config.ProctorConfig + email string + token string + proctorConfig config.ProctorConfig } func (context *testContext) setUp(t *testing.T) { diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index 1790e062..bebf67ea 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -69,6 +69,7 @@ func NewRouter() (*mux.Router, error) { authenticationMiddleware := securityMiddleware.NewAuthenticationMiddleware(_securityService) authorizationMiddleware := securityMiddleware.NewAuthorizationMiddleware(_securityService, metadataStore) + adminAuthorizationMiddleware := securityMiddleware.NewAdminAuthorizationMiddleware(_securityService) pingRoute := router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { _, _ = fmt.Fprintf(w, "pong") @@ -94,8 +95,8 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/metadata", jobMetadataHandler.GetAll()).Methods("GET") - router.HandleFunc("/metadata", jobMetadataHandler.Post()).Methods("POST") - router.HandleFunc("/secret", jobSecretsHandler.Post()).Methods("POST") + adminAuthorizationMiddleware.Secure(router, "/metadata", jobMetadataHandler.Post()).Methods("POST") + adminAuthorizationMiddleware.Secure(router, "/secret", jobSecretsHandler.Post()).Methods("POST") authorizationMiddleware.Secure(router, "/schedule", scheduleHandler.Post()).Methods("POST") router.HandleFunc("/schedule", scheduleHandler.GetAll()).Methods("GET") diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index cc12609e..f21e9e38 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -9,5 +9,5 @@ type UserDetail struct { Name string Email string Active bool - Groups []string + Groups []string } diff --git a/plugins/gate-auth-plugin/gate/status/status.go b/plugins/gate-auth-plugin/gate/status/status.go index 6b8b9312..9482b568 100644 --- a/plugins/gate-auth-plugin/gate/status/status.go +++ b/plugins/gate-auth-plugin/gate/status/status.go @@ -2,5 +2,5 @@ package status const ( TokenAuthenticationFailedMessage = "authentication failed, please check your access token" - UserNotFoundMessage = "user not found for email %s" + UserNotFoundMessage = "user not found for email %s" ) From a8a86c500dd7119a5bb49bfddcbba9977298ed85 Mon Sep 17 00:00:00 2001 From: mazbergaz Date: Mon, 23 Sep 2019 12:01:04 +0700 Subject: [PATCH 232/268] update README for local setup --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index a33f2e8c..fcb7fc89 100644 --- a/README.md +++ b/README.md @@ -30,12 +30,15 @@ For proctor service * Copy `.env.sample` into `.env` file. Please refer [here](#proctor-service-configuration-explanation) for configuration explanation * Make sure you set correct value in `.env` for Kubernetes, Postgresql, and Redis * Export value of `.env` by running `source .env` +* Run `make db.setup` to setup local postgresql and migration * Run `./_output/bin/server s` to start proctor service For proctor cli * Run `./_output/bin/cli config PROCTOR_HOST=` to point you proctor cli to local proctor service * Run `./_output/bin/cli` to see complete usage of proctor cli +* Run `make ftest.update.metadata` to generate sample available command +* Test the client with `./_output/bin/cli list` ## Proctor Components Here's the overview of proctor components. From 2ec9c09e83152e5b585c4409bfaa587d4ff6136f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:16:14 +0700 Subject: [PATCH 233/268] [Jasoet] Enable Support for Yaml Config --- .env.sample | 33 ------ .env.test | 9 +- Dockerfile | 3 +- Makefile | 5 +- config.yaml | 51 +++++++++ .../service/execution_integration_test.go | 2 +- internal/app/service/infra/config/config.go | 103 ++++++++++-------- .../app/service/infra/config/config_test.go | 75 ++++++------- .../app/service/infra/kubernetes/client.go | 2 +- .../app/service/server/middleware/newrelic.go | 2 +- 10 files changed, 153 insertions(+), 132 deletions(-) delete mode 100644 .env.sample create mode 100644 config.yaml diff --git a/.env.sample b/.env.sample deleted file mode 100644 index 7a02c8d3..00000000 --- a/.env.sample +++ /dev/null @@ -1,33 +0,0 @@ -export PROCTOR_KUBE_CONFIG="out-of-cluster" -export PROCTOR_LOG_LEVEL="debug" -export PROCTOR_APP_PORT="5000" -export PROCTOR_DEFAULT_NAMESPACE="default" -export PROCTOR_REDIS_ADDRESS="localhost:6379" -export PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS="10" -export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS="60" -export PROCTOR_KUBE_SERVICE_ACCOUNT_NAME="default" -export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE="140" -export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE="4096" -export PROCTOR_KUBE_CLUSTER_HOST_NAME="localhost:8001" -export PROCTOR_KUBE_POD_LIST_WAIT_TIME="5" -export PROCTOR_KUBE_CA_CERT_ENCODED="LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCmNlcnRpZmljYXRlCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" -export PROCTOR_KUBE_BASIC_AUTH_ENCODED="Y2EtY2VydAo=" -export PROCTOR_POSTGRES_USER="postgres" -export PROCTOR_POSTGRES_PASSWORD="" -export PROCTOR_POSTGRES_HOST="localhost" -export PROCTOR_POSTGRES_PORT="5432" -export PROCTOR_POSTGRES_DATABASE="proctord_development" -export PROCTOR_POSTGRES_MAX_CONNECTIONS="50" -export PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME="30" -export PROCTOR_NEW_RELIC_APP_NAME="PROCTORD" -export PROCTOR_NEW_RELIC_LICENCE_KEY="nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr" -export PROCTOR_MIN_CLIENT_VERSION="v0.2.0" -export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS="5" -export PROCTOR_MAIL_USERNAME="user@mail.com" -export PROCTOR_MAIL_PASSWORD="password" -export PROCTOR_MAIL_SERVER_HOST="smtp.mail.com" -export PROCTOR_MAIL_SERVER_PORT="123" -export PROCTOR_JOB_POD_ANNOTATIONS="{\"key.one\":\"true\"}" -export PROCTOR_SENTRY_DSN="foo" -export PROCTOR_DOCS_PATH="/path/to/docs/dir" -export PROCTOR_AUTH_ENABLED=false diff --git a/.env.test b/.env.test index 9cf693e7..922869ec 100644 --- a/.env.test +++ b/.env.test @@ -1,4 +1,3 @@ -export ENVIRONMENT=test export PROCTOR_KUBE_CONFIG=out-of-cluster export PROCTOR_KUBE_CONTEXT=minikube export PROCTOR_LOG_LEVEL=debug @@ -12,11 +11,7 @@ export PROCTOR_KUBE_SERVICE_ACCOUNT_NAME=default export PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE=140 export PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE=4096 export PROCTOR_KUBE_WAIT_FOR_RESOURCE_POLL_COUNT=5 -export PROCTOR_KUBE_CLUSTER_HOST_NAME=192.168.99.100:8443 -export PROCTOR_KUBE_POD_LIST_WAIT_TIME=60 export PROCTOR_KUBE_LOG_PROCESS_WAIT_TIME=60 -export PROCTOR_KUBE_CA_CERT_ENCODED=LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwdGFXNXAKYTNWaVpVTkJNQjRYRFRFNU1EY3dPVEF6TkRZeE1Wb1hEVEk1TURjd056QXpORFl4TVZvd0ZURVRNQkVHQTFVRQpBeE1LYldsdWFXdDFZbVZEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTkNkCmpXRUhaTi8wanFRbmE5RVorVUFiT2ZoYzAyTUdQWW1Nd0VFUEZMSlNJQnhONEMzWTNEOFBENkVsZ3NjL1NjY04KYUNyZHBZbkhqdjZnQ2lCNmFtMVRjdkVacy81MHl2QlNjcjVldFdwakNLVmd6NnJBU2d1YUdoaU5obUU3a2xtWgpMQVc5c3R0c3F0N1ZrRjJ4VnBNOTA2Y2FiZ2RFdzBrcTJITWcydzJGRHJyaG1VZU9NUWpxSjBBUFQvc252bGMrCkRYRG9QTDh2aFVUakpsdzZ2OE5WMTlCSjZBYnJqMmxzSXJYbk5saEIwRnBldENZeFRmRXNQWnp3bmgwdEVjZk0KTGNNTFBqV3dLejg5cHJBTCt6Qjd5cWErUXprMUVWWEJlMVl3VnlpRXFiRHAvY2V4WGhPeHUyd25HemxxOXN4NApneWR4b3l2eEdnLzYxUVhjWm9FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUIwR0ExVWRKUVFXCk1CUUdDQ3NHQVFVRkJ3TUNCZ2dyQmdFRkJRY0RBVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDMVlZYWsrY2xHZVlPT21QOUpwWUcrQ3R1TTJXaFZQK0Rad2ZDR0tnNWQ1eWhONThTRwp3Q09seDVZMzAxQ3A1dG9NVmU1a3NISityMFQrb0pzMjQvVFlzcVBneTA2MS9Ja3ViYnRjZWova3VBVUdjbkRkCmFmaWg4L1N0d0N4Y1ZzR1ljR3NBRGZGclZYLzhaZm1GZWdwS1dHMWcxcGNpNVByUWxWR2dMdG0rd0lvcVJPWEQKaktuc1VhNWVGQzN3L0h5cmxRZzd6d29qcGMvaURWYzN6UzlxWXlqUUd5MFpEbWZOYloxNVJvS1M0YytHR2lrSQo4OVRKZHU0SDBkckZBU1RsMTlDQ29zcE1KRFdoRGhaWTU4OENPYjVNeUt1RjBDRVVuZ1hBSmxkaGdncXJocFR3CmkwT1phajBteE0xNmtrbFAwR0tyNW1jVmNnYkgzdTBqOGE4cgotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== -export PROCTOR_KUBE_BASIC_AUTH_ENCODED=YWRtaW46cGFzc3dvcmQK export PROCTOR_POSTGRES_USER=postgres export PROCTOR_POSTGRES_PASSWORD= export PROCTOR_POSTGRES_HOST=localhost @@ -25,7 +20,7 @@ export PROCTOR_POSTGRES_DATABASE=proctord_test export PROCTOR_POSTGRES_MAX_CONNECTIONS=50 export PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 export PROCTOR_NEW_RELIC_APP_NAME=PROCTORD -export PROCTOR_NEW_RELIC_LICENCE_KEY=0123456789012345678901234567890123456789 +export PROCTOR_NEW_RELIC_LICENSE_KEY=0123456789012345678901234567890123456789 export PROCTOR_MIN_CLIENT_VERSION=v2.0.0 export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS=5 export PROCTOR_MAIL_USERNAME=user@mail.com @@ -33,10 +28,10 @@ export PROCTOR_MAIL_PASSWORD=password export PROCTOR_MAIL_SERVER_HOST=smtp.mail.com export PROCTOR_MAIL_SERVER_PORT=123 export PROCTOR_JOB_POD_ANNOTATIONS={\"key.one\":\"true\"} -export PROCTOR_SENTRY_DSN=foo export PROCTOR_DOCS_PATH=/path/to/docs/dir export PROCTOR_AUTH_PLUGIN_BINARY= export PROCTOR_AUTH_PLUGIN_EXPORTED=GateAuth export PROCTOR_AUTH_ENABLED=false export PROCTOR_NOTIFICATION_PLUGIN_BINARY= export PROCTOR_NOTIFICATION_PLUGIN_EXPORTED=SlackNotification +export PROCTOR_REQUIRED_ADMIN_GROUP=proctor_admin diff --git a/Dockerfile b/Dockerfile index fc2b4eeb..8aa78a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM golang:1.12 AS builder WORKDIR /go/src/app COPY . . RUN make plugin.auth +RUN make plugin.slack RUN make server FROM ubuntu:latest @@ -10,8 +11,8 @@ RUN apt-get install -y ca-certificates WORKDIR /app/ COPY --from=builder /go/src/app/_output/bin/server . COPY --from=builder /go/src/app/_output/bin/plugin/auth.so . +COPY --from=builder /go/src/app/_output/bin/plugin/slack.so . COPY --from=builder /go/src/app/migrations ./migrations -ENV PROCTOR_AUTH_PLUGIN_BINARY=/app/auth.so ENTRYPOINT ["./server"] CMD ["s"] diff --git a/Makefile b/Makefile index 07dd9ecf..f17fd36c 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ SHELL := /bin/bash #!make -include .env.test -export $(shell sed 's/=.*//' .env.test) +#include .env.test +#export $(shell sed 's/=.*//' .env.test) .EXPORT_ALL_VARIABLES: SRC_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) @@ -14,6 +14,7 @@ FTEST_DIR := test/procs CONFIG_DIR := test/config GOPROXY ?= https://proxy.golang.org GO111MODULE := on +CONFIG_LOCATION := $(SRC_DIR) $(@info $(shell mkdir -p $(OUT_DIR) $(BIN_DIR) $(PLUGIN_DIR)) diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..a1552389 --- /dev/null +++ b/config.yaml @@ -0,0 +1,51 @@ +kube: + config: out-of-cluster + context: minikube + job: + active.deadline.seconds: 60 + retries: 0 + service.account.name: default + wait.for.resource.poll.count: 5 + log.process.wait.time: 60 +log.level: debug +app.port: 5000 +default.namespace: default +redis: + address: localhost:6379 + max.active.connections: 10 +logs.stream: + read.buffer.size: 140 + write.buffer.size: 4096 +postgres: + user: postgres + password: + host: localhost + port: 5432 + database: proctord_test + max.connections: 50 + connections.max.lifetime: 30 +new.relic: + app.name: proctor-service + license.key: +min.client.version: v2.0.0 +scheduled.jobs.fetch.interval.in.mins: 5 +mail: + username: user@mail.com + password: password + server: + host: smtp.mail.com + port: 123 +job.pod.annotations: "{\"key.one\":\"true\"}" +docs.path: /path/to/docs/dir +auth: + enabled: false + plugin: + binary: + exported: GateAuth + required.admin.group: proctor_admin +notification.plugin: + binary: + exported: SlackNotification + + + diff --git a/internal/app/service/execution/service/execution_integration_test.go b/internal/app/service/execution/service/execution_integration_test.go index 0e9cd2c4..e5b88515 100644 --- a/internal/app/service/execution/service/execution_integration_test.go +++ b/internal/app/service/execution/service/execution_integration_test.go @@ -98,7 +98,7 @@ func (suite *TestExecutionIntegrationSuite) TestStreamLogsSuccess() { executedJobname, err := suite.kubernetesClient.ExecuteJobWithCommand(sampleImageName, envVarsForContainer, []string{"echo", "Bimo Horizon"}) assert.NoError(t, err) - waitTime := config.Config().KubePodsListWaitTime * time.Second + waitTime := config.Config().KubeLogProcessWaitTime * time.Second logStream, err := suite.service.StreamJobLogs(executedJobname, waitTime) assert.NoError(t, err) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index a050fb90..d0e92f14 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "github.com/spf13/viper" + "os" "strings" "sync" "sync/atomic" @@ -66,7 +67,6 @@ type ProctorConfig struct { RedisMaxActiveConnections int LogsStreamWriteBufferSize int KubeWaitForResourcePollCount int - KubePodsListWaitTime time.Duration KubeLogProcessWaitTime time.Duration KubeJobActiveDeadlineSeconds *int64 KubeJobRetries *int32 @@ -80,7 +80,7 @@ type ProctorConfig struct { PostgresMaxConnections int PostgresConnectionMaxLifetime int NewRelicAppName string - NewRelicLicenceKey string + NewRelicLicenseKey string MinClientVersion string ScheduledJobsFetchIntervalInMins int MailUsername string @@ -88,7 +88,6 @@ type ProctorConfig struct { MailPassword string MailServerPort string JobPodAnnotations map[string]string - SentryDSN string DocsPath string AuthPluginBinary string AuthEnabled bool @@ -97,51 +96,63 @@ type ProctorConfig struct { AuthRequiredAdminGroup []string } -func Load() ProctorConfig { +func load() ProctorConfig { fang := viper.New() - fang.AutomaticEnv() + fang.SetEnvPrefix("PROCTOR") + fang.SetEnvKeyReplacer(strings.NewReplacer(".","_")) + fang.AutomaticEnv() + + value, available := os.LookupEnv("CONFIG_LOCATION") + if available == true { + fang.SetConfigName("config") + fang.AddConfigPath("$HOME/.proctor") + fang.AddConfigPath(value) + err := fang.ReadInConfig() + if err != nil { + panic(fmt.Errorf("Fatal error config file: %s \n", err)) + } + } + proctorConfig := ProctorConfig{ viper: fang, - KubeConfig: fang.GetString("KUBE_CONFIG"), - KubeContext: GetStringDefault(fang, "KUBE_CONTEXT", "default"), - LogLevel: GetStringDefault(fang, "LOG_LEVEL", "DEBUG"), - AppPort: GetStringDefault(fang, "APP_PORT", "5001"), - DefaultNamespace: fang.GetString("DEFAULT_NAMESPACE"), - RedisAddress: fang.GetString("REDIS_ADDRESS"), - RedisMaxActiveConnections: fang.GetInt("REDIS_MAX_ACTIVE_CONNECTIONS"), - LogsStreamReadBufferSize: fang.GetInt("LOGS_STREAM_READ_BUFFER_SIZE"), - LogsStreamWriteBufferSize: fang.GetInt("LOGS_STREAM_WRITE_BUFFER_SIZE"), - KubeWaitForResourcePollCount: fang.GetInt("KUBE_WAIT_FOR_RESOURCE_POLL_COUNT"), - KubePodsListWaitTime: time.Duration(fang.GetInt("KUBE_POD_LIST_WAIT_TIME")), - KubeLogProcessWaitTime: time.Duration(fang.GetInt("KUBE_LOG_PROCESS_WAIT_TIME")), - KubeJobActiveDeadlineSeconds: GetInt64Ref(fang, "KUBE_JOB_ACTIVE_DEADLINE_SECONDS"), - KubeJobRetries: GetInt32Ref(fang, "KUBE_JOB_RETRIES"), - KubeServiceAccountName: fang.GetString("KUBE_SERVICE_ACCOUNT_NAME"), - PostgresUser: fang.GetString("POSTGRES_USER"), - PostgresPassword: fang.GetString("POSTGRES_PASSWORD"), - PostgresHost: fang.GetString("POSTGRES_HOST"), - PostgresPort: fang.GetInt("POSTGRES_PORT"), - PostgresDatabase: fang.GetString("POSTGRES_DATABASE"), - PostgresMaxConnections: fang.GetInt("POSTGRES_MAX_CONNECTIONS"), - PostgresConnectionMaxLifetime: fang.GetInt("POSTGRES_CONNECTIONS_MAX_LIFETIME"), - NewRelicAppName: fang.GetString("NEW_RELIC_APP_NAME"), - NewRelicLicenceKey: fang.GetString("NEW_RELIC_LICENCE_KEY"), - MinClientVersion: fang.GetString("MIN_CLIENT_VERSION"), - ScheduledJobsFetchIntervalInMins: fang.GetInt("SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS"), - MailUsername: fang.GetString("MAIL_USERNAME"), - MailServerHost: fang.GetString("MAIL_SERVER_HOST"), - MailPassword: fang.GetString("MAIL_PASSWORD"), - MailServerPort: fang.GetString("MAIL_SERVER_PORT"), - JobPodAnnotations: GetMapFromJson(fang, "JOB_POD_ANNOTATIONS"), - SentryDSN: fang.GetString("SENTRY_DSN"), - DocsPath: fang.GetString("DOCS_PATH"), - AuthPluginBinary: fang.GetString("AUTH_PLUGIN_BINARY"), - AuthPluginExported: GetStringDefault(fang, "AUTH_PLUGIN_EXPORTED", "Auth"), - AuthEnabled: GetBoolDefault(fang, "AUTH_ENABLED", false), - NotificationPluginBinary: GetArrayString(fang, "NOTIFICATION_PLUGIN_BINARY"), - NotificationPluginExported: GetArrayString(fang, "NOTIFICATION_PLUGIN_EXPORTED"), - AuthRequiredAdminGroup: GetArrayStringDefault(fang, "AUTH_REQUIRED_ADMIN_GROUP", []string{"proctor_admin"}), + KubeConfig: fang.GetString("kube.config"), + KubeContext: GetStringDefault(fang, "kube.context", "default"), + LogLevel: GetStringDefault(fang, "log.level", "DEBUG"), + AppPort: GetStringDefault(fang, "app.port", "5001"), + DefaultNamespace: fang.GetString("default.namespace"), + RedisAddress: fang.GetString("redis.address"), + RedisMaxActiveConnections: fang.GetInt("redis.max.active.connections"), + LogsStreamReadBufferSize: fang.GetInt("logs.stream.read.buffer.size"), + LogsStreamWriteBufferSize: fang.GetInt("logs.stream.write.buffer.size"), + KubeWaitForResourcePollCount: fang.GetInt("kube.wait.for.resource.poll.count"), + KubeLogProcessWaitTime: time.Duration(fang.GetInt("kube.log.process.wait.time")), + KubeJobActiveDeadlineSeconds: GetInt64Ref(fang, "kube.job.active.deadline.seconds"), + KubeJobRetries: GetInt32Ref(fang, "kube.job.retries"), + KubeServiceAccountName: fang.GetString("kube.service.account.name"), + PostgresUser: fang.GetString("postgres.user"), + PostgresPassword: fang.GetString("postgres.password"), + PostgresHost: fang.GetString("postgres.host"), + PostgresPort: fang.GetInt("POSTGRES.PORT"), + PostgresDatabase: fang.GetString("POSTGRES.DATABASE"), + PostgresMaxConnections: fang.GetInt("postgres.max.connections"), + PostgresConnectionMaxLifetime: fang.GetInt("postgres.connections.max.lifetime"), + NewRelicAppName: fang.GetString("new.relic.app.name"), + NewRelicLicenseKey: fang.GetString("new.relic.license.key"), + MinClientVersion: fang.GetString("min.client.version"), + ScheduledJobsFetchIntervalInMins: fang.GetInt("scheduled.jobs.fetch.interval.in.mins"), + MailUsername: fang.GetString("mail.username"), + MailServerHost: fang.GetString("mail.server.host"), + MailPassword: fang.GetString("mail.password"), + MailServerPort: fang.GetString("mail.server.port"), + JobPodAnnotations: GetMapFromJson(fang, "job.pod.annotations"), + DocsPath: fang.GetString("docs.path"), + AuthPluginBinary: fang.GetString("auth.plugin.binary"), + AuthPluginExported: GetStringDefault(fang, "auth.plugin.exported", "Auth"), + AuthEnabled: GetBoolDefault(fang, "auth.enabled", false), + NotificationPluginBinary: GetArrayString(fang, "notification.plugin.binary"), + NotificationPluginExported: GetArrayString(fang, "notification.plugin.exported"), + AuthRequiredAdminGroup: GetArrayStringDefault(fang, "auth.required.admin.group", []string{"proctor.admin"}), } return proctorConfig @@ -176,11 +187,11 @@ func Reset() { func Config() ProctorConfig { once.Do(func() { - config = Load() + config = load() }) if reset.Get() { - config = Load() + config = load() reset.Set(false) } diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 3db88ae9..0fd65d3e 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -13,7 +13,8 @@ func TestEnvironment(t *testing.T) { value := fake.FirstName() _ = os.Setenv("PROCTOR_KUBE_CONFIG", value) - assert.Equal(t, value, Load().KubeConfig) + loadedConfig := load() + assert.Equal(t, value, loadedConfig.KubeConfig) } func TestLogLevel(t *testing.T) { @@ -21,7 +22,7 @@ func TestLogLevel(t *testing.T) { value := fake.FirstName() _ = os.Setenv("PROCTOR_LOG_LEVEL", value) - assert.Equal(t, value, Load().LogLevel) + assert.Equal(t, value, load().LogLevel) } func TestAppPort(t *testing.T) { @@ -29,7 +30,7 @@ func TestAppPort(t *testing.T) { value := strconv.FormatInt(int64(fake.Number(1000, 4000)), 10) _ = os.Setenv("PROCTOR_APP_PORT", value) - assert.Equal(t, value, Load().AppPort) + assert.Equal(t, value, load().AppPort) } func TestDefaultNamespace(t *testing.T) { @@ -37,7 +38,7 @@ func TestDefaultNamespace(t *testing.T) { value := fake.FirstName() _ = os.Setenv("PROCTOR_DEFAULT_NAMESPACE", value) - assert.Equal(t, value, Load().DefaultNamespace) + assert.Equal(t, value, load().DefaultNamespace) } func TestRedisAddress(t *testing.T) { @@ -45,7 +46,7 @@ func TestRedisAddress(t *testing.T) { value := fake.FirstName() _ = os.Setenv("PROCTOR_REDIS_ADDRESS", value) - assert.Equal(t, value, Load().RedisAddress) + assert.Equal(t, value, load().RedisAddress) } func TestRedisMaxActiveConnections(t *testing.T) { @@ -54,178 +55,172 @@ func TestRedisMaxActiveConnections(t *testing.T) { value := strconv.FormatInt(int64(number), 10) _ = os.Setenv("PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS", value) - assert.Equal(t, number, Load().RedisMaxActiveConnections) + assert.Equal(t, number, load().RedisMaxActiveConnections) } func TestLogsStreamReadBufferSize(t *testing.T) { _ = os.Setenv("PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE", "140") - assert.Equal(t, 140, Load().LogsStreamReadBufferSize) + assert.Equal(t, 140, load().LogsStreamReadBufferSize) } func TestLogsStreamWriteBufferSize(t *testing.T) { _ = os.Setenv("PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE", "4096") - assert.Equal(t, 4096, Load().LogsStreamWriteBufferSize) + assert.Equal(t, 4096, load().LogsStreamWriteBufferSize) } func TestKubeJobActiveDeadlineSeconds(t *testing.T) { _ = os.Setenv("PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS", "900") expectedValue := int64(900) - assert.Equal(t, &expectedValue, Load().KubeJobActiveDeadlineSeconds) + assert.Equal(t, &expectedValue, load().KubeJobActiveDeadlineSeconds) } func TestKubeJobRetries(t *testing.T) { _ = os.Setenv("PROCTOR_KUBE_JOB_RETRIES", "0") expectedValue := int32(0) - assert.Equal(t, &expectedValue, Load().KubeJobRetries) + assert.Equal(t, &expectedValue, load().KubeJobRetries) } func TestKubeServiceName(t *testing.T) { _ = os.Setenv("PROCTOR_KUBE_SERVICE_ACCOUNT_NAME", "proctor") expectedValue := "proctor" - assert.Equal(t, expectedValue, Load().KubeServiceAccountName) + assert.Equal(t, expectedValue, load().KubeServiceAccountName) } func TestPostgresUser(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_USER", "postgres") - assert.Equal(t, "postgres", Load().PostgresUser) + assert.Equal(t, "postgres", load().PostgresUser) } func TestPostgresPassword(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_PASSWORD", "ipsum-lorem") - assert.Equal(t, "ipsum-lorem", Load().PostgresPassword) + assert.Equal(t, "ipsum-lorem", load().PostgresPassword) } func TestPostgresHost(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_HOST", "localhost") - assert.Equal(t, "localhost", Load().PostgresHost) + assert.Equal(t, "localhost", load().PostgresHost) } func TestPostgresPort(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_PORT", "5432") - assert.Equal(t, 5432, Load().PostgresPort) + assert.Equal(t, 5432, load().PostgresPort) } func TestPostgresDatabase(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_DATABASE", "proctord_development") - assert.Equal(t, "proctord_development", Load().PostgresDatabase) + assert.Equal(t, "proctord_development", load().PostgresDatabase) } func TestPostgresMaxConnections(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_MAX_CONNECTIONS", "50") - assert.Equal(t, 50, Load().PostgresMaxConnections) + assert.Equal(t, 50, load().PostgresMaxConnections) } func TestPostgresConnectionMaxLifetime(t *testing.T) { _ = os.Setenv("PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME", "30") - assert.Equal(t, 30, Load().PostgresConnectionMaxLifetime) + assert.Equal(t, 30, load().PostgresConnectionMaxLifetime) } func TestNewRelicAppName(t *testing.T) { _ = os.Setenv("PROCTOR_NEW_RELIC_APP_NAME", "PROCTORD") - assert.Equal(t, "PROCTORD", Load().NewRelicAppName) + assert.Equal(t, "PROCTORD", load().NewRelicAppName) } func TestNewRelicLicenceKey(t *testing.T) { - _ = os.Setenv("PROCTOR_NEW_RELIC_LICENCE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") + _ = os.Setenv("PROCTOR_NEW_RELIC_LICENSE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") - assert.Equal(t, "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr", Load().NewRelicLicenceKey) + assert.Equal(t, "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr", load().NewRelicLicenseKey) } func TestMinClientVersion(t *testing.T) { _ = os.Setenv("PROCTOR_MIN_CLIENT_VERSION", "0.2.0") - assert.Equal(t, "0.2.0", Load().MinClientVersion) + assert.Equal(t, "0.2.0", load().MinClientVersion) } func TestScheduledJobsFetchIntervalInMins(t *testing.T) { _ = os.Setenv("PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS", "5") - assert.Equal(t, 5, Load().ScheduledJobsFetchIntervalInMins) + assert.Equal(t, 5, load().ScheduledJobsFetchIntervalInMins) } func TestMailUsername(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_USERNAME", "foo@bar.com") - assert.Equal(t, "foo@bar.com", Load().MailUsername) + assert.Equal(t, "foo@bar.com", load().MailUsername) } func TestMailPassword(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_PASSWORD", "password") - assert.Equal(t, "password", Load().MailPassword) + assert.Equal(t, "password", load().MailPassword) } func TestMailServerHost(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_SERVER_HOST", "127.0.0.1") - assert.Equal(t, "127.0.0.1", Load().MailServerHost) + assert.Equal(t, "127.0.0.1", load().MailServerHost) } func TestMailServerPort(t *testing.T) { _ = os.Setenv("PROCTOR_MAIL_SERVER_PORT", "123") - assert.Equal(t, "123", Load().MailServerPort) + assert.Equal(t, "123", load().MailServerPort) } func TestJobPodAnnotations(t *testing.T) { _ = os.Setenv("PROCTOR_JOB_POD_ANNOTATIONS", "{\"key.one\":\"true\"}") - assert.Equal(t, map[string]string{"key.one": "true"}, Load().JobPodAnnotations) -} - -func TestSentryDSN(t *testing.T) { - _ = os.Setenv("PROCTOR_SENTRY_DSN", "domain") - - assert.Equal(t, "domain", Load().SentryDSN) + assert.Equal(t, map[string]string{"key.one": "true"}, load().JobPodAnnotations) } func TestDocsPath(t *testing.T) { _ = os.Setenv("PROCTOR_DOCS_PATH", "path1") - assert.Equal(t, "path1", Load().DocsPath) + assert.Equal(t, "path1", load().DocsPath) } func TestAuthPluginBinary(t *testing.T) { _ = os.Setenv("PROCTOR_AUTH_PLUGIN_BINARY", "path1") - assert.Equal(t, "path1", Load().AuthPluginBinary) + assert.Equal(t, "path1", load().AuthPluginBinary) } func TestAuthPluginExported(t *testing.T) { _ = os.Setenv("PROCTOR_AUTH_PLUGIN_EXPORTED", "path1") - assert.Equal(t, "path1", Load().AuthPluginExported) + assert.Equal(t, "path1", load().AuthPluginExported) } func TestAuthEnabled(t *testing.T) { _ = os.Setenv("PROCTOR_AUTH_ENABLED", "false") - assert.Equal(t, false, Load().AuthEnabled) + assert.Equal(t, false, load().AuthEnabled) } func TestNotificationPluginBinary(t *testing.T) { _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_BINARY", "path-notification,second-path") expected := []string{"path-notification", "second-path"} - assert.Equal(t, expected, Load().NotificationPluginBinary) + assert.Equal(t, expected, load().NotificationPluginBinary) } func TestNotificationPluginExported(t *testing.T) { _ = os.Setenv("PROCTOR_NOTIFICATION_PLUGIN_EXPORTED", "plugin-notification,second-plugin") expected := []string{"plugin-notification", "second-plugin"} - assert.Equal(t, expected, Load().NotificationPluginExported) + assert.Equal(t, expected, load().NotificationPluginExported) } diff --git a/internal/app/service/infra/kubernetes/client.go b/internal/app/service/infra/kubernetes/client.go index 6c66787f..3bb7fcc9 100644 --- a/internal/app/service/infra/kubernetes/client.go +++ b/internal/app/service/infra/kubernetes/client.go @@ -151,7 +151,7 @@ func (client *kubernetesClient) ExecuteJobWithCommand(imageName string, envMap m podSpec := v1.PodSpec{ Containers: []v1.Container{container}, RestartPolicy: v1.RestartPolicyNever, - ServiceAccountName: config.Load().KubeServiceAccountName, + ServiceAccountName: config.Config().KubeServiceAccountName, } objectMeta := meta.ObjectMeta{ diff --git a/internal/app/service/server/middleware/newrelic.go b/internal/app/service/server/middleware/newrelic.go index dd111bc7..0b0a2c57 100644 --- a/internal/app/service/server/middleware/newrelic.go +++ b/internal/app/service/server/middleware/newrelic.go @@ -11,7 +11,7 @@ var newRelicApp newrelic.Application func InitNewRelic() error { appName := config.Config().NewRelicAppName - licenceKey := config.Config().NewRelicLicenceKey + licenceKey := config.Config().NewRelicLicenseKey newRelicConfig := newrelic.NewConfig(appName, licenceKey) newRelicConfig.Enabled = true app, err := newrelic.NewApplication(newRelicConfig) From bd9bb20635dfb31567ac737016b845db59f43ae8 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:27:31 +0700 Subject: [PATCH 234/268] [Jasoet] Fix load env on makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f17fd36c..e836b63b 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ SHELL := /bin/bash #!make -#include .env.test -#export $(shell sed 's/=.*//' .env.test) +include .env.test +export $(shell sed 's/=.*//' .env.test) .EXPORT_ALL_VARIABLES: SRC_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) From 8bc6d05e4fc7c68ab9733c209a8fc368903674f0 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:30:36 +0700 Subject: [PATCH 235/268] [Jasoet] Add current directory as config path --- internal/app/service/infra/config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index d0e92f14..c410587a 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -108,6 +108,7 @@ func load() ProctorConfig { fang.SetConfigName("config") fang.AddConfigPath("$HOME/.proctor") fang.AddConfigPath(value) + fang.AddConfigPath(".") err := fang.ReadInConfig() if err != nil { panic(fmt.Errorf("Fatal error config file: %s \n", err)) From 4dfb872d79b88ed520dd3253dbe3b46460ef8ddd Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:42:43 +0700 Subject: [PATCH 236/268] [Jasoet] Change config loader logic --- internal/app/service/infra/config/config.go | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index c410587a..0f4b4068 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -100,20 +100,17 @@ func load() ProctorConfig { fang := viper.New() fang.SetEnvPrefix("PROCTOR") - fang.SetEnvKeyReplacer(strings.NewReplacer(".","_")) + fang.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) fang.AutomaticEnv() + fang.SetConfigName("config") + fang.AddConfigPath(".") + fang.AddConfigPath("$HOME/.proctor") value, available := os.LookupEnv("CONFIG_LOCATION") - if available == true { - fang.SetConfigName("config") - fang.AddConfigPath("$HOME/.proctor") + if available { fang.AddConfigPath(value) - fang.AddConfigPath(".") - err := fang.ReadInConfig() - if err != nil { - panic(fmt.Errorf("Fatal error config file: %s \n", err)) - } } + _ = fang.ReadInConfig() proctorConfig := ProctorConfig{ viper: fang, From c5c56d07f17a012f811b68d846180e10468bbe6f Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:52:00 +0700 Subject: [PATCH 237/268] [Jasoet] Revert config to match current yggdrasil --- .env.test | 2 +- config.yaml | 2 +- internal/app/service/infra/config/config.go | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env.test b/.env.test index 922869ec..c5928469 100644 --- a/.env.test +++ b/.env.test @@ -20,7 +20,7 @@ export PROCTOR_POSTGRES_DATABASE=proctord_test export PROCTOR_POSTGRES_MAX_CONNECTIONS=50 export PROCTOR_POSTGRES_CONNECTIONS_MAX_LIFETIME=30 export PROCTOR_NEW_RELIC_APP_NAME=PROCTORD -export PROCTOR_NEW_RELIC_LICENSE_KEY=0123456789012345678901234567890123456789 +export PROCTOR_NEW_RELIC_LICENCE_KEY=0123456789012345678901234567890123456789 export PROCTOR_MIN_CLIENT_VERSION=v2.0.0 export PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS=5 export PROCTOR_MAIL_USERNAME=user@mail.com diff --git a/config.yaml b/config.yaml index a1552389..2c8408bd 100644 --- a/config.yaml +++ b/config.yaml @@ -26,7 +26,7 @@ postgres: connections.max.lifetime: 30 new.relic: app.name: proctor-service - license.key: + licence.key: min.client.version: v2.0.0 scheduled.jobs.fetch.interval.in.mins: 5 mail: diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 0f4b4068..1e7aeed0 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -131,12 +131,12 @@ func load() ProctorConfig { PostgresUser: fang.GetString("postgres.user"), PostgresPassword: fang.GetString("postgres.password"), PostgresHost: fang.GetString("postgres.host"), - PostgresPort: fang.GetInt("POSTGRES.PORT"), - PostgresDatabase: fang.GetString("POSTGRES.DATABASE"), + PostgresPort: fang.GetInt("postgres.port"), + PostgresDatabase: fang.GetString("postgres.database"), PostgresMaxConnections: fang.GetInt("postgres.max.connections"), PostgresConnectionMaxLifetime: fang.GetInt("postgres.connections.max.lifetime"), NewRelicAppName: fang.GetString("new.relic.app.name"), - NewRelicLicenseKey: fang.GetString("new.relic.license.key"), + NewRelicLicenseKey: fang.GetString("new.relic.licence.key"), MinClientVersion: fang.GetString("min.client.version"), ScheduledJobsFetchIntervalInMins: fang.GetInt("scheduled.jobs.fetch.interval.in.mins"), MailUsername: fang.GetString("mail.username"), From bb47b6a9ac295240b6134569963a05c1a1aebb78 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 25 Sep 2019 16:57:53 +0700 Subject: [PATCH 238/268] [Jasoet] resolve failed test --- internal/app/service/infra/config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 0fd65d3e..702620bf 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -140,7 +140,7 @@ func TestNewRelicAppName(t *testing.T) { } func TestNewRelicLicenceKey(t *testing.T) { - _ = os.Setenv("PROCTOR_NEW_RELIC_LICENSE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") + _ = os.Setenv("PROCTOR_NEW_RELIC_LICENCE_KEY", "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr") assert.Equal(t, "nrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnrnr", load().NewRelicLicenseKey) } From dd0501ee1fc9e33d6c06f6f8e5be306727270b67 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 3 Oct 2019 11:20:45 +0700 Subject: [PATCH 239/268] Add cli feature list on readme --- README.md | 3 + docs/features.md | 295 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 298 insertions(+) create mode 100644 docs/features.md diff --git a/README.md b/README.md index fcb7fc89..f5ef7c6f 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,9 @@ Bundle repetitive task as a automation and turn it into `procs` to make it easie Before we goes deep down into explanation about proctor, you may want to read about [Proctor Glossary](docs/glossary.md) +## Feature list +Section for full features list of proctor CLI is separated [here](./docs/features.md) + ## Installation This section provide installation for unix environment. diff --git a/docs/features.md b/docs/features.md new file mode 100644 index 00000000..ef17d3dd --- /dev/null +++ b/docs/features.md @@ -0,0 +1,295 @@ +# Proctor CLI Feature + +- [proctor config \](#proctor-config) +- [proctor config show](#proctor-config-show) +- [proctor describe \](#proctor-describe-proc) +- [proctor execute \ \](#proctor-execute-proc-params) +- [proctor help](#proctor-help) +- [proctor list](#proctor-list) +- [proctor logs \](#proctor-logs-execution-id) +- [proctor schedule \ \](#proctor-schedule-proc-params) +- [proctor schedule describe \](#proctor-schedule-describe-schedule-id) +- [proctor schedule list](#proctor-schedule-list) +- [proctor schedule remove <\schedule-id\>](#proctor-schedule-remove-schedule-id) +- [proctor status \](#proctor-status-execution-id) +- [proctor template \](#proctor-template-proc) + +## proctor config +Set configuration to run proctor. +Proctor client keep the configuration on ~/.proctor/proctor.yaml + +### Params +| Key | Description | Required | +|:------------- |:------------------------------------- |:--------:| +| PROCTOR_HOST | Host address of proctor service | yes | +| EMAIL_ID | Email account for auth process | no | +| ACCESS_TOKEN | Account access token for auth process | no | + +### Example +```shell script +foo@bar:~$ proctor config PROCTOR_HOST=proctor.com EMAIL_ID=mr.proctor@gmail.com ACCESS_TOKEN=MR_PROCTOR +Proctor client configured successfully +``` + +## proctor config show +Use this command to show current configuration + +### Example +```shell script +foo@bar:~$ proctor config show +PROCTOR_HOST: proctor.com +EMAIL_ID: mr.proctor@gmail.com +ACCESS_TOKEN: MR_PROCTOR +``` + +## proctor describe \ +Use this command to learn more about \ + +### Params +| Key | Description | Required | +|:------------- |:------------------------------------- |:--------:| +| \ | Name of proc | yes | + +### Example +```shell script +foo@bar:~$ proctor describe echo-worker +Description I will echo your name +Contributors Mr.Proctor +Organization Proctor +Authorized Groups [] + +Args +name name to echo +``` + +## proctor execute \ \ +Execute proc on server with specified params + +### Params +| Key | Description | Required | +|:------------- |:------------------------------------- |:--------:| +| \ | Name of proc | yes | + +Other params depend on proc requirements + +### Flags +| Flag | Description | +|:------------- |:------------------------------------- | +| --filename -f | Insert proc parameters from yaml file | + +### Example +```shell script +foo@bar:~$ proctor execute echo-worker name=Mr.Proctor +Executing Proc echo-worker +With Variables +name Mr.Proctor + +Execution Created +ID 1826735143124102 +Name proctor-1289c792631 + +Streaming logs +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor + +Execution completed. +``` + +## proctor help +Show proctor client help text + +### Example +```shell script +foo@bar~:$ proctor help +A command-line interface to run procs + +Usage: + proctor [command] + +Available Commands: + config Configure proctor client + describe Help on executing a proc + execute Execute a proc with given arguments + help Help about any command + list List procs available for execution + logs Get logs of an execution context + schedule Create scheduled procs + status Get status of an execution context + template Get input template of a procs + version Print version of Proctor command-line tool + +Flags: + -h, --help help for proctor + +Use "proctor [command] --help" for more information about a command. +``` + +## proctor list +Show list of available procs + +### Example +```shell script +foo@bar~:$ proctor list +List of Procs: + +echo-worker echo-worker + +For detailed information of any proc, run: +proctor describe +``` + +## proctor logs \ +Stream logs from executed proc, this process will continue until execution complete + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Execution ID generated by execute command | yes | + +Other params depend on proc requirements + +### Example +```shell script +foo@bar:~$ proctor logs 1826735143124102 +Getting logs +ID 1826735143124102 + +Streaming logs +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor +Mr.Proctor + + +Execution completed. +``` + +## proctor schedule \ \ +Run a proc on scheduled basis specified with cron format + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Proc name to scheduled | yes | + +### Flags +| Flag | Description | Required | +|:------------- |:------------------------------------------------------- |:--------:| +| --cron -c | Schedule in cron format ([link](https://crontab.guru/)) | yes | +| --group -g | Group Name | yes | +| --notify -n | Email to notify schedule progress | yes | +| --tags -T | Schedule tags for management purpose | yes | + + +### Example +```shell script +foo@bar:~$ proctor schedule echo-worker name=Mr.Proctor -g proctor -c '0 2 * * *' -T 'proctor' -n mr.proctor@gmail.com +Creating Scheduled Job echo-worker +With Variables +name Mr.Proctor +Scheduled Job UUID : 269349349056612582 +``` + +## proctor schedule describe \ +Get more information about scheduled proc + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Schedule id to learn | yes | + +### Example +```shell script +foo@bar:~$ proctor schedule describe 269349349056612582 +ID 269349349056612582 +PROC NAME echo-worker +GROUP NAME proctor +TAGS proctor +Cron 0 0 2 * * * +Notifier mr.proctor@gmail.com + +Args +name Mr.Proctor +``` + +## proctor schedule list +Get list of scheduled procs + +### Example +```shell script +foo@bar:~$ proctor schedule list +ID PROC NAME GROUP NAME TAGS +269349349056612582 diagnose-vm system-test test,proctor +``` + +## proctor schedule remove <\schedule-id\> +Remove scheduled procs + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Schedule id to remove | yes | + +### Example +```shell script +foo@bar:~$ proctor schedule remove 269349349056612582 +Sucessfully removed the scheduled job ID: 269349349056612582 +``` + +## proctor status \ +Get status of executed proc + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Execution id to get | yes | + +### Example +```shell script +foo@bar:~$ proctor status 1826735143124102 +Getting status +ID 1826735143124102 +Job Name echo-worker +Status FINISHED +Updated At 2019-10-03 04:00:58.083798 +0000 +0000 +Execution completed. +``` + + +## proctor template \ \ +Render yaml template to use with [execute](#proctor-execute-proc-params) command + +### Params +| Key | Description | Required | +|:---------------- |:----------------------------------------- |:--------:| +| \ | Proc name to render | yes | +| \ | Yaml to keep the result | yes | + +### Example +```shell script +foo@bar:~$ proctor template echo-worker echo-worker.yaml + +Args +name [Mandatory] name to echo + +To echo-worker, run: +proctor execute echo-worker -f echo-worker.yaml + +foo@bar:~$ cat echo-worker.yaml +# [Mandatory] name to echo +name: +``` From 8b4aadd178c2f5dcf91f5270c1c77ed465c0a0a9 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 4 Oct 2019 08:59:06 +0700 Subject: [PATCH 240/268] [Jasoet|Dembo] fix required parameter check on CLI --- internal/app/cli/command/schedule/describe/describe.go | 1 + internal/app/cli/command/schedule/remove/remove.go | 1 + 2 files changed, 2 insertions(+) diff --git a/internal/app/cli/command/schedule/describe/describe.go b/internal/app/cli/command/schedule/describe/describe.go index cfaf3322..6eea5c82 100644 --- a/internal/app/cli/command/schedule/describe/describe.go +++ b/internal/app/cli/command/schedule/describe/describe.go @@ -17,6 +17,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { Short: "Describe scheduled job", Long: "This command helps to describe scheduled job", Example: fmt.Sprintf("proctor schedule describe 502376124721"), + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { jobID, err := strconv.ParseUint(args[0], 10, 64) diff --git a/internal/app/cli/command/schedule/remove/remove.go b/internal/app/cli/command/schedule/remove/remove.go index 7d1135f9..f6871035 100644 --- a/internal/app/cli/command/schedule/remove/remove.go +++ b/internal/app/cli/command/schedule/remove/remove.go @@ -14,6 +14,7 @@ func NewCmd(printer io.Printer, proctorDClient daemon.Client) *cobra.Command { Short: "Remove scheduled job", Long: "This command helps to remove scheduled job", Example: fmt.Sprintf("proctor schedule remove D958FCCC-F2B3-49D1-B83A-4E70A2A775A0"), + Args: cobra.MinimumNArgs(1), Run: func(cmd *cobra.Command, args []string) { jobID := args[0] From 48e3276331980ff5d5902abe23717c91cc597eaf Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Fri, 4 Oct 2019 09:01:36 +0700 Subject: [PATCH 241/268] [Jasoet|Dembo] remove documentation for yaml file flag --- docs/features.md | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/docs/features.md b/docs/features.md index ef17d3dd..05a23969 100644 --- a/docs/features.md +++ b/docs/features.md @@ -72,11 +72,6 @@ Execute proc on server with specified params Other params depend on proc requirements -### Flags -| Flag | Description | -|:------------- |:------------------------------------- | -| --filename -f | Insert proc parameters from yaml file | - ### Example ```shell script foo@bar:~$ proctor execute echo-worker name=Mr.Proctor @@ -269,27 +264,3 @@ Updated At 2019-10-03 04:00:58.083798 +0000 +0000 Execution completed. ``` - -## proctor template \ \ -Render yaml template to use with [execute](#proctor-execute-proc-params) command - -### Params -| Key | Description | Required | -|:---------------- |:----------------------------------------- |:--------:| -| \ | Proc name to render | yes | -| \ | Yaml to keep the result | yes | - -### Example -```shell script -foo@bar:~$ proctor template echo-worker echo-worker.yaml - -Args -name [Mandatory] name to echo - -To echo-worker, run: -proctor execute echo-worker -f echo-worker.yaml - -foo@bar:~$ cat echo-worker.yaml -# [Mandatory] name to echo -name: -``` From 27ef13f4fa5a42900a7b19cc43b9b22a88692ad1 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 7 Oct 2019 10:59:01 +0700 Subject: [PATCH 242/268] [Jasoet|Dembo] add missing env variable on README --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f5ef7c6f..1163cb48 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,6 @@ You can read [here](./docs/creating_procs.md) to learn more about creating procs * `PROCTOR_KUBE_JOB_RETRIES` is the number of retries for a kubernetes job (on failure) * `PROCTOR_DEFAULT_NAMESPACE` is the namespace under which jobs will be run in kubernetes cluster. By default, K8s has namespace "default". If you set another value, please create namespace in K8s before deploying `proctord` * `PROCTOR_KUBE_CONTEXT` is used the name of context you want to use when running out of cluster. -* `PROCTOR_KUBE_CLUSTER_HOST_NAME` is address/ip address to api-server of kube cluster. It is used for fetching logs of a pod using https -* `PROCTOR_KUBE_CA_CERT_ENCODED` is the CA cert file encoded in base64. This is used for establishing authority while talking to kubernetes api-server on a public https call -* `PROCTOR_KUBE_BASIC_AUTH_ENCODED` is the base64 encoded authentication of kubernetes. Enocde `username:password` to base64 and set this config. * Before streaming logs of jobs, `PROCTOR_KUBE_POD_LIST_WAIT_TIME` is the time to wait until jobs and pods are in active/successful/failed state * `PROCTOR_POSTGRES_USER`, `PROCTOR_POSTGRES_PASSWORD`, `PROCTOR_POSTGRES_HOST` and `PROCTOR_POSTGRES_PORT` is the username and password to the postgres database you wish to connect to * Set `PROCTOR_POSTGRES_DATABASE` to `proctord_development` for development purpose @@ -153,4 +150,7 @@ You can read [here](./docs/creating_procs.md) to learn more about creating procs * `PROCTOR_SCHEDULED_JOBS_FETCH_INTERVAL_IN_MINS` is the interval at which the scheduler fetches updated jobs from database * `PROCTOR_MAIL_USERNAME`, `PROCTOR_MAIL_PASSWORD`, `PROCTOR_MAIL_SERVER_HOST`, `PROCTOR_MAIL_SERVER_PORT` are the creds required to send notification to users on scheduled jobs execution * `PROCTOR_JOB_POD_ANNOTATIONS` is used to set any kubernetes pod specific annotations. -* `PROCTOR_SENTRY_DSN` is used to set sentry DSN. +* `PROCTOR_AUTH_ENABLED` is used to set whether Authentication is enabled or not. +* `PROCTOR_AUTH_PLUGIN_BINARY` binary location of AUTH Plugin +* `PROCTOR_AUTH_PLUGIN_EXPORTED` variable name exported by the Auth Plugin +* `PROCTOR_REQUIRED_ADMIN_GROUP` list group required by user to access admin features for proctor such as post Metadata and Secrets From d12c05b09dfadc7daa2f6c5341e93b5fe29662aa Mon Sep 17 00:00:00 2001 From: Hadrian Bayanulhaq S Date: Fri, 11 Oct 2019 14:08:55 +0700 Subject: [PATCH 243/268] [hadrian.siregar|bimo.horizon] Inject user email to execution argument in execution post handler --- internal/app/service/execution/handler/http.go | 10 ++++++---- internal/app/service/execution/handler/http_test.go | 10 ++++++++-- internal/pkg/constant/constant.go | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 1d267724..99ec9b31 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -20,6 +20,7 @@ import ( "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" serviceNotification "proctor/internal/app/service/notification/service" + "proctor/internal/pkg/constant" "proctor/internal/pkg/model/execution" "proctor/pkg/notification/event" ) @@ -155,10 +156,10 @@ func (httpHandler *executionHTTPHandler) GetStatus() http.HandlerFunc { response.WriteHeader(http.StatusOK) - responseJson, err := json.Marshal(responseBody) + responseJSON, err := json.Marshal(responseBody) logger.LogErrors(err, "marshal json from: ", responseBody) - _, _ = response.Write(responseJson) + _, _ = response.Write(responseJSON) } } @@ -179,6 +180,7 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { _, _ = response.Write([]byte(status.MalformedRequest)) return } + job.Args[constant.AuthorEmailKey] = userEmail context, executionName, err := httpHandler.service.Execute(job.Name, userEmail, job.Args) @@ -205,10 +207,10 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { response.WriteHeader(http.StatusCreated) - responseJson, err := json.Marshal(responseBody) + responseJSON, err := json.Marshal(responseBody) logger.LogErrors(err, "marshal json from: ", responseBody) - _, _ = response.Write(responseJson) + _, _ = response.Write(responseJSON) return } } diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index 9a7e0c3e..b1b96a7b 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -261,7 +261,10 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHa userEmail := "mrproctor@example.com" job := parameter.Job{ Name: "sample-job-name", - Args: map[string]string{"argOne": "sample-arg"}, + Args: map[string]string{ + "argOne": "sample-arg", + "AUTHOR": userEmail, + }, } context := &model.ExecutionContext{ UserEmail: userEmail, @@ -321,7 +324,10 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestGenericErrorJobExecutionPostHTTP userEmail := "mrproctor@example.com" job := parameter.Job{ Name: "sample-job-name", - Args: map[string]string{"argOne": "sample-arg"}, + Args: map[string]string{ + "argOne": "sample-arg", + "AUTHOR": userEmail, + }, } context := &model.ExecutionContext{ UserEmail: userEmail, diff --git a/internal/pkg/constant/constant.go b/internal/pkg/constant/constant.go index 14476b93..e4a190d3 100644 --- a/internal/pkg/constant/constant.go +++ b/internal/pkg/constant/constant.go @@ -49,3 +49,4 @@ const AccessTokenHeaderKey = "Access-Token" const ClientVersionHeaderKey = "Client-Version" const WorkerEmail = "worker@proctor" +const AuthorEmailKey = "AUTHOR" From 027871314c3806f5198909963588b0406356507f Mon Sep 17 00:00:00 2001 From: Hadrian Bayanulhaq S Date: Fri, 11 Oct 2019 17:02:49 +0700 Subject: [PATCH 244/268] [hadrian.siregar|bimo.horizon] Inject context args to execution args in serice instead doing it in controller --- internal/app/service/execution/handler/http.go | 3 --- .../app/service/execution/model/execution_context.go | 4 +++- internal/app/service/execution/service/execution.go | 10 ++++++++++ .../app/service/execution/service/execution_test.go | 4 ++-- internal/pkg/constant/constant.go | 1 - 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/internal/app/service/execution/handler/http.go b/internal/app/service/execution/handler/http.go index 99ec9b31..e2fe4045 100644 --- a/internal/app/service/execution/handler/http.go +++ b/internal/app/service/execution/handler/http.go @@ -20,7 +20,6 @@ import ( "proctor/internal/app/service/infra/config" "proctor/internal/app/service/infra/logger" serviceNotification "proctor/internal/app/service/notification/service" - "proctor/internal/pkg/constant" "proctor/internal/pkg/model/execution" "proctor/pkg/notification/event" ) @@ -180,8 +179,6 @@ func (httpHandler *executionHTTPHandler) Post() http.HandlerFunc { _, _ = response.Write([]byte(status.MalformedRequest)) return } - job.Args[constant.AuthorEmailKey] = userEmail - context, executionName, err := httpHandler.service.Execute(job.Name, userEmail, job.Args) logger.LogErrors(err, "execute job: ", job) diff --git a/internal/app/service/execution/model/execution_context.go b/internal/app/service/execution/model/execution_context.go index 28d44ccc..43fd0224 100644 --- a/internal/app/service/execution/model/execution_context.go +++ b/internal/app/service/execution/model/execution_context.go @@ -1,10 +1,12 @@ package model import ( + "time" + sqlxTypes "github.com/jmoiron/sqlx/types" + "proctor/internal/app/service/execution/status" dbTypes "proctor/internal/app/service/infra/db/types" - "time" ) type ExecutionContext struct { diff --git a/internal/app/service/execution/service/execution.go b/internal/app/service/execution/service/execution.go index 2bfb4ac6..be8c78b8 100644 --- a/internal/app/service/execution/service/execution.go +++ b/internal/app/service/execution/service/execution.go @@ -104,6 +104,16 @@ func (service *executionService) ExecuteWithCommand(jobName string, userEmail st executionArgs := mergeArgs(args, secret) + contextArgsMap := map[string]string{ + "EXECUTION_ID": fmt.Sprint(context.ExecutionID), + "JOB_NAME": context.JobName, + "EXECUTION_NAME": context.Name, + "USER_EMAIL": context.UserEmail, + "IMAGE_TAG": context.ImageTag, + } + + executionArgs = mergeArgs(executionArgs, contextArgsMap) + context.Status = status.Created executionName, err := service.kubernetesClient.ExecuteJobWithCommand(metadata.ImageName, executionArgs, commands) logger.Info("Executed Job on Kubernetes got ", executionName, " execution jobName and ", err, "errors") diff --git a/internal/app/service/execution/service/execution_test.go b/internal/app/service/execution/service/execution_test.go index cf86b72a..bedc6f28 100644 --- a/internal/app/service/execution/service/execution_test.go +++ b/internal/app/service/execution/service/execution_test.go @@ -105,7 +105,7 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobFailed() { suite.mockMetadataRepository.On("GetByName", jobName).Return(fakeMetadata, nil).Once() suite.mockSecretRepository.On("GetByJobName", jobName).Return(map[string]string{}, nil).Once() suite.mockRepository.On("Insert", mock.Anything).Return(0, nil).Once() - suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, jobArgs, []string{}).Return("", errors.New("Execution Failed")) + suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, mock.Anything, []string{}).Return("", errors.New("Execution Failed")) context, _, err := suite.service.Execute(jobName, userEmail, jobArgs) assert.Error(t, err, "error when executing image") @@ -143,7 +143,7 @@ func (suite *TestExecutionServiceSuite) TestExecuteJobSuccess() { suite.mockRepository.On("GetById", mock.Anything).Return(0, nil).Times(3) executionName := "execution-name" - suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, jobArgs, []string{}).Return(executionName, nil) + suite.mockKubernetesClient.On("ExecuteJobWithCommand", imageName, mock.Anything, []string{}).Return(executionName, nil) suite.mockKubernetesClient.On("WaitForReadyJob", executionName, mock.Anything).Return(nil) podDetail := &v1.Pod{} suite.mockKubernetesClient.On("WaitForReadyPod", executionName, mock.Anything).Return(podDetail, nil) diff --git a/internal/pkg/constant/constant.go b/internal/pkg/constant/constant.go index e4a190d3..14476b93 100644 --- a/internal/pkg/constant/constant.go +++ b/internal/pkg/constant/constant.go @@ -49,4 +49,3 @@ const AccessTokenHeaderKey = "Access-Token" const ClientVersionHeaderKey = "Client-Version" const WorkerEmail = "worker@proctor" -const AuthorEmailKey = "AUTHOR" From e827d703c2d833039f77903b1e81e5b478f785ef Mon Sep 17 00:00:00 2001 From: Hadrian Bayanulhaq S Date: Fri, 11 Oct 2019 17:08:30 +0700 Subject: [PATCH 245/268] remove unnecessary test code --- internal/app/service/execution/handler/http_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/internal/app/service/execution/handler/http_test.go b/internal/app/service/execution/handler/http_test.go index b1b96a7b..c53a4d53 100644 --- a/internal/app/service/execution/handler/http_test.go +++ b/internal/app/service/execution/handler/http_test.go @@ -263,7 +263,6 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestSuccessfulJobExecutionPostHTTPHa Name: "sample-job-name", Args: map[string]string{ "argOne": "sample-arg", - "AUTHOR": userEmail, }, } context := &model.ExecutionContext{ @@ -326,7 +325,6 @@ func (suite *ExecutionHTTPHandlerTestSuite) TestGenericErrorJobExecutionPostHTTP Name: "sample-job-name", Args: map[string]string{ "argOne": "sample-arg", - "AUTHOR": userEmail, }, } context := &model.ExecutionContext{ From 0257fdf1b7580db61c1840eaa73520703d58b55c Mon Sep 17 00:00:00 2001 From: Eko Simanjuntak Date: Wed, 23 Oct 2019 21:00:59 +0700 Subject: [PATCH 246/268] [Eko] Fix typo and hyperlink on plugin.md --- docs/plugin.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plugin.md b/docs/plugin.md index b5f0db63..845e7557 100644 --- a/docs/plugin.md +++ b/docs/plugin.md @@ -36,9 +36,9 @@ In order to use auth plugin you should compile the plugin and fill these environ #### Gate Auth Plugin -In order to use gate auth plugin, you need a running (Gate)[https://github.com/gate-sso/gate] server. +In order to use gate auth plugin, you need a running [Gate](https://github.com/gate-sso/gate) server. Authenticated user mean an user that registered to Gate server. -Autorized user mean an user need to be member of at least one group from groups list specified on authorized_groups metadata for procs. +Authorized user mean an user need to be member of at least one group from groups list specified on authorized_groups metadata for procs. Compile gate auth plugin by running `make plugin.auth` and fill `PROCTOR_AUTH_PLUGIN_BINARY` with generated `auth.so` in `./_output/bin/plugin/auth.so`. @@ -47,6 +47,6 @@ Compile gate auth plugin by running `make plugin.auth` and fill `PROCTOR_AUTH_PL #### Slack Notification Plugin Proctor will send notification to slack when some event happen, see below for a list of events and it's content. -Create a [slack app(https://api.slack.com/incoming-webhooks) then fill `SLACK_PLUGIN_URL` environment variable with incoming webhook url, it should look like `https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX`. +Create a [slack app](https://api.slack.com/incoming-webhooks) then fill `SLACK_PLUGIN_URL` environment variable with incoming webhook url, it should look like `https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX`. Compile slack notification plugin by running `make plugin.slack` and fill `PROCTOR_NOTIFICATION_PLUGIN_BINARY` with generated `slack.so` in `./_output/bin/plugin/slack.so`. From 8ee8810f3568d8878b75bcda21e039ebc60d601d Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 24 Oct 2019 16:30:27 +0700 Subject: [PATCH 247/268] [Dembo|Hadrian.Siregar] Change order of message content for slack --- .../slack/message/execution.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugins/slack-notification-plugin/slack/message/execution.go b/plugins/slack-notification-plugin/slack/message/execution.go index ef4fbd8a..cd8f43fb 100644 --- a/plugins/slack-notification-plugin/slack/message/execution.go +++ b/plugins/slack-notification-plugin/slack/message/execution.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "proctor/pkg/notification/event" + "sort" ) type executionMessage struct { @@ -26,7 +27,14 @@ func (messageObject *executionMessage) JSON() (string, error) { Text: userEmail + " execute job with details:", } section.Fields = []textComponent{} - for key, value := range evt.Content() { + contents := evt.Content() + var keys []string + for key := range contents { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + value := contents[key] keyComponent := textComponent{} keyComponent.Type = "mrkdwn" keyComponent.Text = "*" + key + "*" From 69d8eacab3a7e2b1c1acceb600ed7a1138f2c11b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Thu, 24 Oct 2019 16:49:14 +0700 Subject: [PATCH 248/268] [Dembo|Hadrian.Siregar] Change route order to monitor correct route in new relic --- internal/app/service/server/router.go | 34 +++++++++++++-------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/internal/app/service/server/router.go b/internal/app/service/server/router.go index bebf67ea..58753d7b 100644 --- a/internal/app/service/server/router.go +++ b/internal/app/service/server/router.go @@ -71,24 +71,6 @@ func NewRouter() (*mux.Router, error) { authorizationMiddleware := securityMiddleware.NewAuthorizationMiddleware(_securityService, metadataStore) adminAuthorizationMiddleware := securityMiddleware.NewAdminAuthorizationMiddleware(_securityService) - pingRoute := router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { - _, _ = fmt.Fprintf(w, "pong") - }) - - docsRoute := router.HandleFunc("/docs", docs.APIDocHandler) - docsSubRoute := router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.Config().DocsPath)))) - swaggerRoute := router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, path.Join(config.Config().DocsPath, "swagger.yml")) - }) - - metricsRoute := router.Handle("/metrics", promhttp.Handler()) - - authenticationMiddleware.Exclude(pingRoute, docsRoute, docsSubRoute, swaggerRoute, metricsRoute) - - router = middleware.InstrumentNewRelic(router) - router.Use(middleware.ValidateClientVersion) - router.Use(authenticationMiddleware.MiddlewareFunc) - authorizationMiddleware.Secure(router, "/execution", executionHandler.Post()).Methods("POST") router.HandleFunc("/execution/{contextId}/status", executionHandler.GetStatus()).Methods("GET") router.HandleFunc("/execution/logs", executionHandler.GetLogs()).Methods("GET") @@ -103,5 +85,21 @@ func NewRouter() (*mux.Router, error) { router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Get()).Methods("GET") router.HandleFunc("/schedule/{scheduleID}", scheduleHandler.Delete()).Methods("DELETE") + router = middleware.InstrumentNewRelic(router) + router.Use(middleware.ValidateClientVersion) + + pingRoute := router.HandleFunc("/ping", func(w http.ResponseWriter, req *http.Request) { + _, _ = fmt.Fprintf(w, "pong") + }) + docsRoute := router.HandleFunc("/docs", docs.APIDocHandler) + docsSubRoute := router.PathPrefix("/docs/").Handler(http.StripPrefix("/docs/", http.FileServer(http.Dir(config.Config().DocsPath)))) + swaggerRoute := router.HandleFunc("/swagger.yml", func(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, path.Join(config.Config().DocsPath, "swagger.yml")) + }) + metricsRoute := router.Handle("/metrics", promhttp.Handler()) + + authenticationMiddleware.Exclude(pingRoute, docsRoute, docsSubRoute, swaggerRoute, metricsRoute) + router.Use(authenticationMiddleware.MiddlewareFunc) + return router, nil } From 901c953a50c3c84a232d7a0603c09bd6f9345ecb Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 16 Dec 2019 11:16:25 +0700 Subject: [PATCH 249/268] [Jasoet] fix(version): Fix client version checking --- internal/app/cli/command/version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index f47f6704..b9a291d9 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.0" +const ClientVersion = "v2.0.1" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ @@ -18,7 +18,7 @@ func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Comm Long: `Example: proctor version`, Run: func(cmd *cobra.Command, args []string) { printer.Println(fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset) - release, e := fetcher.LatestRelease("gojektech", "proctor") + release, e := fetcher.LatestRelease("gopaytech", "proctor") if e == nil && release != ClientVersion { printer.Println(fmt.Sprintf("Your version of Proctor client is out of date! The latest version is %s You can update by either running brew upgrade proctor or downloading a release for your OS here: https://proctor/releases", release), color.Reset) } From 757c8d8f09cd734895af3d7f430d61965a572141 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 16 Dec 2019 11:32:06 +0700 Subject: [PATCH 250/268] [Jasoet] fix(version): Client Version to v2.0.2 --- internal/app/cli/command/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index b9a291d9..0621f58f 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.1" +const ClientVersion = "v2.0.2" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ From 4ad8418cf5c7c19895a000eeccc692a263f32d9e Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Mon, 16 Dec 2019 11:34:23 +0700 Subject: [PATCH 251/268] [Jasoet] fix(version): Client Version to v2.0.3 --- internal/app/cli/command/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index 0621f58f..0ef68426 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.2" +const ClientVersion = "v2.0.3" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ From 30c7e6f97f260e2bcb35bc48da34d0507efb4c24 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Tue, 17 Dec 2019 17:24:27 +0700 Subject: [PATCH 252/268] [Jasoet] script(goreleaser): setup binary target --- .goreleaser.yml | 7 +++++++ .travis.yml | 42 ------------------------------------------ 2 files changed, 7 insertions(+), 42 deletions(-) delete mode 100644 .travis.yml diff --git a/.goreleaser.yml b/.goreleaser.yml index 64a57e9f..18c385a2 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -5,12 +5,19 @@ builds: id: "proctor-server" main: ./cmd/server/main.go binary: proctor-server + goos: + - darwin + - linux env: - CGO_ENABLED=0 - id: "proctor-cli" main: ./cmd/cli/main.go binary: proctor2 + goos: + - darwin + - linux + - windows env: - CGO_ENABLED=0 archive: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d1e3f4ba..00000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -dist: trusty -sudo: required -language: go -go: -- "1.12" - -services: - - redis-server - - postgresql - - docker - -env: - global: - - DOCKER_USERNAME=proctorbuilder - - secure: "ZCOM7TMiOsBO55V0VfAI1WQKODLZ4SUq50ngjZBJ2RfrOhHZ9zAqJgjw32bJtzUxuFX7Ez/44UEwd/Z28zbQjnzvIm8jwAwM+zxGH8zPlT9K9nuEN5edCO8oCYb5IdN4uPQkDw2w9FSNkRf8WfgS5Ru53SGiH+SVf2x/DT767HyP8EKhHWXJkKloxvR8wx7dSTr0vyQiNrslhh3EAVGQciZXLh2c6/pkZr0z/I5TpQnvWbvU0w9n9CqhhOaDiYI5zgKrpD/BdnYBmZtxfWO/RDyivacLDgozGcrV3iTJ6FkDbMgmhzrF0sAQL9LSrDOzUq6Yfe476c5BRHPgvh/gBHrpY6y7jmFGQ4vxER+vI9/rVILRLQfUSAYJBwuTAu7Wf/lZulw0GoQfIdaKa8RBCItaEN47HAPglxnPJBcl1ClYLBNc5XaHARdpKRurRz64AtBjCbTmoeUrdw9qktnKHSzxIPGQT1nKGvoizjpLUOURSwCp1T0GXxgcNX/65S/MruSFk5wf6jRz5x7ErVlv8MJLXWORV3EUR8sieXUTL5K0d2jq4NaGYTkbg9v4DwcgxMEm579gn+9MHPWGrrbC/k5xM03HjGYX44MnZb7jLuC0TRaYgnh4V8pe+ffxRY/+FLqDAnGtT7CxJsEONFy9jZrG9wVm2e0wVUxU9KxPvWE=" - -before_script: -- sudo service redis-server start - -stages: - - test - - name: docker publish - if: (branch = master OR tag IS present) and type != pull_request - -jobs: - include: - - stage: test - script: - - make db.setup test - - stage: docker publish - install: skip - script: - - echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin - - docker build -t proctorgojek/proctor-service:latest . - - if [ -n "$TRAVIS_TAG" ]; then - docker tag proctorgojek/proctor-service:latest proctorgojek/proctor-service:$TRAVIS_TAG; - fi - - docker images - - docker push proctorgojek/proctor-service - -#after_success: -# - scripts/release.sh From 1ab8ff9305a4d7508eebc8f9baa0daec500f0c95 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 6 Jan 2020 13:32:30 +0700 Subject: [PATCH 253/268] [william.dembo] test(cli-version): update broken test for version cli --- internal/app/cli/command/version/version_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 6e535f3f..0ce84cd2 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,10 +23,10 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.0" + version := "v2.0.3" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() - githubClient.On("LatestRelease", "gojektech", "proctor").Return(version, nil) + githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) versionCmd.Run(&cobra.Command{}, []string{}) @@ -43,7 +43,7 @@ func TestOldVersionCmd(t *testing.T) { mockPrinter.On("Println", fmt.Sprintf("Your version of Proctor client is out of date!"+ " The latest version is %s You can update by either running brew upgrade proctor or downloading a release for your OS here:"+ " https://proctor/releases", version), color.Reset).Once() - githubClient.On("LatestRelease", "gojektech", "proctor").Return(version, nil) + githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) versionCmd.Run(&cobra.Command{}, []string{}) From a2e8cf977f588f83b1c54db931c2a003412d934f Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 6 Jan 2020 13:35:11 +0700 Subject: [PATCH 254/268] [william.dembo] fix(schedule-worker): fix wrong config for mailer --- internal/app/service/schedule/worker/worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/schedule/worker/worker.go b/internal/app/service/schedule/worker/worker.go index 4e39aa63..1703a284 100644 --- a/internal/app/service/schedule/worker/worker.go +++ b/internal/app/service/schedule/worker/worker.go @@ -140,7 +140,7 @@ func Start() error { return err } kubeClient := kubernetes.NewKubernetesClient(httpClient) - mailer := mail.New(config.Config().MailServerHost, config.Config().AuthPluginExported) + mailer := mail.New(config.Config().MailServerHost, config.Config().MailServerPort) executionSvc := executionService.NewExecutionService(kubeClient, executionContextStore, metadataStore, secretStore) worker := NewWorker(executionSvc, executionContextStore, scheduleStore, scheduleContextStore, mailer) ticker := time.NewTicker(time.Duration(config.Config().ScheduledJobsFetchIntervalInMins) * time.Minute) From a096965e71ae0e8726f7e22f571412a238f7da7b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 6 Jan 2020 13:37:50 +0700 Subject: [PATCH 255/268] [william.dembo] chore(travis): add travis-ci to run test on pipeline --- .travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..1c3c9609 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +dist: trusty +sudo: required +language: go +go: + - "1.12" + +services: + - redis-server + - postgresql + +before_script: + - sudo service redis-server start + +stages: + - test + +jobs: + include: + - stage: test + script: + - make db.setup test From a3d1c74b32ff942a3f9865db2d7af1aa1e981a2b Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 6 Jan 2020 15:55:41 +0700 Subject: [PATCH 256/268] Bump version 2.0.4 --- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index 0ef68426..8e0e502a 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.3" +const ClientVersion = "v2.0.4" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 0ce84cd2..7f587b63 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.3" + version := "v2.0.4" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) From c06e2bd8ab7a061b5f9bd351e8cf268b93e0a0d4 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 13 Jan 2020 17:30:17 +0700 Subject: [PATCH 257/268] [william.dembo] feat(config): add config for redis password --- internal/app/service/infra/config/config.go | 2 ++ internal/app/service/infra/config/config_test.go | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/internal/app/service/infra/config/config.go b/internal/app/service/infra/config/config.go index 1e7aeed0..3bf033df 100644 --- a/internal/app/service/infra/config/config.go +++ b/internal/app/service/infra/config/config.go @@ -63,6 +63,7 @@ type ProctorConfig struct { AppPort string DefaultNamespace string RedisAddress string + RedisPassword string LogsStreamReadBufferSize int RedisMaxActiveConnections int LogsStreamWriteBufferSize int @@ -120,6 +121,7 @@ func load() ProctorConfig { AppPort: GetStringDefault(fang, "app.port", "5001"), DefaultNamespace: fang.GetString("default.namespace"), RedisAddress: fang.GetString("redis.address"), + RedisPassword: fang.GetString("redis.password"), RedisMaxActiveConnections: fang.GetInt("redis.max.active.connections"), LogsStreamReadBufferSize: fang.GetInt("logs.stream.read.buffer.size"), LogsStreamWriteBufferSize: fang.GetInt("logs.stream.write.buffer.size"), diff --git a/internal/app/service/infra/config/config_test.go b/internal/app/service/infra/config/config_test.go index 702620bf..2aa62c47 100644 --- a/internal/app/service/infra/config/config_test.go +++ b/internal/app/service/infra/config/config_test.go @@ -49,6 +49,14 @@ func TestRedisAddress(t *testing.T) { assert.Equal(t, value, load().RedisAddress) } +func TestRedisPassword(t *testing.T) { + fake.Seed(0) + value := fake.FirstName() + _ = os.Setenv("PROCTOR_REDIS_PASSWORD", value) + + assert.Equal(t, value, load().RedisPassword) +} + func TestRedisMaxActiveConnections(t *testing.T) { fake.Seed(0) number := fake.Number(10, 90) From cdfa4cf24c55243e2c61b77952cc3569c9e8edf7 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 13 Jan 2020 17:45:25 +0700 Subject: [PATCH 258/268] [william.demo] feat(redis): able to use password for redis connection --- .env.test | 1 + README.md | 1 + internal/app/service/infra/db/redis/client.go | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.env.test b/.env.test index c5928469..b7bb21ce 100644 --- a/.env.test +++ b/.env.test @@ -4,6 +4,7 @@ export PROCTOR_LOG_LEVEL=debug export PROCTOR_APP_PORT=5000 export PROCTOR_DEFAULT_NAMESPACE=default export PROCTOR_REDIS_ADDRESS=localhost:6379 +export PROCTOR_REDIS_PASSWORD=foobared export PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS=10 export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 export PROCTOR_KUBE_JOB_RETRIES=0 diff --git a/README.md b/README.md index 1163cb48..7bbcac77 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ You can read [here](./docs/creating_procs.md) to learn more about creating procs * `PROCTOR_APP_PORT` is port on which service will run * `PROCTOR_LOG_LEVEL` defines log levels of service. Available options are: `debug`,`info`,`warn`,`error`,`fatal`,`panic` * `PROCTOR_REDIS_ADDRESS` is hostname and port of redis store for jobs configuration and metadata +* `PROCTOR_REDIS_PASSWORD` is password to access redis store for jobs configuration and metadata * `PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS` defines maximum active connections to redis. Maximum idle connections is half of this config * `PROCTOR_LOGS_STREAM_READ_BUFFER_SIZE` and `PROCTOR_LOGS_STREAM_WRITE_BUFFER_SIZE` is the buffer size for websocket connection while streaming logs * `PROCTOR_KUBE_CONFIG` needs to be set only if service is running outside a kubernetes cluster diff --git a/internal/app/service/infra/db/redis/client.go b/internal/app/service/infra/db/redis/client.go index 147fd9bc..53be04e1 100644 --- a/internal/app/service/infra/db/redis/client.go +++ b/internal/app/service/infra/db/redis/client.go @@ -27,11 +27,13 @@ func NewClient() Client { } func newPool() (*redis.Pool, error) { + dialPassword := redis.DialPassword(config.Config().RedisPassword) + dialAddress := config.Config().RedisAddress pool := &redis.Pool{ MaxIdle: config.Config().RedisMaxActiveConnections / 2, MaxActive: config.Config().RedisMaxActiveConnections, IdleTimeout: 5 * time.Second, - Dial: func() (redis.Conn, error) { return redis.Dial("tcp", config.Config().RedisAddress) }, + Dial: func() (redis.Conn, error) { return redis.Dial("tcp", dialAddress, dialPassword) }, TestOnBorrow: func(c redis.Conn, t time.Time) error { if time.Since(t) < time.Minute { return nil From f65e2c47e20b990f2c76dd2d8e18b74fa40a02fe Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Mon, 13 Jan 2020 18:01:29 +0700 Subject: [PATCH 259/268] [william.dembo] chore(env): set default redis password to empty --- .env.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.test b/.env.test index b7bb21ce..fcf3d8ff 100644 --- a/.env.test +++ b/.env.test @@ -4,7 +4,7 @@ export PROCTOR_LOG_LEVEL=debug export PROCTOR_APP_PORT=5000 export PROCTOR_DEFAULT_NAMESPACE=default export PROCTOR_REDIS_ADDRESS=localhost:6379 -export PROCTOR_REDIS_PASSWORD=foobared +export PROCTOR_REDIS_PASSWORD= export PROCTOR_REDIS_MAX_ACTIVE_CONNECTIONS=10 export PROCTOR_KUBE_JOB_ACTIVE_DEADLINE_SECONDS=60 export PROCTOR_KUBE_JOB_RETRIES=0 From 2bd250605169f3261bd4058c73b798d5261384cc Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Tue, 14 Jan 2020 14:32:18 +0700 Subject: [PATCH 260/268] Bump version 2.0.5 --- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index 8e0e502a..d998a464 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.4" +const ClientVersion = "v2.0.5" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 7f587b63..3943045c 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.4" + version := "v2.0.5" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) From 77117f6d67da93a09cb9840bd7cfede3a8898333 Mon Sep 17 00:00:00 2001 From: Deny Prasetyo Date: Wed, 15 Jan 2020 12:30:22 +0700 Subject: [PATCH 261/268] [Jasoet] feat(metadata): Show jobs list now depends on user's authentication group --- internal/app/service/metadata/handler/http.go | 16 +++- .../app/service/metadata/handler/http_test.go | 29 ++++++- .../service/metadata/repository/metadata.go | 50 ++++++++++++ .../metadata/repository/metadata_mock.go | 5 ++ .../metadata/repository/metadata_test.go | 78 +++++++++++++++++++ 5 files changed, 174 insertions(+), 4 deletions(-) diff --git a/internal/app/service/metadata/handler/http.go b/internal/app/service/metadata/handler/http.go index 0d129dc9..1873c67c 100644 --- a/internal/app/service/metadata/handler/http.go +++ b/internal/app/service/metadata/handler/http.go @@ -5,8 +5,10 @@ import ( "net/http" "proctor/internal/app/service/infra/logger" "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/security/middleware" "proctor/internal/pkg/constant" modelMetadata "proctor/internal/pkg/model/metadata" + "proctor/pkg/auth" ) type metadataHTTPHandler struct { @@ -55,7 +57,19 @@ func (handler *metadataHTTPHandler) Post() http.HandlerFunc { func (handler *metadataHTTPHandler) GetAll() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { - metadataSlice, err := handler.repository.GetAll() + userDetailContext := req.Context().Value(middleware.ContextUserDetailKey) + if userDetailContext == nil { + w.WriteHeader(http.StatusUnauthorized) + return + } + + userDetail, ok := userDetailContext.(*auth.UserDetail) + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } + + metadataSlice, err := handler.repository.GetAllByGroups(userDetail.Groups) if err != nil { logger.Error("Error fetching metadata", err.Error()) diff --git a/internal/app/service/metadata/handler/http_test.go b/internal/app/service/metadata/handler/http_test.go index 2987f86c..c54e848b 100644 --- a/internal/app/service/metadata/handler/http_test.go +++ b/internal/app/service/metadata/handler/http_test.go @@ -2,12 +2,15 @@ package handler import ( "bytes" + "context" "encoding/json" "errors" "fmt" "net/http" "net/http/httptest" metadataRepository "proctor/internal/app/service/metadata/repository" + "proctor/internal/app/service/security/middleware" + "proctor/pkg/auth" "testing" "proctor/internal/pkg/model/metadata/env" @@ -121,10 +124,20 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { t := s.T() req := httptest.NewRequest("GET", "/metadata", bytes.NewReader([]byte{})) + groups := []string{"admin", "migratior"} + userDetail := &auth.UserDetail{ + Name: "jasoet", + Email: "jasoet@ambyar.com", + Active: true, + Groups: groups, + } + + ctx := context.WithValue(req.Context(), middleware.ContextUserDetailKey, userDetail) + req = req.WithContext(ctx) responseRecorder := httptest.NewRecorder() - jobsMetadata := []modelMetadata.Metadata{} - s.mockRepository.On("GetAll").Return(jobsMetadata, nil).Once() + var jobsMetadata []modelMetadata.Metadata + s.mockRepository.On("GetAllByGroups", groups).Return(jobsMetadata, nil).Once() s.metadataHTTPHandler.GetAll()(responseRecorder, req) @@ -141,10 +154,20 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { t := s.T() req := httptest.NewRequest("GET", "/metadata", bytes.NewReader([]byte{})) + groups := []string{"admin", "migratior"} + userDetail := &auth.UserDetail{ + Name: "jasoet", + Email: "jasoet@ambyar.com", + Active: true, + Groups: groups, + } + + ctx := context.WithValue(req.Context(), middleware.ContextUserDetailKey, userDetail) + req = req.WithContext(ctx) responseRecorder := httptest.NewRecorder() jobsMetadata := []modelMetadata.Metadata{} - s.mockRepository.On("GetAll").Return(jobsMetadata, errors.New("error")).Once() + s.mockRepository.On("GetAllByGroups", groups).Return(jobsMetadata, errors.New("error")).Once() s.metadataHTTPHandler.GetAll()(responseRecorder, req) diff --git a/internal/app/service/metadata/repository/metadata.go b/internal/app/service/metadata/repository/metadata.go index d2e57abe..2cca83ee 100644 --- a/internal/app/service/metadata/repository/metadata.go +++ b/internal/app/service/metadata/repository/metadata.go @@ -11,6 +11,7 @@ const KeySuffix = "-metadata" type MetadataRepository interface { Save(metadata metadata.Metadata) error GetAll() ([]metadata.Metadata, error) + GetAllByGroups(groups []string) ([]metadata.Metadata, error) GetByName(name string) (*metadata.Metadata, error) } @@ -68,6 +69,55 @@ func (repository *metadataRepository) GetAll() ([]metadata.Metadata, error) { return metadataSlice, nil } +func (repository *metadataRepository) GetAllByGroups(groups []string) ([]metadata.Metadata, error) { + searchKey := "*" + KeySuffix + + keys, err := repository.redisClient.KEYS(searchKey) + if err != nil { + return nil, err + } + + availableKeys := make([]interface{}, len(keys)) + for i := range keys { + availableKeys[i] = keys[i] + } + + values, err := repository.redisClient.MGET(availableKeys...) + if err != nil { + return nil, err + } + + metadataSlice := make([]metadata.Metadata, len(values)) + for i := range values { + err = json.Unmarshal(values[i], &metadataSlice[i]) + if err != nil { + return nil, err + } + } + + var filteredMetadata []metadata.Metadata + for _, meta := range metadataSlice { + if len(meta.AuthorizedGroups) == 0 { + filteredMetadata = append(filteredMetadata, meta) + } else if duplicateItemExists(meta.AuthorizedGroups, groups) { + filteredMetadata = append(filteredMetadata, meta) + } + } + + return filteredMetadata, nil +} + +func duplicateItemExists(first []string, second []string) bool { + for _, firstString := range first { + for _, secondString := range second { + if firstString == secondString { + return true + } + } + } + return false +} + func (repository *metadataRepository) GetByName(name string) (*metadata.Metadata, error) { binaryMetadata, err := repository.redisClient.GET(applySuffix(name)) if err != nil { diff --git a/internal/app/service/metadata/repository/metadata_mock.go b/internal/app/service/metadata/repository/metadata_mock.go index 459d3965..65252887 100644 --- a/internal/app/service/metadata/repository/metadata_mock.go +++ b/internal/app/service/metadata/repository/metadata_mock.go @@ -19,6 +19,11 @@ func (m *MockMetadataRepository) GetAll() ([]modelMetadata.Metadata, error) { return args.Get(0).([]modelMetadata.Metadata), args.Error(1) } +func (m *MockMetadataRepository) GetAllByGroups(group []string) ([]modelMetadata.Metadata, error) { + args := m.Called(group) + return args.Get(0).([]modelMetadata.Metadata), args.Error(1) +} + func (m *MockMetadataRepository) GetByName(name string) (*modelMetadata.Metadata, error) { args := m.Called(name) return args.Get(0).(*modelMetadata.Metadata), args.Error(1) diff --git a/internal/app/service/metadata/repository/metadata_test.go b/internal/app/service/metadata/repository/metadata_test.go index 08e288b1..bc0fdab9 100644 --- a/internal/app/service/metadata/repository/metadata_test.go +++ b/internal/app/service/metadata/repository/metadata_test.go @@ -113,6 +113,84 @@ func (s *MetadataRepositoryTestSuite) TestGetAllFailure() { s.mockRedisClient.AssertExpectations(t) } +func (s *MetadataRepositoryTestSuite) TestGetAllByGroups() { + t := s.T() + + metadata1 := modelMetadata.Metadata{ + Name: "job1", + ImageName: "job1-image-name", + Description: "desc1", + Author: "Test User Date: Wed, 15 Jan 2020 12:53:42 +0700 Subject: [PATCH 262/268] Bump version 2.0.6 --- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index d998a464..bbfae7e4 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.5" +const ClientVersion = "v2.0.6" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 3943045c..1bb864d2 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.5" + version := "v2.0.6" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) From 6e38b4065c9890924c5681aff80158f0f35ba69c Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 15 Jan 2020 13:46:05 +0700 Subject: [PATCH 263/268] [william.dembo] feat(authentication): Add context to check authentication status --- internal/app/service/security/middleware/authentication.go | 7 ++++--- .../app/service/security/middleware/authentication_test.go | 7 ++++++- internal/app/service/security/middleware/middleware.go | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/app/service/security/middleware/authentication.go b/internal/app/service/security/middleware/authentication.go index 20670a03..af164082 100644 --- a/internal/app/service/security/middleware/authentication.go +++ b/internal/app/service/security/middleware/authentication.go @@ -20,12 +20,13 @@ type authenticationMiddleware struct { func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := context.WithValue(r.Context(), ContextAuthEnabled, middleware.enabled) if !middleware.enabled { - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) return } if middleware.isRequestExcluded(r) { - next.ServeHTTP(w, r) + next.ServeHTTP(w, r.WithContext(ctx)) return } token := r.Header.Get(constant.AccessTokenHeaderKey) @@ -40,7 +41,7 @@ func (middleware *authenticationMiddleware) MiddlewareFunc(next http.Handler) ht w.WriteHeader(http.StatusUnauthorized) return } - ctx := context.WithValue(r.Context(), ContextUserDetailKey, userDetail) + ctx = context.WithValue(ctx, ContextUserDetailKey, userDetail) next.ServeHTTP(w, r.WithContext(ctx)) }) } diff --git a/internal/app/service/security/middleware/authentication_test.go b/internal/app/service/security/middleware/authentication_test.go index 2bfe019a..5706bbcf 100644 --- a/internal/app/service/security/middleware/authentication_test.go +++ b/internal/app/service/security/middleware/authentication_test.go @@ -26,6 +26,7 @@ func (context *testContext) setUp(t *testing.T) { context.authMiddleware.service = context.securityService context.authMiddleware.enabled = true fn := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, context.authMiddleware.enabled, r.Context().Value("AUTH_ENABLED")) } context.testHandler = fn } @@ -59,6 +60,7 @@ func TestAuthenticationMiddleware_MiddlewareFuncSuccess(t *testing.T) { authMiddleware := ctx.instance().authMiddleware fn := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, ctx.authMiddleware.enabled, r.Context().Value("AUTH_ENABLED")) assert.Equal(t, userDetail, r.Context().Value("USER_DETAIL")) } testHandler := http.HandlerFunc(fn) @@ -154,7 +156,10 @@ func TestAuthenticationMiddleware_MiddlewareFuncDisabled(t *testing.T) { authMiddleware := ctx.instance().authMiddleware authMiddleware.enabled = false - testHandler := ctx.instance().testHandler + fn := func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, authMiddleware.enabled, r.Context().Value("AUTH_ENABLED")) + } + testHandler := http.HandlerFunc(fn) ts := httptest.NewServer(authMiddleware.MiddlewareFunc(testHandler)) defer ts.Close() diff --git a/internal/app/service/security/middleware/middleware.go b/internal/app/service/security/middleware/middleware.go index 7df2d6b1..fd9c12a2 100644 --- a/internal/app/service/security/middleware/middleware.go +++ b/internal/app/service/security/middleware/middleware.go @@ -6,6 +6,7 @@ import ( ) const ContextUserDetailKey string = "USER_DETAIL" +const ContextAuthEnabled string = "AUTH_ENABLED" type Middleware interface { MiddlewareFunc(http.Handler) http.Handler From ae0f7980d9bf64cb62c4a730d9279cd42aff1c45 Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 15 Jan 2020 14:01:37 +0700 Subject: [PATCH 264/268] [william.dembo] feat(metadata): handle case when authentication is disabled --- internal/app/service/metadata/handler/http.go | 28 +++++++++++-------- .../app/service/metadata/handler/http_test.go | 24 ++++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/internal/app/service/metadata/handler/http.go b/internal/app/service/metadata/handler/http.go index 1873c67c..3069842a 100644 --- a/internal/app/service/metadata/handler/http.go +++ b/internal/app/service/metadata/handler/http.go @@ -56,20 +56,26 @@ func (handler *metadataHTTPHandler) Post() http.HandlerFunc { func (handler *metadataHTTPHandler) GetAll() http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { + var metadataSlice []modelMetadata.Metadata + var err error + authEnabled, ok := req.Context().Value(middleware.ContextAuthEnabled).(bool) + if ok && authEnabled { + userDetailContext := req.Context().Value(middleware.ContextUserDetailKey) + if userDetailContext == nil { + w.WriteHeader(http.StatusUnauthorized) + return + } - userDetailContext := req.Context().Value(middleware.ContextUserDetailKey) - if userDetailContext == nil { - w.WriteHeader(http.StatusUnauthorized) - return - } + userDetail, ok := userDetailContext.(*auth.UserDetail) + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } - userDetail, ok := userDetailContext.(*auth.UserDetail) - if !ok { - w.WriteHeader(http.StatusUnauthorized) - return + metadataSlice, err = handler.repository.GetAllByGroups(userDetail.Groups) + } else { + metadataSlice, err = handler.repository.GetAll() } - - metadataSlice, err := handler.repository.GetAllByGroups(userDetail.Groups) if err != nil { logger.Error("Error fetching metadata", err.Error()) diff --git a/internal/app/service/metadata/handler/http_test.go b/internal/app/service/metadata/handler/http_test.go index c54e848b..5ecc8056 100644 --- a/internal/app/service/metadata/handler/http_test.go +++ b/internal/app/service/metadata/handler/http_test.go @@ -133,6 +133,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { } ctx := context.WithValue(req.Context(), middleware.ContextUserDetailKey, userDetail) + ctx = context.WithValue(ctx, middleware.ContextAuthEnabled, true) req = req.WithContext(ctx) responseRecorder := httptest.NewRecorder() @@ -150,6 +151,28 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplay() { assert.Equal(t, expectedJobDetails, responseRecorder.Body.Bytes()) } +func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayWithoutAuth() { + t := s.T() + + req := httptest.NewRequest("GET", "/metadata", bytes.NewReader([]byte{})) + ctx := context.WithValue(req.Context(), middleware.ContextAuthEnabled, false) + req = req.WithContext(ctx) + responseRecorder := httptest.NewRecorder() + + var jobsMetadata []modelMetadata.Metadata + s.mockRepository.On("GetAll").Return(jobsMetadata, nil).Once() + + s.metadataHTTPHandler.GetAll()(responseRecorder, req) + + s.mockRepository.AssertExpectations(t) + + assert.Equal(t, http.StatusOK, responseRecorder.Code) + + expectedJobDetails, err := json.Marshal(jobsMetadata) + assert.NoError(t, err) + assert.Equal(t, expectedJobDetails, responseRecorder.Body.Bytes()) +} + func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { t := s.T() @@ -163,6 +186,7 @@ func (s *MetadataHandlerTestSuite) TestHandleBulkDisplayStoreFailure() { } ctx := context.WithValue(req.Context(), middleware.ContextUserDetailKey, userDetail) + ctx = context.WithValue(ctx, middleware.ContextAuthEnabled, true) req = req.WithContext(ctx) responseRecorder := httptest.NewRecorder() From dc49f52b8ccccbe2961aa633f9b3d0701e4c683e Mon Sep 17 00:00:00 2001 From: William Albertus Dembo Date: Wed, 15 Jan 2020 14:06:25 +0700 Subject: [PATCH 265/268] Bump version 2.0.7 --- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index bbfae7e4..ced5310d 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.6" +const ClientVersion = "v2.0.7" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 1bb864d2..37995bfb 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.6" + version := "v2.0.7" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) From 8393bef3ce225155ada9f81b4808ddeb4d56242b Mon Sep 17 00:00:00 2001 From: "raymond.ralibi" Date: Fri, 29 May 2020 15:10:27 +0700 Subject: [PATCH 266/268] [Ralibi] fix(schedule-handler) Add optional second before cron validation/parsing --- internal/app/service/schedule/handler/http.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/service/schedule/handler/http.go b/internal/app/service/schedule/handler/http.go index 686a4810..36a13fc3 100644 --- a/internal/app/service/schedule/handler/http.go +++ b/internal/app/service/schedule/handler/http.go @@ -58,6 +58,7 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { return } + schedule.Cron = fmt.Sprintf("0 %s", schedule.Cron) _, err = cron.Parse(schedule.Cron) if err != nil { logger.LogErrors(err, fmt.Sprintf("Cron format is invalid: %s ", schedule.Tags), schedule.JobName, schedule.Cron) @@ -103,7 +104,6 @@ func (httpHandler *scheduleHTTPHandler) Post() http.HandlerFunc { return } - schedule.Cron = fmt.Sprintf("0 %s", schedule.Cron) schedule.ID, err = httpHandler.repository.Insert(schedule) if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint") { From d4fa37e9b63877bf0d8b24bdce825ba449dab926 Mon Sep 17 00:00:00 2001 From: "raymond.ralibi" Date: Fri, 29 May 2020 15:24:10 +0700 Subject: [PATCH 267/268] Bump version 2.0.8 --- internal/app/cli/command/version/version.go | 2 +- internal/app/cli/command/version/version_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/app/cli/command/version/version.go b/internal/app/cli/command/version/version.go index ced5310d..cf4f5163 100644 --- a/internal/app/cli/command/version/version.go +++ b/internal/app/cli/command/version/version.go @@ -9,7 +9,7 @@ import ( "github.com/spf13/cobra" ) -const ClientVersion = "v2.0.7" +const ClientVersion = "v2.0.8" func NewCmd(printer io.Printer, fetcher github.LatestReleaseFetcher) *cobra.Command { return &cobra.Command{ diff --git a/internal/app/cli/command/version/version_test.go b/internal/app/cli/command/version/version_test.go index 37995bfb..997fabb7 100644 --- a/internal/app/cli/command/version/version_test.go +++ b/internal/app/cli/command/version/version_test.go @@ -23,7 +23,7 @@ func TestLatestVersionCmd(t *testing.T) { mockPrinter := &io.MockPrinter{} githubClient := &github.MockClient{} versionCmd := NewCmd(mockPrinter, githubClient) - version := "v2.0.7" + version := "v2.0.8" mockPrinter.On("Println", fmt.Sprintf("Proctor: A Developer Friendly Automation Orchestrator %s", ClientVersion), color.Reset).Once() githubClient.On("LatestRelease", "gopaytech", "proctor").Return(version, nil) From 0293cb8d8bfbf48610c54f0e0a18b8d99ccc67f0 Mon Sep 17 00:00:00 2001 From: "raymond.ralibi" Date: Tue, 2 Jun 2020 11:52:50 +0700 Subject: [PATCH 268/268] [Ralibi] script(goreleaser): Updated config file --- .goreleaser.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 18c385a2..40f032bd 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -20,8 +20,8 @@ builds: - windows env: - CGO_ENABLED=0 -archive: - replacements: +archives: +- replacements: darwin: Darwin linux: Linux windows: Windows