From 585d955905e2d69c1d7f68f5736b5695b0c61191 Mon Sep 17 00:00:00 2001 From: viggy28 Date: Fri, 18 Mar 2022 14:16:39 -0700 Subject: [PATCH] gh-96: rewrite using docker sdk --- api/create.go | 374 +++--------------- api/helpers.go | 76 ---- api/list_cluster.go | 61 +-- backup/backup.go | 25 -- config/config.go | 59 +++ go.mod | 2 +- internal/backup/backup.go | 333 ++++++++++++++++ internal/backup/modify-pghba.sh | 18 + internal/cmd/start.go | 4 +- internal/dockerservice/dockerservice.go | 196 +++++---- internal/internal.go | 120 ------ internal/metastore/metastore.go | 129 ++++++ internal/monitoring/monitoring.go | 61 +++ internal/postgres/postgres.go | 120 ++++++ main.go | 4 - metrics/metrics.go | 44 ++- misc/misc.go | 74 +++- misc/misc_test.go | 39 ++ scripts/install-spinup.sh | 2 +- templates/templates.go | 8 - .../templates/docker-compose-template.yml | 15 - 21 files changed, 1048 insertions(+), 716 deletions(-) delete mode 100644 api/helpers.go delete mode 100644 backup/backup.go create mode 100644 internal/backup/backup.go create mode 100644 internal/backup/modify-pghba.sh delete mode 100644 internal/internal.go create mode 100644 internal/metastore/metastore.go create mode 100644 internal/monitoring/monitoring.go create mode 100644 internal/postgres/postgres.go create mode 100644 misc/misc_test.go delete mode 100644 templates/templates.go delete mode 100644 templates/templates/docker-compose-template.yml diff --git a/api/create.go b/api/create.go index eb57fd3..e6ed390 100644 --- a/api/create.go +++ b/api/create.go @@ -1,77 +1,23 @@ package api import ( - "bytes" - "context" - "database/sql" "encoding/json" "fmt" "io/ioutil" "log" - "net" "net/http" - "os" - "os/exec" - "strings" - "time" + "strconv" "github.com/spinup-host/internal/dockerservice" + "github.com/spinup-host/internal/metastore" + "github.com/spinup-host/internal/monitoring" + "github.com/spinup-host/internal/postgres" + "github.com/spinup-host/misc" - "github.com/docker/docker/client" - "github.com/robfig/cron/v3" - "github.com/spinup-host/backup" "github.com/spinup-host/config" - "github.com/spinup-host/misc" _ "modernc.org/sqlite" ) -type service struct { - Duration time.Duration - UserID string - // one of arm64v8 or arm32v7 or amd64 - Architecture string - //Port uint - Db dbCluster - DockerNetwork string - Version version - BackupEnabled bool - Backup backupConfig -} -type version struct { - Maj uint - Min uint -} -type dbCluster struct { - Name string - ID string - Type string - Port int - Username string - Password string - - Memory string - Storage string - Monitoring string -} - -type backupConfig struct { - // https://man7.org/linux/man-pages/man5/crontab.5.html - Schedule map[string]interface{} - Dest Destination `json:"Dest"` -} - -type Destination struct { - Name string - BucketName string - ApiKeyID string - ApiKeySecret string -} -type serviceResponse struct { - HostName string - Port int - ContainerID string -} - func Hello(w http.ResponseWriter, req *http.Request) { fmt.Fprintf(w, "hello !! Welcome to spinup \n") } @@ -90,8 +36,7 @@ func CreateService(w http.ResponseWriter, req *http.Request) { http.Error(w, "error validating user", http.StatusUnauthorized) return } - - var s service + var s config.Service byteArray, err := ioutil.ReadAll(req.Body) if err != nil { @@ -119,295 +64,64 @@ func CreateService(w http.ResponseWriter, req *http.Request) { http.Error(w, "db type is currently not supported", 500) return } - s.Db.Port, err = portcheck() - s.Architecture = config.Cfg.Common.Architecture - servicePath := config.Cfg.Common.ProjectDir + "/" + s.UserID + "/" + s.Db.Name - s.DockerNetwork = fmt.Sprintf("%s_default", s.Db.Name) // following docker-compose naming format for compatibility - if err = prepareService(s, servicePath); err != nil { - log.Printf("ERROR: preparing service for %s %v", s.UserID, err) - http.Error(w, "Error preparing service", 500) - return - } - var containerID string - if containerID, err = startService(s, servicePath); err != nil { - log.Printf("ERROR: starting service for %s %v", s.UserID, err) - http.Error(w, "Error starting service", 500) - return - } - log.Printf("INFO: created service for user %s", s.UserID) + s.Db.Port, err = misc.Portcheck() if err != nil { - log.Printf("ERROR: getting container id %v", err) - http.Error(w, "Error getting container id", 500) + log.Printf("ERROR: port issue for %s %v", s.UserID, err) + http.Error(w, "port issue", 500) return } - s.Db.ID = containerID - var serRes serviceResponse - //serRes.HostName = s.UserID + "-" + s.Db.Name + ".spinup.host" - serRes.HostName = "localhost" - serRes.Port = s.Db.Port - serRes.ContainerID = containerID - jsonBody, err := json.Marshal(serRes) - if err != nil { - log.Printf("ERROR: marshalling service response struct serviceResponse %v", err) - http.Error(w, "Internal server error ", 500) - return - } - path := config.Cfg.Common.ProjectDir + "/" + s.UserID + "/" + s.UserID + ".db" - db, err := OpenSqliteDB(path) - if err != nil { - ErrorResponse(w, "error accessing database", 500) - return - } - sqlStmt := ` - create table if not exists clusterInfo (id integer not null primary key autoincrement, clusterId text, name text, username text, password text, port integer); - ` - ctx, _ := context.WithTimeout(context.Background(), 3*time.Second) - _, err = db.ExecContext(ctx, sqlStmt) - if err != nil { - log.Printf("%q: %s\n", err, sqlStmt) - } - tx, err := db.Begin() - if err != nil { - log.Fatal(err) - } - stmt, err := tx.PrepareContext(ctx, "insert into clusterInfo(clusterId, name, username, password, port) values(?, ?, ?, ?, ?)") - if err != nil { - log.Fatal(err) - } - defer stmt.Close() - _, err = stmt.ExecContext(ctx, s.Db.ID, s.Db.Name, s.Db.Username, s.Db.Password, s.Db.Port) - if err != nil { - log.Println(err) - } - err = tx.Commit() - if err != nil { - log.Println(err) - } - w.Header().Set("Content-type", "application/json") - w.Write(jsonBody) -} - -func prepareService(s service, path string) error { - err := os.MkdirAll(path, 0755) - if err != nil { - return fmt.Errorf("ERROR: creating project directory at %s", path) - } - if err := CreateDockerComposeFile(path, s); err != nil { - return fmt.Errorf("ERROR: creating service docker-compose file %v", err) - } - return nil -} - -func startService(s service, path string) (serviceContainerID string, err error) { - err = ValidateSystemRequirements() - if err != nil { - return "", err - } - err = ValidateDockerCompose(path) - if err != nil { - return "", err - } - cmd := exec.Command("docker-compose", "-f", path+"/docker-compose.yml", "up", "-d") - // https://stackoverflow.com/questions/18159704/how-to-debug-exit-status-1-error-when-running-exec-command-in-golang/18159705 - // To print the actual error instead of just printing the exit status - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - err = cmd.Run() - if err != nil { - fmt.Println(fmt.Sprint(err) + ": " + stderr.String()) - return "", err - } - serviceContainerID, err = lastContainerID() - if err != nil { - return serviceContainerID, err - } - - if s.Db.Monitoring == "enable" { - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - return serviceContainerID, err - } - - pgExporter := dockerservice.NewPgExporterService(cli, s.DockerNetwork, s.Db.Name, s.Db.Username, s.Db.Password) - if err := pgExporter.Start(); err != nil { - return serviceContainerID, err - } - } - return serviceContainerID, nil -} - -func ValidateDockerCompose(path string) error { - cmd := exec.Command("docker-compose", "-f", path+"/docker-compose.yml", "config") - if err := cmd.Run(); err != nil { - return fmt.Errorf("validating docker-compose file %v", err) - } - return nil -} - -func ValidateSystemRequirements() error { - cmd := exec.Command("which", "docker-compose") - if err := cmd.Run(); err != nil { - return fmt.Errorf("docker-compose doesn't exist %v", err) - } - cmd = exec.Command("which", "docker") - if err := cmd.Run(); err != nil { - return err - } - return nil -} - -func portcheck() (int, error) { - min, endingPort := misc.MinMax(config.Cfg.Common.Ports) - for startingPort := min; startingPort < endingPort; startingPort++ { - target := fmt.Sprintf("%s:%d", "localhost", startingPort) - _, err := net.DialTimeout("tcp", target, 3*time.Second) - if err != nil && !strings.Contains(err.Error(), "connect: connection refused") { - log.Printf("INFO: error on port scanning %d %v", startingPort, err) - return 0, err - } - if err != nil && strings.Contains(err.Error(), "connect: connection refused") { - log.Printf("INFO: port %d is unused", startingPort) - return startingPort, nil - } + s.Architecture = config.Cfg.Common.Architecture + s.DockerNetwork = fmt.Sprintf("%s_default", s.Db.Name) // following docker-compose naming format for compatibility + image := s.Architecture + "/" + s.Db.Type + ":" + strconv.Itoa(int(s.Version.Maj)) + if s.Version.Min > 0 { + image += "." + strconv.Itoa(int(s.Version.Min)) } - log.Printf("WARN: all allocated ports are occupied") - return 0, fmt.Errorf("error all allocated ports are occupied") -} - -func lastContainerID() (string, error) { - cmd := exec.Command("/bin/bash", "-c", "docker ps --last 1 -q") - output, err := cmd.CombinedOutput() + dockerClient, err := dockerservice.NewDocker() if err != nil { - return "", err + fmt.Printf("error creating client %v", err) } - return strings.TrimSuffix(string(output), "\n"), nil -} - -func OpenSqliteDB(path string) (*sql.DB, error) { - db, err := sql.Open("sqlite", path) + postgresContainer, err := postgres.NewPostgresContainer(image, s.Db.Name, s.Db.Username, s.Db.Password) if err != nil { - return nil, err - } - return db, nil -} - -func CreateBackup(w http.ResponseWriter, r *http.Request) { - if (*r).Method != "POST" { - http.Error(w, "Invalid Method", http.StatusMethodNotAllowed) + log.Printf("ERROR: creating new docker service for %s %v", s.UserID, err) + http.Error(w, "Error creating postgres docker service", 500) return } - var s service - byteArray, err := ioutil.ReadAll(r.Body) + postgresContainerBody, err := postgresContainer.Start(dockerClient) if err != nil { - fmt.Printf("error %v", err) - ErrorResponse(w, "error reading from request body", 500) + log.Printf("ERROR: starting new docker service for %s %v", s.UserID, err) + http.Error(w, "Error starting postgres docker service", 500) return } - err = json.Unmarshal(byteArray, &s) + log.Printf("INFO: created service for user %s %s", s.UserID, postgresContainerBody.ID) if err != nil { - fmt.Printf("error %v", err) - ErrorResponse(w, "error reading from readall body", 500) - return - } - fmt.Printf("%+v\n", s) - if !s.BackupEnabled { - ErrorResponse(w, "backup is not enabled", 400) - return - } - if s.Backup.Dest.Name != "AWS" { - http.Error(w, "Destination other than AWS is not supported", http.StatusInternalServerError) - return - } - if s.Backup.Dest.ApiKeyID == "" || s.Backup.Dest.ApiKeySecret == "" { - http.Error(w, "API key id and API key secret is mandatory", http.StatusInternalServerError) - return - } - if s.Backup.Dest.BucketName == "" { - http.Error(w, "bucket name is mandatory", http.StatusInternalServerError) + log.Printf("ERROR: getting container id %v", err) + http.Error(w, "Error getting container id", 500) return } - path := config.Cfg.Common.ProjectDir + "/" + s.UserID + "/" + s.UserID + ".db" - db, err := OpenSqliteDB(path) + db, err := metastore.NewDb(path) if err != nil { - ErrorResponse(w, "error accessing database", 500) + misc.ErrorResponse(w, "error accessing sqlite database", 500) return } - err = db.Ping() - if err != nil { - ErrorResponse(w, "error connecting to database", 500) - return - } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - sqlStmt := ` - create table if not exists backup (id integer not null primary key autoincrement, clusterid text, destination text, bucket text, second integer, minute integer, hour integer, dom integer, month integer, dow integer, foreign key(clusterid) references clusterinfo(clusterid)); - ` - _, err = db.ExecContext(ctx, sqlStmt) - if err != nil { - log.Printf("error: creating table backup %v", err) - ErrorResponse(w, "internal server error", 500) + insertSql := "insert into clusterInfo(clusterId, name, username, password, port, majVersion, minVersion) values(?, ?, ?, ?, ?, ?, ?)" + if err := metastore.InsertServiceIntoMeta(db, insertSql, postgresContainerBody.ID, s.Db.Name, s.Db.Username, s.Db.Password, s.Db.Port, int(s.Version.Maj), int(s.Version.Min)); err != nil { + log.Printf("ERROR: executing insert into cluster info table %v", err) + misc.ErrorResponse(w, "internal server error", 500) return } - tx, err := db.Begin() - if err != nil { - log.Println(err) - } - stmt, err := tx.PrepareContext(ctx, "insert into backup(clusterId, destination, bucket, second, minute, hour, dom, month, dow) values(?, ?, ?, ?, ?, ?, ?, ?, ?)") - if err != nil { - log.Printf("error: preparing insert statement %v", err) - ErrorResponse(w, "internal server error", 500) - return - } - defer stmt.Close() - _, err = stmt.ExecContext(ctx, s.Db.ID, s.Backup.Dest.Name, s.Backup.Dest.BucketName, s.Backup.Schedule["second"], s.Backup.Schedule["minute"], s.Backup.Schedule["hour"], s.Backup.Schedule["dom"], s.Backup.Schedule["month"], s.Backup.Schedule["dow"]) - if err != nil { - log.Printf("error: executing insert statement %v", err) - ErrorResponse(w, "internal server error", 500) - return - } - err = tx.Commit() - if err != nil { - log.Fatal(err) - } - c := cron.New() - var spec string - if minute, ok := s.Backup.Schedule["minute"].(float64); ok { - spec = fmt.Sprintf("%.0f", minute) - } else { - spec += " " + "*" - } - if hour, ok := s.Backup.Schedule["hour"].(float64); ok { - spec += " " + fmt.Sprintf("%.0f", hour) - } else { - spec += " " + "*" - } - if dom, ok := s.Backup.Schedule["dom"].(float64); ok { - spec += " " + fmt.Sprintf("%.0f", dom) - } else { - spec += " " + "*" - } - if month, ok := s.Backup.Schedule["month"].(float64); ok { - spec += " " + fmt.Sprintf("%.0f", month) - } else { - spec += " " + "*" - } - if dow, ok := s.Backup.Schedule["dow"].(float64); ok { - spec += " " + fmt.Sprintf("%.0f", dow) - - } else { - spec += " " + "*" + if s.Db.Monitoring == "enable" { + target := monitoring.Target{ + DockerNetwork: s.DockerNetwork, + ContainerName: postgresContainer.Name, + UserName: s.Db.Username, + Password: s.Db.Password, + } + _, err := target.Enable() + if err != nil { + log.Printf("ERROR: enabling monitoring %v", err) + http.Error(w, "Error enabling monitoring", 500) + return + } } - pgHost := s.Db.Name + "_postgres_1" - networkName := s.Db.Name + "_default" - c.AddFunc(spec, backup.TriggerBackup(networkName, s.Backup.Dest.ApiKeySecret, s.Backup.Dest.ApiKeyID, pgHost, s.Db.Username, s.Db.Password, s.Backup.Dest.BucketName)) - c.Start() - w.WriteHeader(http.StatusOK) -} - -func ErrorResponse(w http.ResponseWriter, msg string, statuscode int) { - w.WriteHeader(statuscode) - w.Write([]byte(msg)) } diff --git a/api/helpers.go b/api/helpers.go deleted file mode 100644 index 6f609bf..0000000 --- a/api/helpers.go +++ /dev/null @@ -1,76 +0,0 @@ -package api - -import ( - "fmt" - "math/rand" - "os" - "path/filepath" - "text/template" - - "github.com/spinup-host/templates" -) - -//TODO: vicky find how to keep the templates/* outside of the api. ie need to figure how to do relative path. -// check: https://stackoverflow.com/questions/66285635/how-do-you-use-go-1-16-embed-features-in-subfolders-packages - -type malformedRequest struct { - status int - msg string -} - -func (mr *malformedRequest) Error() string { - return mr.msg -} - -// https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-go/22892986#22892986 -var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") - -func randSeq(n int) string { - b := make([]rune, n) - for i := range b { - b[i] = letters[rand.Intn(len(letters))] - } - return string(b) -} - -// TODO: To remove the duplication here. We don't need separate function for each file -func CreateDockerComposeFile(absolutepath string, s service) error { - outputPath := filepath.Join(absolutepath, "docker-compose.yml") - // Create the file: - f, err := os.Create(outputPath) - if err != nil { - panic(err) - } - - defer f.Close() // don't forget to close the file when finished. - templ, err := template.ParseFS(templates.DockerTempl, "templates/docker-compose-template.yml") - if err != nil { - return fmt.Errorf("ERROR: parsing template file %v", err) - } - // TODO: not sure is there a better way to pass data to template - // A lot of this data is redundant. Already available in Service struct - data := struct { - UserID string - Architecture string - Type string - Port int - MajVersion uint - MinVersion uint - PostgresUsername string - PostgresPassword string - }{ - s.UserID, - s.Architecture, - s.Db.Type, - s.Db.Port, - s.Version.Maj, - s.Version.Min, - s.Db.Username, - s.Db.Password, - } - err = templ.Execute(f, data) - if err != nil { - return fmt.Errorf("ERROR: executing template file %v", err) - } - return nil -} diff --git a/api/list_cluster.go b/api/list_cluster.go index d5cc1aa..742114e 100644 --- a/api/list_cluster.go +++ b/api/list_cluster.go @@ -4,13 +4,14 @@ import ( "database/sql" "encoding/json" "errors" - "fmt" "io/fs" "log" "net/http" "os" "github.com/spinup-host/config" + "github.com/spinup-host/internal/metastore" + "github.com/spinup-host/misc" _ "modernc.org/sqlite" ) @@ -26,57 +27,29 @@ func ListCluster(w http.ResponseWriter, req *http.Request) { var err error config.Cfg.UserID, err = config.ValidateUser(authHeader, apiKeyHeader) if err != nil { - log.Printf(err.Error()) + log.Println("ERROR: validating user: ", err) http.Error(w, "error validating user", http.StatusUnauthorized) return } - dbPath := config.Cfg.Common.ProjectDir + "/" + config.Cfg.UserID - clusterInfos := ReadClusterInfo(dbPath, config.Cfg.UserID) - clusterByte, err := json.Marshal(clusterInfos) + path := config.Cfg.Common.ProjectDir + "/" + config.Cfg.UserID + "/" + config.Cfg.UserID + ".db" + db, err := metastore.NewDb(path) if err != nil { - log.Printf("ERROR: marshalling clusterInfos %v", err) - http.Error(w, "Internal server error ", 500) + misc.ErrorResponse(w, "error accessing sqlite database", 500) return } - w.Write(clusterByte) -} - -type clusterInfo struct { - ID int `json:"id"` - ClusterID string `json:"cluster_id"` - Name string `json:"name"` - Port int `json:"port"` - Username string `json:"username"` -} - -func ReadClusterInfo(path, dbName string) []clusterInfo { - dsn := path + "/" + dbName + ".db" - if _, err := os.Stat(dsn); errors.Is(err, fs.ErrNotExist) { - log.Printf("INFO: no sqlite database") - return nil - } - db, err := sql.Open("sqlite", dsn) + clustersInfo, err := metastore.ClustersInfo(db) if err != nil { - log.Fatal(err) + log.Println("ERROR: reading from clusterInfo table: ", err) + http.Error(w, "reading from clusterInfo", http.StatusUnauthorized) + return } - defer db.Close() - rows, err := db.Query("select id, clusterId, name, username, port from clusterInfo") + clusterByte, err := json.Marshal(clustersInfo) if err != nil { - log.Fatal(err) - } - defer rows.Close() - var clusterIds []string - var clusterInfos []clusterInfo - var cluster clusterInfo - for rows.Next() { - err = rows.Scan(&cluster.ID, &cluster.ClusterID, &cluster.Name, &cluster.Username, &cluster.Port) - if err != nil { - log.Fatal(err) - } - clusterInfos = append(clusterInfos, cluster) + log.Printf("ERROR: marshalling clusterInfos %v", err) + http.Error(w, "Internal server error ", 500) + return } - fmt.Println(clusterIds) - return clusterInfos + w.Write(clusterByte) } func GetCluster(w http.ResponseWriter, r *http.Request) { @@ -133,8 +106,8 @@ func GetCluster(w http.ResponseWriter, r *http.Request) { }) } -func getClusterFromDb(clusterId string) (clusterInfo, error) { - var ci clusterInfo +func getClusterFromDb(clusterId string) (config.ClusterInfo, error) { + var ci config.ClusterInfo path := config.Cfg.Common.ProjectDir + "/" + config.Cfg.UserID dsn := path + "/" + config.Cfg.UserID + ".db" if _, err := os.Stat(dsn); errors.Is(err, fs.ErrNotExist) { diff --git a/backup/backup.go b/backup/backup.go deleted file mode 100644 index cdb847b..0000000 --- a/backup/backup.go +++ /dev/null @@ -1,25 +0,0 @@ -package backup - -import ( - "fmt" - "github.com/spinup-host/internal/dockerservice" - "log" - - "github.com/docker/docker/client" -) - -func TriggerBackup(networkName, awsAccessKey, awsAccessKeyId, pgHost, pgUsername, pgPassword, walgS3Prefix string) func() { - cli, err := client.NewClientWithOpts(client.FromEnv) - if err != nil { - fmt.Printf("error creating client %v", err) - } - backupSvc := dockerservice.NewPgBackupService(cli, awsAccessKey, awsAccessKeyId, pgHost, walgS3Prefix, networkName, pgUsername, pgPassword) - return func() { - fmt.Println("backup triggered") - err = backupSvc.Start() - if err != nil { - log.Printf("error running backup service %v", err) - } - fmt.Println("backup finished") - } -} diff --git a/config/config.go b/config/config.go index 1eb5cc3..1f35656 100644 --- a/config/config.go +++ b/config/config.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "strings" + "time" "github.com/golang-jwt/jwt" ) @@ -26,6 +27,64 @@ type Configuration struct { var Cfg Configuration +type Service struct { + Duration time.Duration + UserID string + // one of arm64v8 or arm32v7 or amd64 + Architecture string + //Port uint + Db dbCluster + DockerNetwork string + Version version + BackupEnabled bool + Backup backupConfig +} + +type version struct { + Maj uint + Min uint +} +type dbCluster struct { + Name string + ID string + Type string + Port int + Username string + Password string + + Memory string + Storage string + Monitoring string +} + +type backupConfig struct { + // https://man7.org/linux/man-pages/man5/crontab.5.html + Schedule map[string]interface{} + Dest Destination `json:"Dest"` +} + +type Destination struct { + Name string + BucketName string + ApiKeyID string + ApiKeySecret string +} +type serviceResponse struct { + HostName string + Port int + ContainerID string +} + +type ClusterInfo struct { + ID int `json:"id"` + ClusterID string `json:"cluster_id"` + Name string `json:"name"` + Port int `json:"port"` + Username string `json:"username"` + MajVersion int `json:"majversion"` + MinVersion int `json:"minversion"` +} + func ValidateUser(authHeader string, apiKeyHeader string) (string, error) { if authHeader == "" && apiKeyHeader == "" { return "", errors.New("no authorization keys found") diff --git a/go.mod b/go.mod index c92b778..4ac59e7 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.16 require ( github.com/containerd/containerd v1.5.7 // indirect github.com/docker/docker v20.10.9+incompatible - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-connections v0.4.0 github.com/golang-jwt/jwt v3.2.1+incompatible github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.4.2 diff --git a/internal/backup/backup.go b/internal/backup/backup.go new file mode 100644 index 0000000..9ba995e --- /dev/null +++ b/internal/backup/backup.go @@ -0,0 +1,333 @@ +package backup + +import ( + "archive/tar" + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "net/http" + "os" + "strconv" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/go-connections/nat" + "github.com/robfig/cron/v3" + "github.com/spinup-host/config" + "github.com/spinup-host/internal/dockerservice" + "github.com/spinup-host/internal/metastore" + "github.com/spinup-host/internal/postgres" + "github.com/spinup-host/misc" +) + +const PREFIXBACKUPCONTAINER = "spinup-postgres-backup-" + +type BackupData struct { + AwsAccessKeySecret string + AwsAccessKeyId string + WalgS3Prefix string + PgHost string + PgUsername string + PgPassword string + PgDatabase string +} + +// Ideally I would like to keep the modify-pghba.sh script to scripts directory. +// However, Go doesn't support relative directory yet https://github.com/golang/go/issues/46056 !! + +//go:embed modify-pghba.sh +var f embed.FS + +func CreateBackup(w http.ResponseWriter, r *http.Request) { + if (*r).Method != "POST" { + http.Error(w, "Invalid Method", http.StatusMethodNotAllowed) + return + } + var s config.Service + byteArray, err := ioutil.ReadAll(r.Body) + if err != nil { + fmt.Printf("error %v", err) + misc.ErrorResponse(w, "error reading from request body", 500) + return + } + err = json.Unmarshal(byteArray, &s) + if err != nil { + fmt.Printf("error %v", err) + misc.ErrorResponse(w, "error reading from readall body", 500) + return + } + if err = backupDataValidation(&s); err != nil { + l := &logicError{} + if errors.As(err, l) { + misc.ErrorResponse(w, l.Error(), http.StatusBadRequest) + } else { + misc.ErrorResponse(w, err.Error(), http.StatusInternalServerError) + } + return + } + + path := config.Cfg.Common.ProjectDir + "/" + s.UserID + "/" + s.UserID + ".db" + db, err := metastore.NewDb(path) + if err != nil { + misc.ErrorResponse(w, "error accessing database", 500) + return + } + minute, _ := s.Backup.Schedule["minute"].(string) + min, _ := strconv.Atoi(minute) + + hour, _ := s.Backup.Schedule["hour"].(string) + h, _ := strconv.Atoi(hour) + + dom, _ := s.Backup.Schedule["dom"].(string) + domInt, _ := strconv.Atoi(dom) + + month, _ := s.Backup.Schedule["month"].(string) + mon, _ := strconv.Atoi(month) + + dow, _ := s.Backup.Schedule["dow"].(string) + dowInt, _ := strconv.Atoi(dow) + + insertSql := "insert into backup(clusterId, destination, bucket, second, minute, hour, dom, month, dow) values(?, ?, ?, ?, ?, ?, ?, ?, ?)" + if err := metastore.InsertBackupIntoMeta( + db, + insertSql, + s.Db.ID, + s.Backup.Dest.Name, + s.Backup.Dest.BucketName, + 0, + min, + h, + domInt, + mon, + dowInt, + ); err != nil { + log.Printf("ERROR: executing insert into backup table %v", err) + misc.ErrorResponse(w, "internal server error", 500) + return + } + pgHost := postgres.PREFIXPGCONTAINER + s.Db.Name + dockerClient, err := dockerservice.NewDocker() + if err != nil { + fmt.Printf("error creating client %v", err) + } + pgContainer, err := dockerClient.GetContainer(context.Background(), pgHost) + if err != nil { + log.Printf("error: getting container for container name %s %v", pgHost, err) + misc.ErrorResponse(w, "internal server error", 500) + return + } + scriptContent, err := f.ReadFile("modify-pghba.sh") + if err != nil { + log.Printf("error: reading modify-pghba.sh file %v", err) + } + err = updatePghba(pgContainer, dockerClient, scriptContent) + if err != nil { + log.Printf("error: updating pghba %v", err) + misc.ErrorResponse(w, "internal server error", 500) + return + } + cis, err := metastore.ClustersInfo(db) + if err != nil { + misc.ErrorResponse(w, "internal server error", 500) + return + } + ci, err := cis.FilterByName(s.Db.Name) + if err != nil { + misc.ErrorResponse(w, "internal server error", 500) + return + } + execPath := "/usr/lib/postgresql/" + strconv.Itoa(ci.MajVersion) + "/bin/" + if err = postgres.ReloadPostgres(dockerClient, execPath, postgres.PGDATADIR, pgHost); err != nil { + misc.ErrorResponse(w, "internal server error", 500) + return + } + c := cron.New() + var spec string + if minute, ok := s.Backup.Schedule["minute"].(string); ok { + spec = minute + } else { + spec += " " + "*" + } + if hour, ok := s.Backup.Schedule["hour"].(string); ok { + spec += " " + hour + } else { + spec += " " + "*" + } + if dom, ok := s.Backup.Schedule["dom"].(string); ok { + spec += " " + dom + } else { + spec += " " + "*" + } + if month, ok := s.Backup.Schedule["month"].(string); ok { + spec += " " + month + } else { + spec += " " + "*" + } + if dow, ok := s.Backup.Schedule["dow"].(string); ok { + spec += " " + dow + } else { + spec += " " + "*" + } + log.Println("INFO: scheduling backup at:", spec) + + networkName := s.Db.Name + "_default" + backupData := BackupData{ + AwsAccessKeySecret: s.Backup.Dest.ApiKeySecret, + AwsAccessKeyId: s.Backup.Dest.ApiKeyID, + WalgS3Prefix: s.Backup.Dest.BucketName, + PgHost: pgHost, + PgDatabase: "postgres", + PgUsername: s.Db.Username, + PgPassword: s.Db.Password, + } + _, err = c.AddFunc(spec, TriggerBackup(networkName, backupData)) + if err != nil { + log.Printf("error: scheduling database backup") + misc.ErrorResponse(w, "internal server error", 500) + return + } + c.Start() + w.WriteHeader(http.StatusOK) +} + +type logicError struct { + err error +} + +func (l logicError) Error() string { + return fmt.Sprintf("logic error %v", l.err) +} + +func backupDataValidation(s *config.Service) error { + if !s.BackupEnabled { + return logicError{err: errors.New("backup is not enabled")} + } + if s.Backup.Dest.Name != "AWS" { + return logicError{err: errors.New("destination other than AWS is not supported")} + } + if s.Backup.Dest.ApiKeyID == "" || s.Backup.Dest.ApiKeySecret == "" { + return errors.New("api key id and api key secret is mandatory") + } + if s.Backup.Dest.BucketName == "" { + return errors.New("bucket name is mandatory") + } + if s.UserID == "" { + s.UserID = "testuser" + } + return nil +} + +// TriggerBackup returns a closure which is being invoked by the cron +func TriggerBackup(networkName string, backupData BackupData) func() { + var err error + dockerClient, err := dockerservice.NewDocker() + if err != nil { + fmt.Printf("error creating client %v", err) + } + var op container.ContainerCreateCreatedBody + env := []string{} + + // TODO: refactor this if possible. Challenge is functions can't grow a slice. ie. can't append inside another function + env = append(env, misc.StringToDockerEnvVal("AWS_SECRET_ACCESS_KEY", backupData.AwsAccessKeySecret)) + env = append(env, misc.StringToDockerEnvVal("AWS_ACCESS_KEY_ID", backupData.AwsAccessKeyId)) + env = append(env, misc.StringToDockerEnvVal("WALG_S3_PREFIX", backupData.WalgS3Prefix)) + env = append(env, misc.StringToDockerEnvVal("PGHOST", backupData.PgHost)) + env = append(env, misc.StringToDockerEnvVal("PGPASSWORD", backupData.PgPassword)) + env = append(env, misc.StringToDockerEnvVal("PGDATABASE", backupData.PgDatabase)) + env = append(env, misc.StringToDockerEnvVal("PGUSER", backupData.PgUsername)) + + // Ref: https://gist.github.com/viggy28/5b524baf005d029e4bad2ec16cb09dca + // On dealing with container networking and environment variables + // initialized a map + endpointConfig := map[string]*network.EndpointSettings{} + // setting key and value for the map. networkName=$dbname_default (eg: viggy_default) + endpointConfig[networkName] = &network.EndpointSettings{} + nwConfig := network.NetworkingConfig{EndpointsConfig: endpointConfig} + containerOptions := &types.ContainerStartOptions{} + backupContainer := dockerservice.NewContainer( + PREFIXBACKUPCONTAINER+backupData.PgHost, + container.Config{ + Image: "spinuphost/walg:latest", + Env: env, + ExposedPorts: map[nat.Port]struct{}{"5432": {}}, + }, + container.HostConfig{NetworkMode: "default"}, + nwConfig, + containerOptions, + ) + return func() { + // TODO: explicilty ignoring the output of Start. Since i don't know how to use + fmt.Println("INFO: starting backup") + op, err = backupContainer.Start(dockerClient) + if err != nil { + fmt.Println("error starting backup container %w", err) + } + fmt.Println("INFO: created backup container:", op.ID) + fmt.Println("INFO: ending backup") + } +} + +var tarPath = "modify-pghba.tar" + +func updatePghba(c dockerservice.Container, d dockerservice.Docker, content []byte) error { + _, cleanup, err := contentToTar(content) + if err != nil { + return fmt.Errorf("error converting content to tar file %w", err) + } + defer cleanup() + tr, err := os.Open(tarPath) + if err != nil { + return fmt.Errorf("error reading tar file %w", err) + } + defer tr.Close() + err = d.Cli.CopyToContainer(context.Background(), c.ID, "/etc/postgresql", tr, types.CopyToContainerOptions{}) + if err != nil { + return fmt.Errorf("error copying file to container %w", err) + } + hbaFile := postgres.PGDATADIR + "pg_hba.conf" + execConfig := types.ExecConfig{ + User: "postgres", + WorkingDir: "/etc/postgresql", + AttachStderr: true, + AttachStdout: true, + Cmd: []string{"./modify-pghba", hbaFile}, + } + if _, err := c.ExecCommand(context.Background(), d, execConfig); err != nil { + return fmt.Errorf("error executing command %s %w", execConfig.Cmd[0], err) + } + return nil +} + +// contentToTar returns a tar file for given content +// ref https://medium.com/learning-the-go-programming-language/working-with-compressed-tar-files-in-go-e6fe9ce4f51d +func contentToTar(content []byte) (io.Writer, func(), error) { + tarFile, err := os.Create(tarPath) + if err != nil { + log.Fatal(err) + } + defer tarFile.Close() + tw := tar.NewWriter(tarFile) + defer tw.Close() + + hdr := &tar.Header{ + Name: "modify-pghba", + Mode: 0655, + Size: int64(len(content)), + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, nil, err + } + if _, err := tw.Write([]byte(content)); err != nil { + return nil, nil, err + } + rmFunc := func() { + os.Remove(tarPath) + } + return tw, rmFunc, nil +} diff --git a/internal/backup/modify-pghba.sh b/internal/backup/modify-pghba.sh new file mode 100644 index 0000000..7f0cf34 --- /dev/null +++ b/internal/backup/modify-pghba.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +usage() { + echo $0 pg_hba_file_path + + printf 'To replace trust method with md5 for replication connections' +} + +if [ "$1" = "-h" ] || [ "$1" = "--help" ] +then + usage + exit 1 +else + filepath="$1" # the first arg +fi + +sed -i 's/^host replication all 127.0.0.1\/32.*/host replication all all md5/g' $filepath +sed -i 's/^host replication all ::1\/128.*/host replication all all md5/g' $filepath diff --git a/internal/cmd/start.go b/internal/cmd/start.go index a9367b1..0fb6c66 100644 --- a/internal/cmd/start.go +++ b/internal/cmd/start.go @@ -15,6 +15,7 @@ import ( "github.com/spinup-host/api" "github.com/spinup-host/config" + "github.com/spinup-host/internal/backup" "github.com/spinup-host/metrics" "github.com/golang-jwt/jwt" @@ -45,7 +46,8 @@ func apiHandler() http.Handler { mux.HandleFunc("/listcluster", api.ListCluster) mux.HandleFunc("/cluster", api.GetCluster) mux.HandleFunc("/metrics", metrics.HandleMetrics) - mux.HandleFunc("/createbackup", api.CreateBackup) + mux.HandleFunc("/createbackup", backup.CreateBackup) + //mux.HandleFunc("/pghba", backup.UpdatePghba) mux.HandleFunc("/altauth", api.AltAuth) c := cors.New(cors.Options{ AllowedOrigins: []string{"https://app.spinup.host", "http://localhost:3000"}, diff --git a/internal/dockerservice/dockerservice.go b/internal/dockerservice/dockerservice.go index b74d694..f7a7022 100644 --- a/internal/dockerservice/dockerservice.go +++ b/internal/dockerservice/dockerservice.go @@ -2,19 +2,32 @@ package dockerservice import ( "context" + "errors" "fmt" + "log" "os" - "os/exec" - "strings" + "time" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/volume" "github.com/docker/docker/client" - "github.com/rs/zerolog" + "github.com/docker/docker/pkg/stdcopy" + "github.com/spinup-host/misc" ) -type LinePrefixLogger struct { - prefix string - logger zerolog.Logger +type Docker struct { + Cli *client.Client +} + +func NewDocker(opts ...client.Opt) (Docker, error) { + cli, err := client.NewClientWithOpts(opts...) + if err != nil { + fmt.Printf("error creating client %v", err) + } + //TODO: Check. Does this initialize and assign + return Docker{Cli: cli}, nil } type DockerService struct { @@ -29,90 +42,123 @@ type DockerService struct { RemoveContainer bool `yaml:"remove_container"` } -func (ds DockerService) buildArgs() []string { - args := []string{"run", "--net=" + ds.NetworkName, "--name=" + ds.NetworkName + "-" + ds.Name, "--hostname=" + ds.Name} +type Container struct { + ID string + Name string + Options *types.ContainerStartOptions + Ctx context.Context + // portable docker config + Config container.Config + // non-portable docker config + HostConfig container.HostConfig + NetworkConfig network.NetworkingConfig +} - // Environment variables - for name, value := range ds.Environment { - args = append(args, "-e", name+"="+value) +func NewContainer(name string, config container.Config, hostConfig container.HostConfig, networkConfig network.NetworkingConfig, options *types.ContainerStartOptions) *Container { + return &Container{ + Name: name, + Ctx: context.Background(), + Config: config, + HostConfig: hostConfig, + NetworkConfig: networkConfig, + Options: options, } +} - // Published ports - for dst, src := range ds.Ports { - args = append(args, "-p", fmt.Sprintf("%d:%d", dst, src)) +func (d Docker) GetContainer(ctx context.Context, name string) (Container, error) { + c := Container{} + containers, err := d.Cli.ContainerList(ctx, types.ContainerListOptions{}) + if err != nil { + return Container{}, fmt.Errorf("error listing containers %w", err) } - - // Restart policy - args = append(args, "--restart", ds.RestartPolicy) - - // Remove container - if ds.RemoveContainer { - args = append(args, "--rm") + for _, container := range containers { + // TODO: name of the container has prefixed with "/" + // I have hardcoded here; perhaps there is a better way to handle this + if misc.SliceContainsString(container.Names, "/"+name) { + c.ID = container.ID + c.Config.Image = container.Image + break + } } + return c, nil +} - args = append(args, ds.Image) +func (d Docker) LastContainerID(ctx context.Context) (string, error) { + containers, err := d.Cli.ContainerList(ctx, types.ContainerListOptions{Latest: true}) + if err != nil { + return "", err + } + var containerID string + for _, container := range containers { + containerID = container.ID + } + return containerID, nil +} - return args +func (c *Container) Start(d Docker) (container.ContainerCreateCreatedBody, error) { + body, err := d.Cli.ContainerCreate(c.Ctx, &c.Config, &c.HostConfig, &c.NetworkConfig, nil, c.Name) + if err != nil { + log.Println("error creating container") + return container.ContainerCreateCreatedBody{}, err + } + err = d.Cli.ContainerStart(c.Ctx, body.ID, types.ContainerStartOptions{}) + if err != nil { + log.Println("error starting container") + return container.ContainerCreateCreatedBody{}, err + } + return body, nil } -func (ds DockerService) Start() (err error) { - logger := zerolog.New(os.Stderr).With().Str("name", ds.Name).Timestamp().Logger() - _, err = ds.DockerClient.NetworkCreate(context.TODO(), ds.NetworkName, types.NetworkCreate{CheckDuplicate: true}) +// ExecCommand executes a given bash command through execConfig and displays the output in stdout and stderr +// This function doesn't return an error for the failure of the command itself +func (c Container) ExecCommand(ctx context.Context, d Docker, execConfig types.ExecConfig) (types.IDResponse, error) { + if c.ID == "" { + return types.IDResponse{}, errors.New("container id is empty") + } + if _, err := d.Cli.ContainerInspect(ctx, c.ID); err != nil { + return types.IDResponse{}, errors.New("container doesn't exist") + } + execResponse, err := d.Cli.ContainerExecCreate(ctx, c.ID, execConfig) if err != nil { - networkExistsError := fmt.Sprintf("network with name %s already exists", ds.NetworkName) - if !strings.Contains(err.Error(), networkExistsError) { - return err - } + return types.IDResponse{}, fmt.Errorf("creating container exec %w", err) + } + resp, err := d.Cli.ContainerExecAttach(ctx, execResponse.ID, types.ExecStartCheck{Tty: false}) + if err != nil { + return types.IDResponse{}, fmt.Errorf("creating container exec attach %w", err) + } + defer resp.Close() + // show the output to stdout and stderr + if _, err := stdcopy.StdCopy(os.Stdout, os.Stderr, resp.Reader); err != nil { + return types.IDResponse{}, fmt.Errorf("unable to copy the output of container, %w", err) } + if err = d.Cli.ContainerExecStart(ctx, execResponse.ID, types.ExecStartCheck{}); err != nil { + return types.IDResponse{}, fmt.Errorf("starting container exec %v", err) + } + return execResponse, nil +} - cmd := exec.Command("docker", ds.buildArgs()...) - cmd.Stdout = logger - cmd.Stderr = logger +func (c *Container) Stop(d Docker, opts types.ContainerStartOptions) { + timeout := 20 * time.Second + d.Cli.ContainerStop(c.Ctx, c.ID, &timeout) +} - if err = cmd.Start(); err != nil { - return err - } - return nil +func CreateVolume(ctx context.Context, d Docker, opt volume.VolumeCreateBody) (types.Volume, error) { + log.Println("creating volume: ", opt.Name) + return d.Cli.VolumeCreate(ctx, opt) } -func NewPgExporterService(cli *client.Client, networkName, dbName, postgresUsername, postgresPassword string) DockerService { - exporterSvc := DockerService{ - DockerClient: cli, - Name: "postgres_exporter", - NetworkName: networkName, - RestartPolicy: "unless-stopped", - Ports: map[int]int{ - 9187: 9187, - }, - Environment: map[string]string{ - "DATA_SOURCE_NAME": fmt.Sprintf("postgresql://%s:%s@%s:5432/postgres?sslmode=disable", - postgresUsername, - postgresPassword, - dbName+"_postgres_1", - ), - }, - Image: "quay.io/prometheuscommunity/postgres-exporter", - } - return exporterSvc +func CreateNetwork(ctx context.Context, d Docker, name string, opt types.NetworkCreate) (types.NetworkCreateResponse, error) { + networkResponse, err := d.Cli.NetworkCreate(ctx, name, opt) + if err != nil { + return types.NetworkCreateResponse{}, err + } + return networkResponse, nil } -func NewPgBackupService(cli *client.Client, awsAccessKey, awsAccessKeyId, pgHost, walgS3Prefix, networkName, postgresUsername, postgresPassword string) DockerService { - backupSvc := DockerService{ - DockerClient: cli, - Name: "postgres_backup", - NetworkName: networkName, - RestartPolicy: "no", - Environment: map[string]string{ - "AWS_SECRET_ACCESS_KEY": awsAccessKey, - "AWS_ACCESS_KEY_ID": awsAccessKeyId, - "PGHOST": pgHost, - "PGPASSWORD": postgresPassword, - "PGUSER": postgresUsername, - "PGDATABASE": "postgres", - "WALG_S3_PREFIX": walgS3Prefix, - }, - Image: "spinuphost/walg:latest", - RemoveContainer: true, - } - return backupSvc +func NewDockerClient(ops ...client.Opt) (*client.Client, error) { + cli, err := client.NewClientWithOpts(ops...) + if err != nil { + return nil, err + } + return cli, nil } diff --git a/internal/internal.go b/internal/internal.go deleted file mode 100644 index 28e1f0a..0000000 --- a/internal/internal.go +++ /dev/null @@ -1,120 +0,0 @@ -package internal - -import ( - "fmt" - "log" - "os" -) - -// TODO: vicky this entire code to manipulate tunnel client yml should be removed. -// This is just commented for reference. - -/* type ClientTunnel struct { - Server_addr string - Tunnels struct { - Api map[string]string - Prometheus map[string]string `tag:"omitempty"` - Postgres map[string]string `tag:"omitempty"` - Postgresa map[string]string `tag:"omitempty"` - } -} - -type NewClientTunnel struct { - Server_addr string - Tunnels struct { - Api map[string]string - Postgres map[string]string - Postgresa map[string]string `yaml:"postgresa,omitempty"` - Prometheus map[string]string `yaml:"prometheus,omitempty"` - } -} - -func UpdateTunnelClient(port uint) error { - yamlbytes, err := ioutil.ReadFile("/home/pi/tunnel/.tunnel/tunnel.yml") - if err != nil { - return fmt.Errorf("error reading file %v", err) - } - var c ClientTunnel - err = yaml.Unmarshal(yamlbytes, &c) - if err != nil { - log.Printf("error Unmarshal file %v", err) - } - nc := NewClientTunnel{ - Server_addr: c.Server_addr, - } - _, ok := c.Tunnels.Api["addr"] - if ok { - nc.Tunnels.Api = make(map[string]string, 1) - nc.Tunnels.Api["proto"] = c.Tunnels.Api["proto"] - nc.Tunnels.Api["addr"] = c.Tunnels.Api["addr"] - nc.Tunnels.Api["host"] = c.Tunnels.Api["host"] - } - nc.Tunnels.Postgres = make(map[string]string, 1) - nc.Tunnels.Postgres["proto"] = "tcp" - nc.Tunnels.Postgres["addr"] = "10.0.0.244:" + fmt.Sprintf("%d", port) - nc.Tunnels.Postgres["remote_addr"] = "0.0.0.0:" + fmt.Sprintf("%d", port) - outYamlBytes, err := yaml.Marshal(nc) - if err != nil { - log.Printf("error marshalling %v", err) - } - ioutil.WriteFile("/home/pi/tunnel/.tunnel/tunnel.yml", outYamlBytes, 0644) - file, _ := os.OpenFile("/home/pi/tunnel/.tunnel/tunnel.yml", os.O_RDWR, 0644) - file.WriteAt([]byte("postgres"), 0) - return nil -} */ - -func UpdateTunnelClientYml(service string, port int) error { - filePath := "/home/pi/tunnel/.tunnel/tunnel.yml" - fileByte, err := os.ReadFile(filePath) - if err != nil { - log.Printf("failed to read tunnel client file: %s", err) - return err - } - fileLength := len(fileByte) - file, err := os.OpenFile(filePath, os.O_RDWR, 0644) - if err != nil { - log.Printf("failed to open tunnel client file: %s", err) - return err - } - defer file.Close() - bytesWritten, err := file.WriteAt([]byte(" "+service+":"), int64(fileLength)) - if err != nil { - log.Printf("failed to write on tunnel client file: %s", err) - return err - } - fileLength = fileLength + bytesWritten - bytesWritten, err = file.WriteAt([]byte("\n addr: 10.0.0.244:"+fmt.Sprintf("%d", port)), int64(fileLength)) - if err != nil { - log.Printf("failed to write on tunnel client file: %s", err) - return err - } - fileLength = fileLength + bytesWritten - bytesWritten, err = file.WriteAt([]byte("\n proto: tcp"), int64(fileLength)) - if err != nil { - log.Printf("failed to write on tunnel client file: %s", err) - return err - } - fileLength = fileLength + bytesWritten - _, err = file.WriteAt([]byte("\n remote_addr: 0.0.0.0:"+fmt.Sprintf("%d", port)+"\n"), int64(fileLength)) - if err != nil { - log.Printf("failed to write on tunnel client file: %s", err) - return err - } - return nil -} - -/* func RestartTunnelClient() error { - cmd := exec.Command("/bin/sh", "-c", "sudo systemctl restart tunnel-client") - // https://stackoverflow.com/questions/18159704/how-to-debug-exit-status-1-error-when-running-exec-command-in-golang/18159705 - // To print the actual error instead of just printing the exit status - var out bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &out - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - fmt.Println(fmt.Sprint(err) + ": " + stderr.String()) - return err - } - return nil -} */ diff --git a/internal/metastore/metastore.go b/internal/metastore/metastore.go new file mode 100644 index 0000000..e83e414 --- /dev/null +++ b/internal/metastore/metastore.go @@ -0,0 +1,129 @@ +package metastore + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log" + + "github.com/spinup-host/config" +) + +type Db struct { + Client *sql.DB +} + +// clustersInfo type has methods which provide us to filter them by name etc. +type clustersInfo []config.ClusterInfo + +// FilterByName filters cluster by name. Since cluster names are unique as soon as we find a match we return +func (c clustersInfo) FilterByName(name string) (config.ClusterInfo, error) { + for _, clusterInfo := range c { + if clusterInfo.Name == name { + return clusterInfo, nil + } + } + return config.ClusterInfo{}, errors.New("cluster not found") +} + +func NewDb(path string) (Db, error) { + db, err := open(path) + if err != nil { + return Db{}, fmt.Errorf("unable to create a new db sqlite db client %w", err) + } + return Db{Client: db}, nil +} + +func open(path string) (*sql.DB, error) { + return sql.Open("sqlite", path) +} + +// migration creates table +func migration(ctx context.Context, db Db) error { + sqlStatements := []string{ + "create table if not exists clusterInfo (id integer not null primary key autoincrement, clusterId text, name text, username text, password text, port integer, majVersion integer, minVersion integer);", + "create table if not exists backup (id integer not null primary key autoincrement, clusterid text, destination text, bucket text, second integer, minute integer, hour integer, dom integer, month integer, dow integer, foreign key(clusterid) references clusterinfo(clusterid));", + } + tx, err := db.Client.Begin() + if err != nil { + return fmt.Errorf("couldn't begin a transaction %w", err) + + } + for _, sqlStatement := range sqlStatements { + _, err = tx.ExecContext(ctx, sqlStatement) + if err != nil { + return fmt.Errorf("couldn't execute a transaction for %s %w", sqlStatement, err) + } + } + err = tx.Commit() + if err != nil { + return fmt.Errorf("couldn't commit a transaction %w", err) + } + return nil +} + +//TODO: How to write generic functions with varying fields and types? Maybe generics +func InsertServiceIntoMeta(db Db, sql, clusterId, name, username, password string, port, majVersion, minVersion int) error { + tx, err := db.Client.Begin() + if err != nil { + return fmt.Errorf("unable to begin a transaction %w", err) + } + if err = migration(context.Background(), db); err != nil { + return fmt.Errorf("error running a migration %w", err) + } + res, err := tx.ExecContext(context.Background(), sql, clusterId, name, username, password, port, majVersion, minVersion) + if err != nil { + return fmt.Errorf("unable to execute %s %v", sql, err) + } + rows, _ := res.RowsAffected() + log.Println("INFO: rows inserted into clusterInfo table:", rows) + err = tx.Commit() + if err != nil { + return err + } + return nil +} + +func InsertBackupIntoMeta(db Db, sql, clusterId, destination, bucket string, second, minute, hour, dom, month, dow int) error { + tx, err := db.Client.Begin() + if err != nil { + return fmt.Errorf("unable to begin a transaction %w", err) + } + res, err := tx.ExecContext(context.Background(), sql, clusterId, destination, bucket, second, minute, hour, dom, month, dow) + if err != nil { + return fmt.Errorf("unable to execute %s %v", sql, err) + } + rows, _ := res.RowsAffected() + log.Println("INFO: rows inserted into backup table:", rows) + err = tx.Commit() + if err != nil { + return err + } + return nil +} + +// ClustersInfo returns all clusters from clusterinfo table +func ClustersInfo(db Db) (clustersInfo, error) { + if err := db.Client.Ping(); err != nil { + return nil, fmt.Errorf("error pinging sqlite database %w", err) + } + if err := migration(context.Background(), db); err != nil { + return nil, fmt.Errorf("error running a migration %w", err) + } + rows, err := db.Client.Query("select id, clusterId, name, username, port, majversion, minversion from clusterInfo") + if err != nil { + return nil, fmt.Errorf("unable to query clusterinfo") + } + defer rows.Close() + var csi clustersInfo + var cluster config.ClusterInfo + for rows.Next() { + err = rows.Scan(&cluster.ID, &cluster.ClusterID, &cluster.Name, &cluster.Username, &cluster.Port, &cluster.MajVersion, &cluster.MinVersion) + if err != nil { + log.Fatal(err) + } + csi = append(csi, cluster) + } + return csi, nil +} diff --git a/internal/monitoring/monitoring.go b/internal/monitoring/monitoring.go new file mode 100644 index 0000000..8339e8e --- /dev/null +++ b/internal/monitoring/monitoring.go @@ -0,0 +1,61 @@ +package monitoring + +import ( + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/spinup-host/internal/dockerservice" + "github.com/spinup-host/internal/postgres" + "github.com/spinup-host/misc" +) + +// Target represents a postgres host (target) for monitoring +type Target struct { + DockerNetwork string + ContainerName string + UserName string + Password string +} + +func (t Target) Enable() (container.ContainerCreateCreatedBody, error) { + postgresExporterContainer, err := newPostgresExporterContainer(t) + if err != nil { + return container.ContainerCreateCreatedBody{}, err + } + dockerClient, err := dockerservice.NewDocker() + if err != nil { + fmt.Printf("error creating client %v", err) + return container.ContainerCreateCreatedBody{}, err + } + body, err := postgresExporterContainer.Start(dockerClient) + if err != nil { + return container.ContainerCreateCreatedBody{}, err + } + return body, nil +} + +func newPostgresExporterContainer(t Target) (*dockerservice.Container, error) { + networkName := t.ContainerName + "_default" + endpointConfig := map[string]*network.EndpointSettings{} + endpointConfig[networkName] = &network.EndpointSettings{} + nwConfig := network.NetworkingConfig{EndpointsConfig: endpointConfig} + containerName := "spinup-" + t.ContainerName + "-postgres-exporter" + image := "quay.io/prometheuscommunity/postgres-exporter" + env := []string{} + dsn := fmt.Sprintf("postgresql://%s:%s@%s:5432/postgres?sslmode=disable", t.UserName, t.Password, postgres.PREFIXPGCONTAINER+t.ContainerName) + env = append(env, misc.StringToDockerEnvVal("DATA_SOURCE_NAME", dsn)) + config := container.Config{ + Image: image, + Env: env, + } + postgresExporterContainer := dockerservice.NewContainer( + containerName, + config, + container.HostConfig{}, + nwConfig, + &types.ContainerStartOptions{}, + ) + return postgresExporterContainer, nil +} diff --git a/internal/postgres/postgres.go b/internal/postgres/postgres.go new file mode 100644 index 0000000..1067812 --- /dev/null +++ b/internal/postgres/postgres.go @@ -0,0 +1,120 @@ +package postgres + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/api/types/volume" + "github.com/docker/go-connections/nat" + "github.com/spinup-host/internal/dockerservice" + "github.com/spinup-host/misc" +) + +const ( + PREFIXPGCONTAINER = "spinup-postgres-" + PGDATADIR = "/var/lib/postgresql/data/" +) + +func NewPostgresContainer(image, name, postgresUsername, postgresPassword string) (dockerservice.Container, error) { + dockerClient, err := dockerservice.NewDocker() + if err != nil { + fmt.Printf("error creating client %v", err) + } + newVolume, err := dockerservice.CreateVolume(context.Background(), dockerClient, volume.VolumeCreateBody{ + Driver: "local", + Labels: map[string]string{"purpose": "postgres data"}, + Name: name, + }) + if err != nil { + log.Println("error creating volume::", err) + return dockerservice.Container{}, err + } + + _, err = dockerservice.CreateNetwork(context.Background(), dockerClient, name+"_default", types.NetworkCreate{CheckDuplicate: true}) + if err != nil { + return dockerservice.Container{}, err + } + + port, err := misc.Portcheck() + if err != nil { + return dockerservice.Container{}, err + } + containerName := PREFIXPGCONTAINER + name + + newHostPort, err := nat.NewPort("tcp", strconv.Itoa(port)) + if err != nil { + return dockerservice.Container{}, err + } + newContainerport, err := nat.NewPort("tcp", "5432") + if err != nil { + log.Println("error here: ", err) + return dockerservice.Container{}, err + } + + hostConfig := container.HostConfig{ + PortBindings: nat.PortMap{ + newContainerport: []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: newHostPort.Port(), + }, + }, + }, + NetworkMode: "default", + AutoRemove: false, + } + + containerOptions := &types.ContainerStartOptions{} + endpointConfig := map[string]*network.EndpointSettings{} + networkName := name + "_default" + // setting key and value for the map. networkName=$dbname_default (eg: viggy_default) + endpointConfig[networkName] = &network.EndpointSettings{} + nwConfig := network.NetworkingConfig{EndpointsConfig: endpointConfig} + mounts := []mount.Mount{} + mounts = append(mounts, mount.Mount{ + Type: mount.TypeVolume, + Source: newVolume.Name, + Target: "/var/lib/postgresql/data", + }) + env := []string{} + env = append(env, misc.StringToDockerEnvVal("POSTGRES_USER", postgresUsername)) + env = append(env, misc.StringToDockerEnvVal("POSTGRES_PASSWORD", postgresPassword)) + + postgresContainer := dockerservice.NewContainer( + containerName, + container.Config{ + Image: image, + Env: env, + }, + hostConfig, + nwConfig, + containerOptions, + ) + postgresContainer.HostConfig.Mounts = mounts + return *postgresContainer, nil +} + +func ReloadPostgres(d dockerservice.Docker, execpath, datapath, containerName string) error { + execConfig := types.ExecConfig{ + User: "postgres", + Tty: false, + WorkingDir: execpath, + AttachStderr: true, + AttachStdout: true, + Cmd: []string{"pg_ctl", "-D", datapath, "reload"}, + } + pgContainer, err := d.GetContainer(context.Background(), containerName) + if err != nil { + return fmt.Errorf("error getting container for container name %s %v", containerName, err) + } + if _, err := pgContainer.ExecCommand(context.Background(), d, execConfig); err != nil { + return fmt.Errorf("error executing command %s %w", execConfig.Cmd[0], err) + } + return nil +} diff --git a/main.go b/main.go index cfe4590..ee80d93 100644 --- a/main.go +++ b/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "embed" "log" "github.com/spinup-host/internal/cmd" @@ -18,6 +17,3 @@ func main() { log.Fatal(err) } } - -//go:embed templates/* -var DockerTempl embed.FS diff --git a/metrics/metrics.go b/metrics/metrics.go index 2b146f8..10612e5 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -1,15 +1,15 @@ package metrics import ( - "log" + "fmt" "net/http" "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/spinup-host/api" "github.com/spinup-host/config" + "github.com/spinup-host/internal/metastore" ) func HandleMetrics(w http.ResponseWriter, req *http.Request) { @@ -17,16 +17,17 @@ func HandleMetrics(w http.ResponseWriter, req *http.Request) { http.Error(w, "Invalid Method", http.StatusMethodNotAllowed) return } - authHeader := req.Header.Get("Authorization") - apiKeyHeader := req.Header.Get("x-api-key") - var err error - config.Cfg.UserID, err = config.ValidateUser(authHeader, apiKeyHeader) - if err != nil { - log.Printf(err.Error()) - http.Error(w, "error validating user", http.StatusUnauthorized) - return - } - recordMetrics() + /* authHeader := req.Header.Get("Authorization") + apiKeyHeader := req.Header.Get("x-api-key") + var err error + config.Cfg.UserID, err = config.ValidateUser(authHeader, apiKeyHeader) + if err != nil { + log.Printf(err.Error()) + http.Error(w, "error validating user", http.StatusUnauthorized) + return + } */ + errCh := make(chan error, 1) + recordMetrics(errCh) promhttp.Handler().ServeHTTP(w, req) } @@ -37,12 +38,25 @@ var ( }) ) -func recordMetrics() { +var db metastore.Db +var err error + +func recordMetrics(errCh chan error) { + dbPath := config.Cfg.Common.ProjectDir + "/" + config.Cfg.UserID + "/" + config.Cfg.UserID + ".db" + if db.Client == nil { + db, err = metastore.NewDb(dbPath) + if err != nil { + return + } + } go func() { for { time.Sleep(2 * time.Second) - dbPath := config.Cfg.Common.ProjectDir + "/" + config.Cfg.UserID - clusterInfos := api.ReadClusterInfo(dbPath, config.Cfg.UserID) + clusterInfos, err := metastore.ClustersInfo(db) + if err != nil { + errCh <- fmt.Errorf("couldn't read cluster info %w", err) + close(errCh) + } containersCreated.Set(float64(len(clusterInfos))) } }() diff --git a/misc/misc.go b/misc/misc.go index 6343bd5..303eb2c 100644 --- a/misc/misc.go +++ b/misc/misc.go @@ -1,6 +1,19 @@ package misc -func MinMax(array []int) (int, int) { +import ( + "bytes" + "fmt" + "log" + "net" + "net/http" + "os/exec" + "strings" + "time" + + "github.com/spinup-host/config" +) + +func minMax(array []int) (int, int) { var max int = array[0] var min int = array[0] for _, value := range array { @@ -13,3 +26,62 @@ func MinMax(array []int) (int, int) { } return min, max } + +func Portcheck() (int, error) { + min, endingPort := minMax(config.Cfg.Common.Ports) + for port := min; port <= endingPort; port++ { + target := fmt.Sprintf("%s:%d", "localhost", port) + conn, err := net.DialTimeout("tcp", target, 3*time.Second) + if err == nil { + // we were able to connect, post is already used + log.Printf("INFO: port %d in use", port) + continue + } else { + if strings.Contains(err.Error(), "connect: connection refused") { + // could not reach port (probably because port is not in use) + log.Printf("INFO: port %d is unused", port) + return port, nil + } else { + // could not reach port because of some other error + log.Printf("INFO: failed to reach port %d and checking next port: %d", port, port+1) + } + defer conn.Close() + } + } + log.Printf("WARN: all allocated ports are occupied") + return 0, fmt.Errorf("error all allocated ports are occupied") +} + +func GetContainerIdByName(name string) (string, error) { + name = "name=" + name + command := "docker ps -f name=" + name + " --format '{{.ID}}'" + cmd := exec.Command("/bin/bash", "-c", command) + // trying to directly run docker is not working correctly. ref https://stackoverflow.com/questions/53640424/exit-code-125-from-docker-when-trying-to-run-container-programmatically + // cmd := exec.Command("docker", "ps", "-f", name, "--format '{{.ID}}'") + var out bytes.Buffer + cmd.Stdout = &out + if err := cmd.Run(); err != nil { + return "", err + } + return string(out.String()), nil +} + +// SliceContainsString returns true if str present in s +func SliceContainsString(s []string, str string) bool { + for _, v := range s { + if v == str { + return true + } + } + return false +} + +func ErrorResponse(w http.ResponseWriter, msg string, statuscode int) { + w.WriteHeader(statuscode) + w.Write([]byte(msg)) +} + +func StringToDockerEnvVal(key, val string) string { + keyval := key + "=" + val + return keyval +} diff --git a/misc/misc_test.go b/misc/misc_test.go new file mode 100644 index 0000000..781be39 --- /dev/null +++ b/misc/misc_test.go @@ -0,0 +1,39 @@ +package misc + +import "testing" + +type SliceContainsStringData struct { + Slice []string + Contains string + Expected bool +} + +func TestSliceContainsString(t *testing.T) { + /* actual := SliceContainsString([]string{"hello", "world"}, "world") + expected := true + if actual != expected { + t.Errorf("actual %t, expected %t", actual, expected) + } */ + SliceContainsStrings := []SliceContainsStringData{ + { + Slice: []string{"hello", "world"}, + Contains: "world", + Expected: true, + }, + { + Slice: []string{"hello", "world"}, + Contains: "word", + Expected: false, + }, + { + Slice: []string{"/hello", "/world"}, + Contains: "/world", + Expected: true, + }, + } + for _, iter := range SliceContainsStrings { + if actual := SliceContainsString(iter.Slice, iter.Contains); actual != iter.Expected { + t.Errorf("actual %t, expected %t", actual, iter.Expected) + } + } +} diff --git a/scripts/install-spinup.sh b/scripts/install-spinup.sh index 57530ac..922ff15 100755 --- a/scripts/install-spinup.sh +++ b/scripts/install-spinup.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -for req in "docker" "docker-compose" "openssl" "npm"; +for req in "docker" "openssl" "npm"; do if [ ! $(command -v "$req") ]; then echo "Cannot find or execute '$req' command" diff --git a/templates/templates.go b/templates/templates.go deleted file mode 100644 index 11b588a..0000000 --- a/templates/templates.go +++ /dev/null @@ -1,8 +0,0 @@ -package templates - -import ( - "embed" -) - -//go:embed templates/* -var DockerTempl embed.FS diff --git a/templates/templates/docker-compose-template.yml b/templates/templates/docker-compose-template.yml deleted file mode 100644 index 50dc658..0000000 --- a/templates/templates/docker-compose-template.yml +++ /dev/null @@ -1,15 +0,0 @@ -# docker-compose to spin up postgres -version: "3.9" -services: - postgres: - image: {{ .Architecture }}/{{ .Type }}:{{.MajVersion}}.{{.MinVersion}} - restart: unless-stopped - ports: - - "{{ .Port }}:5432" - environment: - POSTGRES_USER: {{ .PostgresUsername }} - POSTGRES_PASSWORD: {{ .PostgresPassword }} - volumes: - - data-volume-{{ .UserID }}:/var/lib/postgresql/data -volumes: - data-volume-{{ .UserID }}: \ No newline at end of file