From 611c5cc61172829475107842191f31c05499700e Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Jan 2024 16:49:28 +0100 Subject: [PATCH 01/10] initial commit --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f15e7bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +web/* \ No newline at end of file From 9d8b80a6147de56a5d07545bee76e09ff33ce272 Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Jan 2024 16:51:40 +0100 Subject: [PATCH 02/10] connection to s3 bucket --- config.go | 72 ++++++++++++++++++++++++++++++++++++ go.mod | 50 +++++++++++++++++++++++++ go.sum | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 13 +++++++ s3.go | 50 +++++++++++++++++++++++++ 5 files changed, 294 insertions(+) create mode 100644 config.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 s3.go diff --git a/config.go b/config.go new file mode 100644 index 0000000..0b096cd --- /dev/null +++ b/config.go @@ -0,0 +1,72 @@ +package main + +import ( + "strings" + + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +func get_config() metadata_S3Config { + parseConfig() + s3_conf := configS3Storage() + return s3_conf + +} + +type metadata_S3Config struct { + URL string + Port int + AccessKey string + SecretKey string + Bucket string + Region string + Chunksize int + Cacert string + Web_metadata_folder string +} + +func configS3Storage() metadata_S3Config { + s3 := metadata_S3Config{} + s3.URL = viper.GetString("metadata_s3.url") + s3.AccessKey = viper.GetString("metadata_s3.accesskey") + s3.SecretKey = viper.GetString("metadata_s3.secretkey") + s3.Bucket = viper.GetString("metadata_s3.bucket") + s3.Port = 9000 + s3.Web_metadata_folder = viper.GetString("metadata_s3.Web_metadata_folder") + if viper.IsSet("s3.port") { + s3.Port = viper.GetInt("metadata_s3.port") + } + + if viper.IsSet("s3.region") { + s3.Region = viper.GetString("metadata_s3.region") + } + + if viper.IsSet("s3.chunksize") { + s3.Chunksize = viper.GetInt("metadata_s3.chunksize") * 1024 * 1024 + } + + if viper.IsSet("s3.cacert") { + s3.Cacert = viper.GetString("metadata_s3.cacert") + } + + return s3 +} + +func parseConfig() { + viper.SetConfigName("config") + viper.AddConfigPath(".") + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + if viper.IsSet("configFile") { + viper.SetConfigFile(viper.GetString("configFile")) + } + if err := viper.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); ok { + log.Infoln("No config file found, using ENVs only") + } else { + log.Fatalf("Error when reading config file: '%s'", err) + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7f673bb --- /dev/null +++ b/go.mod @@ -0,0 +1,50 @@ +module lpg + +go 1.20 + +require ( + github.com/aws/aws-sdk-go-v2 v1.24.1 + github.com/aws/aws-sdk-go-v2/config v1.26.5 + github.com/aws/aws-sdk-go-v2/credentials v1.16.16 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 + github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 + github.com/sirupsen/logrus v1.9.3 + github.com/spf13/viper v1.18.2 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect + github.com/aws/smithy-go v1.19.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b6ab93b --- /dev/null +++ b/go.sum @@ -0,0 +1,109 @@ +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4 h1:OCs21ST2LrepDfD3lwlQiOqIGp6JiEUqG84GzTDoyJs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.4/go.mod h1:usURWEKSNNAcAZuzRn/9ZYPT8aZQkR7xcCtunK/LkJo= +github.com/aws/aws-sdk-go-v2/config v1.26.5 h1:lodGSevz7d+kkFJodfauThRxK9mdJbyutUxGq1NNhvw= +github.com/aws/aws-sdk-go-v2/config v1.26.5/go.mod h1:DxHrz6diQJOc9EwDslVRh84VjjrE17g+pVZXUeSxaDU= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16 h1:8q6Rliyv0aUFAVtzaldUEcS+T5gbadPbWdV1WcAddK8= +github.com/aws/aws-sdk-go-v2/credentials v1.16.16/go.mod h1:UHVZrdUsv63hPXFo1H7c5fEneoVo9UXiz36QG1GEPi0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13 h1:8Nt4LBUEKV0FxLBO2BmRzDKax3hp2LRMKySMBwL4vMc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.13/go.mod h1:t5QEDu/FBJJM4kslbQlTSpYtnhoWDNmHSsgQojIxE0o= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10 h1:5oE2WzJE56/mVveuDZPJESKlg/00AaS2pY2QZcnxg4M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.2.10/go.mod h1:FHbKWQtRBYUz4vO5WBWjzMD2by126ny5y/1EoaWoLfI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10 h1:L0ai8WICYHozIKK+OtPzVJBugL7culcuM4E4JOpIEm8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.2.10/go.mod h1:byqfyxJBshFk0fF9YmK0M0ugIO8OWjzH2T3bPG4eGuA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10 h1:KOxnQeWy5sXyS37fdKEvAsGHOr9fa/qvwxfJurR/BzE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.16.10/go.mod h1:jMx5INQFYFYB3lQD9W0D8Ohgq6Wnl7NYOJ2TQndbulI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0 h1:PJTdBMsyvra6FtED7JZtDpQrIAflYDHFoZAu/sKYkwU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.48.0/go.mod h1:4qXHrG1Ne3VGIMZPCB8OjH/pLFO94sKABIusjh0KWPU= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7 h1:eajuO3nykDPdYicLlP3AGgOyVN3MOlFmZv7WGTuJPow= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.7/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7 h1:QPMJf+Jw8E1l7zqhZmMlFw6w1NmfkfiSK8mS4zOx3BA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.7/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..37c9e52 --- /dev/null +++ b/main.go @@ -0,0 +1,13 @@ +package main + +import ( + log "github.com/sirupsen/logrus" +) + +func main() { + log.Infoln("started app successfully") + mconf := get_config() + m_client := connect_to_s3(mconf) + log.Infof("Connection to the bucket established") + metadata_downloader(m_client) +} diff --git a/s3.go b/s3.go new file mode 100644 index 0000000..ee37cea --- /dev/null +++ b/s3.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "crypto/tls" + "net/http" + + "github.com/aws/aws-sdk-go-v2/aws" + awshttp "github.com/aws/aws-sdk-go-v2/aws/transport/http" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + log "github.com/sirupsen/logrus" +) + +type metadata_backend struct { + Client *s3.Client + Bucket string +} + +func connect_to_s3(mconf metadata_S3Config) *metadata_backend { + httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { + if tr.TLSClientConfig == nil { + tr.TLSClientConfig = &tls.Config{} + } + tr.TLSClientConfig.MinVersion = tls.VersionTLS13 + }) + + cfg, err := config.LoadDefaultConfig(context.TODO(), + config.WithRegion(mconf.Region), + config.WithHTTPClient(httpClient), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(mconf.AccessKey, mconf.SecretKey, "")), + config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{URL: mconf.URL}, nil + })), + ) + + if err != nil { + log.Fatal("Could not connect to metadata S3 bucket: ", err) + } + client := s3.NewFromConfig(cfg) + metadata_client := &metadata_backend{ + + Client: client, + Bucket: mconf.Bucket, + } + + return metadata_client +} From 8402344167a241fe4f06275208bed09cf926573f Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Jan 2024 16:52:41 +0100 Subject: [PATCH 03/10] added dev utils --- dev_utils/compose.yaml | 16 ++++++++++++++++ dev_utils/config.yaml | 9 +++++++++ 2 files changed, 25 insertions(+) create mode 100644 dev_utils/compose.yaml create mode 100644 dev_utils/config.yaml diff --git a/dev_utils/compose.yaml b/dev_utils/compose.yaml new file mode 100644 index 0000000..d99d7cb --- /dev/null +++ b/dev_utils/compose.yaml @@ -0,0 +1,16 @@ +version: "3.7" +services: + minio: + command: server /data + container_name: s3 + environment: + - MINIO_ACCESS_KEY=myaccesskey + - MINIO_SECRET_KEY=mysecretkey + healthcheck: + test: ["CMD", "curl", "-fkq", "https://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 3 + image: minio/minio:RELEASE.2020-10-28T08-16-50Z + ports: + - "9000:9000" \ No newline at end of file diff --git a/dev_utils/config.yaml b/dev_utils/config.yaml new file mode 100644 index 0000000..e555a6a --- /dev/null +++ b/dev_utils/config.yaml @@ -0,0 +1,9 @@ +metadata_s3: + url: "http://127.0.0.1:9000" + accesskey: "myaccesskey" + secretkey: "mysecretkey" + region: "us-east-1" + bucket: "test/" + web_metadata_folder: "web/content/" + + From 44ea9b6142075818fb2fd577ef48805cfa3e5a1e Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Jan 2024 16:53:39 +0100 Subject: [PATCH 04/10] docker file for cluster --- Dockerfile | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..bdcb16b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM golang:1.20.3-alpine3.17 as build +COPY . . +ENV GO111MODULE=on +ENV GOPATH=$PWD +ENV GOOS=linux +RUN go build -o /test . + + +FROM alpine:3.17 +RUN apk add --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community hugo +COPY --from=build /test . +COPY dev_utils . +CMD ["/test"] +COPY /web /web \ No newline at end of file From 9f3aa98fa5acd71c8b5dffd069a6060945474095 Mon Sep 17 00:00:00 2001 From: varshith Date: Thu, 25 Jan 2024 16:54:21 +0100 Subject: [PATCH 05/10] downloader for metadata files from s3 bucket --- download_metadata.go | 66 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 download_metadata.go diff --git a/download_metadata.go b/download_metadata.go new file mode 100644 index 0000000..2efd3fc --- /dev/null +++ b/download_metadata.go @@ -0,0 +1,66 @@ +package main + +import ( + "context" + "os" + "path/filepath" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + log "github.com/sirupsen/logrus" +) + +func metadata_downloader(m_client *metadata_backend) { + + var ( + Bucket = m_client.Bucket // Download from this bucket + Prefix = "datasets/" // Using this key prefix + LocalDirectory = "web/content/" // Into this directory + ) + client := m_client.Client + + manager := manager.NewDownloader(client) + paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ + Bucket: &Bucket, + Prefix: &Prefix, + }) + log.Infoln("Started downloading metadata files") + for paginator.HasMorePages() { + page, err := paginator.NextPage(context.TODO()) + if err != nil { + log.Fatalln("error:", err) + } + for _, obj := range page.Contents { + if err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)); err != nil { + log.Fatalln("error:", err) + } + } + + } + log.Infoln("Completed downloading metadatafiles") + +} + +func downloadToFile(downloader *manager.Downloader, targetDirectory, bucket, key string) error { + // Create the directories in the path + file := filepath.Join(targetDirectory, key) + if err := os.MkdirAll(filepath.Dir(file), 0775); err != nil { + return err + } + + // Set up the local file + fd, err := os.Create(file) + if err != nil { + return err + } + defer fd.Close() + // Download the file using the AWS SDK for Go + //fmt.Printf("Downloading s3://%s/%s to %s...\n", bucket, key, file) + + _, err = downloader.Download(context.TODO(), fd, &s3.GetObjectInput{Bucket: &bucket, Key: &key}) + if err != nil { + log.Infoln("Failed to download metadatafiles") + } + return err +} From 7fe3ff24ac6282014881fdd07afa53276baf15d0 Mon Sep 17 00:00:00 2001 From: varshith Date: Fri, 26 Jan 2024 10:52:01 +0100 Subject: [PATCH 06/10] variable case fixed --- config.go | 10 +++++----- s3.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config.go b/config.go index 0b096cd..cf204c2 100644 --- a/config.go +++ b/config.go @@ -7,14 +7,14 @@ import ( "github.com/spf13/viper" ) -func get_config() metadata_S3Config { +func get_config() metadata_s3_config { parseConfig() s3_conf := configS3Storage() return s3_conf } -type metadata_S3Config struct { +type metadata_s3_config struct { URL string Port int AccessKey string @@ -26,14 +26,14 @@ type metadata_S3Config struct { Web_metadata_folder string } -func configS3Storage() metadata_S3Config { - s3 := metadata_S3Config{} +func configS3Storage() metadata_s3_config { + s3 := metadata_s3_config{} s3.URL = viper.GetString("metadata_s3.url") s3.AccessKey = viper.GetString("metadata_s3.accesskey") s3.SecretKey = viper.GetString("metadata_s3.secretkey") s3.Bucket = viper.GetString("metadata_s3.bucket") s3.Port = 9000 - s3.Web_metadata_folder = viper.GetString("metadata_s3.Web_metadata_folder") + s3.Web_metadata_folder = viper.GetString("metadata_s3.web_metadata_folder") if viper.IsSet("s3.port") { s3.Port = viper.GetInt("metadata_s3.port") } diff --git a/s3.go b/s3.go index ee37cea..9332bb7 100644 --- a/s3.go +++ b/s3.go @@ -18,7 +18,7 @@ type metadata_backend struct { Bucket string } -func connect_to_s3(mconf metadata_S3Config) *metadata_backend { +func connect_to_s3(mconf metadata_s3_config) *metadata_backend { httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { if tr.TLSClientConfig == nil { tr.TLSClientConfig = &tls.Config{} From 1e90a70ba869a2a73d89eb79e69ff816a2afdf46 Mon Sep 17 00:00:00 2001 From: varshith Date: Fri, 26 Jan 2024 11:17:20 +0100 Subject: [PATCH 07/10] variable naming fixed --- config.go | 48 +++++++++++++++++++++---------------------- dev_utils/config.yaml | 4 ++-- download_metadata.go | 10 ++++----- main.go | 6 +++--- s3.go | 14 ++++++------- 5 files changed, 41 insertions(+), 41 deletions(-) diff --git a/config.go b/config.go index cf204c2..6d9e509 100644 --- a/config.go +++ b/config.go @@ -7,47 +7,47 @@ import ( "github.com/spf13/viper" ) -func get_config() metadata_s3_config { +func getConfig() MetadataS3Config { parseConfig() - s3_conf := configS3Storage() - return s3_conf + S3Conf := configS3Storage() + return S3Conf } -type metadata_s3_config struct { - URL string - Port int - AccessKey string - SecretKey string - Bucket string - Region string - Chunksize int - Cacert string - Web_metadata_folder string +type MetadataS3Config struct { + URL string + Port int + AccessKey string + SecretKey string + Bucket string + Region string + Chunksize int + Cacert string + WebMetadataFolder string } -func configS3Storage() metadata_s3_config { - s3 := metadata_s3_config{} - s3.URL = viper.GetString("metadata_s3.url") - s3.AccessKey = viper.GetString("metadata_s3.accesskey") - s3.SecretKey = viper.GetString("metadata_s3.secretkey") - s3.Bucket = viper.GetString("metadata_s3.bucket") +func configS3Storage() MetadataS3Config { + s3 := MetadataS3Config{} + s3.URL = viper.GetString("S3MetadataBucket.url") + s3.AccessKey = viper.GetString("S3MetadataBucket.accesskey") + s3.SecretKey = viper.GetString("S3MetadataBucket.secretkey") + s3.Bucket = viper.GetString("S3MetadataBucket.bucket") s3.Port = 9000 - s3.Web_metadata_folder = viper.GetString("metadata_s3.web_metadata_folder") + s3.WebMetadataFolder = viper.GetString("S3MetadataBucket.WebMetadataFolder") if viper.IsSet("s3.port") { - s3.Port = viper.GetInt("metadata_s3.port") + s3.Port = viper.GetInt("S3MetadataBucket.port") } if viper.IsSet("s3.region") { - s3.Region = viper.GetString("metadata_s3.region") + s3.Region = viper.GetString("S3MetadataBucket.region") } if viper.IsSet("s3.chunksize") { - s3.Chunksize = viper.GetInt("metadata_s3.chunksize") * 1024 * 1024 + s3.Chunksize = viper.GetInt("S3MetadataBucket.chunksize") * 1024 * 1024 } if viper.IsSet("s3.cacert") { - s3.Cacert = viper.GetString("metadata_s3.cacert") + s3.Cacert = viper.GetString("S3MetadataBucket.cacert") } return s3 diff --git a/dev_utils/config.yaml b/dev_utils/config.yaml index e555a6a..efaa0fd 100644 --- a/dev_utils/config.yaml +++ b/dev_utils/config.yaml @@ -1,9 +1,9 @@ -metadata_s3: +S3MetadataBucket: url: "http://127.0.0.1:9000" accesskey: "myaccesskey" secretkey: "mysecretkey" region: "us-east-1" bucket: "test/" - web_metadata_folder: "web/content/" + WebMetadataFolder: "web/content/" diff --git a/download_metadata.go b/download_metadata.go index 2efd3fc..0e72a77 100644 --- a/download_metadata.go +++ b/download_metadata.go @@ -11,14 +11,14 @@ import ( log "github.com/sirupsen/logrus" ) -func metadata_downloader(m_client *metadata_backend) { +func metadataDownloader(Metadataclient *MetadataBackend) { var ( - Bucket = m_client.Bucket // Download from this bucket - Prefix = "datasets/" // Using this key prefix - LocalDirectory = "web/content/" // Into this directory + Bucket = Metadataclient.Bucket // Download from this bucket + Prefix = "datasets/" // Using this key prefix + LocalDirectory = "web/content/" // Into this directory ) - client := m_client.Client + client := Metadataclient.Client manager := manager.NewDownloader(client) paginator := s3.NewListObjectsV2Paginator(client, &s3.ListObjectsV2Input{ diff --git a/main.go b/main.go index 37c9e52..c400ce4 100644 --- a/main.go +++ b/main.go @@ -6,8 +6,8 @@ import ( func main() { log.Infoln("started app successfully") - mconf := get_config() - m_client := connect_to_s3(mconf) + mConf := getConfig() + Metadataclient := connect_to_s3(mConf) log.Infof("Connection to the bucket established") - metadata_downloader(m_client) + metadataDownloader(Metadataclient) } diff --git a/s3.go b/s3.go index 9332bb7..33c7bc6 100644 --- a/s3.go +++ b/s3.go @@ -13,12 +13,12 @@ import ( log "github.com/sirupsen/logrus" ) -type metadata_backend struct { +type MetadataBackend struct { Client *s3.Client Bucket string } -func connect_to_s3(mconf metadata_s3_config) *metadata_backend { +func connect_to_s3(mConf MetadataS3Config) *MetadataBackend { httpClient := awshttp.NewBuildableClient().WithTransportOptions(func(tr *http.Transport) { if tr.TLSClientConfig == nil { tr.TLSClientConfig = &tls.Config{} @@ -27,12 +27,12 @@ func connect_to_s3(mconf metadata_s3_config) *metadata_backend { }) cfg, err := config.LoadDefaultConfig(context.TODO(), - config.WithRegion(mconf.Region), + config.WithRegion(mConf.Region), config.WithHTTPClient(httpClient), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(mconf.AccessKey, mconf.SecretKey, "")), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(mConf.AccessKey, mConf.SecretKey, "")), config.WithEndpointResolverWithOptions(aws.EndpointResolverWithOptionsFunc( func(service, region string, options ...interface{}) (aws.Endpoint, error) { - return aws.Endpoint{URL: mconf.URL}, nil + return aws.Endpoint{URL: mConf.URL}, nil })), ) @@ -40,10 +40,10 @@ func connect_to_s3(mconf metadata_s3_config) *metadata_backend { log.Fatal("Could not connect to metadata S3 bucket: ", err) } client := s3.NewFromConfig(cfg) - metadata_client := &metadata_backend{ + metadata_client := &MetadataBackend{ Client: client, - Bucket: mconf.Bucket, + Bucket: mConf.Bucket, } return metadata_client From e3fc65e017bd8a0ace558781ed3546754f806050 Mon Sep 17 00:00:00 2001 From: varshith Date: Tue, 30 Jan 2024 17:36:28 +0100 Subject: [PATCH 08/10] added error handling to s3 config and testing bucket connection --- s3.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/s3.go b/s3.go index 33c7bc6..c78e7be 100644 --- a/s3.go +++ b/s3.go @@ -37,7 +37,7 @@ func connect_to_s3(mConf MetadataS3Config) *MetadataBackend { ) if err != nil { - log.Fatal("Could not connect to metadata S3 bucket: ", err) + log.Fatalf("Error while setting up s3 config: %v\n ", err) } client := s3.NewFromConfig(cfg) metadata_client := &MetadataBackend{ @@ -45,6 +45,13 @@ func connect_to_s3(mConf MetadataS3Config) *MetadataBackend { Client: client, Bucket: mConf.Bucket, } - + _, err = metadata_client.Client.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(metadata_client.Bucket), + }) + if err != nil { + log.Fatal("Error while connecting to the metadata bucket ", err) + } else { + log.Infoln("Connection established to metadata bucket", metadata_client.Bucket) + } return metadata_client } From 74f84274d79cbcf29574751a85f59e45689e00c8 Mon Sep 17 00:00:00 2001 From: varshith Date: Fri, 2 Feb 2024 13:48:43 +0100 Subject: [PATCH 09/10] fixed docker file and error formatting --- Dockerfile | 6 ++++-- download_metadata.go | 2 +- s3.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index bdcb16b..bf734ba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM golang:1.20.3-alpine3.17 as build +WORKDIR /lp_app COPY . . ENV GO111MODULE=on ENV GOPATH=$PWD @@ -7,8 +8,9 @@ RUN go build -o /test . FROM alpine:3.17 +WORKDIR /gen_app RUN apk add --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community hugo -COPY --from=build /test . +COPY --from=build /test /gen_app/ COPY dev_utils . -CMD ["/test"] +CMD ["/gen_app/test"] COPY /web /web \ No newline at end of file diff --git a/download_metadata.go b/download_metadata.go index 0e72a77..b418279 100644 --- a/download_metadata.go +++ b/download_metadata.go @@ -33,7 +33,7 @@ func metadataDownloader(Metadataclient *MetadataBackend) { } for _, obj := range page.Contents { if err := downloadToFile(manager, LocalDirectory, Bucket, aws.ToString(obj.Key)); err != nil { - log.Fatalln("error:", err) + log.Fatalf("Error while getting next page from ListObjectsV2Paginator: %v", err) } } diff --git a/s3.go b/s3.go index c78e7be..6351f3e 100644 --- a/s3.go +++ b/s3.go @@ -49,7 +49,7 @@ func connect_to_s3(mConf MetadataS3Config) *MetadataBackend { Bucket: aws.String(metadata_client.Bucket), }) if err != nil { - log.Fatal("Error while connecting to the metadata bucket ", err) + log.Fatalf("Error while connecting to the metadata bucket %v\n ", err) } else { log.Infoln("Connection established to metadata bucket", metadata_client.Bucket) } From 22f5cba784ce58cb861c35473c45dcc9ab2ed252 Mon Sep 17 00:00:00 2001 From: varshith Date: Fri, 2 Feb 2024 13:57:48 +0100 Subject: [PATCH 10/10] added go linter --- .github/workflows/go-linter.yaml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .github/workflows/go-linter.yaml diff --git a/.github/workflows/go-linter.yaml b/.github/workflows/go-linter.yaml new file mode 100644 index 0000000..095b72a --- /dev/null +++ b/.github/workflows/go-linter.yaml @@ -0,0 +1,32 @@ +name: golangci-lint +on: + push: + branches: + - master + - main + pull_request: + branches: + - master + +permissions: + contents: read + # Optional: allow read access to pull request. Use with `only-new-issues` option. + # pull-requests: read + +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v4 + with: + go-version: '1.21' + cache: false + - name: golangci-lint + uses: golangci/golangci-lint-action@v3 + with: + # Require: The version of golangci-lint to use. + # When `install-mode` is `binary` (default) the value can be v1.2 or v1.2.3 or `latest` to use the latest version. + # When `install-mode` is `goinstall` the value can be v1.2.3, `latest`, or the hash of a commit. + version: v1.54 \ No newline at end of file