Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

23/add s3 support to all input output files #43

Closed
wants to merge 2 commits into from
Closed

23/add s3 support to all input output files #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d9cbbef
Remove s3-path option
Jun 20, 2019
b2944fc
Clean up file name
Jun 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions command/load.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
"path"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -132,7 +133,7 @@ func cliCommandLoad(cmd *cobra.Command, args []string) {

// Start the loading process
log.Info("🚜 ", aurora.Bold(aurora.Green("Loading the anonymized database")), " 🚜")
if err = load(dbConf, viper.GetString("load.load-file"), viper.GetString("load.s3-file-path")); err != nil {
if err = load(dbConf, viper.GetString("load.load-file")); err != nil {
log.Error(err)
log.Error("❌ Gonymizer did not exit properly. See above for errors ❌")
os.Exit(1)
Expand All @@ -153,15 +154,16 @@ func cliCommandLoad(cmd *cobra.Command, args []string) {
}

// load starts the loading process.
func load(conf gonymizer.PGConfig, loadFile, s3FilePath string) (err error) {
// Check for S3 file here. If it is defined we should download it to loadFile's path and then load it.
if s3FilePath != "" {
log.Infof("🚛 Downloading from S3 '%s' -> %s\n", s3FilePath, loadFile)
func load(conf gonymizer.PGConfig, filePath string) (err error) {
// Check for S3 file here. If it is defined we should download it to loadFile's filePath and then load it.
if strings.HasPrefix(strings.ToLower(filePath), "s3://") {
anonFile := new(gonymizer.S3File)
if err = anonFile.ParseS3Url(s3FilePath); err != nil {
if err = anonFile.ParseS3Url(filePath); err != nil {
return err
}
if err = gonymizer.GetFileFromS3(nil, anonFile, loadFile); err != nil {
tmpPath := "/tmp/" + path.Base(anonFile.FilePath)
log.Infof("🚛 Downloading from S3 '%s' -> %s\n", filePath, tmpPath)
if err = gonymizer.GetFileFromS3(nil, anonFile, tmpPath); err != nil {
return err
}
}
Expand Down
10 changes: 5 additions & 5 deletions command/process.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,30 +125,30 @@ func cliCommandProcess(cmd *cobra.Command, args []string) {
}

// process is the entry point for processing a dump file according to the map file.
func process(dumpFile, mapFile, processedDumpFile string, generateSeed bool, s3file *gonymizer.S3File) (err error) {
func process(dumpFile, mapFile, processedFile string, generateSeed bool) (err error) {
log.Info("Loading map file from: ", mapFile)
columnMap, err := gonymizer.LoadConfigSkeleton(mapFile)
if err != nil {
return err
}

log.Info("Processing dump file: ", dumpFile)
err = gonymizer.ProcessDumpFile(columnMap, dumpFile, processedDumpFile, postProcessFile, generateSeed)
err = gonymizer.ProcessDumpFile(columnMap, dumpFile, processedFile, postProcessFile, generateSeed)
if err != nil {
return err
}

// Upload the processed file to S3 (iff the user selected this option and it is valid)
log.Info("S3 scheme: ", s3file.Scheme)
if s3file.Scheme == "s3" {
log.Infof("🚛 Uploading '%s' => S3: %s\n", processedDumpFile, s3file.URL)
log.Infof("🚛 Uploading '%s' => S3: %s\n", processedFile, s3file.URL)
sess, err := session.NewSession(&aws.Config{Region: aws.String(s3file.Region)})
if err != nil {
return err
}

if err = gonymizer.AddFileToS3(sess, processedDumpFile, s3file); err != nil {
log.Errorf("Unable to upload '%s' => '%s'", processedDumpFile, s3file.URL)
if err = gonymizer.AddFileToS3(sess, processedFile, s3file); err != nil {
log.Errorf("Unable to upload '%s' => '%s'", processedFile, s3file.URL)
return err
}
}
Expand Down