Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] bugfix/data-upload-size #103

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

[WIP] bugfix/data-upload-size #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
464bda5
bugfix/data-upload-size
jguevarra Oct 25, 2023
a613ca5
some file data parsing changes
jguevarra Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 59 additions & 98 deletions handlers/pallidsturgeonhandler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package handlers

import (
"encoding/csv"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"strconv"
Expand Down Expand Up @@ -1003,113 +1007,70 @@ func (sd *PallidSturgeonHandler) GetUploadSessionId(c echo.Context) error {
}

func (sd *PallidSturgeonHandler) Upload(c echo.Context) error {
var err error
uploads := models.Upload{}
if err := c.Bind(&uploads); err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
// Retrieve single uploaded file from the request.
file, err := c.FormFile("files")
log.Printf("file: %v", file)
if err != nil {
return c.JSON(http.StatusBadRequest, &models.Response{
Message: "Failed! No files were submitted",
Status: "Failed",
Data: nil,
})
}

sessionId, err := sd.Store.GetUploadSessionId()
// Open file
fileContent, err := file.Open()
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
return c.JSON(http.StatusBadRequest, &models.Response{
Message: "Failed! Unable to open the file",
Status: "Failed",
Data: nil,
})
}
defer fileContent.Close()

user := c.Get("PSUSER").(models.User)
// Create the CSV reader
csvReader := csv.NewReader(fileContent)
csvReader.FieldsPerRecord = -1

for _, uploadSite := range uploads.SiteUpload.Items {
uploadSite.LastUpdated = time.Now()
uploadSite.UploadedBy = user.FirstName + " " + user.LastName
uploadSite.UploadSessionId = sessionId
uploadSite.EditInitials = uploads.EditInitials
uploadSite.UploadFilename = uploads.SiteUpload.UploadFilename
err = sd.Store.SaveSiteUpload(uploadSite)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadFish := range uploads.FishUpload.Items {
uploadFish.LastUpdated = time.Now()
uploadFish.UploadedBy = user.FirstName + " " + user.LastName
uploadFish.UploadSessionId = sessionId
uploadFish.EditInitials = uploads.EditInitials
uploadFish.UploadFilename = uploads.FishUpload.UploadFilename
err = sd.Store.SaveFishUpload(uploadFish)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadSearch := range uploads.SearchUpload.Items {
uploadSearch.SearchDate = processStringTime(DerefString(uploadSearch.SearchDate), "db")
uploadSearch.LastUpdated = time.Now()
uploadSearch.UploadedBy = user.FirstName + " " + user.LastName
uploadSearch.UploadSessionId = sessionId
uploadSearch.EditInitials = uploads.EditInitials
uploadSearch.UploadFilename = uploads.SearchUpload.UploadFilename
err = sd.Store.SaveSearchUpload(uploadSearch)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadSupplemental := range uploads.SupplementalUpload.Items {
uploadSupplemental.LastUpdated = time.Now()
uploadSupplemental.UploadedBy = user.FirstName + " " + user.LastName
uploadSupplemental.UploadSessionId = sessionId
uploadSupplemental.EditInitials = uploads.EditInitials
uploadSupplemental.UploadFilename = uploads.SupplementalUpload.UploadFilename
err = sd.Store.SaveSupplementalUpload(uploadSupplemental)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadProcedure := range uploads.ProcedureUpload.Items {
uploadProcedure.ProcedureDate = processStringTime(DerefString(uploadProcedure.ProcedureDate), "db")
uploadProcedure.DstStartDate = processStringTime(DerefString(uploadProcedure.DstStartDate), "db")
uploadProcedure.LastUpdated = time.Now()
uploadProcedure.UploadedBy = user.FirstName + " " + user.LastName
uploadProcedure.UploadSessionId = sessionId
uploadProcedure.EditInitials = uploads.EditInitials
uploadProcedure.UploadFilename = uploads.ProcedureUpload.UploadFilename
err = sd.Store.SaveProcedureUpload(uploadProcedure)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadMoriver := range uploads.MoriverUpload.Items {
uploadMoriver.SetDate = processStringTime(DerefString(uploadMoriver.SetDate), "db")
uploadMoriver.LastUpdated = time.Now()
uploadMoriver.UploadedBy = user.FirstName + " " + user.LastName
uploadMoriver.UploadSessionId = sessionId
uploadMoriver.EditInitials = uploads.EditInitials
uploadMoriver.UploadFilename = uploads.MoriverUpload.UploadFilename
err = sd.Store.SaveMoriverUpload(uploadMoriver)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

for _, uploadTelemetry := range uploads.TelemetryUpload.Items {
uploadTelemetry.LastUpdated = time.Now()
uploadTelemetry.UploadedBy = user.FirstName + " " + user.LastName
uploadTelemetry.UploadSessionId = sessionId
uploadTelemetry.EditInitials = uploads.EditInitials
uploadTelemetry.UploadFilename = uploads.TelemetryUpload.UploadFilename
err = sd.Store.SaveTelemetryUpload(uploadTelemetry)
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
}
}

procedureOut, err := sd.Store.CallStoreProcedures(user.FirstName+" "+user.LastName, sessionId)
csvData, err := csvReader.ReadAll()
if err != nil {
return c.JSON(http.StatusInternalServerError, err.Error())
return c.JSON(http.StatusBadRequest, &models.Response{
Message: "Failed! Unable to read CSV file",
Status: "Failed",
Data: nil,
})
}

return c.JSON(http.StatusOK, procedureOut)
var moriverUpload models.UploadMoriver
var moriverUploads []models.UploadMoriver

for _, each := range csvData {
moriverUpload.SiteID, _ = strconv.Atoi(each[0])
moriverUpload.SiteFid = each[1]
moriverUpload.MrFid = each[2]
moriverUpload.SeFieldID = each[3]
moriverUpload.Season = each[4]
moriverUploads = append(moriverUploads, moriverUpload)
}

// Convert to JSON
jsonData, err := json.Marshal(moriverUploads)
if err != nil {
return c.JSON(http.StatusBadRequest, &models.Response{
Message: "Failed! Unable to convert to JSON",
Status: "Failed",
Data: nil,
})
}

fmt.Println(string(jsonData))

return c.JSON(http.StatusBadRequest, &models.Response{
Message: "Upload is finished :)",
Status: "Failed",
Data: nil,
})
}

func (sd *PallidSturgeonHandler) CallStoreProcedures(c echo.Context) error {
Expand Down
4 changes: 3 additions & 1 deletion models/auth.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,9 @@ type KeyCloakRole struct {
}

type Response struct {
Message string `json:"message"`
Message string `json:"message"`
Status string `json:"status"`
Data []string `json:"data"`
}

type Jwks struct {
Expand Down
180 changes: 90 additions & 90 deletions models/pallidsturgeon.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,96 +626,96 @@ type UploadMoriverData struct {
}

type UploadMoriver struct {
SiteID int `db:"site_id" json:"siteId"`
SiteFid string `db:"site_fid" json:"siteFid"`
MrID int `db:"mr_id" json:"mrId"`
MrFid string `db:"mr_fid" json:"mrFid"`
SeFieldID string `db:"se_field_id" json:"seFieldId"`
Season string `db:"season" json:"season"`
SetDate *string `db:"setdate" json:"setdate"`
SetDateTime time.Time `db:"setdate" json:"setDateTime"`
Subsample float64 `db:"subsample" json:"subsample"`
Subsamplepass float64 `db:"subsamplepass" json:"subsamplepass"`
SubsampleROrN string `db:"subsample_r_or_n" json:"subsampleROrN"`
Subsamplen string `db:"subsamplen" json:"subsamplen"`
Recorder string `db:"recorder" json:"recorder"`
Gear string `db:"gear" json:"gear"`
GearType string `db:"gear_type" json:"gearType"`
Temp float64 `db:"temp" json:"temp"`
Turbidity *float64 `db:"turbidity" json:"turbidity"`
Conductivity *float64 `db:"conductivity" json:"conductivity"`
Do *float64 `db:"do" json:"dissolvedOxygen"`
Distance *float64 `db:"distance" json:"distance"`
Width *float64 `db:"width" json:"width"`
Netrivermile *float64 `db:"netrivermile" json:"netrivermile"`
Structurenumber string `db:"structurenumber" json:"structurenumber"`
Usgs string `db:"usgs" json:"usgs"`
Riverstage *float64 `db:"riverstage" json:"riverstage"`
Discharge *float64 `db:"discharge" json:"discharge"`
U1 string `db:"u1" json:"u1"`
U2 string `db:"u2" json:"u2"`
U3 string `db:"u3" json:"u3"`
U4 string `db:"u4" json:"u4"`
U5 string `db:"u5" json:"u5"`
U6 string `db:"u6" json:"u6"`
U7 string `db:"u7" json:"u7"`
Macro string `db:"macro" json:"macro"`
Meso string `db:"meso" json:"meso"`
Habitatrn string `db:"habitatrn" json:"habitatrn"`
Qc string `db:"qc" json:"qc"`
MicroStructure string `db:"micro_structure" json:"microStructure"`
StructureFlow string `db:"structure_flow" json:"structureFlow"`
StructureMod string `db:"structure_mod" json:"structureMod"`
SetSite1 string `db:"set_site_1" json:"setSite1"`
SetSite2 string `db:"set_site_2" json:"setSite2"`
SetSite3 string `db:"set_site_3" json:"setSite3"`
StartTime string `db:"starttime" json:"startTime"`
StartLatitude float64 `db:"startlatitude" json:"startlatitude"`
StartLongitude float64 `db:"startlongitude" json:"startlongitude"`
StopTime string `db:"stoptime" json:"stoptime"`
StopLatitude *float64 `db:"stoplatitude" json:"stoplatitude"`
StopLongitude *float64 `db:"stop_longitude" json:"stoplongitude"`
Depth1 *float64 `db:"depth1" json:"depth1"`
Velocitybot1 *float64 `db:"velocitybot1" json:"velocitybot1"`
Velocity08_1 *float64 `db:"velocity08_1" json:"velocity081"`
Velocity02or06_1 *float64 `db:"velocity02or06_1" json:"velocity02or061"`
Depth2 *float64 `db:"depth2" json:"depth2"`
Velocitybot2 *float64 `db:"velocitybot2" json:"velocitybot2"`
Velocity08_2 *float64 `db:"velocity08_2" json:"velocity082"`
Velocity02or06_2 *float64 `db:"velocity02or06_2" json:"velocity02or062"`
Depth3 *float64 `db:"depth3" json:"depth3"`
Velocitybot3 *float64 `db:"velocitybot3" json:"velocitybot3"`
Velocity08_3 *float64 `db:"velocity08_3" json:"velocity083"`
Velocity02or06_3 *float64 `db:"velocity02or06_3" json:"velocity02or063"`
Watervel *float64 `db:"watervel" json:"watervel"`
Cobble *float64 `db:"cobble" json:"cobble"`
Organic *float64 `db:"organic" json:"organic"`
Silt *float64 `db:"silt" json:"silt"`
Sand *float64 `db:"sand" json:"sand"`
Gravel *float64 `db:"gravel" json:"gravel"`
Comments string `db:"comments" json:"comments"`
Complete *float64 `db:"complete" json:"complete"`
Checkby string `db:"checkby" json:"checkby"`
NoTurbidity string `db:"no_turbidity" json:"noTurbidity"`
NoVelocity string `db:"no_velocity" json:"noVelocity"`
EditInitials string `db:"edit_initials" json:"editInitials"`
LastEditComment string `db:"last_edit_comment" json:"lastEditComment"`
Project *int `db:"PROJECT_ID" json:"project"`
FieldOffice string `db:"FIELDOFFICE" json:"fieldOffice"`
Segment *int `db:"SEGMENT_ID" json:"segment"`
BkgColor string `db:"bkg_color" json:"bkgColor"`
SuppBkgColor string `db:"supp_bkg_color" json:"suppBkgColor"`
ProcBkgColor string `db:"proc_bkg_color" json:"procBkgColor"`
FishCount int `db:"fish_count" json:"fishCount"`
SuppCount int `db:"supp_count" json:"suppCount"`
ProcCount int `db:"proc_count" json:"procCount"`
Bend int `db:"bend" json:"bend"`
BendRn string `db:"bendrn" json:"bendrn"`
BendRiverMile float64 `db:"bendrivermile" json:"bendrivermile"`
LastUpdated time.Time `db:"last_updated" json:"lastUpdated"`
UploadSessionId int `db:"upload_session_id" json:"uploadSessionId"`
UploadedBy string `db:"uploaded_by" json:"uploadedBy"`
UploadFilename string `db:"upload_filename" json:"uploadFilename"`
SiteID int `db:"site_id" json:"siteId" csv:"SITEID"`
SiteFid string `db:"site_fid" json:"siteFid" csv:"SITEFID"`
MrID int `db:"mr_id" json:"mrId" csv:"MRID"`
MrFid string `db:"mr_fid" json:"mrFid" csv:"MRFID"`
SeFieldID string `db:"se_field_id" json:"seFieldId" csv:"SEFIELDID"`
Season string `db:"season" json:"season" csv:"SEASON"`
SetDate *string `db:"setdate" json:"setdate" csv:"SETDATE"`
SetDateTime time.Time `db:"setdate" json:"setDateTime" csv:"SETDATETIME"`
Subsample float64 `db:"subsample" json:"subsample" csv:"SUBSAMPLE"`
Subsamplepass float64 `db:"subsamplepass" json:"subsamplepass" csv:"SUBSAMPLEPASS"`
SubsampleROrN string `db:"subsample_r_or_n" json:"subsampleROrN" csv:"SUBSAMPLERORN"`
Subsamplen string `db:"subsamplen" json:"subsamplen" csv:"SUBSAMPLEN"`
Recorder string `db:"recorder" json:"recorder" csv:"RECORDER"`
Gear string `db:"gear" json:"gear" csv:"GEAR"`
GearType string `db:"gear_type" json:"gearType" csv:"GEARTYPE"`
Temp float64 `db:"temp" json:"temp" csv:"TEMP"`
Turbidity *float64 `db:"turbidity" json:"turbidity" csv:"TURBIDITY"`
Conductivity *float64 `db:"conductivity" json:"conductivity" csv:"CONDUCTIVITY"`
Do *float64 `db:"do" json:"dissolvedOxygen" csv:"DISSOLVEDOXYGEN"`
Distance *float64 `db:"distance" json:"distance" csv:"DISTANCE"`
Width *float64 `db:"width" json:"width" csv:"WIDTH"`
Netrivermile *float64 `db:"netrivermile" json:"netrivermile" csv:"NETRIVERMILE"`
Structurenumber string `db:"structurenumber" json:"structurenumber" csv:"STRUCTURENUMBER"`
Usgs string `db:"usgs" json:"usgs" csv:"USGS"`
Riverstage *float64 `db:"riverstage" json:"riverstage" csv:"RIVERSTAGE"`
Discharge *float64 `db:"discharge" json:"discharge" csv:"DISCHARGE"`
U1 string `db:"u1" json:"u1" csv:"U1"`
U2 string `db:"u2" json:"u2" csv:"U2"`
U3 string `db:"u3" json:"u3" csv:"U3"`
U4 string `db:"u4" json:"u4" csv:"U4"`
U5 string `db:"u5" json:"u5" csv:"U5"`
U6 string `db:"u6" json:"u6" csv:"U6"`
U7 string `db:"u7" json:"u7" csv:"U7"`
Macro string `db:"macro" json:"macro" csv:"MACRO"`
Meso string `db:"meso" json:"meso" csv:"MESO"`
Habitatrn string `db:"habitatrn" json:"habitatrn" csv:"HABITATRN"`
Qc string `db:"qc" json:"qc" csv:"QC"`
MicroStructure string `db:"micro_structure" json:"microStructure" csv:"MICROSTRUCTURE"`
StructureFlow string `db:"structure_flow" json:"structureFlow" csv:"STRUCTUREFLOW"`
StructureMod string `db:"structure_mod" json:"structureMod" csv:"STRUCTUREMOD"`
SetSite1 string `db:"set_site_1" json:"setSite1" csv:"SETSITE1"`
SetSite2 string `db:"set_site_2" json:"setSite2" csv:"SETSITE2"`
SetSite3 string `db:"set_site_3" json:"setSite3" csv:"SETSITE3"`
StartTime string `db:"starttime" json:"startTime" csv:"STARTTIME"`
StartLatitude float64 `db:"startlatitude" json:"startlatitude" csv:"STARTLATITUDE"`
StartLongitude float64 `db:"startlongitude" json:"startlongitude" csv:"STARTLONGITUDE"`
StopTime string `db:"stoptime" json:"stoptime" csv:"STOPTIME"`
StopLatitude *float64 `db:"stoplatitude" json:"stoplatitude" csv:"STOPLATITUDE"`
StopLongitude *float64 `db:"stop_longitude" json:"stoplongitude" csv:"STOPLONGITUDE"`
Depth1 *float64 `db:"depth1" json:"depth1" csv:"DEPTH1"`
Velocitybot1 *float64 `db:"velocitybot1" json:"velocitybot1" csv:"VELOCITYBOT1"`
Velocity08_1 *float64 `db:"velocity08_1" json:"velocity081" csv:"VELOCITY081"`
Velocity02or06_1 *float64 `db:"velocity02or06_1" json:"velocity02or061" csv:"VELOCITY02OR061"`
Depth2 *float64 `db:"depth2" json:"depth2" csv:"DEPTH2"`
Velocitybot2 *float64 `db:"velocitybot2" json:"velocitybot2" csv:"VELOCITYBOT2"`
Velocity08_2 *float64 `db:"velocity08_2" json:"velocity082" csv:"VELOCITY082"`
Velocity02or06_2 *float64 `db:"velocity02or06_2" json:"velocity02or062" csv:"VELOCITY02OR062"`
Depth3 *float64 `db:"depth3" json:"depth3" csv:"DEPTH3"`
Velocitybot3 *float64 `db:"velocitybot3" json:"velocitybot3" csv:"VELOCITYBOT3"`
Velocity08_3 *float64 `db:"velocity08_3" json:"velocity083" csv:"VELOCITY083"`
Velocity02or06_3 *float64 `db:"velocity02or06_3" json:"velocity02or063" csv:"VELOCITY02OR063"`
Watervel *float64 `db:"watervel" json:"watervel" csv:"WATERVEL"`
Cobble *float64 `db:"cobble" json:"cobble" csv:"COBBLE"`
Organic *float64 `db:"organic" json:"organic" csv:"ORGANIC"`
Silt *float64 `db:"silt" json:"silt" csv:"SILT"`
Sand *float64 `db:"sand" json:"sand" csv:"SAND"`
Gravel *float64 `db:"gravel" json:"gravel" csv:"GRAVEL"`
Comments string `db:"comments" json:"comments" csv:"COMMENTS"`
Complete *float64 `db:"complete" json:"complete" csv:"COMPLETE"`
Checkby string `db:"checkby" json:"checkby" csv:"CHECKBY"`
NoTurbidity string `db:"no_turbidity" json:"noTurbidity" csv:"NOTURBIDITY"`
NoVelocity string `db:"no_velocity" json:"noVelocity" csv:"NOVELOCITY"`
EditInitials string `db:"edit_initials" json:"editInitials" csv:"EDITINITIALS"`
LastEditComment string `db:"last_edit_comment" json:"lastEditComment" csv:"LASTEDITCOMMENT"`
Project *int `db:"PROJECT_ID" json:"project" csv:"PROJECT"`
FieldOffice string `db:"FIELDOFFICE" json:"fieldOffice" csv:"FIELDOFFICE"`
Segment *int `db:"SEGMENT_ID" json:"segment" csv:"SEGMENT"`
BkgColor string `db:"bkg_color" json:"bkgColor" csv:"BKGCOLOR"`
SuppBkgColor string `db:"supp_bkg_color" json:"suppBkgColor" csv:"SUPPBKGCOLOR"`
ProcBkgColor string `db:"proc_bkg_color" json:"procBkgColor" csv:"PROCBKGCOLOR"`
FishCount int `db:"fish_count" json:"fishCount" csv:"FISHCOUNT"`
SuppCount int `db:"supp_count" json:"suppCount" csv:"SUPPCOUNT"`
ProcCount int `db:"proc_count" json:"procCount" csv:"PROCCOUNT"`
Bend int `db:"bend" json:"bend" csv:"BEND"`
BendRn string `db:"bendrn" json:"bendrn" csv:"BENDRN"`
BendRiverMile float64 `db:"bendrivermile" json:"bendrivermile" csv:"BENDRIVERMILE"`
LastUpdated time.Time `db:"last_updated" json:"lastUpdated" csv:"LASTUPDATED"`
UploadSessionId int `db:"upload_session_id" json:"uploadSessionId" csv:"UPLOADSESSIONID"`
UploadedBy string `db:"uploaded_by" json:"uploadedBy" csv:"UPLOADEDBY"`
UploadFilename string `db:"upload_filename" json:"uploadFilename" csv:"UPLOADFILENAME"`
}

type TelemetryDataEntryWithCount struct {
Expand Down