From 3ad07e6ce5e2c6c3baec56b2762071123377a387 Mon Sep 17 00:00:00 2001 From: Andre Henriques Date: Tue, 16 Apr 2024 14:09:03 +0100 Subject: [PATCH] add button to add new image to class closes #15 --- logic/db_types/utils.go | 27 +++++- logic/models/data.go | 82 ++++++++++++++++++ logic/utils/handler.go | 56 ++++++++++++- webpage/src/lib/FileUpload.svelte | 1 + webpage/src/lib/requests.svelte.ts | 1 + .../src/routes/models/edit/ModelTable.svelte | 84 +++++++++++++++++-- 6 files changed, 240 insertions(+), 11 deletions(-) diff --git a/logic/db_types/utils.go b/logic/db_types/utils.go index 9f9458e..771f139 100644 --- a/logic/db_types/utils.go +++ b/logic/db_types/utils.go @@ -29,7 +29,7 @@ type BasePackStruct struct { } func (b BasePackStruct) GetHost() string { - return b.Host + return b.Host } func (b BasePackStruct) GetDb() *sql.DB { @@ -315,6 +315,31 @@ func InsertReturnId(c QueryInterface, store interface{}, tablename string, retur return } +func GetDbVar[T interface{}](c QueryInterface, var_to_extract string, tablename string, args ...any) (*T, error) { + db_query, err := c.Prepare(fmt.Sprintf("select %s from %s", var_to_extract, tablename)) + if err != nil { + return nil, err + } + defer db_query.Close() + + rows, err := db_query.Query(args...) + if err != nil { + return nil, err + } + defer rows.Close() + + if !rows.Next() { + return nil, NotFoundError + } + + dat := new(T) + if err = rows.Scan(dat); err != nil { + return nil, err + } + + return dat, nil +} + func GetDBOnce(db QueryInterface, store interface{}, tablename string, args ...any) error { t := reflect.TypeOf(store).Elem() diff --git a/logic/models/data.go b/logic/models/data.go index 09aa640..bdf5d77 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -498,6 +498,88 @@ func handleDataUpload(handle *Handle) { return c.SendJSON(modelClass) }) + type AddNewImage struct { + ClassId string `json:"id" validate:"required"` + } + PostAuthFormJson(handle, "/models/data/class/image", User_Normal, func(c *Context, dat *AddNewImage, file []byte) *Error { + model_id, err := GetDbVar[string](c, "m.id", "model_classes as mc inner join models as m on m.id = mc.model_id where mc.id=$1;", dat.ClassId) + if err == NotFoundError { + return c.JsonBadRequest("Could not find the class") + } else if err != nil { + return c.E500M("Error getting class information", err) + } + + c.Logger.Info("model", "model", *model_id) + + model, err := GetBaseModel(c.Db, *model_id) + if err == ModelNotFoundError { + return c.JsonBadRequest("Could not find the model") + } else if err != nil { + return c.E500M("Error getting model information", err) + } + + // TODO make this work for zip files as well + /*c.Logger.Debug("Processing File", "file", file.Name) + data, err := reader.Open(file.Name) + if err != nil { + c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err) + back_channel <- index + continue + } + defer data.Close() + file_data, err := io.ReadAll(data) + if err != nil { + c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err) + back_channel <- index + continue + }*/ + + // TODO check if the file is a valid photo that matched the defined photo on the database + + //parts := strings.Split(file.Name, "/") + + mode := DATA_POINT_MODE_TRAINING + // TODO add the mode + /*mode := DATA_POINT_MODE_TRAINING + if parts[0] == "testing" { + mode = DATA_POINT_MODE_TESTING + }*/ + + data_point_id, err := model_classes.AddDataPoint(c.Db, dat.ClassId, "id://", mode) + if err != nil { + //c.Logger.Error("Failed to add datapoint", "model", model.Id, "file name", file.Name, "err", err) + c.Logger.Error("Failed to add datapoint", "data_point_id", data_point_id) + return c.E500M("Could not add image to model", err) + } + + file_path := path.Join("savedData", model.Id, "data", data_point_id+"."+model.Format) + f, err := os.Create(file_path) + if err != nil { + //c.Logger.Error("Failed to save datapoint to disk", "model", model.Id, "file name", file.Name, "err", err) + c.Logger.Error("Failed to save datapoint to disk", "model", model.Id, "err", err) + return c.E500M("Could not add image to model", err) + } + defer f.Close() + f.Write(file) + + if !TestImgForModel(c, model, file_path) { + //c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) + c.Logger.Errorf("Image did not have valid format for model %s!", file_path) + c.Logger.Warn("Not failling updating data point to status -1") + message := "Image did not have valid format for the model" + if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { + //c.Logger.Error("Failed to update data point", "model", model.Id, "file name", file.Name, "err", err) + c.Logger.Error("Failed to update data point", "model", model.Id, "err", err) + return c.E500M("Could not update information about the data point", err) + } + return c.JsonBadRequest("Provided file is not a valid image for this model") + } + + return c.SendJSON(struct { + Id string `json:"id"` + }{data_point_id}) + }) + // ------ // ------ CLASS DATA UPLOAD // ------ diff --git a/logic/utils/handler.go b/logic/utils/handler.go index 2e0a96e..647fb87 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -1,9 +1,11 @@ package utils import ( + "bytes" "database/sql" "errors" "fmt" + "io" "net/http" "os" "path" @@ -89,7 +91,7 @@ func (x *Handle) PostAuth(path string, authLevel dbtypes.UserType, fn func(c *Co x.posts = append(x.posts, HandleFunc{path, inner_fn}) } -func PostAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) { +func PostAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, dat *T) *Error) { inner_fn := func(c *Context) *Error { if !c.CheckAuthLevel(authLevel) { return nil @@ -107,6 +109,56 @@ func PostAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserT x.posts = append(x.posts, HandleFunc{path, inner_fn}) } +func PostAuthFormJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, dat *T, file []byte) *Error) { + inner_fn := func(c *Context) *Error { + if !c.CheckAuthLevel(1) { + return nil + } + + read_form, err := c.R.MultipartReader() + if err != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + + var json_data string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + if part.FormName() == "json_data" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + json_data = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + if !c.CheckAuthLevel(authLevel) { + return nil + } + + dat := new(T) + + decoder := json.NewDecoder(strings.NewReader(json_data)) + if err := c.decodeAndValidade(decoder, dat); err != nil { + return err + } + + return fn(c, dat, file) + } + + x.posts = append(x.posts, HandleFunc{path, inner_fn}) +} + func (x *Handle) Delete(path string, fn func(c *Context) *Error) { x.deletes = append(x.deletes, HandleFunc{path, fn}) } @@ -200,7 +252,7 @@ func (c Context) GetLogger() *log.Logger { } func (c Context) GetHost() string { - return c.Handle.Config.Hostname + return c.Handle.Config.Hostname } func (c Context) Query(query string, args ...any) (*sql.Rows, error) { diff --git a/webpage/src/lib/FileUpload.svelte b/webpage/src/lib/FileUpload.svelte index e9fbf38..70967b3 100644 --- a/webpage/src/lib/FileUpload.svelte +++ b/webpage/src/lib/FileUpload.svelte @@ -35,6 +35,7 @@ + @@ -312,7 +342,45 @@ {/if} + +
+
+ +
+ Please provide a file that has the training and testing data
+ The file must have 2 folders one with testing images and one with training images. +
+ Each of the folders will contain the classes of the model. The folders must be the same in testing + and training. The class folders must have the images for the classes. + +
+ + + Upload Zip File or Image +
+ + File selected +
+ +
+ + {#if addFile} + {#await adding} + + {:then} + + {/await} + {/if} + + +