diff --git a/logic/models/classes/main.go b/logic/models/classes/main.go index 9383f43..22ac648 100644 --- a/logic/models/classes/main.go +++ b/logic/models/classes/main.go @@ -3,36 +3,19 @@ package model_classes import ( "database/sql" "errors" - // . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" + + . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) type ModelClass struct { Id string `json:"id"` - ModelId string `json:"model_id"` + ModelId string `json:"model_id" db:"model_id"` Name string `json:"name"` + Status int `json:"status"` } -func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) { - - rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id) - if err != nil { - return - } - defer rows.Close() - - cls = []ModelClass{} - - for rows.Next() { - var model ModelClass - err = rows.Scan(&model.Id, &model.ModelId, &model.Name) - if err != nil { - return - } - - cls = append(cls, model) - } - - return +func ListClasses(c *Context, model_id string) (cls []*ModelClass, err error) { + return GetDbMultitple[ModelClass](c, "model_classes where model_id=$1", model_id) } func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) { diff --git a/logic/models/data.go b/logic/models/data.go index 9ef7783..9825fce 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -151,6 +151,122 @@ func processZipFile(c *Context, model *BaseModel) { ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) } +func processZipFileExpand(c *Context, model *BaseModel) { + var err error + + failed := func(msg string) { + c.Logger.Error(msg, "err", err) + ModelUpdateStatus(c, model.Id, READY_FAILED) + } + + reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip")) + if err != nil { + failed("Faield to proccess zip file failed to open reader\n") + return + } + defer reader.Close() + + training := []string{} + testing := []string{} + + for _, file := range reader.Reader.File { + + paths := strings.Split(file.Name, "/") + + if paths[1] == "" { + continue + } + + if paths[0] != "training" && paths[0] != "testing" { + failed(fmt.Sprintf("Invalid file '%s' TODO add msg to response!!!", file.Name)) + return + } + + if paths[0] != "training" { + training = InsertIfNotPresent(training, paths[1]) + } else if paths[0] != "testing" { + testing = InsertIfNotPresent(testing, paths[1]) + } + } + + if !reflect.DeepEqual(testing, training) { + failed("testing and training are diferent") + return + } + + base_path := path.Join("savedData", model.Id, "data") + if err = os.MkdirAll(base_path, os.ModePerm); err != nil { + failed("Failed to create base_path dir") + return + } + + ids := map[string]string{} + + for i, name := range training { + id, err := model_classes.CreateClass(c.Db, model.Id, i, name) + if err != nil { + failed(fmt.Sprintf("Failed to create class '%s' on db\n", name)) + return + } + ids[name] = id + } + + for _, file := range reader.Reader.File { + if file.Name[len(file.Name)-1] == '/' { + continue + } + + data, err := reader.Open(file.Name) + if err != nil { + failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name)) + return + } + defer data.Close() + file_data, err := io.ReadAll(data) + if err != nil { + failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name)) + return + } + + // TODO check if the file is a valid photo that matched the defined photo on the database + + parts := strings.Split(file.Name, "/") + + mode := model_classes.DATA_POINT_MODE_TRAINING + if parts[0] == "testing" { + mode = model_classes.DATA_POINT_MODE_TESTING + } + + data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode) + if err != nil { + failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id)) + return + } + + file_path := path.Join(base_path, data_point_id+"."+model.Format) + f, err := os.Create(file_path) + if err != nil { + failed(fmt.Sprintf("Could not create file %s\n", file_path)) + return + } + defer f.Close() + f.Write(file_data) + + if !testImgForModel(c, model, file_path) { + c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) + c.Logger.Warn("Not failling updating data point to status -1") + message := "Image did not have valid format for the model" + if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { + failed(fmt.Sprintf("Failed to update data point status")) + return + } + } + } + + c.Logger.Info("Added data to model", "id", model.Id) + ModelUpdateStatus(c, model.Id, READY) +} + func handleDataUpload(handle *Handle) { handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { @@ -267,6 +383,73 @@ func handleDataUpload(handle *Handle) { return nil }) + // ------ + // ------ CLASS DATA UPLOAD + // ------ + handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + + read_form, err := r.MultipartReader() + if err != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + + var id string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + if part.FormName() == "id" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + id = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + c.Logger.Info("Trying to expand model", "id", id) + + model, err := GetBaseModel(handle.Db, id) + if err == ModelNotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found") + } else if err != nil { + return Error500(err) + } + + // TODO work in allowing the model to add new in the pre ready moment + if model.Status != READY { + return c.JsonBadRequest("Model not in the correct state to add a more classes") + } + + // TODO mk this path configurable + dir_path := path.Join("savedData", id) + + f, err := os.Create(path.Join(dir_path, "expand_data.zip")) + if err != nil { + return Error500(err) + } + defer f.Close() + + f.Write(file) + + ModelUpdateStatus(c, id, READY_ALTERATION) + + go processZipFileExpand(c, model) + + return c.SendJSON(model.Id) + }) + handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil @@ -274,7 +457,7 @@ func handleDataUpload(handle *Handle) { if c.Mode == JSON { type ModelData struct { - Id string `json:"id"` + Id string `json:"id"` } var dat ModelData @@ -285,32 +468,52 @@ func handleDataUpload(handle *Handle) { model, err := GetBaseModel(handle.Db, dat.Id) if err == ModelNotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found"); + return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { return Error500(err) } - if model.Status != FAILED_PREPARING_ZIP_FILE { - return c.SendJSONStatus(http.StatusNotFound, "Model not in the correct status") + delete_path := "base_data.zip" + + if model.Status == READY_FAILED { + delete_path = "expand_data.zip" + } else if model.Status != FAILED_PREPARING_ZIP_FILE { + return c.JsonBadRequest("Model not in the correct status") } - err = os.Remove(path.Join("savedData", model.Id, "base_data.zip")) + err = os.Remove(path.Join("savedData", model.Id, delete_path)) if err != nil { return Error500(err) } - err = os.RemoveAll(path.Join("savedData", model.Id, "data")) - if err != nil { - return Error500(err) - } + if model.Status != READY_FAILED { + err = os.RemoveAll(path.Join("savedData", model.Id, "data")) + if err != nil { + return Error500(err) + } + } else { + c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding") + } - _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) - if err != nil { - return Error500(err) - } + if model.Status != READY_FAILED { + _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) + if err != nil { + return Error500(err) + } + } else { + _, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN) + if err != nil { + return Error500(err) + } + } - ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) - return c.SendJSON(model.Id) + if model.Status != READY_FAILED { + ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) + } else { + ModelUpdateStatus(c, model.Id, READY) + } + + return c.SendJSON(model.Id) } f, err := MyParseForm(r) diff --git a/logic/models/edit.go b/logic/models/edit.go index 0baa990..3312e57 100644 --- a/logic/models/edit.go +++ b/logic/models/edit.go @@ -75,7 +75,7 @@ func handleEdit(handle *Handle) { return c.Error500(err) } - cls, err := model_classes.ListClasses(handle.Db, id) + cls, err := model_classes.ListClasses(c, id) if err != nil { return c.Error500(err) } @@ -86,9 +86,9 @@ func handleEdit(handle *Handle) { } type ReturnType struct { - Classes []model_classes.ModelClass `json:"classes"` - HasData bool `json:"has_data"` - NumberOfInvalidImages int `json:"number_of_invalid_images"` + Classes []*model_classes.ModelClass `json:"classes"` + HasData bool `json:"has_data"` + NumberOfInvalidImages int `json:"number_of_invalid_images"` } return c.SendJSON(ReturnType{ @@ -314,7 +314,7 @@ func handleEdit(handle *Handle) { return c.Error500(err) } - cls, err := model_classes.ListClasses(handle.Db, id) + cls, err := model_classes.ListClasses(c, id) if err != nil { return c.Error500(err) } diff --git a/logic/models/run.go b/logic/models/run.go index 394391d..250ecb4 100644 --- a/logic/models/run.go +++ b/logic/models/run.go @@ -103,7 +103,6 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten var predictions = results[0].Value().([][]float32)[0] for i, v := range predictions { - c.Logger.Info("This is test", "v", v) if v > vmax { order = element.Range_start + i vmax = v diff --git a/logic/models/train/train.go b/logic/models/train/train.go index a16d8ad..d91c0d7 100644 --- a/logic/models/train/train.go +++ b/logic/models/train/train.go @@ -102,7 +102,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e } func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) { - _, err = c.Db.Exec("update model_classes set stauts = $1 where "+filter, args...) + _, err = c.Db.Exec(fmt.Sprintf("update model_classes set status=%d where %s", status, filter), args...) return } @@ -485,6 +485,7 @@ func (nf ToRemoveList) Less(i, j int) bool { } func trainModel(c *Context, model *BaseModel) { + definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) if err != nil { c.Logger.Error("Failed to train Model! Err:") @@ -783,9 +784,7 @@ func trainModelExp(c *Context, model *BaseModel) { if len_def == 0 { break - } - - if len_def == 1 { + } else if len_def == 1 { continue } @@ -811,44 +810,38 @@ func trainModelExp(c *Context, model *BaseModel) { } } - rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) + // Set the class status to trained + err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING) if err != nil { - failed("DB: failed to read definition") + failed("Failed to set class status") return } - defer rows.Close() + + var dat JustId - if !rows.Next() { + err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) + if err == NotFoundError { failed("All definitions failed to train!") return - } - - var id string - if err = rows.Scan(&id); err != nil { - failed("Failed to read id") + } else if err != nil { + failed("DB: failed to read definition") return - } + } - if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil { + if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil { failed("Failed to update model definition") return } - to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) - if err != nil { + to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) + if err != nil { failed("Failed to select model_definition to delete") return - } - defer to_delete.Close() + } - for to_delete.Next() { - var id string - if to_delete.Scan(&id); err != nil { - failed("Failed to scan the id of a model_definition to delete") - return - } - os.RemoveAll(path.Join("savedData", model.Id, "defs", id)) - } + for _, d := range(to_delete) { + os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id)) + } // TODO Check if returning also works here if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { @@ -863,7 +856,6 @@ func trainModelExp(c *Context, model *BaseModel) { // There should only be one def availabale def := JustId{} - if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil { return } @@ -1099,7 +1091,7 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe } func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error { - cls, err := model_classes.ListClasses(c.Db, model.Id) + cls, err := model_classes.ListClasses(c, model.Id) if err != nil { ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING) // TODO improve this response @@ -1267,7 +1259,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy // TODO make this json friendy func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error { - cls, err := model_classes.ListClasses(c.Db, model.Id) + cls, err := model_classes.ListClasses(c, model.Id) if err != nil { ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING) // TODO improve this response diff --git a/logic/models/utils/types.go b/logic/models/utils/types.go index 0831d93..62b8985 100644 --- a/logic/models/utils/types.go +++ b/logic/models/utils/types.go @@ -28,6 +28,8 @@ const ( PREPARING_ZIP_FILE = 3 TRAINING = 4 READY = 5 + READY_ALTERATION = 6 + READY_FAILED = -6 ) type ModelDefinitionStatus int @@ -56,8 +58,8 @@ type ModelClassStatus int const ( MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1 - MODEL_CLASS_STATUS_TRAINING = 2 - MODEL_CLASS_STATUS_TRAINED = 3 + MODEL_CLASS_STATUS_TRAINING = 2 + MODEL_CLASS_STATUS_TRAINED = 3 ) var ModelNotFoundError = errors.New("Model not found error") diff --git a/logic/models/utils/utils.go b/logic/models/utils/utils.go index d196376..25c5755 100644 --- a/logic/models/utils/utils.go +++ b/logic/models/utils/utils.go @@ -1,17 +1,14 @@ package models_utils import ( - "fmt" - . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) // TODO make this return and caller handle error func ModelUpdateStatus(c *Context, id string, status int) { - _, err := c.Db.Exec("update models set status = $1 where id = $2", status, id) + _, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id) if err != nil { - fmt.Println("Failed to update model status") - fmt.Println(err) - panic("TODO handle better") + c.Logger.Error("Failed to update model status", "err", err) + c.Logger.Warn("TODO Maybe handle better") } } diff --git a/webpage/src/lib/Tabs.svelte b/webpage/src/lib/Tabs.svelte index 790a526..2bffee5 100644 --- a/webpage/src/lib/Tabs.svelte +++ b/webpage/src/lib/Tabs.svelte @@ -14,7 +14,7 @@
-