diff --git a/logic/models/add.go b/logic/models/add.go index 53c680b..5d65ea5 100644 --- a/logic/models/add.go +++ b/logic/models/add.go @@ -86,90 +86,14 @@ func loadBaseImage(c *Context, id string) { } func handleAdd(handle *Handle) { - handle.GetHTML("/models/add", AnswerTemplate("models/add.html", nil, 1)) - handle.Post("/models/add", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Post("/models/add", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - read_form, err := r.MultipartReader() - if err != nil { - return c.JsonErrorBadRequest(err, "Please provide a valid multipart Reader request") - } - - var name string - var file []byte - - for { - part, err_part := read_form.NextPart() - if err_part == io.EOF { - break - } else if err_part != nil { - return &Error{Code: http.StatusBadRequest} - } - if part.FormName() == "name" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - name = buf.String() - } - if part.FormName() == "file" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - file = buf.Bytes() - } - } - - if name == "" || len(file) == 0 { - return c.JsonBadRequest("Name is empty or file is empty") - } - - row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id) - if err != nil { - return c.Error500(err) - } - - if row.Next() { - return c.JsonBadRequest("Model with that name already exists!") - } - - row, err = handle.Db.Query("insert into models (user_id, name) values ($1, $2) returning id", c.User.Id, name) - if err != nil || !row.Next() { - return c.Error500(err) - } - - var id string - err = row.Scan(&id) - if err != nil { - return c.Error500(err) - } - - // TODO mk this path configurable - dir_path := path.Join("savedData", id) - - err = os.Mkdir(dir_path, os.ModePerm) - if err != nil { - return c.Error500(err) - } - - f, err := os.Create(path.Join(dir_path, "baseimage.png")) - if err != nil { - return c.Error500(err) - } - defer f.Close() - - f.Write(file) - - c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id) - go loadBaseImage(c, id) - - return c.SendJSON(id) - } - - read_form, err := r.MultipartReader() + read_form, err := c.R.MultipartReader() if err != nil { - LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) - return nil + return c.JsonErrorBadRequest(err, "Please provide a valid multipart Reader request") } var name string @@ -195,8 +119,7 @@ func handleAdd(handle *Handle) { } if name == "" || len(file) == 0 { - LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) - return nil + return c.JsonBadRequest("Name is empty or file is empty") } row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id) @@ -205,27 +128,14 @@ func handleAdd(handle *Handle) { } if row.Next() { - LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(AnyMap{ - "NameFoundError": true, - "Name": name, - })) - return nil + return c.JsonBadRequest("Model with that name already exists!") } - _, err = handle.Db.Exec("insert into models (user_id, name) values ($1, $2)", c.User.Id, name) - if err != nil { + row, err = handle.Db.Query("insert into models (user_id, name) values ($1, $2) returning id", c.User.Id, name) + if err != nil || !row.Next() { return c.Error500(err) } - row, err = handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id) - if err != nil { - return c.Error500(err) - } - - if !row.Next() { - return &Error{Code: http.StatusInternalServerError} - } - var id string err = row.Scan(&id) if err != nil { @@ -239,6 +149,7 @@ func handleAdd(handle *Handle) { if err != nil { return c.Error500(err) } + f, err := os.Create(path.Join(dir_path, "baseimage.png")) if err != nil { return c.Error500(err) @@ -250,7 +161,6 @@ func handleAdd(handle *Handle) { c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id) go loadBaseImage(c, id) - Redirect("/models/edit?id="+id, c.Mode, w, r) - return nil + return c.SendJSON(id) }) } diff --git a/logic/models/classes/list.go b/logic/models/classes/list.go index 6762ad9..a40e012 100644 --- a/logic/models/classes/list.go +++ b/logic/models/classes/list.go @@ -1,148 +1,69 @@ package model_classes import ( - "net/http" "strconv" - . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) -func models_data_list_json(w http.ResponseWriter, r *http.Request, c *Context) *Error { - id, err := GetIdFromUrl(r, "id") - if err != nil { - return c.JsonBadRequest("Model Class not found!") - } - - page := 0 - if r.URL.Query().Has("page") { - page_url := r.URL.Query().Get("page") - page_url_number, err := strconv.Atoi(page_url) - if err != nil { - return c.JsonBadRequest("Page is not a number") - } - page = page_url_number - } - - var class_row struct { - Name string - Model_id string - } - - err = utils.GetDBOnce(c, &class_row, "model_classes where id=$1", id) - if err == NotFoundError { - return c.JsonBadRequest("Model Class not found!") - } else if err != nil { - return c.Error500(err) - } - - type baserow struct { - Id string `json:"id"` - File_Path string `json:"file_path"` - Model_Mode int `json:"model_mode"` - Status int `json:"status"` - } - - rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10) - if err != nil { - return c.Error500(err) - } - - type ReturnType struct { - ImageList []*baserow `json:"image_list"` - Page int `json:"page"` - ShowNext bool `json:"showNext"` - } - - max_len := min(11, len(rows)) - - return c.SendJSON(ReturnType{ - ImageList: rows[0:max_len], - Page: page, - ShowNext: len(rows) == 11, - }) -} - func HandleList(handle *Handle) { - handle.Get("/models/data/list", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Get("/models/data/list", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - return models_data_list_json(w, r, c) - } - id, err := GetIdFromUrl(r, "id") + id, err := GetIdFromUrl(c, "id") if err != nil { - return ErrorCode(err, 400, c.AddMap(nil)) + return c.JsonBadRequest("Model Class not found!") } page := 0 - if r.URL.Query().Has("page") { - page_url := r.URL.Query().Get("page") + if c.R.URL.Query().Has("page") { + page_url := c.R.URL.Query().Get("page") page_url_number, err := strconv.Atoi(page_url) if err != nil { - return ErrorCode(err, http.StatusBadRequest, c.AddMap(nil)) + return c.JsonBadRequest("Page is not a number") } page = page_url_number } - class_rows, err := handle.Db.Query("select name, model_id from model_classes where id=$1;", id) - if err != nil { - return Error500(err) - } - defer class_rows.Close() - - if !class_rows.Next() { - return ErrorCode(nil, 404, c.AddMap(nil)) + var class_row struct { + Name string + Model_id string } - name := "" - model_id := "" - if err = class_rows.Scan(&name, &model_id); err != nil { - return Error500(nil) + err = utils.GetDBOnce(c, &class_row, "model_classes where id=$1", id) + if err == NotFoundError { + return c.JsonBadRequest("Model Class not found!") + } else if err != nil { + return c.Error500(err) } - model, err := GetBaseModel(c.Db, model_id) - if err != nil { - return Error500(err) - } - - rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 11 offset $2;", id, page*10) - if err != nil { - return Error500(err) - } - defer rows.Close() - type baserow struct { - Id string - FilePath string - Mode int - Status int + Id string `json:"id"` + File_Path string `json:"file_path"` + Model_Mode int `json:"model_mode"` + Status int `json:"status"` } - got := []baserow{} - - for rows.Next() { - nrow := baserow{} - err = rows.Scan(&nrow.Id, &nrow.FilePath, &nrow.Mode, &nrow.Status) - if err != nil { - return Error500(err) - } - got = append(got, nrow) + rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10) + if err != nil { + return c.Error500(err) } - max_len := min(11, len(got)) + type ReturnType struct { + ImageList []*baserow `json:"image_list"` + Page int `json:"page"` + ShowNext bool `json:"showNext"` + } - LoadDefineTemplate(w, "/models/edit.html", "data-model-create-class-table-table", c.AddMap(AnyMap{ - "List": got[0:max_len], - "Page": page, - "Id": id, - "Name": name, - "Model": model, - "ShowNext": len(got) == 11, - })) - return nil + max_len := min(11, len(rows)) + + return c.SendJSON(ReturnType{ + ImageList: rows[0:max_len], + Page: page, + ShowNext: len(rows) == 11, + }) }) } diff --git a/logic/models/data.go b/logic/models/data.go index 9825fce..a04c888 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -152,8 +152,8 @@ func processZipFile(c *Context, model *BaseModel) { } func processZipFileExpand(c *Context, model *BaseModel) { - var err error - + var err error + failed := func(msg string) { c.Logger.Error(msg, "err", err) ModelUpdateStatus(c, model.Id, READY_FAILED) @@ -161,7 +161,7 @@ func processZipFileExpand(c *Context, model *BaseModel) { reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip")) if err != nil { - failed("Faield to proccess zip file failed to open reader\n") + failed("Faield to proccess zip file failed to open reader\n") return } defer reader.Close() @@ -257,8 +257,8 @@ func processZipFileExpand(c *Context, model *BaseModel) { c.Logger.Warn("Not failling updating data point to status -1") message := "Image did not have valid format for the model" if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { - failed(fmt.Sprintf("Failed to update data point status")) - return + failed(fmt.Sprintf("Failed to update data point status")) + return } } } @@ -268,130 +268,12 @@ func processZipFileExpand(c *Context, model *BaseModel) { } func handleDataUpload(handle *Handle) { - handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Post("/models/data/upload", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - read_form, err := r.MultipartReader() - if err != nil { - return c.JsonBadRequest("Please provide a valid form data request!") - } - - var id string - var file []byte - - for { - part, err_part := read_form.NextPart() - if err_part == io.EOF { - break - } else if err_part != nil { - return c.JsonBadRequest("Please provide a valid form data request!") - } - if part.FormName() == "id" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - id = buf.String() - } - if part.FormName() == "file" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - file = buf.Bytes() - } - } - - model, err := GetBaseModel(handle.Db, id) - if err == ModelNotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } else if err != nil { - return Error500(err) - } - - // TODO mk this path configurable - dir_path := path.Join("savedData", id) - - f, err := os.Create(path.Join(dir_path, "base_data.zip")) - if err != nil { - return Error500(err) - } - defer f.Close() - - f.Write(file) - - ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) - - go processZipFile(c, model) - - return c.SendJSON(model.Id) - } - - read_form, err := r.MultipartReader() - if err != nil { - LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) - return nil - } - - var id string - var file []byte - - for { - part, err_part := read_form.NextPart() - if err_part == io.EOF { - break - } else if err_part != nil { - return &Error{Code: http.StatusBadRequest} - } - if part.FormName() == "id" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - id = buf.String() - } - if part.FormName() == "file" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - file = buf.Bytes() - } - } - - model, err := GetBaseModel(handle.Db, id) - if err == ModelNotFoundError { - return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) - } else if err != nil { - return Error500(err) - } - - // TODO mk this path configurable - dir_path := path.Join("savedData", id) - - f, err := os.Create(path.Join(dir_path, "base_data.zip")) - if err != nil { - return Error500(err) - } - defer f.Close() - - f.Write(file) - - ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) - - go processZipFile(c, model) - - Redirect("/models/edit?id="+id, c.Mode, w, r) - return nil - }) - - // ------ - // ------ CLASS DATA UPLOAD - // ------ - handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { - return nil - } - - read_form, err := r.MultipartReader() + read_form, err := c.R.MultipartReader() if err != nil { return c.JsonBadRequest("Please provide a valid form data request!") } @@ -418,26 +300,86 @@ func handleDataUpload(handle *Handle) { } } - c.Logger.Info("Trying to expand model", "id", id) + model, err := GetBaseModel(handle.Db, id) + if err == ModelNotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found") + } else if err != nil { + return c.Error500(err) + } + + // TODO mk this path configurable + dir_path := path.Join("savedData", id) + + f, err := os.Create(path.Join(dir_path, "base_data.zip")) + if err != nil { + return c.Error500(err) + } + defer f.Close() + + f.Write(file) + + ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) + + go processZipFile(c, model) + + return c.SendJSON(model.Id) + }) + + // ------ + // ------ CLASS DATA UPLOAD + // ------ + handle.Post("/models/data/class/upload", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { + return nil + } + + read_form, err := c.R.MultipartReader() + if err != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + + var id string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + if part.FormName() == "id" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + id = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + c.Logger.Info("Trying to expand model", "id", id) model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { - return Error500(err) + return c.Error500(err) + } + + // TODO work in allowing the model to add new in the pre ready moment + if model.Status != READY { + return c.JsonBadRequest("Model not in the correct state to add a more classes") } - - // TODO work in allowing the model to add new in the pre ready moment - if model.Status != READY { - return c.JsonBadRequest("Model not in the correct state to add a more classes") - } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "expand_data.zip")) if err != nil { - return Error500(err) + return c.Error500(err) } defer f.Close() @@ -450,115 +392,68 @@ func handleDataUpload(handle *Handle) { return c.SendJSON(model.Id) }) - handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Delete("/models/data/delete-zip-file", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - type ModelData struct { - Id string `json:"id"` - } - - var dat ModelData - - if err := c.ToJSON(r, &dat); err != nil { - return err - } - - model, err := GetBaseModel(handle.Db, dat.Id) - if err == ModelNotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } else if err != nil { - return Error500(err) - } - - delete_path := "base_data.zip" - - if model.Status == READY_FAILED { - delete_path = "expand_data.zip" - } else if model.Status != FAILED_PREPARING_ZIP_FILE { - return c.JsonBadRequest("Model not in the correct status") - } - - err = os.Remove(path.Join("savedData", model.Id, delete_path)) - if err != nil { - return Error500(err) - } - - if model.Status != READY_FAILED { - err = os.RemoveAll(path.Join("savedData", model.Id, "data")) - if err != nil { - return Error500(err) - } - } else { - c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding") - } - - if model.Status != READY_FAILED { - _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) - if err != nil { - return Error500(err) - } - } else { - _, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN) - if err != nil { - return Error500(err) - } - } - - if model.Status != READY_FAILED { - ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) - } else { - ModelUpdateStatus(c, model.Id, READY) - } - - return c.SendJSON(model.Id) + type ModelData struct { + Id string `json:"id"` } - f, err := MyParseForm(r) - if err != nil { - return ErrorCode(err, 400, c.AddMap(nil)) + var dat ModelData + + if err := c.ToJSON(&dat); err != nil { + return err } - if !CheckId(f, "id") { - return ErrorCode(err, 400, c.AddMap(nil)) - } - - id := f.Get("id") - - model, err := GetBaseModel(handle.Db, id) + model, err := GetBaseModel(handle.Db, dat.Id) if err == ModelNotFoundError { - return ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) + return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { - return Error500(err) + return c.Error500(err) } - if model.Status != FAILED_PREPARING_ZIP_FILE { - // TODO add message - return ErrorCode(nil, 400, c.AddMap(nil)) + delete_path := "base_data.zip" + + if model.Status == READY_FAILED { + delete_path = "expand_data.zip" + } else if model.Status != FAILED_PREPARING_ZIP_FILE { + return c.JsonBadRequest("Model not in the correct status") } - err = os.Remove(path.Join("savedData", id, "base_data.zip")) + err = os.Remove(path.Join("savedData", model.Id, delete_path)) if err != nil { - return Error500(err) + return c.Error500(err) } - err = os.RemoveAll(path.Join("savedData", id, "data")) - if err != nil { - return Error500(err) + if model.Status != READY_FAILED { + err = os.RemoveAll(path.Join("savedData", model.Id, "data")) + if err != nil { + return c.Error500(err) + } + } else { + c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding") } - _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) - if err != nil { - return Error500(err) + if model.Status != READY_FAILED { + _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) + if err != nil { + return c.Error500(err) + } + } else { + _, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN) + if err != nil { + return c.Error500(err) + } } - ModelUpdateStatus(c, id, CONFIRM_PRE_TRAINING) - Redirect("/models/edit?id="+id, c.Mode, w, r) - return nil + if model.Status != READY_FAILED { + ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) + } else { + ModelUpdateStatus(c, model.Id, READY) + } + + return c.SendJSON(model.Id) }) } diff --git a/logic/models/delete.go b/logic/models/delete.go index 7d3b55b..4a25149 100644 --- a/logic/models/delete.go +++ b/logic/models/delete.go @@ -4,40 +4,12 @@ import ( "net/http" "os" "path" - "strconv" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" utils "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) -func deleteModel(handle *Handle, id string, w http.ResponseWriter, c *Context, model BaseModel) { - c.Logger.Warnf("Removing model with id: %s", id) - _, err := handle.Db.Exec("delete from models where id=$1;", id) - if err != nil { - c.Logger.Error(err) - panic("TODO handle better deleteModel failed delete database query") - } - - model_path := path.Join("./savedData", id) - c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path) - err = os.RemoveAll(model_path) - if err != nil { - c.Logger.Error(err) - panic("TODO handle better deleteModel failed to delete folder") - } - - if c.Mode == HTML { - // TODO move this to a constant so i don't forget - w.WriteHeader(309) - c.Mode = HTMLFULL - } - - LoadBasedOnAnswer(c.Mode, w, "/models/delete.html", c.AddMap(AnyMap{ - "Model": model, - })) -} - func deleteModelJSON(c *Context, id string) *Error { c.Logger.Warnf("Removing model with id: %s", id) _, err := c.Db.Exec("delete from models where id=$1;", id) @@ -56,94 +28,29 @@ func deleteModelJSON(c *Context, id string) *Error { } func handleDelete(handle *Handle) { - handle.Delete("/models/delete", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - - if c.Mode == JSON { - - var dat struct { - Id string `json:"id" validate:"required"` - Name *string `json:"name,omitempty"` - } - - if err_ := c.ToJSON(r, &dat); err_ != nil { - return err_ - } - - var model struct { - Id string - Name string - Status int - } - - err := utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id) - if err == NotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found!") - } else if err != nil { - return c.Error500(err) - } - - switch model.Status { - case FAILED_TRAINING: - fallthrough - case FAILED_PREPARING_ZIP_FILE: - fallthrough - case FAILED_PREPARING_TRAINING: - fallthrough - case FAILED_PREPARING: - return deleteModelJSON(c, dat.Id) - - case READY: - fallthrough - case CONFIRM_PRE_TRAINING: - if dat.Name == nil { - return c.JsonBadRequest("Provided name does not match the model name") - } - - if *dat.Name != model.Name { - return c.JsonBadRequest("Provided name does not match the model name") - } - - return deleteModelJSON(c, dat.Id) - default: - c.Logger.Warn("Do not know how to handle model in status", "status", model.Status) - return c.JsonBadRequest("Model in invalid status") - } + handle.Delete("/models/delete", func(c *Context) *Error { + if c.CheckAuthLevel(1) { + return nil + } + var dat struct { + Id string `json:"id" validate:"required"` + Name *string `json:"name,omitempty"` } - // This is required to parse delete forms with bodies - f, err := MyParseForm(r) - if err != nil { - return c.ErrorCode(err, 400, nil) + if err_ := c.ToJSON(&dat); err_ != nil { + return err_ } - if !CheckId(f, "id") { - return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) + var model struct { + Id string + Name string + Status int } - id := f.Get("id") - - // TODO handle admin users - rows, err := handle.Db.Query("select name, status from models where id=$1 and user_id=$2;", id, c.User.Id) - if err != nil { - return c.ErrorCode(err, http.StatusInternalServerError, nil) - } - defer rows.Close() - - if !rows.Next() { - c.Logger.Warn("Could not find model for", id, c.User.Id) - return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) - } - - model := BaseModel{} - model.Id = id - - if err = rows.Scan(&model.Name, &model.Status); err != nil { + err := utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id) + if err == NotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found!") + } else if err != nil { return c.Error500(err) } @@ -155,32 +62,23 @@ func handleDelete(handle *Handle) { case FAILED_PREPARING_TRAINING: fallthrough case FAILED_PREPARING: - deleteModel(handle, id, w, c, model) - return nil + return deleteModelJSON(c, dat.Id) case READY: fallthrough case CONFIRM_PRE_TRAINING: - if CheckEmpty(f, "name") { - return c.Error400(nil, "Name is empty", w, "/models/edit.html", "delete-model-card", AnyMap{ - "NameDoesNotMatch": true, - "Model": model, - }) + if dat.Name == nil { + return c.JsonBadRequest("Provided name does not match the model name") } - name := f.Get("name") - if name != model.Name { - LoadDefineTemplate(w, "/models/edit.html", "delete-model-card", c.AddMap(AnyMap{ - "NameDoesNotMatch": true, - "Model": model, - })) - return nil + if *dat.Name != model.Name { + return c.JsonBadRequest("Provided name does not match the model name") } - deleteModel(handle, id, w, c, model) - return nil + return deleteModelJSON(c, dat.Id) default: - panic("Do not know how to handle model in status:" + strconv.Itoa(model.Status)) + c.Logger.Warn("Do not know how to handle model in status", "status", model.Status) + return c.JsonBadRequest("Model in invalid status") } }) } diff --git a/logic/models/edit.go b/logic/models/edit.go index 3312e57..f9a12e2 100644 --- a/logic/models/edit.go +++ b/logic/models/edit.go @@ -2,7 +2,6 @@ package models import ( "fmt" - "net/http" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" @@ -10,79 +9,30 @@ import ( . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) -func handleJson(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { - return nil - } - - id, err := GetIdFromUrl(r, "id") - if err != nil { - return c.JsonBadRequest("Model not found") - } - - type rowmodel struct { - Name string `json:"name"` - Status int `json:"status"` - Id string `json:"id"` - Width *int `json:"width"` - Height *int `json:"height"` - Color_mode *string `json:"color_mode"` - Format string `json:"format"` - Model_type int `json:"model_type"` - } - - var model rowmodel = rowmodel{} - err = utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2", id, c.User.Id) - if err == NotFoundError { - return c.SendJSONStatus(404, "Model not found") - } else if err != nil { - return c.Error500(err) - } - - return c.SendJSON(model) - - /* - - switch model.Status { - case TRAINING: - } - */ -} - func handleEdit(handle *Handle) { - handle.Get("/models/edit/classes", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Get("/models/edit/classes", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode != JSON { - return c.ErrorCode(nil, 400, AnyMap{}) - } - id, err := GetIdFromUrl(r, "id") - if err != nil { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } - - model, err := GetBaseModel(c.Db, id) - if err == ModelNotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } else if err != nil { - return c.Error500(err) - } + model, err_ := c.GetModelFromId("id") + if err_ != nil { + return err_ + } wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id) if err != nil { return c.Error500(err) } - cls, err := model_classes.ListClasses(c, id) + cls, err := model_classes.ListClasses(c, model.Id) if err != nil { return c.Error500(err) } - has_data, err := model_classes.ModelHasDataPoints(handle.Db, id) + has_data, err := model_classes.ModelHasDataPoints(handle.Db, model.Id) if err != nil { - return Error500(err) + return c.Error500(err) } type ReturnType struct { @@ -98,25 +48,15 @@ func handleEdit(handle *Handle) { }) }) - handle.Get("/models/edit/definitions", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Get("/models/edit/definitions", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode != JSON { - return c.ErrorCode(nil, 400, AnyMap{}) - } - id, err := GetIdFromUrl(r, "id") - if err != nil { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } - - model, err := GetBaseModel(c.Db, id) - if err == ModelNotFoundError { - return c.SendJSONStatus(http.StatusNotFound, "Model not found") - } else if err != nil { - return c.Error500(err) - } + model, err_ := c.GetModelFromId("id") + if err_ != nil { + return err_ + } type defrow struct { Id string @@ -243,217 +183,39 @@ func handleEdit(handle *Handle) { return c.SendJSON(defsToReturn) }) - handle.Get("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Get("/models/edit", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - return handleJson(w, r, c) + if !c.CheckAuthLevel(1) { + return nil } - id, err := GetIdFromUrl(r, "id") + id, err := GetIdFromUrl(c, "id") if err != nil { - return ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) - } - - // TODO handle admin users - rows, err := handle.Db.Query("select name, status, width, height, color_mode, format, model_type from models where id=$1 and user_id=$2;", id, c.User.Id) - if err != nil { - return Error500(err) - } - defer rows.Close() - - if !rows.Next() { - return ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) + return c.JsonBadRequest("Model not found") } type rowmodel struct { - Name string - Status int - Id string - Width *int - Height *int - Color_mode *string - Format string - Type int + Name string `json:"name"` + Status int `json:"status"` + Id string `json:"id"` + Width *int `json:"width"` + Height *int `json:"height"` + Color_mode *string `json:"color_mode"` + Format string `json:"format"` + Model_type int `json:"model_type"` } var model rowmodel = rowmodel{} - model.Id = id - - err = rows.Scan(&model.Name, &model.Status, &model.Width, &model.Height, &model.Color_mode, &model.Format, &model.Type) - if err != nil { - return Error500(err) + err = utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2", id, c.User.Id) + if err == NotFoundError { + return c.SendJSONStatus(404, "Model not found") + } else if err != nil { + return c.Error500(err) } - // Handle errors - // All errors will be negative - if model.Status < 0 { - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - })) - return nil - } - - switch model.Status { - case PREPARING: - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - })) - case CONFIRM_PRE_TRAINING: - - wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id) - if err != nil { - return c.Error500(err) - } - - cls, err := model_classes.ListClasses(c, id) - if err != nil { - return c.Error500(err) - } - - has_data, err := model_classes.ModelHasDataPoints(handle.Db, id) - if err != nil { - return Error500(err) - } - - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - "Classes": cls, - "HasData": has_data, - "NumberOfInvalidImages": wrong_number, - })) - case READY: - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - })) - case TRAINING: - - type defrow struct { - Id string - Status int - EpochProgress int - Epoch int - Accuracy float64 - } - - defs := []defrow{} - - if model.Type == 2 { - def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id) - if err != nil { - return c.Error500(err) - } - defer def_rows.Close() - - for def_rows.Next() { - var def defrow - err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) - if err != nil { - return c.Error500(err) - } - defs = append(defs, def) - } - } else { - def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id) - if err != nil { - return c.Error500(err) - } - defer def_rows.Close() - - for def_rows.Next() { - var def defrow - err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) - if err != nil { - return c.Error500(err) - } - defs = append(defs, def) - } - } - - type layerdef struct { - id string - LayerType int - Shape string - } - - layers := []layerdef{} - - for _, def := range defs { - if def.Status == MODEL_DEFINITION_STATUS_TRAINING { - rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id) - if err != nil { - return c.Error500(err) - } - defer rows.Close() - - for rows.Next() { - var layerdef layerdef - err = rows.Scan(&layerdef.id, &layerdef.LayerType, &layerdef.Shape) - if err != nil { - return c.Error500(err) - } - layers = append(layers, layerdef) - } - - if model.Type == 2 { - - type lastLayerType struct { - Id string - Range_start int - Range_end int - } - - var lastLayer lastLayerType - - err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id) - if err != nil { - return c.Error500(err) - } - - layers = append(layers, layerdef{ - id: lastLayer.Id, - LayerType: LAYER_DENSE, - Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1), - }) - } - - break - } - } - - sep_mod := 100 - if len(layers) > 8 { - sep_mod = 100 - (len(layers)-8)*10 - } - - if sep_mod < 10 { - sep_mod = 10 - } - - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - "Defs": defs, - "Layers": layers, - "SepMod": sep_mod, - })) - case PREPARING_ZIP_FILE: - LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ - "Model": model, - })) - default: - fmt.Printf("Unkown Status: %d\n", model.Status) - return Error500(nil) - } - - return nil + return c.SendJSON(model) }) - } diff --git a/logic/models/list.go b/logic/models/list.go index 016eda0..8094217 100644 --- a/logic/models/list.go +++ b/logic/models/list.go @@ -1,66 +1,26 @@ package models import ( - "net/http" - "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) func handleList(handle *Handle) { - handle.Get("/models", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Get("/models", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - // TODO handle admin - if c.Mode == JSON { - rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id) - if err != nil { - return Error500(err) - } - defer rows.Close() - - type Row struct { - Name string `json:"name"` - Id string `json:"id"` - } - - got, err := utils.GetDbMultitple[Row](c, "models where user_id=$1", c.User.Id); - if err != nil { - c.Logger.Warn("HERE 6") - return c.Error500(nil) - } - - return c.SendJSON(got) + type Row struct { + Name string `json:"name"` + Id string `json:"id"` } - rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id) + got, err := utils.GetDbMultitple[Row](c, "models where user_id=$1", c.User.Id) if err != nil { - return Error500(err) - } - defer rows.Close() - - type row struct { - Name string - Id string + return c.Error500(nil) } - got := []row{} - - for rows.Next() { - var r row - err = rows.Scan(&r.Id, &r.Name) - if err != nil { - return Error500(err) - } - got = append(got, r) - } - - LoadBasedOnAnswer(c.Mode, w, "/models/list.html", c.AddMap(AnyMap{ - "List": got, - })) - return nil + return c.SendJSON(got) }) - } diff --git a/logic/models/run.go b/logic/models/run.go index 250ecb4..1255dd2 100644 --- a/logic/models/run.go +++ b/logic/models/run.go @@ -2,9 +2,7 @@ package models import ( "bytes" - "fmt" "io" - "net/http" "os" "path" @@ -118,143 +116,14 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten } func handleRun(handle *Handle) { - handle.Post("/models/run", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Post("/models/run", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - read_form, err := r.MultipartReader() - if err != nil { - // TODO improve message - return ErrorCode(nil, 400, nil) - } - - var id string - var file []byte - - for { - part, err_part := read_form.NextPart() - if err_part == io.EOF { - break - } else if err_part != nil { - return c.JsonBadRequest("Invalid multipart data") - } - if part.FormName() == "id" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - id = buf.String() - } - if part.FormName() == "file" { - buf := new(bytes.Buffer) - buf.ReadFrom(part) - file = buf.Bytes() - } - } - - model, err := GetBaseModel(handle.Db, id) - if err == ModelNotFoundError { - return c.JsonBadRequest("Models not found") - } else if err != nil { - return c.Error500(err) - } - - if model.Status != READY { - return c.JsonBadRequest("Model not ready to run images") - } - - def := JustId{} - err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id) - if err == NotFoundError { - return c.JsonBadRequest("Could not find definition") - } else if err != nil { - return c.Error500(err) - } - - def_id := def.Id - - // TODO create a database table with tasks - run_path := path.Join("/tmp", model.Id, "runs") - os.MkdirAll(run_path, os.ModePerm) - img_path := path.Join(run_path, "img."+model.Format) - - img_file, err := os.Create(img_path) - if err != nil { - return c.Error500(err) - } - defer img_file.Close() - img_file.Write(file) - - if !testImgForModel(c, model, img_path) { - return c.JsonBadRequest("Provided image does not match the model") - } - - root := tg.NewRoot() - - var tf_img *image.Image = nil - - switch model.Format { - case "png": - tf_img = ReadPNG(root, img_path, int64(model.ImageMode)) - case "jpeg": - tf_img = ReadJPG(root, img_path, int64(model.ImageMode)) - default: - panic("Not sure what to do with '" + model.Format + "'") - } - - exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{}) - inputImage, err := tf.NewTensor(exec_results[0].Value()) - if err != nil { - return c.Error500(err) - } - - vi := -1 - var confidence float32 = 0 - - if model.ModelType == 2 { - c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, confidence, err = runModelExp(c, model, def_id, inputImage) - if err != nil { - return c.Error500(err) - } - } else { - c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, confidence, err = runModelNormal(c, model, def_id, inputImage) - if err != nil { - return c.Error500(err) - } - } - - os.RemoveAll(run_path) - - rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi) - if err != nil { - return c.Error500(err) - } - if !rows.Next() { - return c.SendJSON(nil) - } - - var name string - if err = rows.Scan(&name); err != nil { - return c.Error500(err) - } - - returnValue := struct { - Class string `json:"class"` - Confidence float32 `json:"confidence"` - }{ - Class: name, - Confidence: confidence, - } - - return c.SendJSON(returnValue) - } - - read_form, err := r.MultipartReader() + read_form, err := c.R.MultipartReader() if err != nil { - // TODO improve message - return ErrorCode(nil, 400, nil) + return c.JsonBadRequest("Invalid muilpart body") } var id string @@ -265,7 +134,7 @@ func handleRun(handle *Handle) { if err_part == io.EOF { break } else if err_part != nil { - return &Error{Code: http.StatusBadRequest} + return c.JsonBadRequest("Invalid multipart data") } if part.FormName() == "id" { buf := new(bytes.Buffer) @@ -281,27 +150,21 @@ func handleRun(handle *Handle) { model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { - return ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) + return c.JsonBadRequest("Models not found") } else if err != nil { - return Error500(err) + return c.Error500(err) } if model.Status != READY { - // TODO improve this - return ErrorCode(nil, 400, c.AddMap(nil)) + return c.JsonBadRequest("Model not ready to run images") } def := JustId{} err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id) if err == NotFoundError { - // TODO improve this - fmt.Printf("Could not find definition\n") - return ErrorCode(nil, 400, c.AddMap(nil)) + return c.JsonBadRequest("Could not find definition") } else if err != nil { - return Error500(err) + return c.Error500(err) } def_id := def.Id @@ -313,19 +176,13 @@ func handleRun(handle *Handle) { img_file, err := os.Create(img_path) if err != nil { - return Error500(err) + return c.Error500(err) } defer img_file.Close() img_file.Write(file) if !testImgForModel(c, model, img_path) { - LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{ - "Model": model, - "NotFound": false, - "Result": nil, - "ImageError": true, - })) - return nil + return c.JsonBadRequest("Provided image does not match the model") } root := tg.NewRoot() @@ -344,7 +201,7 @@ func handleRun(handle *Handle) { exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{}) inputImage, err := tf.NewTensor(exec_results[0].Value()) if err != nil { - return Error500(err) + return c.Error500(err) } vi := -1 @@ -368,27 +225,25 @@ func handleRun(handle *Handle) { rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi) if err != nil { - return Error500(err) + return c.Error500(err) } if !rows.Next() { - LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{ - "Model": model, - "NotFound": true, - "Result": nil, - "Confidence": confidence, - })) - return nil + return c.SendJSON(nil) } var name string if err = rows.Scan(&name); err != nil { - return nil + return c.Error500(err) } - LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{ - "Model": model, - "Result": name, - })) - return nil + returnValue := struct { + Class string `json:"class"` + Confidence float32 `json:"confidence"` + }{ + Class: name, + Confidence: confidence, + } + + return c.SendJSON(returnValue) }) } diff --git a/logic/models/train/main.go b/logic/models/train/main.go index befdb5b..2c9af45 100644 --- a/logic/models/train/main.go +++ b/logic/models/train/main.go @@ -7,7 +7,4 @@ import ( func HandleTrainEndpoints(handle *Handle) { handleTrain(handle) handleRest(handle) - - //TODO remove - handleTest(handle) } diff --git a/logic/models/train/reset.go b/logic/models/train/reset.go index 1fde63a..2bdd759 100644 --- a/logic/models/train/reset.go +++ b/logic/models/train/reset.go @@ -1,7 +1,6 @@ package models_train import ( - "net/http" "os" "path" @@ -10,82 +9,39 @@ import ( ) func handleRest(handle *Handle) { - handle.Delete("/models/train/reset", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Delete("/models/train/reset", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - var dat struct { - Id string `json:"id"` - } - - if err := c.ToJSON(r, &dat); err != nil { - return err; - } - - model, err := GetBaseModel(c.Db, dat.Id) - if err == ModelNotFoundError { - return c.JsonBadRequest("Model not found"); - } else if err != nil { - // TODO improve response - return c.Error500(err) - } - - if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING { - return c.JsonBadRequest("Model is not in status that be reset") - } - - os.RemoveAll(path.Join("savedData", model.Id, "defs")) - - _, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id) - if err != nil { - // TODO improve response - return c.Error500(err) - } - - ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) - return c.SendJSON(model.Id) + var dat struct { + Id string `json:"id"` } - f, err := MyParseForm(r) - if err != nil { - // TODO improve response - return c.ErrorCode(nil, 400, c.AddMap(nil)) + if err := c.ToJSON(&dat); err != nil { + return err } - if !CheckId(f, "id") { - // TODO improve response - return c.ErrorCode(nil, 400, c.AddMap(nil)) - } - - id := f.Get("id") - - model, err := GetBaseModel(handle.Db, id) + model, err := GetBaseModel(c.Db, dat.Id) if err == ModelNotFoundError { - return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - }) + return c.JsonBadRequest("Model not found") } else if err != nil { // TODO improve response return c.Error500(err) } if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING { - // TODO improve response - return c.ErrorCode(nil, 400, c.AddMap(nil)) + return c.JsonBadRequest("Model is not in status that be reset") } os.RemoveAll(path.Join("savedData", model.Id, "defs")) - _, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id) + _, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id) if err != nil { // TODO improve response return c.Error500(err) } ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) - Redirect("/models/edit?id="+model.Id, c.Mode, w, r) - return nil + return c.SendJSON(model.Id) }) } diff --git a/logic/models/train/tensorflow-test.go b/logic/models/train/tensorflow-test.go deleted file mode 100644 index e6d0a77..0000000 --- a/logic/models/train/tensorflow-test.go +++ /dev/null @@ -1,175 +0,0 @@ -package models_train - -import ( - "fmt" - "io" - "net/http" - "os" - "os/exec" - "path" - "strconv" - "strings" - "text/template" - - . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" -) - -/*import ( - tf "github.com/galeone/tensorflow/tensorflow/go" - tg "github.com/galeone/tfgo" - "github.com/galeone/tfgo/image" - "github.com/galeone/tfgo/image/filter" - "github.com/galeone/tfgo/image/padding" -)*/ - -func getDir() string { - dir, err := os.Getwd() - if err != nil { - panic(err) - } - return dir -} - -func shapeToSize(shape string) string { - split := strings.Split(shape, ",") - return strings.Join(split[:len(split) - 1], ",") -} - -func handleTest(handle *Handle) { - handle.Post("/models/train/test", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { - return nil - } - - id, err := GetIdFromUrl(r, "id") - if err != nil { - return ErrorCode(err, 400, c.AddMap(nil)) - } - - rows, err := handle.Db.Query("select mc.name, mdp.file_path from model_classes as mc join model_data_point as mdp on mdp.class_id = mc.id where mdp.model_mode = 1 and mc.model_id = $1 limit 1;", id) - if err != nil { - return Error500(err) - } - defer rows.Close() - - if !rows.Next() { - return Error500(err) - } - - var name string - var file_path string - err = rows.Scan(&name, &file_path) - if err != nil { - return Error500(err) - } - - file_path = strings.Replace(file_path, "file://", "", 1) - - img_path := path.Join("savedData", id, "data", "training", name, file_path) - - fmt.Printf("%s\n", img_path) - - definitions, err := handle.Db.Query("select id from model_definition where model_id=$1 and status=2 limit 1;", id) - if err != nil { - return Error500(err) - } - defer definitions.Close() - - if !definitions.Next() { - fmt.Println("Did not find definition") - return Error500(nil) - } - - var definition_id string - - if err = definitions.Scan(&definition_id); err != nil { - return Error500(err) - } - - layers, err := handle.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id) - if err != nil { - return Error500(err) - } - defer layers.Close() - - type layerrow struct { - LayerType int - Shape string - } - - got := []layerrow{} - - for layers.Next() { - var row = layerrow{} - if err = layers.Scan(&row.LayerType, &row.Shape); err != nil { - return Error500(err) - } - row.Shape = shapeToSize(row.Shape) - got = append(got, row) - } - - // Generate folder - - run_path := path.Join("/tmp", id, "defs", definition_id) - - err = os.MkdirAll(run_path, os.ModePerm) - if err != nil { - return Error500(err) - } - - f, err := os.Create(path.Join(run_path, "run.py")) - if err != nil { - return Error500(err) - } - defer f.Close() - - fmt.Printf("Using path: %s\n", run_path) - - tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py") - if err != nil { - return Error500(err) - } - - if err = tmpl.Execute(f, AnyMap{ - "Layers": got, - "Size": got[0].Shape, - "DataDir": path.Join(getDir(), "savedData", id, "data", "training"), - }); err != nil { - return Error500(err) - } - - cmd := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)) - _, err = cmd.Output() - if err != nil { - return Error500(err) - } - - result_path := path.Join("savedData", id, "defs", definition_id) - - os.MkdirAll(result_path, os.ModePerm) - - err = exec.Command("cp", path.Join(run_path, "model.keras"), path.Join(result_path, "model.keras")).Run() - if err != nil { - return Error500(err) - } - - accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val")) - if err != nil { - return Error500(err) - } - defer accuracy_file.Close() - - accuracy_file_bytes, err := io.ReadAll(accuracy_file) - if err != nil { - return Error500(err) - } - - accuracy, err := strconv.ParseFloat(string(accuracy_file_bytes), 64) - if err != nil { - return Error500(err) - } - - w.Write([]byte(strconv.FormatFloat(accuracy, 'f', -1, 64))) - return nil - }) -} diff --git a/logic/models/train/train.go b/logic/models/train/train.go index d91c0d7..d8be72d 100644 --- a/logic/models/train/train.go +++ b/logic/models/train/train.go @@ -6,12 +6,12 @@ import ( "fmt" "io" "math" - "net/http" "os" "os/exec" "path" "sort" "strconv" + "strings" "text/template" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" @@ -23,6 +23,19 @@ import ( const EPOCH_PER_RUN = 20 const MAX_EPOCH = 100 +func shapeToSize(shape string) string { + split := strings.Split(shape, ",") + return strings.Join(split[:len(split)-1], ",") +} + +func getDir() string { + dir, err := os.Getwd() + if err != nil { + panic(err) + } + return dir +} + func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) { id = "" rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy) @@ -702,13 +715,13 @@ func trainModelExp(c *Context, model *BaseModel) { ModelUpdateStatus(c, model.Id, FAILED_TRAINING) } - var definitions TrainModelRowUsables + var definitions TrainModelRowUsables - definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) - if err != nil { - failed("Failed to get definitions"); - return - } + definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) + if err != nil { + failed("Failed to get definitions") + return + } if len(definitions) == 0 { failed("No Definitions defined!") return @@ -810,38 +823,38 @@ func trainModelExp(c *Context, model *BaseModel) { } } - // Set the class status to trained + // Set the class status to trained err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING) if err != nil { failed("Failed to set class status") return } - - var dat JustId - err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) - if err == NotFoundError { + var dat JustId + + err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) + if err == NotFoundError { failed("All definitions failed to train!") return - } else if err != nil { + } else if err != nil { failed("DB: failed to read definition") return - } + } if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil { failed("Failed to update model definition") return } - to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) - if err != nil { + to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) + if err != nil { failed("Failed to select model_definition to delete") return - } + } - for _, d := range(to_delete) { + for _, d := range to_delete { os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id)) - } + } // TODO Check if returning also works here if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { @@ -1293,163 +1306,81 @@ func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy return nil } -func handle_models_train_json(w http.ResponseWriter, r *http.Request, c *Context) *Error { - var dat struct { - Id string `json:"id"` - ModelType string `json:"model_type"` - NumberOfModels int `json:"number_of_models"` - Accuracy int `json:"accuracy"` - } - - if err_ := c.ToJSON(r, &dat); err_ != nil { - return err_ - } - - if dat.Id == "" { - return c.JsonBadRequest("Please provide a id") - } - - modelTypeId := 1 - if dat.ModelType == "expandable" { - modelTypeId = 2 - } else if dat.ModelType != "simple" { - return c.JsonBadRequest("Invalid model type!") - } - - model, err := GetBaseModel(c.Db, dat.Id) - if err == ModelNotFoundError { - return c.JsonBadRequest("Model not found") - } else if err != nil { - return c.Error500(err) - } - - if model.Status != CONFIRM_PRE_TRAINING { - return c.JsonBadRequest("Model in invalid status for training") - } - - if modelTypeId == 2 { - full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels) - if full_error != nil { - return full_error - } - } else { - full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels) - if full_error != nil { - return full_error - } - } - - if modelTypeId == 2 { - go trainModelExp(c, model) - } else { - go trainModel(c, model) - } - - _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id) - if err != nil { - fmt.Println("Failed to update model status") - fmt.Println(err) - // TODO improve this response - return Error500(err) - } - - return c.SendJSON(model.Id) -} - func handleTrain(handle *Handle) { - handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { + handle.Post("/models/train", func(c *Context) *Error { + if !c.CheckAuthLevel(1) { return nil } - if c.Mode == JSON { - return handle_models_train_json(w, r, c) + var dat struct { + Id string `json:"id"` + ModelType string `json:"model_type"` + NumberOfModels int `json:"number_of_models"` + Accuracy int `json:"accuracy"` } - r.ParseForm() - f := r.Form - - number_of_models := 0 - accuracy := 0 - - if !CheckId(f, "id") || CheckEmpty(f, "model_type") || !CheckNumber(f, "number_of_models", &number_of_models) || !CheckNumber(f, "accuracy", &accuracy) { - // TODO improve this response - return ErrorCode(nil, 400, c.AddMap(nil)) + if err_ := c.ToJSON(&dat); err_ != nil { + return err_ } - id := f.Get("id") - model_type_id := 1 - model_type_form := f.Get("model_type") - - if model_type_form == "expandable" { - model_type_id = 2 - } else if model_type_form != "simple" { - return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{ - "HasData": true, - "ErrorMessage": "Invalid model type!", - }) + if dat.Id == "" { + return c.JsonBadRequest("Please provide a id") } - model, err := GetBaseModel(handle.Db, id) + modelTypeId := 1 + if dat.ModelType == "expandable" { + modelTypeId = 2 + } else if dat.ModelType != "simple" { + return c.JsonBadRequest("Invalid model type!") + } + + model, err := GetBaseModel(c.Db, dat.Id) if err == ModelNotFoundError { - return ErrorCode(nil, http.StatusNotFound, c.AddMap(AnyMap{ - "NotFoundMessage": "Model not found", - "GoBackLink": "/models", - })) + return c.JsonBadRequest("Model not found") } else if err != nil { - // TODO improve this response - return Error500(err) + return c.Error500(err) } if model.Status != CONFIRM_PRE_TRAINING { - // TODO improve this response - return ErrorCode(nil, 400, c.AddMap(nil)) + return c.JsonBadRequest("Model in invalid status for training") } - if model_type_id == 2 { - full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models) + if modelTypeId == 2 { + full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels) if full_error != nil { return full_error } } else { - full_error := generateDefinitions(c, model, accuracy, number_of_models) + full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels) if full_error != nil { return full_error } } - if model_type_id == 2 { + if modelTypeId == 2 { go trainModelExp(c, model) } else { go trainModel(c, model) } - _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id) + _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id) if err != nil { fmt.Println("Failed to update model status") fmt.Println(err) // TODO improve this response - return Error500(err) + return c.Error500(err) } - Redirect("/models/edit?id="+model.Id, c.Mode, w, r) - return nil + + return c.SendJSON(model.Id) }) - handle.Get("/model/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - // TODO check auth level - if c.Mode != NORMAL { - // This should only handle normal requests - c.Logger.Warn("This function only works with normal") - return c.UnsafeErrorCode(nil, 400, nil) - } - - f := r.URL.Query() + handle.Get("/model/epoch/update", func(c *Context) *Error { + f := c.R.URL.Query() accuracy := 0.0 if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) { - c.Logger.Warn("Invalid: model_id or definition or epoch or accuracy") - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy") } accuracy = accuracy * 100 @@ -1458,9 +1389,7 @@ func handleTrain(handle *Handle) { def_id := f.Get("definition") epoch, err := strconv.Atoi(f.Get("epoch")) if err != nil { - c.Logger.Warn("Epoch is not a number") - // No need to improve message because this function is only called internaly - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Epoch is not a number") } rows, err := c.Db.Query("select md.status from model_definition as md where md.model_id=$1 and md.id=$2", model_id, def_id) @@ -1482,8 +1411,7 @@ func handleTrain(handle *Handle) { if status != 3 { c.Logger.Warn("Definition not on status 3(training)", "status", status) - // No need to improve message because this function is only called internaly - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Definition not on status 3(training)") } c.Logger.Info("Updated model_definition!", "model", model_id, "progress", epoch, "accuracy", accuracy) @@ -1495,21 +1423,13 @@ func handleTrain(handle *Handle) { return nil }) - handle.Get("/model/head/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - // TODO check auth level - if c.Mode != NORMAL { - // This should only handle normal requests - c.Logger.Warn("This function only works with normal") - return c.UnsafeErrorCode(nil, 400, nil) - } - - f := r.URL.Query() + handle.Get("/model/head/epoch/update", func(c *Context) *Error { + f := c.R.URL.Query() accuracy := 0.0 if !CheckId(f, "head_id") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) { - c.Logger.Warn("Invalid: model_id or head_id or epoch or accuracy") - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy") } accuracy = accuracy * 100 @@ -1517,9 +1437,7 @@ func handleTrain(handle *Handle) { head_id := f.Get("head_id") epoch, err := strconv.Atoi(f.Get("epoch")) if err != nil { - c.Logger.Warn("Epoch is not a number") - // No need to improve message because this function is only called internaly - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Epoch is not a number") } rows, err := c.Db.Query("select hd.status from exp_model_head as hd where hd.id=$1;", head_id) @@ -1541,8 +1459,7 @@ func handleTrain(handle *Handle) { if status != 3 { c.Logger.Warn("Head not on status 3(training)", "status", status) - // No need to improve message because this function is only called internaly - return c.UnsafeErrorCode(nil, 400, nil) + return c.JsonBadRequest("Head not on status 3(training)") } c.Logger.Info("Updated model_head!", "head", head_id, "progress", epoch, "accuracy", accuracy) diff --git a/logic/models/utils/utils.go b/logic/models/utils/utils.go deleted file mode 100644 index 25c5755..0000000 --- a/logic/models/utils/utils.go +++ /dev/null @@ -1,14 +0,0 @@ -package models_utils - -import ( - . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" -) - -// TODO make this return and caller handle error -func ModelUpdateStatus(c *Context, id string, status int) { - _, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id) - if err != nil { - c.Logger.Error("Failed to update model status", "err", err) - c.Logger.Warn("TODO Maybe handle better") - } -} diff --git a/logic/utils/handler.go b/logic/utils/handler.go index 8f91479..055238f 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -13,194 +13,28 @@ import ( "time" dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types" + . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" "github.com/charmbracelet/log" "github.com/goccy/go-json" ) -func Mul(n1 int, n2 int) int { - return n1 * n2 -} - -func Add(n1 int, n2 int) int { - return n1 + n2 -} - -func baseLoadTemplate(base string, path string) (*template.Template, any) { - funcs := map[string]any{ - "startsWith": strings.HasPrefix, - "replace": strings.Replace, - "mul": Mul, - "add": Add, - } - return template.New(base).Funcs(funcs).ParseFiles( - "./views/"+base, - "./views/"+path, - "./views/partials/header.html", - ) -} - -func loadTemplate(path string) (*template.Template, any) { - return baseLoadTemplate("layout.html", path) -} - -func LoadView(writer http.ResponseWriter, path string, data interface{}) { - tmpl, err := loadTemplate(path) - if err != nil { - fmt.Printf("Failed to load view %s\n", path) - fmt.Println(err) - if path == "500.html" { - writer.Write([]byte("
- File Path - | -- Mode - | -- - | -- - | -
---|---|---|---|
- {{ if eq .FilePath "id://" }} - Managed - {{ else }} - {{.FilePath}} - {{ end }} - | -- {{ if (eq .Mode 2) }} - Testing - {{ else }} - Training - {{ end }} - | -- {{ if startsWith .FilePath "id://" }} - - {{ else }} - TODO - img {{ .FilePath }} - {{ end }} - | -- {{ if eq .Status 1 }} - - {{ else }} - - {{ end }} - | -
- You need to upload data so the model can train. -
-- You need to upload data so the model can train. -
- {{ if gt .NumberOfInvalidImages 0 }} -- There are images {{ .NumberOfInvalidImages }} that were loaded that do not have the correct format. These images will be delete when the model trains. -
- {{ end }} -- Done Progress - | -- Training Round Progress - | -- Accuracy - | -- Status - | -
---|---|---|---|
- {{.Epoch}} - | -- {{.EpochProgress}}/20 - | -- {{.Accuracy}}% - | -- {{ if (eq .Status 2) }} - - {{ else if (eq .Status 3) }} - - {{ else if (eq .Status 6) }} - - {{ else if (eq .Status -3) }} - - {{ else }} - {{.Status}} - {{ end }} - | -
- - | -
- Name - | -- - | -
---|---|
- {{.Name}} - | -- - Edit - - | -