From b6afecc68202c47bfa68c0688a0cc4997fe08601 Mon Sep 17 00:00:00 2001 From: Andre Henriques Date: Tue, 10 Oct 2023 12:28:49 +0100 Subject: [PATCH] chore: started working on #38 --- logic/models/classes/data_point.go | 27 ++-- logic/models/classes/main.go | 11 ++ logic/models/data.go | 10 +- logic/models/edit.go | 9 +- logic/models/train/train.go | 231 +++++++++++++++++------------ sql/models.sql | 6 +- views/models/edit.html | 64 ++++---- views/styles/main.css | 4 + 8 files changed, 225 insertions(+), 137 deletions(-) diff --git a/logic/models/classes/data_point.go b/logic/models/classes/data_point.go index 14b5ab0..eb38bf7 100644 --- a/logic/models/classes/data_point.go +++ b/logic/models/classes/data_point.go @@ -8,16 +8,21 @@ import ( var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error") func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) { - id = "" - result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode) - if err != nil { - return - } - defer result.Close() - if !result.Next() { - err = FailedToGetIdAfterInsertError - return - } - err = result.Scan(&id) + id = "" + result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode, status) values ($1, $2, $3, 1) returning id;", class_id, file_path, mode) + if err != nil { + return + } + defer result.Close() + if !result.Next() { + err = FailedToGetIdAfterInsertError + return + } + err = result.Scan(&id) + return +} + +func UpdateDataPointStatus(db *sql.DB, data_point_id string, status int, message *string) (err error) { + _, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id) return } diff --git a/logic/models/classes/main.go b/logic/models/classes/main.go index 2921248..1381609 100644 --- a/logic/models/classes/main.go +++ b/logic/models/classes/main.go @@ -71,3 +71,14 @@ func CreateClass(db *sql.DB, model_id string, order int, name string) (id string err = rows.Scan(&id) return } + +func GetNumberOfWrongDataPoints(db *sql.DB, model_id string) (number int, err error) { + number = 0 + rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id) + if err != nil { return } + defer rows.Close() + // TODO not an error because if there is no result means that there is no need to count + if !rows.Next() { return } + err = rows.Scan(&number) + return +} diff --git a/logic/models/data.go b/logic/models/data.go index 737047a..a7cf3fe 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -137,9 +137,13 @@ func processZipFile(c *Context, model *BaseModel) { f.Write(file_data) if !testImgForModel(c, model, file_path) { - c.Logger.Errorf("Image did not have valid format for model %s\n", file_path) - ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) - return + c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) + c.Logger.Warn("Not failling updating data point to status -1") + message := "Image did not have valid format for the model" + if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { + c.Logger.Error("Failed to update data point status") + ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) + } } } diff --git a/logic/models/edit.go b/logic/models/edit.go index facc8a2..ab3b4bd 100644 --- a/logic/models/edit.go +++ b/logic/models/edit.go @@ -69,10 +69,12 @@ func handleEdit(handle *Handle) { "Model": model, })) case CONFIRM_PRE_TRAINING: + + wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id) + if err != nil { return c.Error500(err) } + cls, err := model_classes.ListClasses(handle.Db, id) - if err != nil { - return Error500(err) - } + if err != nil { return c.Error500(err) } has_data, err := model_classes.ModelHasDataPoints(handle.Db, id) if err != nil { @@ -83,6 +85,7 @@ func handleEdit(handle *Handle) { "Model": model, "Classes": cls, "HasData": has_data, + "NumberOfInvalidImages": wrong_number, })) case READY: LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ diff --git a/logic/models/train/train.go b/logic/models/train/train.go index 3af8261..ee24dfd 100644 --- a/logic/models/train/train.go +++ b/logic/models/train/train.go @@ -19,12 +19,16 @@ import ( func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) { id = "" - rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy) - if err != nil { return } - defer rows.Close() - if !rows.Next() { return id, errors.New("Something wrong!") } + rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy) + if err != nil { + return + } + defer rows.Close() + if !rows.Next() { + return id, errors.New("Something wrong!") + } err = rows.Scan(&id) - return + return } type ModelDefinitionStatus int @@ -41,9 +45,9 @@ const ( type LayerType int const ( - LAYER_INPUT LayerType = 1 - LAYER_DENSE = 2 - LAYER_FLATTEN = 3 + LAYER_INPUT LayerType = 1 + LAYER_DENSE = 2 + LAYER_FLATTEN = 3 ) func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionStatus) (err error) { @@ -56,36 +60,48 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType, return } -func generateCvs(c *Context, run_path string, model_id string) (count int, err error) { +func generateCvs(c *Context, run_path string, model_id string) (count int, err error) { - classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id) - if err != nil { return } - defer classes.Close() - if !classes.Next() { return } - if err = classes.Scan(&count); err != nil { return } + classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id) + if err != nil { + return + } + defer classes.Close() + if !classes.Next() { + return + } + if err = classes.Scan(&count); err != nil { + return + } - data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id) - if err != nil { return } - defer data.Close() + data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id) + if err != nil { + return + } + defer data.Close() - f, err := os.Create(path.Join(run_path, "train.csv")) - if err != nil { return } - defer f.Close() - f.Write([]byte("Id,Index\n")) - - for data.Next() { - var id string - var class_order int - var file_path string - if err = data.Scan(&id, &class_order, &file_path); err != nil { return } - if file_path == "id://" { - f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n")) - } else { - return count, errors.New("TODO generateCvs to file_path " + file_path) - } - } + f, err := os.Create(path.Join(run_path, "train.csv")) + if err != nil { + return + } + defer f.Close() + f.Write([]byte("Id,Index\n")) - return + for data.Next() { + var id string + var class_order int + var file_path string + if err = data.Scan(&id, &class_order, &file_path); err != nil { + return + } + if file_path == "id://" { + f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n")) + } else { + return count, errors.New("TODO generateCvs to file_path " + file_path) + } + } + + return } func trainDefinition(c *Context, model *BaseModel, definition_id string) (accuracy float64, err error) { @@ -120,8 +136,10 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura return } - _, err = generateCvs(c, run_path, model.Id) - if err != nil { return } + _, err = generateCvs(c, run_path, model.Id) + if err != nil { + return + } // Create python script f, err := os.Create(path.Join(run_path, "run.py")) @@ -136,19 +154,19 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura } if err = tmpl.Execute(f, AnyMap{ - "Layers": got, - "Size": got[0].Shape, - "DataDir": path.Join(getDir(), "savedData", model.Id, "data"), - "RunPath": run_path, - "ColorMode": model.ImageMode, + "Layers": got, + "Size": got[0].Shape, + "DataDir": path.Join(getDir(), "savedData", model.Id, "data"), + "RunPath": run_path, + "ColorMode": model.ImageMode, }); err != nil { return } // Run the command - out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output() - if err != nil { - fmt.Println(string(out)) + out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output() + if err != nil { + fmt.Println(string(out)) return } @@ -174,14 +192,14 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura return } - fmt.Println(string(accuracy_file_bytes)) + fmt.Println(string(accuracy_file_bytes)) accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64) if err != nil { return } - - os.RemoveAll(run_path) + + os.RemoveAll(run_path) return } @@ -205,8 +223,8 @@ func trainModel(c *Context, model *BaseModel) { for definitionsRows.Next() { var rowv row if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy); err != nil { - c.Logger.Error("Failed to train Model Could not read definition from db!Err:") - c.Logger.Error(err) + c.Logger.Error("Failed to train Model Could not read definition from db!Err:") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return } @@ -214,7 +232,7 @@ func trainModel(c *Context, model *BaseModel) { } if len(definitions) == 0 { - c.Logger.Error("No Definitions defined!") + c.Logger.Error("No Definitions defined!") ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return } @@ -222,8 +240,8 @@ func trainModel(c *Context, model *BaseModel) { for _, def := range definitions { accuracy, err := trainDefinition(c, model, def.id) if err != nil { - c.Logger.Error("Failed to train definition!Err:") - c.Logger.Error(err) + c.Logger.Error("Failed to train definition!Err:") + c.Logger.Error(err) ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) continue } @@ -247,66 +265,90 @@ func trainModel(c *Context, model *BaseModel) { rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) if err != nil { - c.Logger.Error("DB: failed to read definition") - c.Logger.Error(err) + c.Logger.Error("DB: failed to read definition") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return } - defer rows.Close() + defer rows.Close() - if !rows.Next() { - // TODO Make the Model status have a message - c.Logger.Error("All definitions failed to train!") + if !rows.Next() { + // TODO Make the Model status have a message + c.Logger.Error("All definitions failed to train!") ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return - } + } - var id string - if err = rows.Scan(&id); err != nil { - c.Logger.Error("Failed to read id:") - c.Logger.Error(err) + var id string + if err = rows.Scan(&id); err != nil { + c.Logger.Error("Failed to read id:") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return - } + } - if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil { - c.Logger.Error("Failed to update model definition") - c.Logger.Error(err) + if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil { + c.Logger.Error("Failed to update model definition") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return - } + } - to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) - if err != nil { - c.Logger.Error("Failed to select model_definition to delete") - c.Logger.Error(err) + to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) + if err != nil { + c.Logger.Error("Failed to select model_definition to delete") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return - } - defer to_delete.Close() + } + defer to_delete.Close() - for to_delete.Next() { - var id string - if to_delete.Scan(&id);err != nil { - c.Logger.Error("Failed to scan the id of a model_definition to delete") - c.Logger.Error(err) - ModelUpdateStatus(c, model.Id, FAILED_TRAINING) - return - } - os.RemoveAll(path.Join("savedData", model.Id, "defs", id)) - } - - // TODO Check if returning also works here - if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { - c.Logger.Error("Failed to delete model_definition") - c.Logger.Error(err) + for to_delete.Next() { + var id string + if to_delete.Scan(&id); err != nil { + c.Logger.Error("Failed to scan the id of a model_definition to delete") + c.Logger.Error(err) + ModelUpdateStatus(c, model.Id, FAILED_TRAINING) + return + } + os.RemoveAll(path.Join("savedData", model.Id, "defs", id)) + } + + // TODO Check if returning also works here + if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { + c.Logger.Error("Failed to delete model_definition") + c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return - } + } ModelUpdateStatus(c, model.Id, READY) } +func removeFailedDataPoints(db *sql.DB, model *BaseModel) (err error) { + rows, err := db.Query("select id from model_data_point as mdp join model_classes as mc on mc.id=mpd.class_id where mc.model_id=$1 and mdp.status=-1;", model.Id) + if err != nil { + return + } + defer rows.Close() + + base_path := path.Join("savedData", model.Id, "data") + + for rows.Next() { + var dataPointId string + err = rows.Scan(&dataPointId) + if err != nil { + return + } + err = os.RemoveAll(path.Join(base_path, dataPointId + model.Format)) + if err != nil { + return + } + } + + return +} + func handleTrain(handle *Handle) { handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { @@ -376,6 +418,11 @@ func handleTrain(handle *Handle) { return c.Error500(err) } + err = removeFailedDataPoints(c.Db, model.Id) + if err != nil { + return c.Error500(err) + } + var fid string for i := 0; i < number_of_models; i++ { def_id, err := MakeDefenition(handle.Db, model.Id, accuracy) @@ -402,13 +449,13 @@ func handleTrain(handle *Handle) { // TODO improve this response return c.Error500(err) } - err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3)) + err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)*3)) if err != nil { ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING) // TODO improve this response return c.Error500(err) } - // Using sparce + // Using sparce err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d, 1", len(cls))) if err != nil { ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING) diff --git a/sql/models.sql b/sql/models.sql index 7003eb0..3dae684 100644 --- a/sql/models.sql +++ b/sql/models.sql @@ -33,7 +33,11 @@ create table if not exists model_data_point ( file_path text not null, -- 1 training -- 2 testing - model_mode integer default 1 + model_mode integer default 1, + -- -1 Error on creation + -- 1 OK + status integer not null, + status_message text ); -- drop table if exists model_definition; diff --git a/views/models/edit.html b/views/models/edit.html index f55b26e..e862c7c 100644 --- a/views/models/edit.html +++ b/views/models/edit.html @@ -247,6 +247,11 @@

You need to upload data so the model can train.

+ {{ if gt .NumberOfInvalidImages 0 }} +

+ There are images that were loaded that do not have the correct format. These images will be delete when the model trains. +

+ {{ end }}
+ {{ if gt .NumberOfInvalidImages 0 }} +

+ There are images that were loaded that do not have the correct format. These images will be delete when the model trains. +

+ {{ end }} + {{/* TODO expading mode */}} + +
+ + Model Type + +
+ + +
+
+ {{/* TODO allow more models to be created */}} +
+ + +
+ {{/* TODO to Change the acc */}} +
+ + +
+ {{/* TODO allow to chose the base of the model */}} + {{/* TODO allow to change the shape of the model */}} + {{ else }}

- Please provide data to the model first + To train the model please provide data to the model first

{{ end }} diff --git a/views/styles/main.css b/views/styles/main.css index 498b35e..8e2e54b 100644 --- a/views/styles/main.css +++ b/views/styles/main.css @@ -66,6 +66,10 @@ main { font-size: 1.1rem; } +.danger { + color: red; +} + /* Generic */ .button, button {