chore: started working on #38

logic/models/classes/data_point.go

@@ -9,7 +9,7 @@ var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Er
 
 func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
 	id = ""
-    result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode)
+	result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode, status) values ($1, $2, $3, 1) returning id;", class_id, file_path, mode)
 	if err != nil {
 		return
 	}
@@ -21,3 +21,8 @@ func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT
 	err = result.Scan(&id)
 	return
 }
+
+func UpdateDataPointStatus(db *sql.DB, data_point_id string, status int, message *string) (err error) {
+    _, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id)
+    return
+}

logic/models/classes/main.go

@@ -71,3 +71,14 @@ func CreateClass(db *sql.DB, model_id string, order int, name string) (id string
     err = rows.Scan(&id)
     return
 }
+
+func GetNumberOfWrongDataPoints(db *sql.DB, model_id string) (number int, err error) {
+    number = 0
+    rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id) 
+    if err != nil { return }
+    defer rows.Close()
+    // TODO not an error because if there is no result means that there is no need to count
+    if !rows.Next() { return }
+    err = rows.Scan(&number)
+    return
+}

logic/models/data.go

@@ -137,9 +137,13 @@ func processZipFile(c *Context, model *BaseModel) {
 		f.Write(file_data)
 
 		if !testImgForModel(c, model, file_path) {
-			c.Logger.Errorf("Image did not have valid format for model %s\n", file_path)
+            c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
+            c.Logger.Warn("Not failling updating data point to status -1")
+            message := "Image did not have valid format for the model"
+            if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
+                c.Logger.Error("Failed to update data point status")
 			    ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
-			return
+            }
 		}
 	}
 

logic/models/edit.go

@@ -69,10 +69,12 @@ func handleEdit(handle *Handle) {
 				"Model": model,
 			}))
 		case CONFIRM_PRE_TRAINING:
+
+            wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id)
+            if err != nil { return c.Error500(err) }
+
             cls, err := model_classes.ListClasses(handle.Db, id)
-            if err != nil {
+            if err != nil { return c.Error500(err) }
-                return Error500(err)
-            }
 
             has_data, err := model_classes.ModelHasDataPoints(handle.Db, id)
             if err != nil {
@@ -83,6 +85,7 @@ func handleEdit(handle *Handle) {
 				"Model": model,
                 "Classes": cls,
                 "HasData": has_data,
+                "NumberOfInvalidImages": wrong_number,
 			}))
 		case READY:
 			LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{

logic/models/train/train.go

@@ -20,9 +20,13 @@ import (
 func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) {
 	id = ""
 	rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
-	if err != nil { return }
+	if err != nil {
+		return
+	}
 	defer rows.Close()
-	if !rows.Next() { return id, errors.New("Something wrong!") }
+	if !rows.Next() {
+		return id, errors.New("Something wrong!")
+	}
 	err = rows.Scan(&id)
 	return
 }
@@ -59,17 +63,27 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
 func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
 
 	classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
-    if err != nil { return }
+	if err != nil {
+		return
+	}
 	defer classes.Close()
-    if !classes.Next() { return }
+	if !classes.Next() {
-    if err = classes.Scan(&count); err != nil { return }
+		return
+	}
+	if err = classes.Scan(&count); err != nil {
+		return
+	}
 
 	data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
-    if err != nil { return }
+	if err != nil {
+		return
+	}
 	defer data.Close()
 
 	f, err := os.Create(path.Join(run_path, "train.csv"))
-    if err != nil { return }
+	if err != nil {
+		return
+	}
 	defer f.Close()
 	f.Write([]byte("Id,Index\n"))
 
@@ -77,7 +91,9 @@ func generateCvs(c *Context, run_path string, model_id string)  (count int, err
 		var id string
 		var class_order int
 		var file_path string
-        if err = data.Scan(&id, &class_order, &file_path); err != nil { return }
+		if err = data.Scan(&id, &class_order, &file_path); err != nil {
+			return
+		}
 		if file_path == "id://" {
 			f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
 		} else {
@@ -121,7 +137,9 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
 	}
 
 	_, err = generateCvs(c, run_path, model.Id)
-    if err != nil { return }
+	if err != nil {
+		return
+	}
 
 	// Create python script
 	f, err := os.Create(path.Join(run_path, "run.py"))
@@ -287,7 +305,7 @@ func trainModel(c *Context, model *BaseModel) {
 
 	for to_delete.Next() {
 		var id string
-        if to_delete.Scan(&id);err != nil {
+		if to_delete.Scan(&id); err != nil {
 			c.Logger.Error("Failed to scan the id of a model_definition to delete")
 			c.Logger.Error(err)
 			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
@@ -307,6 +325,30 @@ func trainModel(c *Context, model *BaseModel) {
 	ModelUpdateStatus(c, model.Id, READY)
 }
 
+func removeFailedDataPoints(db *sql.DB, model *BaseModel) (err error) {
+	rows, err := db.Query("select id from model_data_point as mdp join model_classes as mc on mc.id=mpd.class_id where mc.model_id=$1 and mdp.status=-1;", model.Id)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+    base_path := path.Join("savedData", model.Id, "data")
+
+	for rows.Next() {
+		var dataPointId string
+		err = rows.Scan(&dataPointId)
+		if err != nil {
+			return
+		}
+        err = os.RemoveAll(path.Join(base_path, dataPointId + model.Format))
+        if err != nil {
+            return
+        }
+	}
+
+	return
+}
+
 func handleTrain(handle *Handle) {
 	handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
 		if !CheckAuthLevel(1, w, r, c) {
@@ -376,6 +418,11 @@ func handleTrain(handle *Handle) {
 			return c.Error500(err)
 		}
 
+		err = removeFailedDataPoints(c.Db, model.Id)
+		if err != nil {
+			return c.Error500(err)
+		}
+
 		var fid string
 		for i := 0; i < number_of_models; i++ {
 			def_id, err := MakeDefenition(handle.Db, model.Id, accuracy)
@@ -402,7 +449,7 @@ func handleTrain(handle *Handle) {
 				// TODO improve this response
 				return c.Error500(err)
 			}
-			err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3))
+			err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)*3))
 			if err != nil {
 				ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
 				// TODO improve this response

sql/models.sql

@@ -33,7 +33,11 @@ create table if not exists model_data_point (
     file_path text not null,
     -- 1 training
     -- 2 testing
-    model_mode integer default 1
+    model_mode integer default 1,
+    -- -1 Error on creation
+    -- 1 OK
+    status integer not null,
+    status_message text
 );
 
 -- drop table if exists model_definition; 

views/models/edit.html

@@ -247,6 +247,11 @@
             <p>
                 You need to upload data so the model can train.
             </p>
+            {{ if gt .NumberOfInvalidImages 0 }}
+                <p class="danger">
+                    There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
+                </p>
+            {{ end }}
             <div class="tabs">
                 <button class="tab" data-tab="create_class">
                     Create Class
@@ -268,6 +273,11 @@
 {{ define "train-model-card" }}
     <form hx-post="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
         {{ if .HasData }}
+            {{ if gt .NumberOfInvalidImages 0 }}
+                <p class="danger">
+                    There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
+                </p>
+            {{ end }}
             {{/* TODO expading mode */}}
             <input type="hidden" value="{{ .Model.Id }}" name="id" />
             <fieldset>
@@ -296,7 +306,7 @@
             </button>
         {{ else }}
             <h2>
-                Please provide data to the model first
+                To train the model please provide data to the model first
             </h2>
         {{ end }}
     </form>

views/styles/main.css

@@ -66,6 +66,10 @@ main {
     font-size: 1.1rem;
 }
 
+.danger {
+    color: red;
+}
+
 /* Generic */
 .button,
 button {