chore: started working on #38

logic/models/classes/data_point.go

@@ -8,16 +8,21 @@ import (
 var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")
 
 func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
-    id = ""
+	id = ""
-    result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode)
+	result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode, status) values ($1, $2, $3, 1) returning id;", class_id, file_path, mode)
-    if err != nil {
+	if err != nil {
-        return
+		return
-    }
+	}
-    defer result.Close()
+	defer result.Close()
-    if !result.Next() {
+	if !result.Next() {
-        err =  FailedToGetIdAfterInsertError
+		err = FailedToGetIdAfterInsertError
-        return
+		return
-    }
+	}
-    err = result.Scan(&id)
+	err = result.Scan(&id)
+	return
+}
+
+func UpdateDataPointStatus(db *sql.DB, data_point_id string, status int, message *string) (err error) {
+    _, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id)
     return
 }

logic/models/classes/main.go

@@ -71,3 +71,14 @@ func CreateClass(db *sql.DB, model_id string, order int, name string) (id string
     err = rows.Scan(&id)
     return
 }
+
+func GetNumberOfWrongDataPoints(db *sql.DB, model_id string) (number int, err error) {
+    number = 0
+    rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id) 
+    if err != nil { return }
+    defer rows.Close()
+    // TODO not an error because if there is no result means that there is no need to count
+    if !rows.Next() { return }
+    err = rows.Scan(&number)
+    return
+}

logic/models/data.go

@@ -137,9 +137,13 @@ func processZipFile(c *Context, model *BaseModel) {
 		f.Write(file_data)
 
 		if !testImgForModel(c, model, file_path) {
-			c.Logger.Errorf("Image did not have valid format for model %s\n", file_path)
+            c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
-			ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
+            c.Logger.Warn("Not failling updating data point to status -1")
-			return
+            message := "Image did not have valid format for the model"
+            if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
+                c.Logger.Error("Failed to update data point status")
+			    ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
+            }
 		}
 	}
 

logic/models/edit.go

@@ -69,10 +69,12 @@ func handleEdit(handle *Handle) {
 				"Model": model,
 			}))
 		case CONFIRM_PRE_TRAINING:
+
+            wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id)
+            if err != nil { return c.Error500(err) }
+
             cls, err := model_classes.ListClasses(handle.Db, id)
-            if err != nil {
+            if err != nil { return c.Error500(err) }
-                return Error500(err)
-            }
 
             has_data, err := model_classes.ModelHasDataPoints(handle.Db, id)
             if err != nil {
@@ -83,6 +85,7 @@ func handleEdit(handle *Handle) {
 				"Model": model,
                 "Classes": cls,
                 "HasData": has_data,
+                "NumberOfInvalidImages": wrong_number,
 			}))
 		case READY:
 			LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{

logic/models/train/train.go

@@ -19,12 +19,16 @@ import (
 
 func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) {
 	id = ""
-    rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
+	rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
-	if err != nil { return }
+	if err != nil {
-    defer rows.Close()
+		return
-	if !rows.Next() { return id, errors.New("Something wrong!") }
+	}
+	defer rows.Close()
+	if !rows.Next() {
+		return id, errors.New("Something wrong!")
+	}
 	err = rows.Scan(&id)
-    return
+	return
 }
 
 type ModelDefinitionStatus int
@@ -41,9 +45,9 @@ const (
 type LayerType int
 
 const (
-    LAYER_INPUT LayerType = 1
+	LAYER_INPUT   LayerType = 1
-    LAYER_DENSE = 2
+	LAYER_DENSE             = 2
-    LAYER_FLATTEN = 3
+	LAYER_FLATTEN           = 3
 )
 
 func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionStatus) (err error) {
@@ -56,36 +60,48 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
 	return
 }
 
-func generateCvs(c *Context, run_path string, model_id string)  (count int, err error) {
+func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
 
-    classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
+	classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
-    if err != nil { return }
+	if err != nil {
-    defer classes.Close()
+		return
-    if !classes.Next() { return }
+	}
-    if err = classes.Scan(&count); err != nil { return }
+	defer classes.Close()
+	if !classes.Next() {
+		return
+	}
+	if err = classes.Scan(&count); err != nil {
+		return
+	}
 
-    data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
+	data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
-    if err != nil { return }
+	if err != nil {
-    defer data.Close()
+		return
+	}
+	defer data.Close()
 
-    f, err := os.Create(path.Join(run_path, "train.csv"))
+	f, err := os.Create(path.Join(run_path, "train.csv"))
-    if err != nil { return }
+	if err != nil {
-    defer f.Close()
+		return
-    f.Write([]byte("Id,Index\n"))
+	}
-    
+	defer f.Close()
-    for data.Next() {
+	f.Write([]byte("Id,Index\n"))
-        var id string
-        var class_order int
-        var file_path string
-        if err = data.Scan(&id, &class_order, &file_path); err != nil { return }
-        if file_path == "id://" {
-            f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
-        } else {
-            return count, errors.New("TODO generateCvs to file_path " + file_path)
-        }
-    }
 
-    return
+	for data.Next() {
+		var id string
+		var class_order int
+		var file_path string
+		if err = data.Scan(&id, &class_order, &file_path); err != nil {
+			return
+		}
+		if file_path == "id://" {
+			f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
+		} else {
+			return count, errors.New("TODO generateCvs to file_path " + file_path)
+		}
+	}
+
+	return
 }
 
 func trainDefinition(c *Context, model *BaseModel, definition_id string) (accuracy float64, err error) {
@@ -120,8 +136,10 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
 		return
 	}
 
-    _, err = generateCvs(c, run_path, model.Id)
+	_, err = generateCvs(c, run_path, model.Id)
-    if err != nil { return }
+	if err != nil {
+		return
+	}
 
 	// Create python script
 	f, err := os.Create(path.Join(run_path, "run.py"))
@@ -136,19 +154,19 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
 	}
 
 	if err = tmpl.Execute(f, AnyMap{
-		"Layers":  got,
+		"Layers":    got,
-		"Size":    got[0].Shape,
+		"Size":      got[0].Shape,
-		"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
+		"DataDir":   path.Join(getDir(), "savedData", model.Id, "data"),
-        "RunPath": run_path,
+		"RunPath":   run_path,
-        "ColorMode": model.ImageMode,
+		"ColorMode": model.ImageMode,
 	}); err != nil {
 		return
 	}
 
 	// Run the command
-    out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
+	out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
-    if err != nil {
+	if err != nil {
-        fmt.Println(string(out))
+		fmt.Println(string(out))
 		return
 	}
 
@@ -174,14 +192,14 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
 		return
 	}
 
-    fmt.Println(string(accuracy_file_bytes))
+	fmt.Println(string(accuracy_file_bytes))
 
 	accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64)
 	if err != nil {
 		return
 	}
-    
+
-    os.RemoveAll(run_path)
+	os.RemoveAll(run_path)
 	return
 }
 
@@ -205,8 +223,8 @@ func trainModel(c *Context, model *BaseModel) {
 	for definitionsRows.Next() {
 		var rowv row
 		if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy); err != nil {
-            c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
+			c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
-            c.Logger.Error(err)
+			c.Logger.Error(err)
 			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 			return
 		}
@@ -214,7 +232,7 @@ func trainModel(c *Context, model *BaseModel) {
 	}
 
 	if len(definitions) == 0 {
-        c.Logger.Error("No Definitions defined!")
+		c.Logger.Error("No Definitions defined!")
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
 	}
@@ -222,8 +240,8 @@ func trainModel(c *Context, model *BaseModel) {
 	for _, def := range definitions {
 		accuracy, err := trainDefinition(c, model, def.id)
 		if err != nil {
-            c.Logger.Error("Failed to train definition!Err:")
+			c.Logger.Error("Failed to train definition!Err:")
-            c.Logger.Error(err)
+			c.Logger.Error(err)
 			ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
 			continue
 		}
@@ -247,66 +265,90 @@ func trainModel(c *Context, model *BaseModel) {
 
 	rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
 	if err != nil {
-        c.Logger.Error("DB: failed to read definition")
+		c.Logger.Error("DB: failed to read definition")
-        c.Logger.Error(err)
+		c.Logger.Error(err)
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
 	}
-    defer rows.Close()
+	defer rows.Close()
 
-    if !rows.Next() {
+	if !rows.Next() {
-        // TODO Make the Model status have a message
+		// TODO Make the Model status have a message
-        c.Logger.Error("All definitions failed to train!")
+		c.Logger.Error("All definitions failed to train!")
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
-    }
+	}
 
-    var id string
+	var id string
-    if err = rows.Scan(&id); err != nil {
+	if err = rows.Scan(&id); err != nil {
-        c.Logger.Error("Failed to read id:")
+		c.Logger.Error("Failed to read id:")
-        c.Logger.Error(err)
+		c.Logger.Error(err)
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
-    }
+	}
 
-    if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
+	if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
-        c.Logger.Error("Failed to update model definition")
+		c.Logger.Error("Failed to update model definition")
-        c.Logger.Error(err)
+		c.Logger.Error(err)
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
-    }
+	}
 
-    to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
+	to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
-    if err != nil {
+	if err != nil {
-        c.Logger.Error("Failed to select model_definition to delete")
+		c.Logger.Error("Failed to select model_definition to delete")
-        c.Logger.Error(err)
+		c.Logger.Error(err)
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
-    }
+	}
-    defer to_delete.Close()
+	defer to_delete.Close()
 
-    for to_delete.Next() {
+	for to_delete.Next() {
-        var id string
+		var id string
-        if to_delete.Scan(&id);err != nil {
+		if to_delete.Scan(&id); err != nil {
-            c.Logger.Error("Failed to scan the id of a model_definition to delete")
+			c.Logger.Error("Failed to scan the id of a model_definition to delete")
-            c.Logger.Error(err)
+			c.Logger.Error(err)
-            ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
-            return
+			return
-        }
+		}
-        os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
+		os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
-    }
+	}
-    
+
-    // TODO Check if returning also works here
+	// TODO Check if returning also works here
-    if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
+	if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
-        c.Logger.Error("Failed to delete model_definition")
+		c.Logger.Error("Failed to delete model_definition")
-        c.Logger.Error(err)
+		c.Logger.Error(err)
 		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
 		return
-    }
+	}
 
 	ModelUpdateStatus(c, model.Id, READY)
 }
 
+func removeFailedDataPoints(db *sql.DB, model *BaseModel) (err error) {
+	rows, err := db.Query("select id from model_data_point as mdp join model_classes as mc on mc.id=mpd.class_id where mc.model_id=$1 and mdp.status=-1;", model.Id)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+    base_path := path.Join("savedData", model.Id, "data")
+
+	for rows.Next() {
+		var dataPointId string
+		err = rows.Scan(&dataPointId)
+		if err != nil {
+			return
+		}
+        err = os.RemoveAll(path.Join(base_path, dataPointId + model.Format))
+        if err != nil {
+            return
+        }
+	}
+
+	return
+}
+
 func handleTrain(handle *Handle) {
 	handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
 		if !CheckAuthLevel(1, w, r, c) {
@@ -376,6 +418,11 @@ func handleTrain(handle *Handle) {
 			return c.Error500(err)
 		}
 
+		err = removeFailedDataPoints(c.Db, model.Id)
+		if err != nil {
+			return c.Error500(err)
+		}
+
 		var fid string
 		for i := 0; i < number_of_models; i++ {
 			def_id, err := MakeDefenition(handle.Db, model.Id, accuracy)
@@ -402,13 +449,13 @@ func handleTrain(handle *Handle) {
 				// TODO improve this response
 				return c.Error500(err)
 			}
-			err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3))
+			err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)*3))
 			if err != nil {
 				ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
 				// TODO improve this response
 				return c.Error500(err)
 			}
-            // Using sparce
+			// Using sparce
 			err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d, 1", len(cls)))
 			if err != nil {
 				ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)

sql/models.sql

@@ -33,7 +33,11 @@ create table if not exists model_data_point (
     file_path text not null,
     -- 1 training
     -- 2 testing
-    model_mode integer default 1
+    model_mode integer default 1,
+    -- -1 Error on creation
+    -- 1 OK
+    status integer not null,
+    status_message text
 );
 
 -- drop table if exists model_definition; 

views/models/edit.html

@@ -247,6 +247,11 @@
             <p>
                 You need to upload data so the model can train.
             </p>
+            {{ if gt .NumberOfInvalidImages 0 }}
+                <p class="danger">
+                    There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
+                </p>
+            {{ end }}
             <div class="tabs">
                 <button class="tab" data-tab="create_class">
                     Create Class
@@ -268,35 +273,40 @@
 {{ define "train-model-card" }}
     <form hx-post="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
         {{ if .HasData }}
-        {{/* TODO expading mode */}}
+            {{ if gt .NumberOfInvalidImages 0 }}
-        <input type="hidden" value="{{ .Model.Id }}" name="id" />
+                <p class="danger">
-        <fieldset>
+                    There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
-            <legend>
+                </p>
-                Model Type
+            {{ end }}
-            </legend>
+            {{/* TODO expading mode */}}
-            <div class="input-radial">
+            <input type="hidden" value="{{ .Model.Id }}" name="id" />
-                <input id="model_type_simple" value="simple" name="model_type" type="radio" checked  />
+            <fieldset>
-                <label for="model_type_simple">Simple</label>
+                <legend>
-            </div>
+                    Model Type
-        </fieldset>
+                </legend>
-        {{/* TODO allow more models to be created */}}
+                <div class="input-radial">
-        <fieldset>
+                    <input id="model_type_simple" value="simple" name="model_type" type="radio" checked  />
-            <label for="number_of_models">Number of Models</label>
+                    <label for="model_type_simple">Simple</label>
-            <input id="number_of_models" type="number" name="number_of_models" value="1" />
+                </div>
-        </fieldset>
+            </fieldset>
-        {{/* TODO to Change the acc */}}
+            {{/* TODO allow more models to be created */}}
-        <fieldset>
+            <fieldset>
-            <label for="accuracy">Target accuracy</label>
+                <label for="number_of_models">Number of Models</label>
-            <input id="accuracy" type="number" name="accuracy" value="95" />
+                <input id="number_of_models" type="number" name="number_of_models" value="1" />
-        </fieldset>
+            </fieldset>
-        {{/* TODO allow to chose the base of the model */}}
+            {{/* TODO to Change the acc */}}
-        {{/* TODO allow to change the shape of the model */}}
+            <fieldset>
-        <button>
+                <label for="accuracy">Target accuracy</label>
-            Train
+                <input id="accuracy" type="number" name="accuracy" value="95" />
-        </button>
+            </fieldset>
+            {{/* TODO allow to chose the base of the model */}}
+            {{/* TODO allow to change the shape of the model */}}
+            <button>
+                Train
+            </button>
         {{ else }}
             <h2>
-                Please provide data to the model first
+                To train the model please provide data to the model first
             </h2>
         {{ end }}
     </form>

views/styles/main.css

@@ -66,6 +66,10 @@ main {
     font-size: 1.1rem;
 }
 
+.danger {
+    color: red;
+}
+
 /* Generic */
 .button,
 button {