more work on the expadable models

auto_reload.sh

@@ -0,0 +1,5 @@
+#!/bin/fish
+
+cd $(dirname "$0")  
+
+go run .

logic/models/train/train.go

@@ -17,6 +17,7 @@ import (
 	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	"github.com/charmbracelet/log"
 )
 
 const EPOCH_PER_RUN = 20
@@ -198,6 +199,151 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
 	return
 }
 
+func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
+	accuracy = 0
+
+	c.Logger.Warn("About to start training definition")
+
+	// Get untrained models heads
+
+	// Status = 2 (INIT)
+	rows, err := c.Db.Query("select id, range_start, range_end exp_model_head where def_id=$1 and status = 2", definition_id)
+	if err != nil {
+		return
+	}
+    defer rows.Close()
+
+	type ExpHead struct {
+		id          string
+		start int
+		end   int
+	}
+
+    exp := ExpHead{}
+
+    if rows.Next() {
+        if err = rows.Scan(&exp.id, &exp.start, &exp.end); err == nil {
+            return
+        }
+    } else {
+        log.Error("Failed to get the exp head of the model")
+        err = errors.New("Failed to get the exp head of the model")
+        return
+    }
+
+    if rows.Next() {
+        log.Error("This training function can only train one model at the time")
+        err = errors.New("This training function can only train one model at the time")
+        return
+    }
+
+	layers, err := c.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
+	if err != nil {
+		return
+	}
+	defer layers.Close()
+
+	type layerrow struct {
+		LayerType int
+		Shape     string
+	}
+
+	got := []layerrow{}
+
+	for layers.Next() {
+		var row = layerrow{}
+		if err = layers.Scan(&row.LayerType, &row.Shape); err != nil {
+			return
+		}
+		row.Shape = shapeToSize(row.Shape)
+		got = append(got, row)
+	}
+
+    got = append(got, layerrow{
+        LayerType: LAYER_DENSE,
+        Shape: fmt.Sprintf("%d", exp.end - exp.start),
+    })
+
+	// Generate run folder
+	run_path := path.Join("/tmp", model.Id, "defs", definition_id)
+
+	err = os.MkdirAll(run_path, os.ModePerm)
+	if err != nil {
+		return
+	}
+	defer os.RemoveAll(run_path)
+
+	_, err = generateCvs(c, run_path, model.Id)
+	if err != nil {
+		return
+	}
+
+    // TODO update the run script
+
+	// Create python script
+	f, err := os.Create(path.Join(run_path, "run.py"))
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
+	if err != nil {
+		return
+	}
+
+	// Copy result around
+	result_path := path.Join("savedData", model.Id, "defs", definition_id)
+
+	if err = tmpl.Execute(f, AnyMap{
+		"Layers":           got,
+		"Size":             got[0].Shape,
+		"DataDir":          path.Join(getDir(), "savedData", model.Id, "data"),
+		"RunPath":          run_path,
+		"ColorMode":        model.ImageMode,
+		"Model":            model,
+		"EPOCH_PER_RUN":    EPOCH_PER_RUN,
+		"DefId":            definition_id,
+		"LoadPrev":         load_prev,
+		"LastModelRunPath": path.Join(getDir(), result_path, "model.keras"),
+		"SaveModelPath":    path.Join(getDir(), result_path),
+	}); err != nil {
+		return
+	}
+
+	// Run the command
+	out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput()
+	if err != nil {
+		c.Logger.Debug(string(out))
+		return
+	}
+
+	c.Logger.Info("Python finished running")
+
+	if err = os.MkdirAll(result_path, os.ModePerm); err != nil {
+		return
+	}
+
+	accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val"))
+	if err != nil {
+		return
+	}
+	defer accuracy_file.Close()
+
+	accuracy_file_bytes, err := io.ReadAll(accuracy_file)
+	if err != nil {
+		return
+	}
+
+	accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64)
+	if err != nil {
+		return
+	}
+
+	c.Logger.Info("Model finished training!", "accuracy", accuracy)
+	return
+}
+
 func remove[T interface{}](lst []T, i int) []T {
 	lng := len(lst)
 	if i >= lng {
@@ -433,7 +579,7 @@ func trainModel(c *Context, model *BaseModel) {
 }
 
 func trainModelExp(c *Context, model *BaseModel) {
-    var err error = nil
+	var err error = nil
 
 	failed := func(msg string) {
 		c.Logger.Error(msg, "err", err)
@@ -442,7 +588,7 @@ func trainModelExp(c *Context, model *BaseModel) {
 
 	definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
 	if err != nil {
-        failed("Failed to trainModel!")
+		failed("Failed to trainModel!")
 		return
 	}
 	defer definitionsRows.Close()
@@ -474,7 +620,7 @@ func trainModelExp(c *Context, model *BaseModel) {
 		var toRemove ToRemoveList = []int{}
 		for i, def := range definitions {
 			ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING)
-			accuracy, err := trainDefinition(c, model, def.id, !firstRound)
+			accuracy, err := trainDefinitionExp(c, model, def.id, !firstRound)
 			if err != nil {
 				c.Logger.Error("Failed to train definition!Err:", "err", err)
 				ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
@@ -797,23 +943,23 @@ func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, numb
 }
 
 func CreateExpModelHead(c *Context, def_id string, range_start int, range_end int, status ModelDefinitionStatus) (id string, err error) {
-    rows, err := c.Db.Query("insert into exp_model_head (def_id, range_start, range_end) values ($1, $2, $3, $4) returning id", def_id, range_start, range_end, status)
+	rows, err := c.Db.Query("insert into exp_model_head (def_id, range_start, range_end) values ($1, $2, $3, $4) returning id", def_id, range_start, range_end, status)
 
 	if err != nil {
 		return
 	}
 	defer rows.Close()
 
-    if !rows.Next() {
+	if !rows.Next() {
-        c.Logger.Error("Could not get status of model definition")
+		c.Logger.Error("Could not get status of model definition")
-        err = errors.New("Could not get status of model definition")
+		err = errors.New("Could not get status of model definition")
-        return 
+		return
-    }
+	}
 
 	err = rows.Scan(&id)
-    if err != nil {
+	if err != nil {
-        return
+		return
-    }
+	}
 
 	return
 }
@@ -833,7 +979,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
 	}
 
 	if complexity == 0 {
-        return failed()
+		return failed()
 	}
 
 	def_id, err := MakeDefenition(c.Db, model.Id, target_accuracy)
@@ -843,8 +989,8 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
 
 	order := 1
 
-    width := model.Width
+	width := model.Width
-    height := model.Height
+	height := model.Height
 
 	// Note the shape of the first layer defines the import size
 	if complexity == 2 {
@@ -853,62 +999,62 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
 		height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
 		c.Logger.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
 
-	}        
+	}
 
-    err = MakeLayerExpandable(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1)
+	err = MakeLayerExpandable(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1)
 
 	order++
-    
-    // handle the errors inside the pervious if block
-    if err != nil {
-        return failed()
-    }
-    
-    // Create the blocks
-    loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
-    if loop == 0 {
-        loop = 1
-    }
 
-    for i := 0; i < loop; i++ {
+	// handle the errors inside the pervious if block
-        err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1)
+	if err != nil {
-        order++
+		return failed()
-        if err != nil {
+	}
-            return failed()
-        }
-    }
-    
-    // Flatten the blocks into dense   
-    err = MakeLayerExpandable(c.Db, def_id, order, LAYER_FLATTEN, "", 1)
-    if err != nil {
-        return failed()
-    }
-    order++
 
-    // Flatten the blocks into dense   
+	// Create the blocks
-    err = MakeLayerExpandable(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes * 2), 1)
+	loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
-    if err != nil {
+	if loop == 0 {
-        return failed()
+		loop = 1
-    }
+	}
-    order++
 
-    loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
+	for i := 0; i < loop; i++ {
-    if loop == 0 {
+		err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1)
-        loop = 1
+		order++
-    }
+		if err != nil {
+			return failed()
+		}
+	}
 
-    for i := 0; i < loop; i++ {
+	// Flatten the blocks into dense
-        err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
+	err = MakeLayerExpandable(c.Db, def_id, order, LAYER_FLATTEN, "", 1)
-        order++
+	if err != nil {
-        if err != nil {
+		return failed()
-            return failed()
+	}
-        }
+	order++
-    }
 
-    _, err = CreateExpModelHead(c, def_id, 0, number_of_classes - 1, MODEL_DEFINITION_STATUS_INIT)
+	// Flatten the blocks into dense
-    if err != nil {
+	err = MakeLayerExpandable(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*2), 1)
-        return failed()
+	if err != nil {
-    }
+		return failed()
+	}
+	order++
+
+	loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
+	if loop == 0 {
+		loop = 1
+	}
+
+	for i := 0; i < loop; i++ {
+		err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
+		order++
+		if err != nil {
+			return failed()
+		}
+	}
+
+	_, err = CreateExpModelHead(c, def_id, 0, number_of_classes-1, MODEL_DEFINITION_STATUS_INIT)
+	if err != nil {
+		return failed()
+	}
 
 	err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
 	if err != nil {
@@ -980,15 +1126,15 @@ func handleTrain(handle *Handle) {
 		if model_type_form == "expandable" {
 			model_type_id = 2
 			c.Logger.Warn("TODO: handle expandable")
-            return c.Error400(nil, "TODO: handle expandable!", w, "/models/edit.html", "train-model-card", AnyMap{
+			return c.Error400(nil, "TODO: handle expandable!", w, "/models/edit.html", "train-model-card", AnyMap{
-                "HasData": true,
+				"HasData":      true,
-                "ErrorMessage": "TODO: handle expandable!",
+				"ErrorMessage": "TODO: handle expandable!",
-            })
+			})
 		} else if model_type_form != "simple" {
-            return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{
+			return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{
-                "HasData": true,
+				"HasData":      true,
-                "ErrorMessage": "Invalid model type!",
+				"ErrorMessage": "Invalid model type!",
-            })
+			})
 		}
 
 		model, err := GetBaseModel(handle.Db, id)
@@ -1007,28 +1153,31 @@ func handleTrain(handle *Handle) {
 			return ErrorCode(nil, 400, c.AddMap(nil))
 		}
 
-        if model_type_id == 2 {
+		if model_type_id == 2 {
-            full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models)
+			full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models)
-            if full_error != nil {
+			if full_error != nil {
-                return full_error
+				return full_error
-            }
+			}
-        } else {
+		} else {
-            full_error := generateDefinitions(c, model, accuracy, number_of_models)
+			full_error := generateDefinitions(c, model, accuracy, number_of_models)
-            if full_error != nil {
+			if full_error != nil {
-                return full_error
+				return full_error
-            }
+			}
-        }
+		}
 
+		if model_type_id == 2 {
+			go trainModelExp(c, model)
+		} else {
+			go trainModel(c, model)
+		}
 
-		go trainModel(c, model)
+		_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id)
-
+		if err != nil {
-        _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id)
+			fmt.Println("Failed to update model status")
-        if err != nil {
+			fmt.Println(err)
-            fmt.Println("Failed to update model status")
-            fmt.Println(err)
 			// TODO improve this response
 			return Error500(err)
-        }
+		}
 		Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
 		return nil
 	})

logic/utils/handler.go

@@ -636,7 +636,7 @@ func NewHandler(db *sql.DB) *Handle {
 }
 
 func (x Handle) Startup() {
-	fmt.Printf("Starting up!\n")
+	log.Info("Starting up!\n")
 
 	port := os.Getenv("PORT")
 	if port == "" {

main.go

@@ -31,7 +31,7 @@ func main() {
 		panic(err)
 	}
 	defer db.Close()
-	fmt.Println("Starting server on :8000!")
+	log.Info("Starting server on :8000!")
 
 	//TODO check if file structure exists to save data
 	handle := NewHandler(db)