diff --git a/auto_reload.sh b/auto_reload.sh new file mode 100755 index 0000000..d59f1f1 --- /dev/null +++ b/auto_reload.sh @@ -0,0 +1,5 @@ +#!/bin/fish + +cd $(dirname "$0") + +go run . diff --git a/logic/models/train/train.go b/logic/models/train/train.go index a9db993..ed5136f 100644 --- a/logic/models/train/train.go +++ b/logic/models/train/train.go @@ -17,6 +17,7 @@ import ( model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" + "github.com/charmbracelet/log" ) const EPOCH_PER_RUN = 20 @@ -198,6 +199,151 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr return } +func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) { + accuracy = 0 + + c.Logger.Warn("About to start training definition") + + // Get untrained models heads + + // Status = 2 (INIT) + rows, err := c.Db.Query("select id, range_start, range_end exp_model_head where def_id=$1 and status = 2", definition_id) + if err != nil { + return + } + defer rows.Close() + + type ExpHead struct { + id string + start int + end int + } + + exp := ExpHead{} + + if rows.Next() { + if err = rows.Scan(&exp.id, &exp.start, &exp.end); err == nil { + return + } + } else { + log.Error("Failed to get the exp head of the model") + err = errors.New("Failed to get the exp head of the model") + return + } + + if rows.Next() { + log.Error("This training function can only train one model at the time") + err = errors.New("This training function can only train one model at the time") + return + } + + layers, err := c.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id) + if err != nil { + return + } + defer layers.Close() + + type layerrow struct { + LayerType int + Shape string + } + + got := []layerrow{} + + for layers.Next() { + var row = layerrow{} + if err = layers.Scan(&row.LayerType, &row.Shape); err != nil { + return + } + row.Shape = shapeToSize(row.Shape) + got = append(got, row) + } + + got = append(got, layerrow{ + LayerType: LAYER_DENSE, + Shape: fmt.Sprintf("%d", exp.end - exp.start), + }) + + // Generate run folder + run_path := path.Join("/tmp", model.Id, "defs", definition_id) + + err = os.MkdirAll(run_path, os.ModePerm) + if err != nil { + return + } + defer os.RemoveAll(run_path) + + _, err = generateCvs(c, run_path, model.Id) + if err != nil { + return + } + + // TODO update the run script + + // Create python script + f, err := os.Create(path.Join(run_path, "run.py")) + if err != nil { + return + } + defer f.Close() + + tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py") + if err != nil { + return + } + + // Copy result around + result_path := path.Join("savedData", model.Id, "defs", definition_id) + + if err = tmpl.Execute(f, AnyMap{ + "Layers": got, + "Size": got[0].Shape, + "DataDir": path.Join(getDir(), "savedData", model.Id, "data"), + "RunPath": run_path, + "ColorMode": model.ImageMode, + "Model": model, + "EPOCH_PER_RUN": EPOCH_PER_RUN, + "DefId": definition_id, + "LoadPrev": load_prev, + "LastModelRunPath": path.Join(getDir(), result_path, "model.keras"), + "SaveModelPath": path.Join(getDir(), result_path), + }); err != nil { + return + } + + // Run the command + out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput() + if err != nil { + c.Logger.Debug(string(out)) + return + } + + c.Logger.Info("Python finished running") + + if err = os.MkdirAll(result_path, os.ModePerm); err != nil { + return + } + + accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val")) + if err != nil { + return + } + defer accuracy_file.Close() + + accuracy_file_bytes, err := io.ReadAll(accuracy_file) + if err != nil { + return + } + + accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64) + if err != nil { + return + } + + c.Logger.Info("Model finished training!", "accuracy", accuracy) + return +} + func remove[T interface{}](lst []T, i int) []T { lng := len(lst) if i >= lng { @@ -433,7 +579,7 @@ func trainModel(c *Context, model *BaseModel) { } func trainModelExp(c *Context, model *BaseModel) { - var err error = nil + var err error = nil failed := func(msg string) { c.Logger.Error(msg, "err", err) @@ -442,7 +588,7 @@ func trainModelExp(c *Context, model *BaseModel) { definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) if err != nil { - failed("Failed to trainModel!") + failed("Failed to trainModel!") return } defer definitionsRows.Close() @@ -474,7 +620,7 @@ func trainModelExp(c *Context, model *BaseModel) { var toRemove ToRemoveList = []int{} for i, def := range definitions { ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING) - accuracy, err := trainDefinition(c, model, def.id, !firstRound) + accuracy, err := trainDefinitionExp(c, model, def.id, !firstRound) if err != nil { c.Logger.Error("Failed to train definition!Err:", "err", err) ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) @@ -797,23 +943,23 @@ func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, numb } func CreateExpModelHead(c *Context, def_id string, range_start int, range_end int, status ModelDefinitionStatus) (id string, err error) { - rows, err := c.Db.Query("insert into exp_model_head (def_id, range_start, range_end) values ($1, $2, $3, $4) returning id", def_id, range_start, range_end, status) + rows, err := c.Db.Query("insert into exp_model_head (def_id, range_start, range_end) values ($1, $2, $3, $4) returning id", def_id, range_start, range_end, status) if err != nil { return } defer rows.Close() - if !rows.Next() { - c.Logger.Error("Could not get status of model definition") - err = errors.New("Could not get status of model definition") - return - } + if !rows.Next() { + c.Logger.Error("Could not get status of model definition") + err = errors.New("Could not get status of model definition") + return + } err = rows.Scan(&id) - if err != nil { - return - } + if err != nil { + return + } return } @@ -833,7 +979,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy } if complexity == 0 { - return failed() + return failed() } def_id, err := MakeDefenition(c.Db, model.Id, target_accuracy) @@ -843,8 +989,8 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy order := 1 - width := model.Width - height := model.Height + width := model.Width + height := model.Height // Note the shape of the first layer defines the import size if complexity == 2 { @@ -853,62 +999,62 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0)))) c.Logger.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height) - } + } - err = MakeLayerExpandable(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1) + err = MakeLayerExpandable(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1) order++ - - // handle the errors inside the pervious if block - if err != nil { - return failed() - } - - // Create the blocks - loop := int((math.Log(float64(model.Width)) / math.Log(float64(10)))) - if loop == 0 { - loop = 1 - } - for i := 0; i < loop; i++ { - err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1) - order++ - if err != nil { - return failed() - } - } - - // Flatten the blocks into dense - err = MakeLayerExpandable(c.Db, def_id, order, LAYER_FLATTEN, "", 1) - if err != nil { - return failed() - } - order++ + // handle the errors inside the pervious if block + if err != nil { + return failed() + } - // Flatten the blocks into dense - err = MakeLayerExpandable(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes * 2), 1) - if err != nil { - return failed() - } - order++ + // Create the blocks + loop := int((math.Log(float64(model.Width)) / math.Log(float64(10)))) + if loop == 0 { + loop = 1 + } - loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2) - if loop == 0 { - loop = 1 - } + for i := 0; i < loop; i++ { + err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1) + order++ + if err != nil { + return failed() + } + } - for i := 0; i < loop; i++ { - err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i))) - order++ - if err != nil { - return failed() - } - } + // Flatten the blocks into dense + err = MakeLayerExpandable(c.Db, def_id, order, LAYER_FLATTEN, "", 1) + if err != nil { + return failed() + } + order++ - _, err = CreateExpModelHead(c, def_id, 0, number_of_classes - 1, MODEL_DEFINITION_STATUS_INIT) - if err != nil { - return failed() - } + // Flatten the blocks into dense + err = MakeLayerExpandable(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*2), 1) + if err != nil { + return failed() + } + order++ + + loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2) + if loop == 0 { + loop = 1 + } + + for i := 0; i < loop; i++ { + err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i))) + order++ + if err != nil { + return failed() + } + } + + _, err = CreateExpModelHead(c, def_id, 0, number_of_classes-1, MODEL_DEFINITION_STATUS_INIT) + if err != nil { + return failed() + } err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT) if err != nil { @@ -980,15 +1126,15 @@ func handleTrain(handle *Handle) { if model_type_form == "expandable" { model_type_id = 2 c.Logger.Warn("TODO: handle expandable") - return c.Error400(nil, "TODO: handle expandable!", w, "/models/edit.html", "train-model-card", AnyMap{ - "HasData": true, - "ErrorMessage": "TODO: handle expandable!", - }) + return c.Error400(nil, "TODO: handle expandable!", w, "/models/edit.html", "train-model-card", AnyMap{ + "HasData": true, + "ErrorMessage": "TODO: handle expandable!", + }) } else if model_type_form != "simple" { - return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{ - "HasData": true, - "ErrorMessage": "Invalid model type!", - }) + return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{ + "HasData": true, + "ErrorMessage": "Invalid model type!", + }) } model, err := GetBaseModel(handle.Db, id) @@ -1007,28 +1153,31 @@ func handleTrain(handle *Handle) { return ErrorCode(nil, 400, c.AddMap(nil)) } - if model_type_id == 2 { - full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models) - if full_error != nil { - return full_error - } - } else { - full_error := generateDefinitions(c, model, accuracy, number_of_models) - if full_error != nil { - return full_error - } - } + if model_type_id == 2 { + full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models) + if full_error != nil { + return full_error + } + } else { + full_error := generateDefinitions(c, model, accuracy, number_of_models) + if full_error != nil { + return full_error + } + } + if model_type_id == 2 { + go trainModelExp(c, model) + } else { + go trainModel(c, model) + } - go trainModel(c, model) - - _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id) - if err != nil { - fmt.Println("Failed to update model status") - fmt.Println(err) + _, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id) + if err != nil { + fmt.Println("Failed to update model status") + fmt.Println(err) // TODO improve this response return Error500(err) - } + } Redirect("/models/edit?id="+model.Id, c.Mode, w, r) return nil }) diff --git a/logic/utils/handler.go b/logic/utils/handler.go index f70d073..a03b137 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -636,7 +636,7 @@ func NewHandler(db *sql.DB) *Handle { } func (x Handle) Startup() { - fmt.Printf("Starting up!\n") + log.Info("Starting up!\n") port := os.Getenv("PORT") if port == "" { diff --git a/main.go b/main.go index d02aa6b..27348e7 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ func main() { panic(err) } defer db.Close() - fmt.Println("Starting server on :8000!") + log.Info("Starting server on :8000!") //TODO check if file structure exists to save data handle := NewHandler(db)