sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

moved code arround

Browse Source
This commit is contained in:
Andre Henriques
2024-02-05 16:42:23 +00:00
parent cbfd49c5fb
commit 6a0ac457d7
3 changed files with 90 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

119
logic/models/train/train.go
View File

@@ -42,6 +42,11 @@ func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionSt
return
}
func UpdateStatus (c *Context, table string, id string, status int) (err error) {
_, err = c.Db.Exec(fmt.Sprintf("update %s set status = $1 where id = $2", table), status, id)
return
}
func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType, shape string) (err error) {
_, err = db.Exec("insert into model_definition_layer (def_id, layer_order, layer_type, shape) values ($1, $2, $3, $4)", def_id, layer_order, layer_type, shape)
return
@@ -108,9 +113,11 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
type layerrow struct {
LayerType int
Shape string
LayerNum int
}
got := []layerrow{}
i := 1
for layers.Next() {
var row = layerrow{}
@@ -118,7 +125,9 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
return
}
row.Shape = shapeToSize(row.Shape)
row.LayerNum = 1
got = append(got, row)
i = i + 1
}
// Generate run folder
@@ -207,7 +216,7 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
// Get untrained models heads
// Status = 2 (INIT)
rows, err := c.Db.Query("select id, range_start, range_end exp_model_head where def_id=$1 and status = 2", definition_id)
rows, err := c.Db.Query("select id, range_start, range_end from exp_model_head where def_id=$1 and status = 2", definition_id)
if err != nil {
return
}
@@ -237,6 +246,8 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
return
}
UpdateStatus(c, "exp_model_head", exp.id, MODEL_DEFINITION_STATUS_TRANIED)
layers, err := c.Db.Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil {
return
@@ -300,7 +311,7 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
}
defer f.Close()
tmpl, err := template.New("python_model_template-exp.py").ParseFiles("views/py/python_model_template.py")
tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
if err != nil {
return
}
@@ -312,6 +323,7 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"HeadId": exp.id,
"RunPath": run_path,
"ColorMode": model.ImageMode,
"Model": model,
@@ -613,17 +625,14 @@ func trainModelExp(c *Context, model *BaseModel) {
var rowv TrainModelRow
rowv.acuracy = 0
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to train Model Could not read definition from db!")
return
}
definitions = append(definitions, rowv)
}
if len(definitions) == 0 {
c.Logger.Error("No Definitions defined!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("No Definitions defined!")
return
}
@@ -652,15 +661,13 @@ func trainModelExp(c *Context, model *BaseModel) {
c.Logger.Info("Found a definition that reaches target_accuracy!")
_, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to train definition!")
return
}
_, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to train definition!")
return
}
@@ -677,8 +684,7 @@ func trainModelExp(c *Context, model *BaseModel) {
_, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to train definition!")
return
}
}
@@ -731,40 +737,30 @@ func trainModelExp(c *Context, model *BaseModel) {
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err != nil {
c.Logger.Error("DB: failed to read definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("DB: failed to read definition")
return
}
defer rows.Close()
if !rows.Next() {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("All definitions failed to train!")
return
}
var id string
if err = rows.Scan(&id); err != nil {
c.Logger.Error("Failed to read id:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to read id")
return
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
c.Logger.Error("Failed to update model definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to update model definition")
return
}
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.Logger.Error("Failed to select model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to select model_definition to delete")
return
}
defer to_delete.Close()
@@ -772,9 +768,7 @@ func trainModelExp(c *Context, model *BaseModel) {
for to_delete.Next() {
var id string
if to_delete.Scan(&id); err != nil {
c.Logger.Error("Failed to scan the id of a model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to scan the id of a model_definition to delete")
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@@ -782,9 +776,7 @@ func trainModelExp(c *Context, model *BaseModel) {
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
c.Logger.Error("Failed to delete model_definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
failed("Failed to delete model_definition")
return
}
@@ -1255,4 +1247,63 @@ func handleTrain(handle *Handle) {
}
return nil
})
handle.Get("/model/head/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
// TODO check auth level
if c.Mode != NORMAL {
// This should only handle normal requests
c.Logger.Warn("This function only works with normal")
return c.UnsafeErrorCode(nil, 400, nil)
}
f := r.URL.Query()
accuracy := 0.0
if !CheckId(f, "head_id") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or head_id or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
}
accuracy = accuracy * 100
head_id := f.Get("head_id")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
c.Logger.Warn("Epoch is not a number")
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
}
rows, err := c.Db.Query("select hd.status from exp_model_head as hd where hd.id=$1;", head_id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
if !rows.Next() {
c.Logger.Error("Could not get status of model head")
return c.Error500(nil)
}
var status int
err = rows.Scan(&status)
if err != nil {
return c.Error500(err)
}
if status != 3 {
c.Logger.Warn("Head not on status 3(training)", "status", status)
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
}
c.Logger.Info("Updated model_head!", "head", head_id, "progress", epoch, "accuracy", accuracy)
_, err = c.Db.Exec("update exp_model_head set epoch_progress=$1, accuracy=$2 where id=$3", epoch, accuracy, head_id)
if err != nil {
return c.Error500(err)
}
return nil
})
}