sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

started working on runner

Browse Source
This commit is contained in:
Andre Henriques
2024-05-06 01:10:58 +01:00
parent 7d346ba2ce
commit e22df8adc9
38 changed files with 4134 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

162
logic/models/train/train.go
View File

@@ -39,16 +39,6 @@ func getDir() string {
return dir
}
// This function creates a new model_definition
func MakeDefenition(db db.Db, model_id string, target_accuracy int) (id string, err error) {
var NewDefinition = struct {
ModelId string `db:"model_id"`
TargetAccuracy int `db:"target_accuracy"`
}{ModelId: model_id, TargetAccuracy: target_accuracy}
return InsertReturnId(db, &NewDefinition, "model_definition", "id")
}
func ModelDefinitionUpdateStatus(c BasePack, id string, status ModelDefinitionStatus) (err error) {
_, err = c.GetDb().Exec("update model_definition set status = $1 where id = $2", status, id)
return
@@ -118,14 +108,14 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
count = co.Count
if count == 0 {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
}
@@ -137,7 +127,7 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
return generateCvsExp(c, run_path, model_id, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@@ -287,7 +277,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@@ -296,7 +286,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
count := co.Count
if count == 0 {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
} else if doPanic {
@@ -305,7 +295,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
return generateCvsExpandExp(c, run_path, model_id, offset, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@@ -339,7 +329,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
// This is to load some extra data so that the model has more things to train on
//
data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINED, count*10)
if err != nil {
return
}
@@ -737,7 +727,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err != nil {
l.Error("Failed to train Model! Err:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer definitionsRows.Close()
@@ -750,7 +740,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
l.Error("Failed to train Model Could not read definition from db!Err:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
definitions = append(definitions, rowv)
@@ -758,7 +748,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if len(definitions) == 0 {
l.Error("No Definitions defined!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@@ -788,14 +778,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
@@ -813,7 +803,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
}
@@ -868,7 +858,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err != nil {
l.Error("DB: failed to read definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer rows.Close()
@@ -876,7 +866,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if !rows.Next() {
// TODO Make the Model status have a message
l.Error("All definitions failed to train!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@@ -884,14 +874,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err = rows.Scan(&id); err != nil {
l.Error("Failed to read id:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
if _, err = db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
l.Error("Failed to update model definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@@ -899,7 +889,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err != nil {
l.Error("Failed to select model_definition to delete")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer to_delete.Close()
@@ -909,7 +899,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err = to_delete.Scan(&id); err != nil {
l.Error("Failed to scan the id of a model_definition to delete")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@@ -919,7 +909,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
l.Error("Failed to delete model_definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@@ -1066,7 +1056,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
err = GetDBOnce(db, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("All definitions failed to train! And Failed to set class status")
return err
@@ -1101,7 +1091,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
}
if err = splitModel(c, model); err != nil {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to split the model! And Failed to set class status")
return err
@@ -1112,7 +1102,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
}
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to set class status")
return err
@@ -1272,7 +1262,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
db := c.GetDb()
l := c.GetLogger()
def_id, err := MakeDefenition(db, model.Id, target_accuracy)
def, err := MakeDefenition(db, model.Id, target_accuracy)
if err != nil {
failed()
return
@@ -1281,60 +1271,68 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
order := 1
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height))
if err != nil {
failed()
return
}
order++
} else {
err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
if err != nil {
failed()
return
}
order++
}
loop := max(int((math.Log(float64(model.Width)) / math.Log(float64(10)))), 1)
for i := 0; i < loop; i++ {
err = MakeLayer(db, def_id, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
failed()
return
}
}
err = MakeLayer(db, def_id, order, LAYER_FLATTEN, "")
//_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(model.Width, model.Height, model.ImageMode))
_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height))
if err != nil {
failed()
return
}
order++
loop = max(int((math.Log(float64(number_of_classes))/math.Log(float64(10)))/2), 1)
for i := 0; i < loop; i++ {
err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++
if complexity == 0 {
_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
if err != nil {
failed()
return
}
}
order++
err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
if err != nil {
loop := int(math.Log2(float64(number_of_classes)))
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
order++
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
return
}
}
} else if complexity == 1 || complexity == 2 {
loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
failed()
return
}
}
_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
if err != nil {
failed()
return
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
order++
if err != nil {
failed()
return
}
}
} else {
l.Error("Unkown complexity", "complexity", complexity)
failed()
return
}
return nil
return def.UpdateStatus(db, DEFINITION_STATUS_INIT)
}
func generateDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
@@ -1393,12 +1391,14 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
return
}
def_id, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
def, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
if err != nil {
failed()
return
}
def_id := def.Id
order := 1
width := model.Width
@@ -1533,7 +1533,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
}
func ResetClasses(c BasePack, model *BaseModel) {
_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TO_TRAIN, MODEL_CLASS_STATUS_TRAINING, model.Id)
_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
c.GetLogger().Error("Error while reseting the classes", "error", err)
}
@@ -1544,7 +1544,7 @@ func trainExpandable(c *Context, model *BaseModel) {
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
ResetClasses(c, model)
}
@@ -1588,7 +1588,7 @@ func trainExpandable(c *Context, model *BaseModel) {
}
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
@@ -1648,7 +1648,7 @@ func RunTaskTrain(b BasePack, task Task) (err error) {
if err != nil {
l.Error("Failed to train model", "err", err)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
ModelUpdateStatus(b, model.Id, int(FAILED_TRAINING))
return
}
@@ -1731,7 +1731,7 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
l.Info("Model updaded")
_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINED, MODEL_CLASS_STATUS_TRAINING, model.Id)
_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINED, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
l.Error("Error while updating the classes", "error", err)
failed()
@@ -1861,7 +1861,7 @@ func handleTrain(handle *Handle) {
c,
"model_classes where model_id=$1 and status=$2 order by class_order asc",
model.Id,
MODEL_CLASS_STATUS_TO_TRAIN,
CLASS_STATUS_TO_TRAIN,
)
if err != nil {
_err := c.RollbackTx()
@@ -1882,7 +1882,7 @@ func handleTrain(handle *Handle) {
//Update the classes
{
_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINING, MODEL_CLASS_STATUS_TO_TRAIN, model.Id)
_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINING, CLASS_STATUS_TO_TRAIN, model.Id)
if err != nil {
_err := c.RollbackTx()
if _err != nil {