added some nice graphs to the tasks

logic/models/run.go

@@ -118,6 +118,13 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
 }
 
 func ClassifyTask(base BasePack, task Task) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			base.GetLogger().Error("Task failed due to", "error", r)
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Task failed running")
+		}
+	}()
+
 	task.UpdateStatusLog(base, TASK_RUNNING, "Runner running task")
 
 	model, err := GetBaseModel(base.GetDb(), *task.ModelId)

logic/models/train/train.go

@@ -384,7 +384,6 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
 		log.Error("Failed to get the exp head of the model")
 		return
 	} else if len(heads) != 1 {
-		log.Error("This training function can only train one model at the time")
 		err = errors.New("This training function can only train one model at the time")
 		return
 	}
@@ -1450,9 +1449,9 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 	// Create the blocks
 	loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
 
-	if model.Width < 50 && model.Height < 50 {
-		loop = 0
-	}
+	/*if model.Width < 50 && model.Height < 50 {
+	loop = 0
+	}*/
 
 	log.Info("Size of the simple block", "loop", loop)
 
@@ -1541,7 +1540,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 		if model.Width > 100 && model.Height > 100 {
 			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		} else {
-			generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
+			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		}
 	} else if number_of_models == 3 {
 		for i := 0; i < number_of_models; i++ {
@@ -1550,7 +1549,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 	} else {
 		// TODO handle incrisea the complexity
 		for i := 0; i < number_of_models; i++ {
-			generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
+			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		}
 	}
 
@@ -1702,21 +1701,39 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
 
 	task.UpdateStatusLog(b, TASK_RUNNING, "Model retraining")
 
-	defId, err := GetDbVar[string](db, "md.id", "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
+	var defData struct {
+		Id             string  `db:"md.id"`
+		TargetAcuuracy float64 `db:"md.target_accuracy"`
+	}
+
+	err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
 	if err != nil {
 		failed()
 		return
 	}
 
-	// This is something I have to check
-	acc, err := trainDefinitionExpandExp(b, model, *defId, false)
-	if err != nil {
+	var acc float64 = 0
+	var epocs = 0
+	// TODO make max epochs come from db
+	for acc*100 < defData.TargetAcuuracy && epocs < 20 {
+		// This is something I have to check
+		acc, err = trainDefinitionExpandExp(b, model, defData.Id, epocs > 0)
+		if err != nil {
+			failed()
+			return
+		}
+
+		l.Info("Retrained model", "accuracy", acc, "target", defData.TargetAcuuracy)
+
+		epocs += 1
+	}
+
+	if acc*100 < defData.TargetAcuuracy {
+		l.Error("Model never achived targetd accuracy", "acc", acc*100, "target", defData.TargetAcuuracy)
 		failed()
 		return
 	}
 
-	l.Info("Retrained model", "accuracy", acc)
-
 	// TODO check accuracy
 	err = UpdateStatus(db, "models", model.Id, READY)
 	if err != nil {