Merge branch 'main' into runner-go

more worker on go-runner
more work on the rust runner
2024-05-07 01:18:25 +01:00 · 2024-05-07 01:16:38 +01:00 · 2024-05-06 12:48:02 +01:00 · 2024-05-06 01:10:58 +01:00
22 changed files with 1086 additions and 139 deletions
--- a/go.mod
+++ b/go.mod
@@ -9,10 +9,11 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
 	golang.org/x/crypto v0.19.0
+	github.com/BurntSushi/toml v1.3.2
+	github.com/goccy/go-json v0.10.2
 )

 require (
-	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/lipgloss v0.9.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@@ -20,7 +21,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.19.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx v3.6.2+incompatible // indirect
--- a/logic/db_types/classes.go
+++ b/logic/db_types/classes.go
@@ -6,3 +6,19 @@ const (
 	DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
 	DATA_POINT_MODE_TESTING                  = 2
 )
+
+type ModelClassStatus int
+
+const (
+	CLASS_STATUS_TO_TRAIN ModelClassStatus = iota + 1
+	CLASS_STATUS_TRAINING
+	CLASS_STATUS_TRAINED
+)
+
+type ModelClass struct {
+	Id         string `db:"mc.id" json:"id"`
+	ModelId    string `db:"mc.model_id" json:"model_id"`
+	Name       string `db:"mc.name" json:"name"`
+	ClassOrder int    `db:"mc.class_order" json:"class_order"`
+	Status     int    `db:"mc.status" json:"status"`
+}
--- a/logic/db_types/definitions.go
+++ b/logic/db_types/definitions.go
@@ -0,0 +1,95 @@
+package dbtypes
+
+import (
+	"time"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+)
+
+type DefinitionStatus int
+
+const (
+	DEFINITION_STATUS_CANCELD_TRAINING DefinitionStatus = -4
+	DEFINITION_STATUS_FAILED_TRAINING                   = -3
+	DEFINITION_STATUS_PRE_INIT                          = 1
+	DEFINITION_STATUS_INIT                              = 2
+	DEFINITION_STATUS_TRAINING                          = 3
+	DEFINITION_STATUS_PAUSED_TRAINING                   = 6
+	DEFINITION_STATUS_TRANIED                           = 4
+	DEFINITION_STATUS_READY                             = 5
+)
+
+type Definition struct {
+	Id             string    `db:"md.id" json:"id"`
+	ModelId        string    `db:"md.model_id" json:"model_id"`
+	Accuracy       float64   `db:"md.accuracy" json:"accuracy"`
+	TargetAccuracy int       `db:"md.target_accuracy" json:"target_accuracy"`
+	Epoch          int       `db:"md.epoch" json:"epoch"`
+	Status         int       `db:"md.status" json:"status"`
+	CreatedOn      time.Time `db:"md.created_on" json:"created"`
+	EpochProgress  int       `db:"md.epoch_progress" json:"epoch_progress"`
+}
+
+type SortByAccuracyDefinitions []*Definition
+
+func (nf SortByAccuracyDefinitions) Len() int      { return len(nf) }
+func (nf SortByAccuracyDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
+func (nf SortByAccuracyDefinitions) Less(i, j int) bool {
+	return nf[i].Accuracy < nf[j].Accuracy
+}
+
+func GetDefinition(db db.Db, definition_id string) (definition Definition, err error) {
+	err = GetDBOnce(db, &definition, "model_definition as md where id=$1;", definition_id)
+	return
+}
+
+func MakeDefenition(db db.Db, model_id string, target_accuracy int) (definition Definition, err error) {
+	var NewDefinition = struct {
+		ModelId        string `db:"model_id"`
+		TargetAccuracy int    `db:"target_accuracy"`
+	}{ModelId: model_id, TargetAccuracy: target_accuracy}
+
+	id, err := InsertReturnId(db, &NewDefinition, "model_definition", "id")
+	if err != nil {
+		return
+	}
+	return GetDefinition(db, id)
+}
+
+func (d Definition) UpdateStatus(db db.Db, status DefinitionStatus) (err error) {
+	_, err = db.Exec("update model_definition set status=$1 where id=$2", status, d.Id)
+	return
+}
+
+func (d Definition) MakeLayer(db db.Db, layer_order int, layer_type LayerType, shape string) (layer Layer, err error) {
+	var NewLayer = struct {
+		DefinitionId string    `db:"def_id"`
+		LayerOrder   int       `db:"layer_order"`
+		LayerType    LayerType `db:"layer_type"`
+		Shape        string    `db:"shape"`
+	}{
+		DefinitionId: d.Id,
+		LayerOrder:   layer_order,
+		LayerType:    layer_type,
+		Shape:        shape,
+	}
+
+	id, err := InsertReturnId(db, &NewLayer, "model_definition_layer", "id")
+	if err != nil {
+		return
+	}
+
+	return GetLayer(db, id)
+}
+
+func (d Definition) GetLayers(db db.Db, filter string, args ...any) (layer []*Layer, err error) {
+	args = append(args, d.Id)
+	return GetDbMultitple[Layer](db, "model_definition_layer as mdl where mdl.def_id=$1 "+filter, args...)
+}
+
+func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64, epoch int) (err error) {
+	d.Accuracy = accuracy
+	d.Epoch += epoch
+	_, err = db.Exec("update model_definition set epoch=$1, accuracy=$2 where id=$3", d.Epoch, d.Accuracy, d.Id)
+	return
+}
--- a/logic/db_types/layer.go
+++ b/logic/db_types/layer.go
@@ -0,0 +1,50 @@
+package dbtypes
+
+import (
+	"encoding/json"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+)
+
+type LayerType int
+
+const (
+	LAYER_INPUT        LayerType = 1
+	LAYER_DENSE                  = 2
+	LAYER_FLATTEN                = 3
+	LAYER_SIMPLE_BLOCK           = 4
+)
+
+type Layer struct {
+	Id           string    `db:"mdl.id" json:"id"`
+	DefinitionId string    `db:"mdl.def_id" json:"definition_id"`
+	LayerOrder   string    `db:"mdl.layer_order" json:"layer_order"`
+	LayerType    LayerType `db:"mdl.layer_type" json:"layer_type"`
+	Shape        string    `db:"mdl.shape" json:"shape"`
+	ExpType      string    `db:"mdl.exp_type" json:"exp_type"`
+}
+
+func ShapeToString(args ...int) string {
+	text, err := json.Marshal(args)
+	if err != nil {
+		panic("Could not generate Shape")
+	}
+	return string(text)
+}
+
+func StringToShape(str string) (shape []int64) {
+	err := json.Unmarshal([]byte(str), &shape)
+	if err != nil {
+		panic("Could not parse Shape")
+	}
+	return
+}
+
+func (l Layer) GetShape() []int64 {
+	return StringToShape(l.Shape)
+}
+
+func GetLayer(db db.Db, layer_id string) (layer Layer, err error) {
+	err = GetDBOnce(db, &layer, "model_definition_layer as mdl where mdl.id=$1", layer_id)
+	return
+}
--- a/logic/db_types/types.go
+++ b/logic/db_types/types.go
@@ -2,23 +2,26 @@ package dbtypes

 import (
 	"errors"
+	"fmt"
+	"path"

 	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
 )

-const (
-	FAILED_TRAINING           = -4
-	FAILED_PREPARING_TRAINING = -3
-	FAILED_PREPARING_ZIP_FILE = -2
-	FAILED_PREPARING          = -1
+type ModelStatus int

-	PREPARING               = 1
-	CONFIRM_PRE_TRAINING    = 2
-	PREPARING_ZIP_FILE      = 3
-	TRAINING                = 4
-	READY                   = 5
-	READY_ALTERATION        = 6
-	READY_ALTERATION_FAILED = -6
+const (
+	FAILED_TRAINING           ModelStatus = -4
+	FAILED_PREPARING_TRAINING             = -3
+	FAILED_PREPARING_ZIP_FILE             = -2
+	FAILED_PREPARING                      = -1
+	PREPARING                             = 1
+	CONFIRM_PRE_TRAINING                  = 2
+	PREPARING_ZIP_FILE                    = 3
+	TRAINING                              = 4
+	READY                                 = 5
+	READY_ALTERATION                      = 6
+	READY_ALTERATION_FAILED               = -6

 	READY_RETRAIN        = 7
 	READY_RETRAIN_FAILED = -7
@@ -26,15 +29,6 @@ const (

 type ModelDefinitionStatus int

-type LayerType int
-
-const (
-	LAYER_INPUT        LayerType = 1
-	LAYER_DENSE                  = 2
-	LAYER_FLATTEN                = 3
-	LAYER_SIMPLE_BLOCK           = 4
-)
-
 const (
 	MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
 	MODEL_DEFINITION_STATUS_FAILED_TRAINING                        = -3
@@ -46,14 +40,6 @@ const (
 	MODEL_DEFINITION_STATUS_READY                                  = 5
 )

-type ModelClassStatus int
-
-const (
-	MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
-	MODEL_CLASS_STATUS_TRAINING                  = 2
-	MODEL_CLASS_STATUS_TRAINED                   = 3
-)
-
 type ModelHeadStatus int

 const (
@@ -97,6 +83,61 @@ func (m BaseModel) CanEval() bool {
 	return true
 }

+// DO NOT Pass un filtered data on filters
+func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
+	n_args := []any{m.Id}
+	n_args = append(n_args, args...)
+	return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
+}
+
+func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
+	n_args := []any{m.Id}
+	n_args = append(n_args, args...)
+	return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
+}
+
+func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
+	_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
+	return
+}
+
+type DataPoint struct {
+	Class int    `json:"class"`
+	Path  string `json:"path"`
+}
+
+func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
+	rows, err := db.Query(
+		"select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner "+
+			"join model_classes as mc on mc.id = mdp.class_id "+
+			"where mc.model_id = $1 and mdp.model_mode=$2;",
+		m.Id, mode)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	data = []DataPoint{}
+
+	for rows.Next() {
+		var id string
+		var class_order int
+		var file_path string
+		if err = rows.Scan(&id, &class_order, &file_path); err != nil {
+			return
+		}
+		if file_path == "id://" {
+			data = append(data, DataPoint{
+				Path:  path.Join("./savedData", m.Id, "data", id+"."+m.Format),
+				Class: class_order,
+			})
+		} else {
+			panic("TODO remote file path")
+		}
+	}
+	return
+}
+
 func StringToImageMode(colorMode string) int {
 	switch colorMode {
 	case "greyscale":
--- a/logic/models/classes/main.go
+++ b/logic/models/classes/main.go
@@ -7,15 +7,15 @@ import (
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
 )

-type ModelClass struct {
+type ModelClassJSON struct {
 	Id      string `json:"id"`
 	ModelId string `json:"model_id" db:"model_id"`
 	Name    string `json:"name"`
 	Status  int    `json:"status"`
 }

-func ListClasses(c BasePack, model_id string) (cls []*ModelClass, err error) {
-	return GetDbMultitple[ModelClass](c.GetDb(), "model_classes where model_id=$1", model_id)
+func ListClasses(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
+	return GetDbMultitple[ModelClassJSON](c.GetDb(), "model_classes where model_id=$1", model_id)
 }

 func ModelHasDataPoints(db db.Db, model_id string) (result bool, err error) {
--- a/logic/models/data.go
+++ b/logic/models/data.go
@@ -495,7 +495,7 @@ func handleDataUpload(handle *Handle) {
 			return c.E500M("Could not create class", err)
 		}

-		var modelClass model_classes.ModelClass
+		var modelClass model_classes.ModelClassJSON
 		err = GetDBOnce(c, &modelClass, "model_classes where id=$1;", id)
 		if err != nil {
 			return c.E500M("Failed to get class information but class was creted", err)
@@ -704,7 +704,7 @@ func handleDataUpload(handle *Handle) {
 				return c.Error500(err)
 			}
 		} else {
-			_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
+			_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TO_TRAIN)
 			if err != nil {
 				return c.Error500(err)
 			}
--- a/logic/models/delete.go
+++ b/logic/models/delete.go
@@ -51,7 +51,7 @@ func handleDelete(handle *Handle) {
 			return c.E500M("Faield to get model", err)
 		}

-		switch model.Status {
+		switch ModelStatus(model.Status) {
 		case FAILED_TRAINING:
 			fallthrough
 		case FAILED_PREPARING_ZIP_FILE:
--- a/logic/models/edit.go
+++ b/logic/models/edit.go
@@ -35,9 +35,9 @@ func handleEdit(handle *Handle) {
 		}

 		type ReturnType struct {
-			Classes               []*model_classes.ModelClass `json:"classes"`
-			HasData               bool                        `json:"has_data"`
-			NumberOfInvalidImages int                         `json:"number_of_invalid_images"`
+			Classes               []*model_classes.ModelClassJSON `json:"classes"`
+			HasData               bool                            `json:"has_data"`
+			NumberOfInvalidImages int                             `json:"number_of_invalid_images"`
 		}

 		c.ShowMessage = false
--- a/logic/models/train/remote_train.go
+++ b/logic/models/train/remote_train.go
@@ -0,0 +1,79 @@
+package models_train
+
+import (
+	"errors"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	"github.com/goccy/go-json"
+)
+
+func PrepareTraining(handler *Handle, b BasePack, task Task, runner_id string) (err error) {
+	l := b.GetLogger()
+
+	model, err := GetBaseModel(b.GetDb(), *task.ModelId)
+	if err != nil {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
+		l.Error("Failed to get model information", "err", err)
+		return err
+	}
+
+	if model.Status != TRAINING {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
+		return errors.New("Model not in the right status")
+	}
+
+	// TODO do this when the runner says it's OK
+	//task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
+
+	// TODO move this to the runner part as well
+	var dat struct {
+		NumberOfModels int
+		Accuracy       int
+	}
+
+	err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
+	if err != nil {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
+	}
+
+	if model.ModelType == 2 {
+		panic("TODO")
+		full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
+		if full_error != nil {
+			l.Error("Failed to generate defintions", "err", full_error)
+			task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
+			return errors.New("Failed to generate definitions")
+		}
+	} else {
+		error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
+		if error != nil {
+			task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
+			return errors.New("Failed to generate definitions")
+		}
+	}
+
+	runners := handler.DataMap["runners"].(map[string]interface{})
+	runner := runners[runner_id].(map[string]interface{})
+	runner["task"] = &task
+
+	runners[runner_id] = runner
+	handler.DataMap["runners"] = runners
+
+	return
+}
+
+func CleanUpFailed(b BasePack, task *Task) {
+	db := b.GetDb()
+	l := b.GetLogger()
+	model, err := GetBaseModel(db, *task.ModelId)
+	if err != nil {
+		l.Error("Failed to get model", "err", err)
+	} else {
+		err = model.UpdateStatus(db, FAILED_TRAINING)
+		if err != nil {
+			l.Error("Failed to get status", err)
+		}
+	}
+}
--- a/logic/models/train/reset.go
+++ b/logic/models/train/reset.go
@@ -17,7 +17,7 @@ func handleRest(handle *Handle) {
 			return c.E500M("Failed to get model", err)
 		}

-		if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
+		if model.Status != FAILED_PREPARING_TRAINING && model.Status != int(FAILED_TRAINING) {
 			return c.JsonBadRequest("Model is not in status that be reset")
 		}

--- a/logic/models/train/train.go
+++ b/logic/models/train/train.go
@@ -39,16 +39,6 @@ func getDir() string {
 	return dir
 }

-// This function creates a new model_definition
-func MakeDefenition(db db.Db, model_id string, target_accuracy int) (id string, err error) {
-	var NewDefinition = struct {
-		ModelId        string `db:"model_id"`
-		TargetAccuracy int    `db:"target_accuracy"`
-	}{ModelId: model_id, TargetAccuracy: target_accuracy}
-
-	return InsertReturnId(db, &NewDefinition, "model_definition", "id")
-}
-
 func ModelDefinitionUpdateStatus(c BasePack, id string, status ModelDefinitionStatus) (err error) {
 	_, err = c.GetDb().Exec("update model_definition set status = $1 where id = $2", status, id)
 	return
@@ -118,14 +108,14 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
 	var co struct {
 		Count int `db:"count(*)"`
 	}
-	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
+	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
 	count = co.Count

 	if count == 0 {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
+		err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
 		if err != nil {
 			return
 		}
@@ -137,7 +127,7 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
 		return generateCvsExp(c, run_path, model_id, true)
 	}

-	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
+	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -287,7 +277,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	var co struct {
 		Count int `db:"count(*)"`
 	}
-	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
+	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -296,7 +286,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	count := co.Count

 	if count == 0 {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
+		err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
 		if err != nil {
 			return
 		} else if doPanic {
@@ -305,7 +295,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 		return generateCvsExpandExp(c, run_path, model_id, offset, true)
 	}

-	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
+	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -339,7 +329,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	// This is to load some extra data so that the model has more things to train on
 	//

-	data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
+	data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINED, count*10)
 	if err != nil {
 		return
 	}
@@ -737,7 +727,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("Failed to train Model! Err:")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer definitionsRows.Close()
@@ -750,7 +740,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 		if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
 			l.Error("Failed to train Model Could not read definition from db!Err:")
 			l.Error(err)
-			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+			ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 			return
 		}
 		definitions = append(definitions, rowv)
@@ -758,7 +748,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {

 	if len(definitions) == 0 {
 		l.Error("No Definitions defined!")
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}

@@ -788,14 +778,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 				_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
 				if err != nil {
 					l.Error("Failed to train definition!Err:\n", "err", err)
-					ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+					ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 					return err
 				}

 				_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
 				if err != nil {
 					l.Error("Failed to train definition!Err:\n", "err", err)
-					ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+					ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 					return err
 				}

@@ -813,7 +803,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 			_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
 			if err != nil {
 				l.Error("Failed to train definition!Err:\n", "err", err)
-				ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+				ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 				return err
 			}
 		}
@@ -868,7 +858,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("DB: failed to read definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer rows.Close()
@@ -876,7 +866,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if !rows.Next() {
 		// TODO Make the Model status have a message
 		l.Error("All definitions failed to train!")
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}

@@ -884,14 +874,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err = rows.Scan(&id); err != nil {
 		l.Error("Failed to read id:")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}

 	if _, err = db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
 		l.Error("Failed to update model definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}

@@ -899,7 +889,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("Failed to select model_definition to delete")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer to_delete.Close()
@@ -909,7 +899,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 		if err = to_delete.Scan(&id); err != nil {
 			l.Error("Failed to scan the id of a model_definition to delete")
 			l.Error(err)
-			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+			ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 			return
 		}
 		os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@@ -919,7 +909,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
 		l.Error("Failed to delete model_definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}

@@ -1066,7 +1056,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	err = GetDBOnce(db, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
 	if err == NotFoundError {
 		// Set the class status to trained
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+		err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 		if err != nil {
 			l.Error("All definitions failed to train! And Failed to set class status")
 			return err
@@ -1101,7 +1091,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	}

 	if err = splitModel(c, model); err != nil {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+		err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 		if err != nil {
 			l.Error("Failed to split the model! And Failed to set class status")
 			return err
@@ -1112,7 +1102,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	}

 	// Set the class status to trained
-	err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+	err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		l.Error("Failed to set class status")
 		return err
@@ -1272,7 +1262,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 	db := c.GetDb()
 	l := c.GetLogger()

-	def_id, err := MakeDefenition(db, model.Id, target_accuracy)
+	def, err := MakeDefenition(db, model.Id, target_accuracy)
 	if err != nil {
 		failed()
 		return
@@ -1281,60 +1271,77 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 	order := 1

 	// Note the shape of the first layer defines the import size
-	if complexity == 2 {
-		// Note the shape for now is no used
-		width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
-		height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
-		l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
-		err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height))
-		if err != nil {
-			failed()
-			return
-		}
-		order++
-	} else {
-		err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
-		if err != nil {
-			failed()
-			return
-		}
-		order++
-	}
-
-	loop := max(int((math.Log(float64(model.Width)) / math.Log(float64(10)))), 1)
-	for i := 0; i < loop; i++ {
-		err = MakeLayer(db, def_id, order, LAYER_SIMPLE_BLOCK, "")
-		order++
-		if err != nil {
-			failed()
-			return
-		}
-	}
-
-	err = MakeLayer(db, def_id, order, LAYER_FLATTEN, "")
+	//_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(model.Width, model.Height, model.ImageMode))
+	_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height))
 	if err != nil {
 		failed()
 		return
 	}
 	order++

-	loop = max(int((math.Log(float64(number_of_classes))/math.Log(float64(10)))/2), 1)
-	for i := 0; i < loop; i++ {
-		err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
-		order++
+	if complexity == 0 {
+		/*
+			_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
+			if err != nil {
+				failed()
+				return
+			}
+			order++
+		*/
+
+		_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
 		if err != nil {
 			failed()
 			return
 		}
-	}
+		order++

-	err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
-	if err != nil {
+		loop := int(math.Log2(float64(number_of_classes)))
+		for i := 0; i < loop; i++ {
+			_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
+			order++
+			if err != nil {
+				ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
+				return
+			}
+		}
+	} else if complexity == 1 || complexity == 2 {
+		loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
+		for i := 0; i < loop; i++ {
+			_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
+			order++
+			if err != nil {
+				failed()
+				return
+			}
+		}
+
+		_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
+		if err != nil {
+			failed()
+			return
+		}
+		order++
+
+		loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
+		if loop == 0 {
+			loop = 1
+		}
+		for i := 0; i < loop; i++ {
+			_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
+			order++
+			if err != nil {
+				failed()
+				return
+			}
+		}
+	} else {
+		l.Error("Unkown complexity", "complexity", complexity)
 		failed()
 		return
 	}

-	return nil
+	return def.UpdateStatus(db, DEFINITION_STATUS_INIT)
 }

 func generateDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
@@ -1393,12 +1400,14 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 		return
 	}

-	def_id, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
+	def, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
 	if err != nil {
 		failed()
 		return
 	}

+	def_id := def.Id
+
 	order := 1

 	width := model.Width
@@ -1533,7 +1542,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 }

 func ResetClasses(c BasePack, model *BaseModel) {
-	_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TO_TRAIN, MODEL_CLASS_STATUS_TRAINING, model.Id)
+	_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING, model.Id)
 	if err != nil {
 		c.GetLogger().Error("Error while reseting the classes", "error", err)
 	}
@@ -1544,7 +1553,7 @@ func trainExpandable(c *Context, model *BaseModel) {

 	failed := func(msg string) {
 		c.Logger.Error(msg, "err", err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		ResetClasses(c, model)
 	}

@@ -1588,7 +1597,7 @@ func trainExpandable(c *Context, model *BaseModel) {
 	}

 	// Set the class status to trained
-	err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+	err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		failed("Failed to set class status")
 		return
@@ -1648,7 +1657,7 @@ func RunTaskTrain(b BasePack, task Task) (err error) {
 	if err != nil {
 		l.Error("Failed to train model", "err", err)
 		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
-		ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(b, model.Id, int(FAILED_TRAINING))
 		return
 	}

@@ -1731,7 +1740,7 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {

 	l.Info("Model updaded")

-	_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINED, MODEL_CLASS_STATUS_TRAINING, model.Id)
+	_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINED, CLASS_STATUS_TRAINING, model.Id)
 	if err != nil {
 		l.Error("Error while updating the classes", "error", err)
 		failed()
@@ -1861,7 +1870,7 @@ func handleTrain(handle *Handle) {
 			c,
 			"model_classes where model_id=$1 and status=$2 order by class_order asc",
 			model.Id,
-			MODEL_CLASS_STATUS_TO_TRAIN,
+			CLASS_STATUS_TO_TRAIN,
 		)
 		if err != nil {
 			_err := c.RollbackTx()
@@ -1882,7 +1891,7 @@ func handleTrain(handle *Handle) {

 		//Update the classes
 		{
-			_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINING, MODEL_CLASS_STATUS_TO_TRAIN, model.Id)
+			_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINING, CLASS_STATUS_TO_TRAIN, model.Id)
 			if err != nil {
 				_err := c.RollbackTx()
 				if _err != nil {
--- a/logic/tasks/README.md
+++ b/logic/tasks/README.md
@@ -0,0 +1,7 @@
+# Runner Protocol
+
+```
+    				   /----\
+    				  \/	|
+Register -> Init -> Active ---> Ready -> Info
+```
--- a/logic/tasks/index.go
+++ b/logic/tasks/index.go
@@ -8,4 +8,5 @@ func HandleTasks(handle *Handle) {
 	handleUpload(handle)
 	handleList(handle)
 	handleRequests(handle)
+	handleRemoteRunner(handle)
 }
--- a/logic/tasks/runner.go
+++ b/logic/tasks/runner.go
@@ -0,0 +1,533 @@
+package tasks
+
+import (
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func verifyRunner(c *Context, dat *JustId) (runner *Runner, e *Error) {
+	runner, err := GetRunner(c, dat.Id)
+	if err == NotFoundError {
+		e = c.JsonBadRequest("Could not find runner, please register runner first")
+		return
+	} else if err != nil {
+		e = c.E500M("Failed to get information about the runner", err)
+		return
+	}
+
+	if runner.Token != *c.Token {
+		return nil, c.SendJSONStatus(401, "Only runners can use this funcion")
+	}
+
+	return
+}
+
+type VerifyTask struct {
+	Id     string `json:"id" validate:"required"`
+	TaskId string `json:"taskId" validate:"required"`
+}
+
+func verifyTask(x *Handle, c *Context, dat *VerifyTask) (task *Task, error *Error) {
+	mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+	if runners[dat.Id] == nil {
+		return nil, c.JsonBadRequest("Runner not active")
+	}
+
+	var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+
+	if runner_data["task"] == nil {
+		return nil, c.SendJSONStatus(404, "No active task")
+	}
+
+	return runner_data["task"].(*Task), nil
+}
+
+func handleRemoteRunner(x *Handle) {
+
+	type RegisterRunner struct {
+		Token string     `json:"token" validate:"required"`
+		Type  RunnerType `json:"type" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/register", User_Normal, func(c *Context, dat *RegisterRunner) *Error {
+		if *c.Token != dat.Token {
+			// TODO do admin
+			return c.E500M("Please make sure that the token is the same that is being registered", nil)
+		}
+
+		c.Logger.Info("test", "dat", dat)
+
+		var runner Runner
+		err := GetDBOnce(c, &runner, "remote_runner as ru where token=$1", dat.Token)
+		if err != NotFoundError && err != nil {
+			return c.E500M("Failed to get information remote runners", err)
+		}
+		if err != NotFoundError {
+			return c.JsonBadRequest("Token is already registered by a runner")
+		}
+
+		// TODO get id from token passed by when doing admin
+		var userId = c.User.Id
+
+		var new_runner = struct {
+			Type   RunnerType
+			UserId string `db:"user_id"`
+			Token  string
+		}{
+			Type:   dat.Type,
+			Token:  dat.Token,
+			UserId: userId,
+		}
+
+		id, err := InsertReturnId(c, &new_runner, "remote_runner", "id")
+		if err != nil {
+			return c.E500M("Failed to create remote runner", err)
+		}
+
+		return c.SendJSON(struct {
+			Id string `json:"id"`
+		}{
+			Id: id,
+		})
+	})
+
+	// TODO remove runner
+
+	PostAuthJson(x, "/tasks/runner/init", User_Normal, func(c *Context, dat *JustId) *Error {
+		runner, error := verifyRunner(c, dat)
+		if error != nil {
+			return error
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		if runners[dat.Id] != nil {
+			c.Logger.Info("Logger trying to register but already registerd")
+			c.ShowMessage = false
+			return c.SendJSON("Ok")
+		}
+
+		var new_runner = map[string]interface{}{}
+		new_runner["last_time_check"] = time.Now()
+		new_runner["runner_info"] = runner
+
+		runners[dat.Id] = new_runner
+
+		x.DataMap["runners"] = runners
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/tasks/runner/active", User_Normal, func(c *Context, dat *JustId) *Error {
+		_, error := verifyRunner(c, dat)
+		if error != nil {
+			return error
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		if runners[dat.Id] == nil {
+			return c.JsonBadRequest("Runner not active")
+		}
+
+		var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+
+		if runner_data["task"] == nil {
+			c.ShowMessage = false
+			return c.SendJSONStatus(404, "No active task")
+		}
+
+		c.ShowMessage = false
+		// This should be a task obj
+		return c.SendJSON(runner_data["task"])
+	})
+
+	PostAuthJson(x, "/tasks/runner/ready", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		err := task.UpdateStatus(c, TASK_RUNNING, "Task Running on Runner")
+		if err != nil {
+			return c.E500M("Failed to set task status", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	type TaskFail struct {
+		Id     string `json:"id" validate:"required"`
+		TaskId string `json:"taskId" validate:"required"`
+		Reason string `json:"reason" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/fail", User_Normal, func(c *Context, dat *TaskFail) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		err := task.UpdateStatus(c, TASK_FAILED_RUNNING, dat.Reason)
+		if err != nil {
+			return c.E500M("Failed to set task status", err)
+		}
+
+		// Do extra clean up on tasks
+		switch task.TaskType {
+		case int(TASK_TYPE_TRAINING):
+			CleanUpFailed(c, task)
+		default:
+			panic("Do not know how to handle this")
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+		runner_data["task"] = nil
+
+		runners[dat.Id] = runner_data
+		x.DataMap["runners"] = runners
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/defs", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		defs, err := model.GetDefinitions(c, "and md.status=$2", DEFINITION_STATUS_INIT)
+		if err != nil {
+			return c.E500M("Failed to get the model definitions", err)
+		}
+
+		return c.SendJSON(defs)
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/classes", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TO_TRAIN)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+
+		return c.SendJSON(classes)
+	})
+
+	type RunnerTrainDefStatus struct {
+		Id     string           `json:"id" validate:"required"`
+		TaskId string           `json:"taskId" validate:"required"`
+		DefId  string           `json:"defId" validate:"required"`
+		Status DefinitionStatus `json:"status" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/def/status", User_Normal, func(c *Context, dat *RunnerTrainDefStatus) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		err = def.UpdateStatus(c, dat.Status)
+		if err != nil {
+			return c.E500M("Failed to update model status", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	type RunnerTrainDefLayers struct {
+		Id     string `json:"id" validate:"required"`
+		TaskId string `json:"taskId" validate:"required"`
+		DefId  string `json:"defId" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/def/layers", User_Normal, func(c *Context, dat *RunnerTrainDefLayers) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		layers, err := def.GetLayers(c, " order by layer_order asc")
+		if err != nil {
+			return c.E500M("Failed to get layers", err)
+		}
+
+		return c.SendJSON(layers)
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/datapoints", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		training_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+		testing_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+
+		return c.SendJSON(struct {
+			Testing  []DataPoint `json:"testing"`
+			Training []DataPoint `json:"training"`
+		}{
+			Testing:  testing_points,
+			Training: training_points,
+		})
+	})
+
+	type RunnerTrainDefEpoch struct {
+		Id       string  `json:"id" validate:"required"`
+		TaskId   string  `json:"taskId" validate:"required"`
+		DefId    string  `json:"defId" validate:"required"`
+		Epoch    int     `json:"epoch" validate:"required"`
+		Accuracy float64 `json:"accuracy" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/epoch", User_Normal, func(c *Context, dat *RunnerTrainDefEpoch) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{
+			Id:     dat.Id,
+			TaskId: dat.TaskId,
+		})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		err = def.UpdateAfterEpoch(c, dat.Accuracy, dat.Epoch)
+		if err != nil {
+			return c.E500M("Failed to update model", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/task/runner/train/mark-failed", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{
+			Id:     dat.Id,
+			TaskId: dat.TaskId,
+		})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		_, err := c.Exec(
+			"update model_definition set status=$1 "+
+				"where model_id=$2 and status in ($3, $4)",
+			MODEL_DEFINITION_STATUS_CANCELD_TRAINING,
+			task.ModelId,
+			MODEL_DEFINITION_STATUS_TRAINING,
+			MODEL_DEFINITION_STATUS_PAUSED_TRAINING,
+		)
+		if err != nil {
+			return c.E500M("Failed to mark definition as failed", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/task/runner/train/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			c.Logger.Error("Failed to get model", "err", err)
+			return c.E500M("Failed to get mode", err)
+		}
+
+		var def Definition
+		err = GetDBOnce(c, &def, "from model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
+		if err == NotFoundError {
+			// TODO Make the Model status have a message
+			c.Logger.Error("All definitions failed to train!")
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
+			return c.SendJSON("Ok")
+		} else if err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
+			return c.E500M("Failed to get model definition", err)
+		}
+
+		if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
+			return c.E500M("Failed to update model definition", err)
+		}
+
+		to_delete, err := c.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
+		if err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+			return c.E500M("Failed to delete unsed definitions", err)
+		}
+		defer to_delete.Close()
+
+		for to_delete.Next() {
+			var id string
+			if err = to_delete.Scan(&id); err != nil {
+				model.UpdateStatus(c, FAILED_TRAINING)
+				task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+				return c.E500M("Failed to delete unsed definitions", err)
+			}
+			os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
+		}
+
+		// TODO Check if returning also works here
+		if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+			return c.E500M("Failed to delete unsed definitions", err)
+		}
+
+		model.UpdateStatus(c, READY)
+
+		return c.SendJSON("Ok")
+	})
+}
--- a/logic/tasks/runner/runner.go
+++ b/logic/tasks/runner/runner.go
@@ -5,6 +5,7 @@ import (
 	"math"
 	"os"
 	"runtime/debug"
+	"sync"
 	"time"

 	"github.com/charmbracelet/log"
@@ -90,6 +91,45 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 	}
 }

+/**
+* Handle remote runner
+ */
+func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Task) {
+	logger := log.NewWithOptions(os.Stdout, log.Options{
+		ReportCaller:    true,
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          fmt.Sprintf("Runner pre %s", runner_id),
+	})
+	defer func() {
+		if r := recover(); r != nil {
+			logger.Error("Runner failed to setup for runner", "due to", r, "stack", string(debug.Stack()))
+			// TODO maybe create better failed task
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to setup task for runner")
+		}
+	}()
+
+	err := task.UpdateStatus(base, TASK_PICKED_UP, "Failed to setup task for runner")
+	if err != nil {
+		logger.Error("Failed to mark task as PICK UP")
+		return
+	}
+
+	mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	switch task.TaskType {
+	case int(TASK_TYPE_TRAINING):
+		if err := PrepareTraining(handler, base, task, runner_id); err != nil {
+			logger.Error("Failed to prepare for training", "err", err)
+		}
+	default:
+		logger.Error("Not sure what to do panicing", "taskType", task.TaskType)
+		panic("not sure what to do")
+	}
+}
+
 /**
 * Tells the orcchestator to look at the task list from time to time
 */
@@ -125,7 +165,7 @@ func attentionSeeker(config Config, back_channel chan int) {
 /**
 * Manages what worker should to Work
 */
-func RunnerOrchestrator(db db.Db, config Config) {
+func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 	logger := log.NewWithOptions(os.Stdout, log.Options{
 		ReportCaller:    true,
 		ReportTimestamp: true,
@@ -133,6 +173,16 @@ func RunnerOrchestrator(db db.Db, config Config) {
 		Prefix:          "Runner Orchestrator Logger",
 	})

+	// Setup vars
+	handler.DataMap["runners"] = map[string]interface{}{}
+	handler.DataMap["runners_mutex"] = &sync.Mutex{}
+
+	base := BasePackStruct{
+		Db:     db,
+		Logger: logger,
+		Host:   config.Hostname,
+	}
+
 	gpu_workers := config.GpuWorker.NumberOfWorkers

 	logger.Info("Starting runners")
@@ -149,7 +199,7 @@ func RunnerOrchestrator(db db.Db, config Config) {
 				close(task_runners[x])
 			}
 			close(back_channel)
-			go RunnerOrchestrator(db, config)
+			go RunnerOrchestrator(db, config, handler)
 		}
 	}()

@@ -198,19 +248,45 @@ func RunnerOrchestrator(db db.Db, config Config) {
 		}

 		if task_to_dispatch != nil {
-			for i := 0; i < len(task_runners_used); i += 1 {
-				if !task_runners_used[i] {
-					task_runners[i] <- *task_to_dispatch
-					task_runners_used[i] = true
+
+			// Only let CPU tasks be done by the local users
+			if task_to_dispatch.TaskType == int(TASK_TYPE_DELETE_USER) {
+				for i := 0; i < len(task_runners_used); i += 1 {
+					if !task_runners_used[i] {
+						task_runners[i] <- *task_to_dispatch
+						task_runners_used[i] = true
+						task_to_dispatch = nil
+						break
+					}
+				}
+				continue
+			}
+
+			mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
+			mutex.Lock()
+			remote_runners := handler.DataMap["runners"].(map[string]interface{})
+
+			for k, v := range remote_runners {
+				runner_data := v.(map[string]interface{})
+				runner_info := runner_data["runner_info"].(*Runner)
+
+				if runner_data["task"] != nil {
+					continue
+				}
+
+				if runner_info.UserId == task_to_dispatch.UserId {
+					go handleRemoteTask(handler, base, k, *task_to_dispatch)
 					task_to_dispatch = nil
 					break
 				}
 			}
+
+			mutex.Unlock()
 		}

 	}
 }

-func StartRunners(db db.Db, config Config) {
-	go RunnerOrchestrator(db, config)
+func StartRunners(db db.Db, config Config, handler *Handle) {
+	go RunnerOrchestrator(db, config, handler)
 }
--- a/logic/tasks/utils/runner.go
+++ b/logic/tasks/utils/runner.go
@@ -0,0 +1,29 @@
+package tasks_utils
+
+import (
+	"time"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+	dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+)
+
+type RunnerType int64
+
+const (
+	RUNNER_TYPE_GPU RunnerType = iota + 1
+)
+
+type Runner struct {
+	Id       string     `json:"id" db:"ru.id"`
+	UserId   string     `json:"user_id" db:"ru.user_id"`
+	Token    string     `json:"token" db:"ru.token"`
+	Type     RunnerType `json:"type" db:"ru.type"`
+	CreateOn time.Time  `json:"createOn" db:"ru.created_on"`
+}
+
+func GetRunner(db db.Db, id string) (ru *Runner, err error) {
+	var runner Runner
+	err = dbtypes.GetDBOnce(db, &runner, "remote_runner as ru where ru.id=$1", id)
+	ru = &runner
+	return
+}
--- a/logic/utils/handler.go
+++ b/logic/utils/handler.go
@@ -374,7 +374,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
 	c.SetReportCaller(true)
 	c.Logger.Warn("Request failed with a bad request", "dat", dat)
 	c.SetReportCaller(false)
-	return c.ErrorCode(nil, 404, dat)
+	return c.SendJSONStatus(http.StatusBadRequest, dat)
 }

 func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
--- a/main.go
+++ b/main.go
@@ -36,11 +36,11 @@ func main() {
 	log.Info("Config loaded!", "config", config)
 	config.GenerateToken(db)

-	StartRunners(db, config)
-
 	//TODO check if file structure exists to save data
 	handle := NewHandler(db, config)

+	StartRunners(db, config, handle)
+
 	config.Cleanup(db)

 	// TODO Handle this in other way
--- a/nginx.dev.conf
+++ b/nginx.dev.conf
@@ -13,7 +13,7 @@ http {
    server {
        listen 8000;

-        client_max_body_size 1G;
+        client_max_body_size 5G;

        location / {
            proxy_http_version 1.1;
--- a/sql/tasks.sql
+++ b/sql/tasks.sql
@@ -38,3 +38,14 @@ create table if not exists tasks_dependencies (
    main_id uuid references tasks (id) on delete cascade not null,
    dependent_id uuid references tasks (id) on delete cascade not null
 );
+
+create table if not exists remote_runner (
+	id uuid primary key default gen_random_uuid(),
+	user_id uuid references users (id) on delete cascade not null,
+	token text not null,
+
+	-- 1: GPU
+	type integer,
+
+	created_on timestamp default current_timestamp
+);
--- a/views/py/python_model_template.py
+++ b/views/py/python_model_template.py
@@ -135,9 +135,9 @@ def addBlock(
        model.add(layers.ReLU())
    if top:
        if pooling_same:
-            model.add(pool_func(padding="same", strides=(1, 1)))
+            model.add(pool_func(pool_size=(2,2), padding="same", strides=(1, 1)))
        else:
-            model.add(pool_func())
+            model.add(pool_func(pool_size=(2,2)))
        model.add(layers.BatchNormalization())
        model.add(layers.LeakyReLU())
        model.add(layers.Dropout(0.4))
@@ -172,7 +172,7 @@ model.compile(

 his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
                NotifyServerCallback(),
-                tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
+                tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)])

 acc = his.history["accuracy"]
Author	SHA1	Message	Date
Andre Henriques	712393cf9e	Merge branch 'main' into runner-go	2024-05-07 01:18:25 +01:00
Andre Henriques	29846012e7	more worker on go-runner	2024-05-07 01:16:38 +01:00
Andre Henriques	b1e4211e6a	more work on the rust runner	2024-05-06 12:48:02 +01:00
Andre Henriques	e22df8adc9	started working on runner	2024-05-06 01:10:58 +01:00