Merge branch 'main' into runner-go

DockerfileServer

@@ -0,0 +1,37 @@
+FROM docker.io/nvidia/cuda:12.3.2-devel-ubuntu22.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Sometimes you have to get update twice because ?
+RUN apt-get update
+RUN apt-get update
+
+RUN apt-get install -y wget unzip python3-pip vim python3 python3-pip curl
+
+RUN wget https://go.dev/dl/go1.22.2.linux-amd64.tar.gz
+RUN tar -xvf go1.22.2.linux-amd64.tar.gz -C /usr/local
+ENV PATH=$PATH:/usr/local/go/bin
+ENV GOPATH=/go
+
+RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.9.1.tar.gz" | tar -C /usr/local -xz'
+RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.15.0.tar.gz" | tar -C /usr/local -xz'
+RUN ldconfig
+
+RUN ln -s /usr/bin/python3 /usr/bin/python
+RUN python -m pip install nvidia-pyindex
+ADD requirements.txt .
+RUN python -m pip install -r requirements.txt
+
+ENV CUDNN_PATH=/usr/local/lib/python3.10/dist-packages/nvidia/cudnn
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.10/dist-packages/nvidia/cudnn/lib
+
+WORKDIR /app
+
+ADD  go.mod .
+ADD  go.sum .
+ADD  main.go .
+ADD  logic logic
+
+RUN go install || true
+
+CMD ["go", "run", "."]

go.mod

@@ -9,10 +9,11 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/lib/pq v1.10.9
 	golang.org/x/crypto v0.19.0
+	github.com/BurntSushi/toml v1.3.2
+	github.com/goccy/go-json v0.10.2
 )
 
 require (
-	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/charmbracelet/lipgloss v0.9.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@@ -20,7 +21,6 @@ require (
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.19.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx v3.6.2+incompatible // indirect

logic/db_types/classes.go

@@ -6,3 +6,19 @@ const (
 	DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
 	DATA_POINT_MODE_TESTING                  = 2
 )
+
+type ModelClassStatus int
+
+const (
+	CLASS_STATUS_TO_TRAIN ModelClassStatus = iota + 1
+	CLASS_STATUS_TRAINING
+	CLASS_STATUS_TRAINED
+)
+
+type ModelClass struct {
+	Id         string `db:"mc.id" json:"id"`
+	ModelId    string `db:"mc.model_id" json:"model_id"`
+	Name       string `db:"mc.name" json:"name"`
+	ClassOrder int    `db:"mc.class_order" json:"class_order"`
+	Status     int    `db:"mc.status" json:"status"`
+}

logic/db_types/definitions.go

@@ -0,0 +1,95 @@
+package dbtypes
+
+import (
+	"time"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+)
+
+type DefinitionStatus int
+
+const (
+	DEFINITION_STATUS_CANCELD_TRAINING DefinitionStatus = -4
+	DEFINITION_STATUS_FAILED_TRAINING                   = -3
+	DEFINITION_STATUS_PRE_INIT                          = 1
+	DEFINITION_STATUS_INIT                              = 2
+	DEFINITION_STATUS_TRAINING                          = 3
+	DEFINITION_STATUS_PAUSED_TRAINING                   = 6
+	DEFINITION_STATUS_TRANIED                           = 4
+	DEFINITION_STATUS_READY                             = 5
+)
+
+type Definition struct {
+	Id             string    `db:"md.id" json:"id"`
+	ModelId        string    `db:"md.model_id" json:"model_id"`
+	Accuracy       float64   `db:"md.accuracy" json:"accuracy"`
+	TargetAccuracy int       `db:"md.target_accuracy" json:"target_accuracy"`
+	Epoch          int       `db:"md.epoch" json:"epoch"`
+	Status         int       `db:"md.status" json:"status"`
+	CreatedOn      time.Time `db:"md.created_on" json:"created"`
+	EpochProgress  int       `db:"md.epoch_progress" json:"epoch_progress"`
+}
+
+type SortByAccuracyDefinitions []*Definition
+
+func (nf SortByAccuracyDefinitions) Len() int      { return len(nf) }
+func (nf SortByAccuracyDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
+func (nf SortByAccuracyDefinitions) Less(i, j int) bool {
+	return nf[i].Accuracy < nf[j].Accuracy
+}
+
+func GetDefinition(db db.Db, definition_id string) (definition Definition, err error) {
+	err = GetDBOnce(db, &definition, "model_definition as md where id=$1;", definition_id)
+	return
+}
+
+func MakeDefenition(db db.Db, model_id string, target_accuracy int) (definition Definition, err error) {
+	var NewDefinition = struct {
+		ModelId        string `db:"model_id"`
+		TargetAccuracy int    `db:"target_accuracy"`
+	}{ModelId: model_id, TargetAccuracy: target_accuracy}
+
+	id, err := InsertReturnId(db, &NewDefinition, "model_definition", "id")
+	if err != nil {
+		return
+	}
+	return GetDefinition(db, id)
+}
+
+func (d Definition) UpdateStatus(db db.Db, status DefinitionStatus) (err error) {
+	_, err = db.Exec("update model_definition set status=$1 where id=$2", status, d.Id)
+	return
+}
+
+func (d Definition) MakeLayer(db db.Db, layer_order int, layer_type LayerType, shape string) (layer Layer, err error) {
+	var NewLayer = struct {
+		DefinitionId string    `db:"def_id"`
+		LayerOrder   int       `db:"layer_order"`
+		LayerType    LayerType `db:"layer_type"`
+		Shape        string    `db:"shape"`
+	}{
+		DefinitionId: d.Id,
+		LayerOrder:   layer_order,
+		LayerType:    layer_type,
+		Shape:        shape,
+	}
+
+	id, err := InsertReturnId(db, &NewLayer, "model_definition_layer", "id")
+	if err != nil {
+		return
+	}
+
+	return GetLayer(db, id)
+}
+
+func (d Definition) GetLayers(db db.Db, filter string, args ...any) (layer []*Layer, err error) {
+	args = append(args, d.Id)
+	return GetDbMultitple[Layer](db, "model_definition_layer as mdl where mdl.def_id=$1 "+filter, args...)
+}
+
+func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64, epoch int) (err error) {
+	d.Accuracy = accuracy
+	d.Epoch += epoch
+	_, err = db.Exec("update model_definition set epoch=$1, accuracy=$2 where id=$3", d.Epoch, d.Accuracy, d.Id)
+	return
+}

logic/db_types/layer.go

@@ -0,0 +1,50 @@
+package dbtypes
+
+import (
+	"encoding/json"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+)
+
+type LayerType int
+
+const (
+	LAYER_INPUT        LayerType = 1
+	LAYER_DENSE                  = 2
+	LAYER_FLATTEN                = 3
+	LAYER_SIMPLE_BLOCK           = 4
+)
+
+type Layer struct {
+	Id           string    `db:"mdl.id" json:"id"`
+	DefinitionId string    `db:"mdl.def_id" json:"definition_id"`
+	LayerOrder   string    `db:"mdl.layer_order" json:"layer_order"`
+	LayerType    LayerType `db:"mdl.layer_type" json:"layer_type"`
+	Shape        string    `db:"mdl.shape" json:"shape"`
+	ExpType      string    `db:"mdl.exp_type" json:"exp_type"`
+}
+
+func ShapeToString(args ...int) string {
+	text, err := json.Marshal(args)
+	if err != nil {
+		panic("Could not generate Shape")
+	}
+	return string(text)
+}
+
+func StringToShape(str string) (shape []int64) {
+	err := json.Unmarshal([]byte(str), &shape)
+	if err != nil {
+		panic("Could not parse Shape")
+	}
+	return
+}
+
+func (l Layer) GetShape() []int64 {
+	return StringToShape(l.Shape)
+}
+
+func GetLayer(db db.Db, layer_id string) (layer Layer, err error) {
+	err = GetDBOnce(db, &layer, "model_definition_layer as mdl where mdl.id=$1", layer_id)
+	return
+}

logic/db_types/types.go

@@ -2,23 +2,26 @@ package dbtypes
 
 import (
 	"errors"
+	"fmt"
+	"path"
 
 	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
 )
 
-const (
+type ModelStatus int
-	FAILED_TRAINING           = -4
-	FAILED_PREPARING_TRAINING = -3
-	FAILED_PREPARING_ZIP_FILE = -2
-	FAILED_PREPARING          = -1
 
-	PREPARING               = 1
+const (
-	CONFIRM_PRE_TRAINING    = 2
+	FAILED_TRAINING           ModelStatus = -4
-	PREPARING_ZIP_FILE      = 3
+	FAILED_PREPARING_TRAINING             = -3
-	TRAINING                = 4
+	FAILED_PREPARING_ZIP_FILE             = -2
-	READY                   = 5
+	FAILED_PREPARING                      = -1
-	READY_ALTERATION        = 6
+	PREPARING                             = 1
-	READY_ALTERATION_FAILED = -6
+	CONFIRM_PRE_TRAINING                  = 2
+	PREPARING_ZIP_FILE                    = 3
+	TRAINING                              = 4
+	READY                                 = 5
+	READY_ALTERATION                      = 6
+	READY_ALTERATION_FAILED               = -6
 
 	READY_RETRAIN        = 7
 	READY_RETRAIN_FAILED = -7
@@ -26,15 +29,6 @@ const (
 
 type ModelDefinitionStatus int
 
-type LayerType int
-
-const (
-	LAYER_INPUT        LayerType = 1
-	LAYER_DENSE                  = 2
-	LAYER_FLATTEN                = 3
-	LAYER_SIMPLE_BLOCK           = 4
-)
-
 const (
 	MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
 	MODEL_DEFINITION_STATUS_FAILED_TRAINING                        = -3
@@ -46,14 +40,6 @@ const (
 	MODEL_DEFINITION_STATUS_READY                                  = 5
 )
 
-type ModelClassStatus int
-
-const (
-	MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
-	MODEL_CLASS_STATUS_TRAINING                  = 2
-	MODEL_CLASS_STATUS_TRAINED                   = 3
-)
-
 type ModelHeadStatus int
 
 const (
@@ -97,6 +83,61 @@ func (m BaseModel) CanEval() bool {
 	return true
 }
 
+// DO NOT Pass un filtered data on filters
+func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
+	n_args := []any{m.Id}
+	n_args = append(n_args, args...)
+	return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
+}
+
+func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
+	n_args := []any{m.Id}
+	n_args = append(n_args, args...)
+	return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
+}
+
+func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
+	_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
+	return
+}
+
+type DataPoint struct {
+	Class int    `json:"class"`
+	Path  string `json:"path"`
+}
+
+func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
+	rows, err := db.Query(
+		"select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner "+
+			"join model_classes as mc on mc.id = mdp.class_id "+
+			"where mc.model_id = $1 and mdp.model_mode=$2;",
+		m.Id, mode)
+	if err != nil {
+		return
+	}
+	defer rows.Close()
+
+	data = []DataPoint{}
+
+	for rows.Next() {
+		var id string
+		var class_order int
+		var file_path string
+		if err = rows.Scan(&id, &class_order, &file_path); err != nil {
+			return
+		}
+		if file_path == "id://" {
+			data = append(data, DataPoint{
+				Path:  path.Join("./savedData", m.Id, "data", id+"."+m.Format),
+				Class: class_order,
+			})
+		} else {
+			panic("TODO remote file path")
+		}
+	}
+	return
+}
+
 func StringToImageMode(colorMode string) int {
 	switch colorMode {
 	case "greyscale":

logic/models/classes/main.go

@@ -7,15 +7,15 @@ import (
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
 )
 
-type ModelClass struct {
+type ModelClassJSON struct {
 	Id      string `json:"id"`
 	ModelId string `json:"model_id" db:"model_id"`
 	Name    string `json:"name"`
 	Status  int    `json:"status"`
 }
 
-func ListClasses(c BasePack, model_id string) (cls []*ModelClass, err error) {
+func ListClasses(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
-	return GetDbMultitple[ModelClass](c.GetDb(), "model_classes where model_id=$1", model_id)
+	return GetDbMultitple[ModelClassJSON](c.GetDb(), "model_classes where model_id=$1", model_id)
 }
 
 func ModelHasDataPoints(db db.Db, model_id string) (result bool, err error) {

logic/models/data.go

@@ -495,7 +495,7 @@ func handleDataUpload(handle *Handle) {
 			return c.E500M("Could not create class", err)
 		}
 
-		var modelClass model_classes.ModelClass
+		var modelClass model_classes.ModelClassJSON
 		err = GetDBOnce(c, &modelClass, "model_classes where id=$1;", id)
 		if err != nil {
 			return c.E500M("Failed to get class information but class was creted", err)
@@ -704,7 +704,7 @@ func handleDataUpload(handle *Handle) {
 				return c.Error500(err)
 			}
 		} else {
-			_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
+			_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TO_TRAIN)
 			if err != nil {
 				return c.Error500(err)
 			}

logic/models/delete.go

@@ -51,7 +51,7 @@ func handleDelete(handle *Handle) {
 			return c.E500M("Faield to get model", err)
 		}
 
-		switch model.Status {
+		switch ModelStatus(model.Status) {
 		case FAILED_TRAINING:
 			fallthrough
 		case FAILED_PREPARING_ZIP_FILE:

logic/models/edit.go

@@ -35,9 +35,9 @@ func handleEdit(handle *Handle) {
 		}
 
 		type ReturnType struct {
-			Classes               []*model_classes.ModelClass `json:"classes"`
+			Classes               []*model_classes.ModelClassJSON `json:"classes"`
-			HasData               bool                        `json:"has_data"`
+			HasData               bool                            `json:"has_data"`
-			NumberOfInvalidImages int                         `json:"number_of_invalid_images"`
+			NumberOfInvalidImages int                             `json:"number_of_invalid_images"`
 		}
 
 		c.ShowMessage = false

logic/models/run.go

@@ -8,6 +8,7 @@ import (
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
 
+	"github.com/charmbracelet/log"
 	tf "github.com/galeone/tensorflow/tensorflow/go"
 	"github.com/galeone/tensorflow/tensorflow/go/op"
 	tg "github.com/galeone/tfgo"
@@ -19,6 +20,7 @@ func ReadPNG(scope *op.Scope, imagePath string, channels int64) *image.Image {
 	contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
 	output := op.DecodePng(scope.SubScope("DecodePng"), contents, op.DecodePngChannels(channels))
 	output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
+	output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
 	image := &image.Image{
 		Tensor: tg.NewTensor(scope, output)}
 	return image.Scale(0, 255)
@@ -29,6 +31,7 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
 	contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
 	output := op.DecodePng(scope.SubScope("DecodeJpeg"), contents, op.DecodePngChannels(channels))
 	output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
+	output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
 	image := &image.Image{
 		Tensor: tg.NewTensor(scope, output)}
 	return image.Scale(0, 255)
@@ -49,6 +52,8 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
 	var vmax float32 = 0.0
 	var predictions = results[0].Value().([][]float32)[0]
 
+	log.Info("preds", "preds", predictions)
+
 	for i, v := range predictions {
 		if v > vmax {
 			order = i
@@ -62,10 +67,13 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
 }
 
 func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
+	log := base.GetLogger()
 
 	err = nil
 	order = 0
 
+	log.Info("Running base")
+
 	base_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "base", "model"), []string{"serve"}, nil)
 
 	//results := base_model.Exec([]tf.Output{
@@ -86,7 +94,7 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
 		return
 	}
 
-	base.GetLogger().Info("test", "count", len(heads))
+	log.Info("Running heads", "heads", heads)
 
 	var vmax float32 = 0.0
 

logic/models/train/remote_train.go

@@ -0,0 +1,79 @@
+package models_train
+
+import (
+	"errors"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	"github.com/goccy/go-json"
+)
+
+func PrepareTraining(handler *Handle, b BasePack, task Task, runner_id string) (err error) {
+	l := b.GetLogger()
+
+	model, err := GetBaseModel(b.GetDb(), *task.ModelId)
+	if err != nil {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
+		l.Error("Failed to get model information", "err", err)
+		return err
+	}
+
+	if model.Status != TRAINING {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
+		return errors.New("Model not in the right status")
+	}
+
+	// TODO do this when the runner says it's OK
+	//task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
+
+	// TODO move this to the runner part as well
+	var dat struct {
+		NumberOfModels int
+		Accuracy       int
+	}
+
+	err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
+	if err != nil {
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
+	}
+
+	if model.ModelType == 2 {
+		panic("TODO")
+		full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
+		if full_error != nil {
+			l.Error("Failed to generate defintions", "err", full_error)
+			task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
+			return errors.New("Failed to generate definitions")
+		}
+	} else {
+		error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
+		if error != nil {
+			task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
+			return errors.New("Failed to generate definitions")
+		}
+	}
+
+	runners := handler.DataMap["runners"].(map[string]interface{})
+	runner := runners[runner_id].(map[string]interface{})
+	runner["task"] = &task
+
+	runners[runner_id] = runner
+	handler.DataMap["runners"] = runners
+
+	return
+}
+
+func CleanUpFailed(b BasePack, task *Task) {
+	db := b.GetDb()
+	l := b.GetLogger()
+	model, err := GetBaseModel(db, *task.ModelId)
+	if err != nil {
+		l.Error("Failed to get model", "err", err)
+	} else {
+		err = model.UpdateStatus(db, FAILED_TRAINING)
+		if err != nil {
+			l.Error("Failed to get status", err)
+		}
+	}
+}

logic/models/train/reset.go

@@ -17,7 +17,7 @@ func handleRest(handle *Handle) {
 			return c.E500M("Failed to get model", err)
 		}
 
-		if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
+		if model.Status != FAILED_PREPARING_TRAINING && model.Status != int(FAILED_TRAINING) {
 			return c.JsonBadRequest("Model is not in status that be reset")
 		}
 

logic/models/train/train.go

@@ -39,16 +39,6 @@ func getDir() string {
 	return dir
 }
 
-// This function creates a new model_definition
-func MakeDefenition(db db.Db, model_id string, target_accuracy int) (id string, err error) {
-	var NewDefinition = struct {
-		ModelId        string `db:"model_id"`
-		TargetAccuracy int    `db:"target_accuracy"`
-	}{ModelId: model_id, TargetAccuracy: target_accuracy}
-
-	return InsertReturnId(db, &NewDefinition, "model_definition", "id")
-}
-
 func ModelDefinitionUpdateStatus(c BasePack, id string, status ModelDefinitionStatus) (err error) {
 	_, err = c.GetDb().Exec("update model_definition set status = $1 where id = $2", status, id)
 	return
@@ -118,14 +108,14 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
 	var co struct {
 		Count int `db:"count(*)"`
 	}
-	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
+	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
 	count = co.Count
 
 	if count == 0 {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
+		err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
 		if err != nil {
 			return
 		}
@@ -137,7 +127,7 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
 		return generateCvsExp(c, run_path, model_id, true)
 	}
 
-	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
+	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -287,7 +277,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	var co struct {
 		Count int `db:"count(*)"`
 	}
-	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
+	err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -296,7 +286,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	count := co.Count
 
 	if count == 0 {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
+		err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
 		if err != nil {
 			return
 		} else if doPanic {
@@ -305,7 +295,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 		return generateCvsExpandExp(c, run_path, model_id, offset, true)
 	}
 
-	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
+	data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
 	if err != nil {
 		return
 	}
@@ -339,7 +329,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
 	// This is to load some extra data so that the model has more things to train on
 	//
 
-	data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
+	data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINED, count*10)
 	if err != nil {
 		return
 	}
@@ -737,7 +727,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("Failed to train Model! Err:")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer definitionsRows.Close()
@@ -750,7 +740,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 		if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
 			l.Error("Failed to train Model Could not read definition from db!Err:")
 			l.Error(err)
-			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+			ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 			return
 		}
 		definitions = append(definitions, rowv)
@@ -758,7 +748,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 
 	if len(definitions) == 0 {
 		l.Error("No Definitions defined!")
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
@@ -788,14 +778,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 				_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
 				if err != nil {
 					l.Error("Failed to train definition!Err:\n", "err", err)
-					ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+					ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 					return err
 				}
 
 				_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
 				if err != nil {
 					l.Error("Failed to train definition!Err:\n", "err", err)
-					ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+					ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 					return err
 				}
 
@@ -813,7 +803,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 			_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
 			if err != nil {
 				l.Error("Failed to train definition!Err:\n", "err", err)
-				ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+				ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 				return err
 			}
 		}
@@ -868,7 +858,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("DB: failed to read definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer rows.Close()
@@ -876,7 +866,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if !rows.Next() {
 		// TODO Make the Model status have a message
 		l.Error("All definitions failed to train!")
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
@@ -884,14 +874,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err = rows.Scan(&id); err != nil {
 		l.Error("Failed to read id:")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
 	if _, err = db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
 		l.Error("Failed to update model definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
@@ -899,7 +889,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if err != nil {
 		l.Error("Failed to select model_definition to delete")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 	defer to_delete.Close()
@@ -909,7 +899,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 		if err = to_delete.Scan(&id); err != nil {
 			l.Error("Failed to scan the id of a model_definition to delete")
 			l.Error(err)
-			ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+			ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 			return
 		}
 		os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@@ -919,7 +909,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
 	if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
 		l.Error("Failed to delete model_definition")
 		l.Error(err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
@@ -1066,7 +1056,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	err = GetDBOnce(db, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
 	if err == NotFoundError {
 		// Set the class status to trained
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+		err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 		if err != nil {
 			l.Error("All definitions failed to train! And Failed to set class status")
 			return err
@@ -1101,7 +1091,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	}
 
 	if err = splitModel(c, model); err != nil {
-		err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+		err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 		if err != nil {
 			l.Error("Failed to split the model! And Failed to set class status")
 			return err
@@ -1112,7 +1102,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
 	}
 
 	// Set the class status to trained
-	err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+	err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		l.Error("Failed to set class status")
 		return err
@@ -1185,7 +1175,14 @@ func splitModel(c BasePack, model *BaseModel) (err error) {
 	count := -1
 
 	for layers.Next() {
+		var layerrow layerrow
+		if err = layers.Scan(&layerrow.ExpType); err != nil {
+			return
+		}
 		count += 1
+		if layerrow.ExpType == 2 {
+			break
+		}
 	}
 
 	if count == -1 {
@@ -1265,7 +1262,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 	db := c.GetDb()
 	l := c.GetLogger()
 
-	def_id, err := MakeDefenition(db, model.Id, target_accuracy)
+	def, err := MakeDefenition(db, model.Id, target_accuracy)
 	if err != nil {
 		failed()
 		return
@@ -1274,28 +1271,25 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 	order := 1
 
 	// Note the shape of the first layer defines the import size
-	if complexity == 2 {
+	//_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(model.Width, model.Height, model.ImageMode))
-		// Note the shape for now is no used
+	_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height))
-		width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
+	if err != nil {
-		height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
+		failed()
-		l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
+		return
-		err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height))
-		if err != nil {
-			failed()
-			return
-		}
-		order++
-	} else {
-		err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
-		if err != nil {
-			failed()
-			return
-		}
-		order++
 	}
+	order++
 
 	if complexity == 0 {
-		err = MakeLayer(db, def_id, order, LAYER_FLATTEN, "")
+		/*
+			_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
+			if err != nil {
+				failed()
+				return
+			}
+			order++
+		*/
+
+		_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
 		if err != nil {
 			failed()
 			return
@@ -1304,22 +1298,17 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 
 		loop := int(math.Log2(float64(number_of_classes)))
 		for i := 0; i < loop; i++ {
-			err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
+			_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
 			order++
 			if err != nil {
 				ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
 				return
 			}
 		}
-
 	} else if complexity == 1 || complexity == 2 {
-
+		loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
-		loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
-		if loop == 0 {
-			loop = 1
-		}
 		for i := 0; i < loop; i++ {
-			err = MakeLayer(db, def_id, order, LAYER_SIMPLE_BLOCK, "")
+			_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
 			order++
 			if err != nil {
 				failed()
@@ -1327,7 +1316,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 			}
 		}
 
-		err = MakeLayer(db, def_id, order, LAYER_FLATTEN, "")
+		_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
 		if err != nil {
 			failed()
 			return
@@ -1339,7 +1328,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 			loop = 1
 		}
 		for i := 0; i < loop; i++ {
-			err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
+			_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
 			order++
 			if err != nil {
 				failed()
@@ -1347,18 +1336,12 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 			}
 		}
 	} else {
-		log.Error("Unkown complexity", "complexity", complexity)
+		l.Error("Unkown complexity", "complexity", complexity)
 		failed()
 		return
 	}
 
-	err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
+	return def.UpdateStatus(db, DEFINITION_STATUS_INIT)
-	if err != nil {
-		failed()
-		return
-	}
-
-	return nil
 }
 
 func generateDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
@@ -1417,12 +1400,14 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 		return
 	}
 
-	def_id, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
+	def, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
 	if err != nil {
 		failed()
 		return
 	}
 
+	def_id := def.Id
+
 	order := 1
 
 	width := model.Width
@@ -1486,10 +1471,10 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 
 	log.Info("Size of the dense layers", "loop", loop)
 
-	// loop = max(loop, 3)
+	loop = max(loop, 3)
 
 	for i := 0; i < loop; i++ {
-		err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
+		err = MakeLayerExpandable(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)*2), 2)
 		order++
 		if err != nil {
 			failed()
@@ -1557,7 +1542,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 }
 
 func ResetClasses(c BasePack, model *BaseModel) {
-	_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TO_TRAIN, MODEL_CLASS_STATUS_TRAINING, model.Id)
+	_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING, model.Id)
 	if err != nil {
 		c.GetLogger().Error("Error while reseting the classes", "error", err)
 	}
@@ -1568,7 +1553,7 @@ func trainExpandable(c *Context, model *BaseModel) {
 
 	failed := func(msg string) {
 		c.Logger.Error(msg, "err", err)
-		ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
 		ResetClasses(c, model)
 	}
 
@@ -1612,7 +1597,7 @@ func trainExpandable(c *Context, model *BaseModel) {
 	}
 
 	// Set the class status to trained
-	err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
+	err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
 	if err != nil {
 		failed("Failed to set class status")
 		return
@@ -1672,7 +1657,7 @@ func RunTaskTrain(b BasePack, task Task) (err error) {
 	if err != nil {
 		l.Error("Failed to train model", "err", err)
 		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
-		ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
+		ModelUpdateStatus(b, model.Id, int(FAILED_TRAINING))
 		return
 	}
 
@@ -1712,10 +1697,22 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
 		return
 	}
 
+	failed = func() {
+		ResetClasses(b, model)
+		ModelUpdateStatus(b, model.Id, READY_RETRAIN_FAILED)
+		task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model failed retraining")
+		_, err_ := db.Exec("delete from exp_model_head where def_id=$1 and status in (2,3)", defData.Id)
+		if err_ != nil {
+			panic(err_)
+		}
+		l.Error("Failed to retrain", "err", err)
+	}
+
 	var acc float64 = 0
 	var epocs = 0
 	// TODO make max epochs come from db
-	for acc*100 < defData.TargetAcuuracy && epocs < 20 {
+	// TODO re increase the target accuracy
+	for acc*100 < defData.TargetAcuuracy-5 && epocs < 10 {
 		// This is something I have to check
 		acc, err = trainDefinitionExpandExp(b, model, defData.Id, epocs > 0)
 		if err != nil {
@@ -1743,7 +1740,7 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
 
 	l.Info("Model updaded")
 
-	_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINED, MODEL_CLASS_STATUS_TRAINING, model.Id)
+	_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINED, CLASS_STATUS_TRAINING, model.Id)
 	if err != nil {
 		l.Error("Error while updating the classes", "error", err)
 		failed()
@@ -1873,7 +1870,7 @@ func handleTrain(handle *Handle) {
 			c,
 			"model_classes where model_id=$1 and status=$2 order by class_order asc",
 			model.Id,
-			MODEL_CLASS_STATUS_TO_TRAIN,
+			CLASS_STATUS_TO_TRAIN,
 		)
 		if err != nil {
 			_err := c.RollbackTx()
@@ -1894,7 +1891,7 @@ func handleTrain(handle *Handle) {
 
 		//Update the classes
 		{
-			_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINING, MODEL_CLASS_STATUS_TO_TRAIN, model.Id)
+			_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINING, CLASS_STATUS_TO_TRAIN, model.Id)
 			if err != nil {
 				_err := c.RollbackTx()
 				if _err != nil {

logic/tasks/README.md

@@ -0,0 +1,7 @@
+# Runner Protocol
+
+```
+    				   /----\
+    				  \/	|
+Register -> Init -> Active ---> Ready -> Info
+```

logic/tasks/index.go

@@ -8,4 +8,5 @@ func HandleTasks(handle *Handle) {
 	handleUpload(handle)
 	handleList(handle)
 	handleRequests(handle)
+	handleRemoteRunner(handle)
 }

logic/tasks/runner.go

@@ -0,0 +1,533 @@
+package tasks
+
+import (
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func verifyRunner(c *Context, dat *JustId) (runner *Runner, e *Error) {
+	runner, err := GetRunner(c, dat.Id)
+	if err == NotFoundError {
+		e = c.JsonBadRequest("Could not find runner, please register runner first")
+		return
+	} else if err != nil {
+		e = c.E500M("Failed to get information about the runner", err)
+		return
+	}
+
+	if runner.Token != *c.Token {
+		return nil, c.SendJSONStatus(401, "Only runners can use this funcion")
+	}
+
+	return
+}
+
+type VerifyTask struct {
+	Id     string `json:"id" validate:"required"`
+	TaskId string `json:"taskId" validate:"required"`
+}
+
+func verifyTask(x *Handle, c *Context, dat *VerifyTask) (task *Task, error *Error) {
+	mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+	if runners[dat.Id] == nil {
+		return nil, c.JsonBadRequest("Runner not active")
+	}
+
+	var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+
+	if runner_data["task"] == nil {
+		return nil, c.SendJSONStatus(404, "No active task")
+	}
+
+	return runner_data["task"].(*Task), nil
+}
+
+func handleRemoteRunner(x *Handle) {
+
+	type RegisterRunner struct {
+		Token string     `json:"token" validate:"required"`
+		Type  RunnerType `json:"type" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/register", User_Normal, func(c *Context, dat *RegisterRunner) *Error {
+		if *c.Token != dat.Token {
+			// TODO do admin
+			return c.E500M("Please make sure that the token is the same that is being registered", nil)
+		}
+
+		c.Logger.Info("test", "dat", dat)
+
+		var runner Runner
+		err := GetDBOnce(c, &runner, "remote_runner as ru where token=$1", dat.Token)
+		if err != NotFoundError && err != nil {
+			return c.E500M("Failed to get information remote runners", err)
+		}
+		if err != NotFoundError {
+			return c.JsonBadRequest("Token is already registered by a runner")
+		}
+
+		// TODO get id from token passed by when doing admin
+		var userId = c.User.Id
+
+		var new_runner = struct {
+			Type   RunnerType
+			UserId string `db:"user_id"`
+			Token  string
+		}{
+			Type:   dat.Type,
+			Token:  dat.Token,
+			UserId: userId,
+		}
+
+		id, err := InsertReturnId(c, &new_runner, "remote_runner", "id")
+		if err != nil {
+			return c.E500M("Failed to create remote runner", err)
+		}
+
+		return c.SendJSON(struct {
+			Id string `json:"id"`
+		}{
+			Id: id,
+		})
+	})
+
+	// TODO remove runner
+
+	PostAuthJson(x, "/tasks/runner/init", User_Normal, func(c *Context, dat *JustId) *Error {
+		runner, error := verifyRunner(c, dat)
+		if error != nil {
+			return error
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		if runners[dat.Id] != nil {
+			c.Logger.Info("Logger trying to register but already registerd")
+			c.ShowMessage = false
+			return c.SendJSON("Ok")
+		}
+
+		var new_runner = map[string]interface{}{}
+		new_runner["last_time_check"] = time.Now()
+		new_runner["runner_info"] = runner
+
+		runners[dat.Id] = new_runner
+
+		x.DataMap["runners"] = runners
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/tasks/runner/active", User_Normal, func(c *Context, dat *JustId) *Error {
+		_, error := verifyRunner(c, dat)
+		if error != nil {
+			return error
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		if runners[dat.Id] == nil {
+			return c.JsonBadRequest("Runner not active")
+		}
+
+		var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+
+		if runner_data["task"] == nil {
+			c.ShowMessage = false
+			return c.SendJSONStatus(404, "No active task")
+		}
+
+		c.ShowMessage = false
+		// This should be a task obj
+		return c.SendJSON(runner_data["task"])
+	})
+
+	PostAuthJson(x, "/tasks/runner/ready", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		err := task.UpdateStatus(c, TASK_RUNNING, "Task Running on Runner")
+		if err != nil {
+			return c.E500M("Failed to set task status", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	type TaskFail struct {
+		Id     string `json:"id" validate:"required"`
+		TaskId string `json:"taskId" validate:"required"`
+		Reason string `json:"reason" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/fail", User_Normal, func(c *Context, dat *TaskFail) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		err := task.UpdateStatus(c, TASK_FAILED_RUNNING, dat.Reason)
+		if err != nil {
+			return c.E500M("Failed to set task status", err)
+		}
+
+		// Do extra clean up on tasks
+		switch task.TaskType {
+		case int(TASK_TYPE_TRAINING):
+			CleanUpFailed(c, task)
+		default:
+			panic("Do not know how to handle this")
+		}
+
+		mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
+		mutex.Lock()
+		defer mutex.Unlock()
+
+		var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
+		var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
+		runner_data["task"] = nil
+
+		runners[dat.Id] = runner_data
+		x.DataMap["runners"] = runners
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/defs", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		defs, err := model.GetDefinitions(c, "and md.status=$2", DEFINITION_STATUS_INIT)
+		if err != nil {
+			return c.E500M("Failed to get the model definitions", err)
+		}
+
+		return c.SendJSON(defs)
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/classes", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TO_TRAIN)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+
+		return c.SendJSON(classes)
+	})
+
+	type RunnerTrainDefStatus struct {
+		Id     string           `json:"id" validate:"required"`
+		TaskId string           `json:"taskId" validate:"required"`
+		DefId  string           `json:"defId" validate:"required"`
+		Status DefinitionStatus `json:"status" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/def/status", User_Normal, func(c *Context, dat *RunnerTrainDefStatus) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		err = def.UpdateStatus(c, dat.Status)
+		if err != nil {
+			return c.E500M("Failed to update model status", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	type RunnerTrainDefLayers struct {
+		Id     string `json:"id" validate:"required"`
+		TaskId string `json:"taskId" validate:"required"`
+		DefId  string `json:"defId" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/def/layers", User_Normal, func(c *Context, dat *RunnerTrainDefLayers) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		layers, err := def.GetLayers(c, " order by layer_order asc")
+		if err != nil {
+			return c.E500M("Failed to get layers", err)
+		}
+
+		return c.SendJSON(layers)
+	})
+
+	PostAuthJson(x, "/tasks/runner/train/datapoints", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			return c.E500M("Failed to get model information", err)
+		}
+
+		training_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+		testing_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
+		if err != nil {
+			return c.E500M("Failed to get the model classes", err)
+		}
+
+		return c.SendJSON(struct {
+			Testing  []DataPoint `json:"testing"`
+			Training []DataPoint `json:"training"`
+		}{
+			Testing:  testing_points,
+			Training: training_points,
+		})
+	})
+
+	type RunnerTrainDefEpoch struct {
+		Id       string  `json:"id" validate:"required"`
+		TaskId   string  `json:"taskId" validate:"required"`
+		DefId    string  `json:"defId" validate:"required"`
+		Epoch    int     `json:"epoch" validate:"required"`
+		Accuracy float64 `json:"accuracy" validate:"required"`
+	}
+	PostAuthJson(x, "/tasks/runner/train/epoch", User_Normal, func(c *Context, dat *RunnerTrainDefEpoch) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{
+			Id:     dat.Id,
+			TaskId: dat.TaskId,
+		})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		def, err := GetDefinition(c, dat.DefId)
+		if err != nil {
+			return c.E500M("Failed to get definition information", err)
+		}
+
+		err = def.UpdateAfterEpoch(c, dat.Accuracy, dat.Epoch)
+		if err != nil {
+			return c.E500M("Failed to update model", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/task/runner/train/mark-failed", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, &VerifyTask{
+			Id:     dat.Id,
+			TaskId: dat.TaskId,
+		})
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		_, err := c.Exec(
+			"update model_definition set status=$1 "+
+				"where model_id=$2 and status in ($3, $4)",
+			MODEL_DEFINITION_STATUS_CANCELD_TRAINING,
+			task.ModelId,
+			MODEL_DEFINITION_STATUS_TRAINING,
+			MODEL_DEFINITION_STATUS_PAUSED_TRAINING,
+		)
+		if err != nil {
+			return c.E500M("Failed to mark definition as failed", err)
+		}
+
+		return c.SendJSON("Ok")
+	})
+
+	PostAuthJson(x, "/task/runner/train/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
+		_, error := verifyRunner(c, &JustId{Id: dat.Id})
+		if error != nil {
+			return error
+		}
+
+		task, error := verifyTask(x, c, dat)
+		if error != nil {
+			return error
+		}
+
+		if task.TaskType != int(TASK_TYPE_TRAINING) {
+			c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
+			return c.JsonBadRequest("Task is not the right type go get the definitions")
+		}
+
+		model, err := GetBaseModel(c, *task.ModelId)
+		if err != nil {
+			c.Logger.Error("Failed to get model", "err", err)
+			return c.E500M("Failed to get mode", err)
+		}
+
+		var def Definition
+		err = GetDBOnce(c, &def, "from model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
+		if err == NotFoundError {
+			// TODO Make the Model status have a message
+			c.Logger.Error("All definitions failed to train!")
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
+			return c.SendJSON("Ok")
+		} else if err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
+			return c.E500M("Failed to get model definition", err)
+		}
+
+		if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
+			return c.E500M("Failed to update model definition", err)
+		}
+
+		to_delete, err := c.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
+		if err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+			return c.E500M("Failed to delete unsed definitions", err)
+		}
+		defer to_delete.Close()
+
+		for to_delete.Next() {
+			var id string
+			if err = to_delete.Scan(&id); err != nil {
+				model.UpdateStatus(c, FAILED_TRAINING)
+				task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+				return c.E500M("Failed to delete unsed definitions", err)
+			}
+			os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
+		}
+
+		// TODO Check if returning also works here
+		if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
+			model.UpdateStatus(c, FAILED_TRAINING)
+			task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
+			return c.E500M("Failed to delete unsed definitions", err)
+		}
+
+		model.UpdateStatus(c, READY)
+
+		return c.SendJSON("Ok")
+	})
+}

logic/tasks/runner/runner.go

@@ -5,6 +5,7 @@ import (
 	"math"
 	"os"
 	"runtime/debug"
+	"sync"
 	"time"
 
 	"github.com/charmbracelet/log"
@@ -90,6 +91,45 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 	}
 }
 
+/**
+* Handle remote runner
+ */
+func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Task) {
+	logger := log.NewWithOptions(os.Stdout, log.Options{
+		ReportCaller:    true,
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          fmt.Sprintf("Runner pre %s", runner_id),
+	})
+	defer func() {
+		if r := recover(); r != nil {
+			logger.Error("Runner failed to setup for runner", "due to", r, "stack", string(debug.Stack()))
+			// TODO maybe create better failed task
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to setup task for runner")
+		}
+	}()
+
+	err := task.UpdateStatus(base, TASK_PICKED_UP, "Failed to setup task for runner")
+	if err != nil {
+		logger.Error("Failed to mark task as PICK UP")
+		return
+	}
+
+	mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	switch task.TaskType {
+	case int(TASK_TYPE_TRAINING):
+		if err := PrepareTraining(handler, base, task, runner_id); err != nil {
+			logger.Error("Failed to prepare for training", "err", err)
+		}
+	default:
+		logger.Error("Not sure what to do panicing", "taskType", task.TaskType)
+		panic("not sure what to do")
+	}
+}
+
 /**
 * Tells the orcchestator to look at the task list from time to time
  */
@@ -125,7 +165,7 @@ func attentionSeeker(config Config, back_channel chan int) {
 /**
 * Manages what worker should to Work
  */
-func RunnerOrchestrator(db db.Db, config Config) {
+func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 	logger := log.NewWithOptions(os.Stdout, log.Options{
 		ReportCaller:    true,
 		ReportTimestamp: true,
@@ -133,6 +173,16 @@ func RunnerOrchestrator(db db.Db, config Config) {
 		Prefix:          "Runner Orchestrator Logger",
 	})
 
+	// Setup vars
+	handler.DataMap["runners"] = map[string]interface{}{}
+	handler.DataMap["runners_mutex"] = &sync.Mutex{}
+
+	base := BasePackStruct{
+		Db:     db,
+		Logger: logger,
+		Host:   config.Hostname,
+	}
+
 	gpu_workers := config.GpuWorker.NumberOfWorkers
 
 	logger.Info("Starting runners")
@@ -149,7 +199,7 @@ func RunnerOrchestrator(db db.Db, config Config) {
 				close(task_runners[x])
 			}
 			close(back_channel)
-			go RunnerOrchestrator(db, config)
+			go RunnerOrchestrator(db, config, handler)
 		}
 	}()
 
@@ -198,19 +248,45 @@ func RunnerOrchestrator(db db.Db, config Config) {
 		}
 
 		if task_to_dispatch != nil {
-			for i := 0; i < len(task_runners_used); i += 1 {
+
-				if !task_runners_used[i] {
+			// Only let CPU tasks be done by the local users
-					task_runners[i] <- *task_to_dispatch
+			if task_to_dispatch.TaskType == int(TASK_TYPE_DELETE_USER) {
-					task_runners_used[i] = true
+				for i := 0; i < len(task_runners_used); i += 1 {
+					if !task_runners_used[i] {
+						task_runners[i] <- *task_to_dispatch
+						task_runners_used[i] = true
+						task_to_dispatch = nil
+						break
+					}
+				}
+				continue
+			}
+
+			mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
+			mutex.Lock()
+			remote_runners := handler.DataMap["runners"].(map[string]interface{})
+
+			for k, v := range remote_runners {
+				runner_data := v.(map[string]interface{})
+				runner_info := runner_data["runner_info"].(*Runner)
+
+				if runner_data["task"] != nil {
+					continue
+				}
+
+				if runner_info.UserId == task_to_dispatch.UserId {
+					go handleRemoteTask(handler, base, k, *task_to_dispatch)
 					task_to_dispatch = nil
 					break
 				}
 			}
+
+			mutex.Unlock()
 		}
 
 	}
 }
 
-func StartRunners(db db.Db, config Config) {
+func StartRunners(db db.Db, config Config, handler *Handle) {
-	go RunnerOrchestrator(db, config)
+	go RunnerOrchestrator(db, config, handler)
 }

logic/tasks/utils/runner.go

@@ -0,0 +1,29 @@
+package tasks_utils
+
+import (
+	"time"
+
+	"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
+	dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+)
+
+type RunnerType int64
+
+const (
+	RUNNER_TYPE_GPU RunnerType = iota + 1
+)
+
+type Runner struct {
+	Id       string     `json:"id" db:"ru.id"`
+	UserId   string     `json:"user_id" db:"ru.user_id"`
+	Token    string     `json:"token" db:"ru.token"`
+	Type     RunnerType `json:"type" db:"ru.type"`
+	CreateOn time.Time  `json:"createOn" db:"ru.created_on"`
+}
+
+func GetRunner(db db.Db, id string) (ru *Runner, err error) {
+	var runner Runner
+	err = dbtypes.GetDBOnce(db, &runner, "remote_runner as ru where ru.id=$1", id)
+	ru = &runner
+	return
+}

logic/users/users.go

@@ -412,6 +412,18 @@ func UsersEndpints(db db.Db, handle *Handle) {
 		return c.SendJSON("Ok")
 	})
 
+	handle.DeleteAuth("/user/token/logoff", User_Normal, func(c *Context) *Error {
+		if c.Token == nil {
+			return c.JsonBadRequest("Failed to get token")
+		}
+		_, err := c.Db.Exec("delete from tokens where token=$1;", c.Token)
+		if err != nil {
+			return c.E500M("Failed to delete token", err)
+		}
+
+		return c.SendJSON("OK")
+	})
+
 	type DeleteUser struct {
 		Id       string `json:"id" validate:"required"`
 		Password string `json:"password" validate:"required"`

logic/utils/handler.go

@@ -175,7 +175,7 @@ func (x *Handle) DeleteAuth(path string, authLevel dbtypes.UserType, fn func(c *
 		}
 		return fn(c)
 	}
-	x.posts = append(x.posts, HandleFunc{path, inner_fn})
+	x.deletes = append(x.deletes, HandleFunc{path, inner_fn})
 }
 
 func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
@@ -374,7 +374,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
 	c.SetReportCaller(true)
 	c.Logger.Warn("Request failed with a bad request", "dat", dat)
 	c.SetReportCaller(false)
-	return c.ErrorCode(nil, 404, dat)
+	return c.SendJSONStatus(http.StatusBadRequest, dat)
 }
 
 func (c Context) JsonErrorBadRequest(err error, dat any) *Error {

main.go

@@ -36,11 +36,11 @@ func main() {
 	log.Info("Config loaded!", "config", config)
 	config.GenerateToken(db)
 
-	StartRunners(db, config)
-
 	//TODO check if file structure exists to save data
 	handle := NewHandler(db, config)
 
+	StartRunners(db, config, handle)
+
 	config.Cleanup(db)
 
 	// TODO Handle this in other way

nginx.dev.conf

@@ -13,7 +13,7 @@ http {
     server {
         listen 8000;
 
-        client_max_body_size 1G;
+        client_max_body_size 5G;
 
         location / {
             proxy_http_version 1.1;

requirements.txt

@@ -0,0 +1,5 @@
+tensorflow[and-cuda] == 2.15.1
+pandas
+# Make sure to install the nvidia pyindex first
+# nvidia-pyindex
+nvidia-cudnn

run.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+podman run --rm --network host --gpus all --name fyp-server -it -v $(pwd):/app -e "TERM=xterm-256color" fyp-server bash

sql/tasks.sql

@@ -38,3 +38,14 @@ create table if not exists tasks_dependencies (
     main_id uuid references tasks (id) on delete cascade not null,
     dependent_id uuid references tasks (id) on delete cascade not null
 );
+
+create table if not exists remote_runner (
+	id uuid primary key default gen_random_uuid(),
+	user_id uuid references users (id) on delete cascade not null,
+	token text not null,
+
+	-- 1: GPU
+	type integer,
+
+	created_on timestamp default current_timestamp
+);

views/py/python_model_template.py

@@ -9,9 +9,9 @@ import requests
 class NotifyServerCallback(tf.keras.callbacks.Callback):
     def on_epoch_end(self, epoch, log, *args, **kwargs):
         {{ if .HeadId }}
-        requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
+        requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
         {{ else }}
-        requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}')
+        requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["val_accuracy"]}&definition={{.DefId}}')
         {{end}}
 
 
@@ -82,7 +82,7 @@ def prepare_dataset(ds: tf.data.Dataset, size: int) -> tf.data.Dataset:
 
 def filterDataset(path):
   path = tf.strings.regex_replace(path, DATA_DIR_PREPARE, "")
-  
+
   {{ if eq .Model.Format "png" }}
   path = tf.strings.regex_replace(path, ".png", "")
   {{ else if eq .Model.Format "jpeg" }}
@@ -90,7 +90,7 @@ def filterDataset(path):
   {{ else }}
   ERROR
   {{ end }}
-  
+
   return tf.reshape(table.lookup(tf.strings.as_string([path])), []) != -1
 
 seed = random.randint(0, 100000000)
@@ -135,9 +135,9 @@ def addBlock(
         model.add(layers.ReLU())
     if top:
         if pooling_same:
-            model.add(pool_func(padding="same", strides=(1, 1)))
+            model.add(pool_func(pool_size=(2,2), padding="same", strides=(1, 1)))
         else:
-            model.add(pool_func())
+            model.add(pool_func(pool_size=(2,2)))
         model.add(layers.BatchNormalization())
         model.add(layers.LeakyReLU())
         model.add(layers.Dropout(0.4))
@@ -172,7 +172,7 @@ model.compile(
 
 his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
                 NotifyServerCallback(),
-                tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
+                tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)])
 
 acc = his.history["accuracy"]
 

views/py/python_model_template_expand.py

@@ -10,7 +10,7 @@ import numpy as np
 
 class NotifyServerCallback(tf.keras.callbacks.Callback):
     def on_epoch_end(self, epoch, log, *args, **kwargs):
-        requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
+        requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
 
 
 DATA_DIR = "{{ .DataDir }}"

webpage/src/routes/UserStore.svelte.ts

@@ -1,4 +1,5 @@
 import { goto } from '$app/navigation';
+import { rdelete } from '$lib/requests.svelte';
 
 type User = {
 	token: string;
@@ -33,6 +34,10 @@ export function createUserStore() {
 			if (value) {
 				localStorage.setItem('user', JSON.stringify(value));
 			} else {
+				if (user) {
+					// Request the deletion of the token
+					rdelete('/user/token/logoff', {});
+				}
 				localStorage.removeItem('user');
 			}
 			user = value;