feat: add tasks closes #74

config.toml

@@ -3,3 +3,9 @@ PORT=5002
 HOSTNAME="https://testing.andr3h3nriqu3s.com"
 
 NUMBER_OF_WORKERS=20
+
+SUPRESS_CUDA=1
+
+[Worker]
+PULLING_TIME="500ms"
+NUMBER_OF_WORKERS=1

logic/models/data.go

@@ -89,7 +89,7 @@ func fileProcessor(
 		defer f.Close()
 		f.Write(file_data)
 
-		if !testImgForModel(c, model, file_path) {
+		if !TestImgForModel(c, model, file_path) {
 			c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
 			c.Logger.Warn("Not failling updating data point to status -1")
 			message := "Image did not have valid format for the model"

logic/models/index.go

@@ -18,7 +18,6 @@ func HandleModels (handle *Handle) {
     model_classes.HandleList(handle)
     
     // Train endpoints
-    handleRun(handle)
     models_train.HandleTrainEndpoints(handle)
 }
 

logic/models/run.go

@@ -1,12 +1,12 @@
 package models
 
 import (
-	"bytes"
+	"errors"
-	"io"
 	"os"
 	"path"
 
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
 
 	tf "github.com/galeone/tensorflow/tensorflow/go"
@@ -35,7 +35,7 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
 	return image.Scale(0, 255)
 }
 
-func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
+func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
 	order = 0
 	err = nil
 
@@ -62,7 +62,7 @@ func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.
 	return
 }
 
-func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
+func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
 
 	err = nil
 	order = 0
@@ -82,12 +82,12 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
 		Range_start int
 	}
 
-	heads, err := GetDbMultitple[head](c, "exp_model_head where def_id=$1;", def_id)
+	heads, err := GetDbMultitple[head](base.GetDb(), "exp_model_head where def_id=$1;", def_id)
 	if err != nil {
 		return
 	}
 
-    c.Logger.Info("test", "count", len(heads))
+	base.GetLogger().Info("test", "count", len(heads))
 
 	var vmax float32 = 0.0
 
@@ -102,9 +102,8 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
 
 		var predictions = results[0].Value().([][]float32)[0]
 
-
 		for i, v := range predictions {
-            c.Logger.Info("predictions", "class", i, "preds", v)
+			base.GetLogger().Debug("predictions", "class", i, "preds", v)
 			if v > vmax {
 				order = element.Range_start + i
 				vmax = v
@@ -115,60 +114,29 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
 	// TODO runthe head model
 	confidence = vmax
 
-	c.Logger.Info("Got", "heads", len(heads), "order", order, "vmax", vmax)
+	base.GetLogger().Debug("Got", "heads", len(heads), "order", order, "vmax", vmax)
 	return
 }
 
-func handleRun(handle *Handle) {
+func ClassifyTask(base BasePack, task Task) (err error) {
-	handle.Post("/models/run", func(c *Context) *Error {
+    task.UpdateStatusLog(base, TASK_RUNNING, "Runner running task")
-		if !c.CheckAuthLevel(1) {
-			return nil
-		}
 
-		read_form, err := c.R.MultipartReader()
+	model, err := GetBaseModel(base.GetDb(), task.ModelId)
 	if err != nil {
-			return c.JsonBadRequest("Invalid muilpart body")
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to obtain the model")
+		return err
 	}
 
-		var id string
+	if !model.CanEval() {
-		var file []byte
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to obtain the model")
-
+		return errors.New("Model not in the right state for evaluation")
-		for {
-			part, err_part := read_form.NextPart()
-			if err_part == io.EOF {
-				break
-			} else if err_part != nil {
-				return c.JsonBadRequest("Invalid multipart data")
-			}
-			if part.FormName() == "id" {
-				buf := new(bytes.Buffer)
-				buf.ReadFrom(part)
-				id = buf.String()
-			}
-			if part.FormName() == "file" {
-				buf := new(bytes.Buffer)
-				buf.ReadFrom(part)
-				file = buf.Bytes()
-			}
-		}
-
-		model, err := GetBaseModel(handle.Db, id)
-		if err == ModelNotFoundError {
-			return c.JsonBadRequest("Models not found")
-		} else if err != nil {
-			return c.Error500(err)
-		}
-
-		if model.Status != READY && model.Status != READY_RETRAIN && model.Status != READY_RETRAIN_FAILED && model.Status != READY_ALTERATION && model.Status != READY_ALTERATION_FAILED {
-			return c.JsonBadRequest("Model not ready to run images")
 	}
 
 	def := JustId{}
-		err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id)
+	err = GetDBOnce(base.GetDb(), &def, "model_definition where model_id=$1", model.Id)
-		if err == NotFoundError {
+	if err != nil {
-			return c.JsonBadRequest("Could not find definition")
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to obtain the model")
-		} else if err != nil {
+		return
-			return c.Error500(err)
 	}
 
 	def_id := def.Id
@@ -176,18 +144,8 @@ func handleRun(handle *Handle) {
 	// TODO create a database table with tasks
 	run_path := path.Join("/tmp", model.Id, "runs")
 	os.MkdirAll(run_path, os.ModePerm)
-		img_path := path.Join(run_path, "img."+model.Format)
 
-		img_file, err := os.Create(img_path)
+	img_path := path.Join("savedData", model.Id, "tasks", task.Id+"."+model.Format)
-		if err != nil {
-			return c.Error500(err)
-		}
-		defer img_file.Close()
-		img_file.Write(file)
-
-		if !testImgForModel(c, model, img_path) {
-			return c.JsonBadRequest("Provided image does not match the model")
-		}
 
 	root := tg.NewRoot()
 
@@ -199,55 +157,62 @@ func handleRun(handle *Handle) {
 	case "jpeg":
 		tf_img = ReadJPG(root, img_path, int64(model.ImageMode))
 	default:
-			panic("Not sure what to do with '" + model.Format + "'")
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to obtain the model")
 	}
 
 	exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{})
 	inputImage, err := tf.NewTensor(exec_results[0].Value())
 	if err != nil {
-			return c.Error500(err)
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
+		return
 	}
 
 	vi := -1
 	var confidence float32 = 0
 
 	if model.ModelType == 2 {
-			c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
+		base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
-			vi, confidence, err = runModelExp(c, model, def_id, inputImage)
+		vi, confidence, err = runModelExp(base, model, def_id, inputImage)
 		if err != nil {
-				return c.Error500(err)
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
+			return
 		}
 	} else {
-			c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
+		base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
-			vi, confidence, err = runModelNormal(c, model, def_id, inputImage)
+		vi, confidence, err = runModelNormal(base, model, def_id, inputImage)
 		if err != nil {
-				return c.Error500(err)
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
+			return
 		}
 	}
 
-		os.RemoveAll(run_path)
+	var GetName struct {
-
+		Name string
-		rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi)
+		Id   string
+	}
+	err = GetDBOnce(base.GetDb(), &GetName, "model_classes where model_id=$1 and class_order=$2;", model.Id, vi)
 	if err != nil {
-			return c.Error500(err)
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to obtain model results")
-		}
+		return
-		if !rows.Next() {
-			return c.SendJSON(nil)
-		}
-
-		var name string
-		if err = rows.Scan(&name); err != nil {
-			return c.Error500(err)
 	}
 
 	returnValue := struct {
+		ClassId    string  `json:"class_id"`
 		Class      string  `json:"class"`
 		Confidence float32 `json:"confidence"`
 	}{
-			Class:      name,
+		Class:      GetName.Name,
+		ClassId:    GetName.Id,
 		Confidence: confidence,
 	}
 
-		return c.SendJSON(returnValue)
+    err = task.SetResult(base, returnValue)
-	})
+    if err != nil {
+		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to save model results")
+		return
+    }
+
+	task.UpdateStatusLog(base, TASK_DONE, "Model ran successfully")
+
+    return
 }

logic/models/test.go

@@ -10,7 +10,7 @@ import (
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
 )
 
-func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
+func TestImgForModel(c *Context, model *BaseModel, path string) (result bool) {
 	result = false
 
 	infile, err := os.Open(path)

logic/models/train/train.go

@@ -58,11 +58,6 @@ func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionSt
 	return
 }
 
-func UpdateStatus(c *Context, table string, id string, status int) (err error) {
-	_, err = c.Db.Exec(fmt.Sprintf("update %s set status = $1 where id = $2", table), status, id)
-	return
-}
-
 func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType, shape string) (err error) {
 	_, err = db.Exec("insert into model_definition_layer (def_id, layer_order, layer_type, shape) values ($1, $2, $3, $4)", def_id, layer_order, layer_type, shape)
 	return
@@ -341,7 +336,7 @@ func generateCvsExpandExp(c *Context, run_path string, model_id string, offset i
 	// This is to load some extra data so that the model has more things to train on
 	//
 
-	data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count)
+	data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count * 10)
 	if err != nil {
 		return
 	}

logic/models/utils/types.go

@@ -5,18 +5,6 @@ import (
 	"errors"
 )
 
-type BaseModel struct {
-	Name   string
-	Status int
-	Id     string
-
-	ModelType int
-	ImageMode int
-	Width     int
-	Height    int
-	Format    string
-}
-
 const (
 	FAILED_TRAINING           = -4
 	FAILED_PREPARING_TRAINING = -3
@@ -75,6 +63,18 @@ const (
 	MODEL_HEAD_STATUS_READY                    = 5
 )
 
+type BaseModel struct {
+	Name   string
+	Status int
+	Id     string
+
+	ModelType int
+	ImageMode int
+	Width     int
+	Height    int
+	Format    string
+}
+
 var ModelNotFoundError = errors.New("Model not found error")
 
 func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
@@ -99,6 +99,13 @@ func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
 	return
 }
 
+func (m BaseModel) CanEval() bool {
+    if m.Status != READY && m.Status != READY_RETRAIN && m.Status != READY_RETRAIN_FAILED && m.Status != READY_ALTERATION && m.Status != READY_ALTERATION_FAILED {
+        return false
+    }
+    return true
+}
+
 func StringToImageMode(colorMode string) int {
 	switch colorMode {
 	case "greyscale":

logic/tasks/handleUpload.go

@@ -0,0 +1,123 @@
+package tasks
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"os"
+	"path"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func handleUpload(handler *Handle) {
+	handler.PostAuth("/tasks/start/image", 1, func(c *Context) *Error {
+
+		read_form, err := c.R.MultipartReader()
+		if err != nil {
+			return c.JsonBadRequest("Please provide a valid form data request!")
+		}
+
+		var json_data string
+		var file []byte
+
+		for {
+			part, err_part := read_form.NextPart()
+			if err_part == io.EOF {
+				break
+			} else if err_part != nil {
+				return c.JsonBadRequest("Please provide a valid form data request!")
+			}
+			if part.FormName() == "json_data" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				json_data = buf.String()
+			}
+			if part.FormName() == "file" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				file = buf.Bytes()
+			}
+		}
+
+		var requestData struct {
+			ModelId string `json:"id" validate:"required"`
+		}
+        
+        _err := c.ParseJson(&requestData, json_data)
+		if _err != nil {
+			return _err
+		}
+
+		model, err := GetBaseModel(c.Db, requestData.ModelId)
+		if err != nil {
+			return c.Error500(err)
+		}
+
+		switch model.Status {
+		case READY:
+		case READY_RETRAIN:
+		case READY_ALTERATION:
+		case READY_ALTERATION_FAILED:
+		case READY_RETRAIN_FAILED:
+			// Model can run
+
+		default:
+			return c.SendJSONStatus(http.StatusBadRequest, "Model not in the correct status to be able to evaludate a model")
+		}
+
+		// TODO Check if the user can use this model
+
+		type CreateNewTask struct {
+			UserId   string `db:"user_id"`
+			ModelId  string `db:"model_id"`
+			TaskType int    `db:"task_type"`
+			Status   int    `db:"status"`
+		}
+
+		newTask := CreateNewTask{
+			UserId:  c.User.Id,
+			ModelId: model.Id,
+			// TODO move this to an enum
+			TaskType: 1,
+			Status:   0,
+		}
+
+		id, err := InsertReturnId(c, &newTask, "tasks", "id")
+        if err != nil {
+			return c.E500M("Error 500", err)
+        }
+
+		save_path := path.Join("savedData", model.Id, "tasks")
+		os.MkdirAll(save_path, os.ModePerm)
+
+		img_path := path.Join(save_path, id+"."+model.Format)
+
+		img_file, err := os.Create(img_path)
+		if err != nil {
+            if _err := UpdateTaskStatus(c,id, -1, "Failed to create the file"); _err != nil {
+                c.Logger.Error("Failed to update tasks")
+            }
+			return c.E500M("Failed to create the file", err)
+		}
+		defer img_file.Close()
+		img_file.Write(file)
+
+		if !TestImgForModel(c, model, img_path) {
+            if _err := UpdateTaskStatus(c, id, -1, "The provided image is not a valid image for this model"); _err != nil {
+                c.Logger.Error("Failed to update tasks")
+            }
+			return c.JsonBadRequest(struct {
+                Message string `json:"message"`
+                Id string `json:"task_id"`
+            } { "Provided image does not match the model", id})
+		}
+        
+        UpdateStatus(c, "tasks", id, 1)
+
+        return c.SendJSON(struct {Id string `json:"id"`}{id})
+	})
+}

logic/tasks/index.go

@@ -0,0 +1,11 @@
+package tasks
+
+import (
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func HandleTasks (handle *Handle) {
+    handleUpload(handle)
+    handleList(handle)
+}
+

logic/tasks/list.go

@@ -0,0 +1,61 @@
+package tasks
+
+import (
+	dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func handleList(handler *Handle) {
+	handler.PostAuth("/tasks/list", 1, func(c *Context) *Error {
+        var err error = nil
+
+		var requestData struct {
+			ModelId string `json:"model_id"`
+            Page int `json:"page"`
+		}
+
+		if _err := c.ToJSON(&requestData); _err != nil {
+			return _err
+		}
+
+		if requestData.ModelId == "" && c.User.UserType < int(dbtypes.User_Admin) {
+			return c.SendJSONStatus(400, "Please provide a model_id")
+		}
+
+		if requestData.ModelId != "" {
+			_, err := GetBaseModel(c.Db, requestData.ModelId)
+			if err == ModelNotFoundError {
+				return c.SendJSONStatus(404, "Model not found!")
+			} else if err != nil {
+				return c.Error500(err)
+			}
+		}
+
+        var rows []*Task = nil
+
+		if requestData.ModelId != "" {
+			rows, err = GetDbMultitple[Task](c, "tasks where model_id=$1 order by created_on desc limit 11 offset $2", requestData.ModelId, requestData.Page * 10)
+			if err != nil {
+				return c.Error500(err)
+			}
+		} else {
+			rows, err = GetDbMultitple[Task](c, "tasks order by created_on desc limit 11 offset $1", requestData.Page * 10)
+			if err != nil {
+				return c.Error500(err)
+			}
+		}
+
+		max_len := min(11, len(rows))
+        
+        c.ShowMessage = false
+		return c.SendJSON(struct {
+            TaskList []*Task `json:"task_list"`
+            ShowNext bool `json:"show_next"`
+        } {
+            rows[0:max_len],
+            len(rows) > 10,
+        })
+	})
+}

logic/tasks/runner/runner.go

@@ -0,0 +1,160 @@
+package task_runner
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/charmbracelet/log"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+    . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
+)
+
+/**
+* Actually runs the code
+ */
+func runner(db *sql.DB, task_channel chan Task, index int, back_channel chan int) {
+	logger := log.NewWithOptions(os.Stdout, log.Options{
+		ReportCaller:    true,
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          fmt.Sprintf("Runner %d", index),
+	})
+
+	defer func() {
+		if r := recover(); r != nil {
+			logger.Error("Recovered in file processor", "processor id", index, "due to", r)
+			back_channel <- -index
+		}
+	}()
+
+	logger.Info("Started up")
+
+    var err error
+
+    base := BasePackStruct{
+        Db: db, 
+        Logger: logger,
+    }
+
+	for task := range task_channel {
+		logger.Info("Got task", "task", task)
+        
+        if task.TaskType == int(TASK_TYPE_CLASSIFICATION) {
+            logger.Info("Classification Task")
+            if err = ClassifyTask(base, task); err != nil {
+                logger.Error("Classification task failed", "error", "err")
+            }
+
+            back_channel <- index
+            continue
+        }
+
+
+        logger.Error("Do not know how to route task", "task", task)
+        back_channel <- index
+	}
+}
+
+/**
+* Tells the orcchestator to look at the task list from time to time
+ */
+func attentionSeeker(config Config, back_channel chan int) {
+	logger := log.NewWithOptions(os.Stdout, log.Options{
+		ReportCaller:    true,
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Runner Orchestrator Logger [Attention]",
+	})
+
+	logger.Info("Started up")
+
+    t, err := time.ParseDuration(config.GpuWorker.Pulling)
+    if err != nil {
+        logger.Error("Failed to load", "error", err)
+        return
+    }
+
+	for true {
+		back_channel <- 0
+        
+        time.Sleep(t)
+	}
+}
+
+/**
+* Manages what worker should to Work
+ */
+func RunnerOrchestrator(db *sql.DB, config Config) {
+	logger := log.NewWithOptions(os.Stdout, log.Options{
+		ReportCaller:    true,
+		ReportTimestamp: true,
+		TimeFormat:      time.Kitchen,
+		Prefix:          "Runner Orchestrator Logger",
+	})
+
+	gpu_workers := config.GpuWorker.NumberOfWorkers
+
+	logger.Info("Starting runners")
+
+	task_runners := make([]chan Task, gpu_workers)
+	task_runners_used := make([]bool, gpu_workers)
+	// One more to accomudate the Attention Seeker channel
+	back_channel := make(chan int, gpu_workers+1)
+
+	go attentionSeeker(config, back_channel)
+
+	// Start the runners
+	for i := 0; i < gpu_workers; i++ {
+		task_runners[i] = make(chan Task, 10)
+		task_runners_used[i] = false
+		go runner(db, task_runners[i], i+1, back_channel)
+	}
+
+    var task_to_dispatch *Task = nil
+
+	for i := range back_channel {
+
+		if i > 0 {
+			logger.Info("Runner freed", "runner", i)
+			task_runners_used[i-1] = false
+		} else if i < 0 {
+			logger.Error("Runner died! Restarting!", "runner", i)
+			task_runners_used[i-1] = false
+			go runner(db, task_runners[i-1], i, back_channel)
+		}
+
+        if task_to_dispatch == nil {
+            var task Task
+            err := GetDBOnce(db, &task, "tasks where status=$1 limit 1", TASK_TODO)
+            if err != NotFoundError && err != nil{
+                log.Error("Failed to get tasks from db")
+                continue
+            }
+            if err == NotFoundError {
+                task_to_dispatch = nil
+            } else {
+                task_to_dispatch = &task
+            }
+        }
+
+        if task_to_dispatch != nil {
+            for i := 0; i < len(task_runners_used); i += 1 {
+                if !task_runners_used[i] {
+                    task_runners[i] <- *task_to_dispatch
+                    task_runners_used[i] = true
+                    task_to_dispatch = nil
+                    break
+                }
+            }
+        }
+
+	}
+}
+
+func StartRunners(db *sql.DB, config Config) {
+	go RunnerOrchestrator(db, config)
+}

logic/tasks/utils/utils.go

@@ -0,0 +1,68 @@
+package tasks_utils
+
+import (
+	"time"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	"github.com/goccy/go-json"
+)
+
+type Task struct {
+	Id            string    `db:"id" json:"id"`
+	UserId        string    `db:"user_id" json:"user_id"`
+	ModelId       string    `db:"model_id" json:"model_id"`
+	Status        int       `db:"status" json:"status"`
+	StatusMessage string    `db:"status_message" json:"status_message"`
+	UserConfirmed int       `db:"user_confirmed" json:"user_confirmed"`
+	Compacted     int       `db:"compacted" json:"compacted"`
+	TaskType      int       `db:"task_type" json:"type"`
+	Result        string    `db:"result" json:"result"`
+	CreatedOn     time.Time `db:"created_on" json:"created"`
+}
+
+type TaskStatus int
+
+const (
+	TASK_FAILED_RUNNING  TaskStatus = -2
+	TASK_FAILED_CREATION            = -1
+	TASK_PREPARING                  = 0
+	TASK_TODO                       = 1
+	TASK_PICKED_UP                  = 2
+	TASK_RUNNING                    = 3
+	TASK_DONE                       = 4
+)
+
+type TaskType int
+
+const (
+	TASK_TYPE_CLASSIFICATION TaskType = 1
+)
+
+func (t Task) UpdateStatus(base BasePack, status TaskStatus, message string) (err error) {
+	return UpdateTaskStatus(base, t.Id, status, message)
+}
+
+/**
+* Call the UpdateStatus function and logs on the case of failure!
+* This varient does not return any error message
+ */
+func (t Task) UpdateStatusLog(base BasePack, status TaskStatus, message string) {
+	err := t.UpdateStatus(base, status, message)
+	if err != nil {
+		base.GetLogger().Error("Failed to update task status", "error", err, "task", t.Id)
+	}
+}
+
+func UpdateTaskStatus(base BasePack, id string, status TaskStatus, message string) (err error) {
+	_, err = base.GetDb().Exec("update tasks set status=$1, status_message=$2 where id=$3", status, message, id)
+	return
+}
+
+func (t Task) SetResult(base BasePack, result any) (err error) {
+	text, err := json.Marshal(result)
+	if err != nil {
+		return
+	}
+	_, err = base.GetDb().Exec("update tasks set result=$1 where id=$2", text, t.Id)
+	return
+}

logic/utils/config.go

@@ -7,10 +7,18 @@ import (
 	"github.com/charmbracelet/log"
 )
 
+type WorkerConfig struct {
+	NumberOfWorkers int    `toml:"number_of_workers"`
+	Pulling         string `toml:"pulling_time"`
+}
+
 type Config struct {
 	Hostname        string
 	Port            int
 	NumberOfWorkers int `toml:"number_of_workers"`
+	SupressCuda     int `toml:"supress_cuda"`
+
+	GpuWorker WorkerConfig `toml:"Worker"`
 }
 
 func LoadConfig() Config {
@@ -25,10 +33,21 @@ func LoadConfig() Config {
 			Hostname:        "localhost",
 			Port:            8000,
 			NumberOfWorkers: 10,
+			GpuWorker: WorkerConfig{
+				NumberOfWorkers: 1,
+				Pulling:         "500ms",
+			},
 		}
 	}
 
 	var conf Config
 	_, err = toml.Decode(string(dat), &conf)
+
+    if conf.SupressCuda == 1 {
+        log.Warn("Supressing Cuda Messages!")
+        os.Setenv("TF_CPP_MIN_VLOG_LEVEL", "3") 
+        os.Setenv("TF_CPP_MIN_LOG_LEVEL", "3")
+    }
+
 	return conf
 }

logic/utils/handler.go

@@ -67,7 +67,7 @@ func handleError(err *Error, c *Context) {
 			e = c.SendJSON(500)
 		}
 		if e != nil {
-			c.Logger.Error("Something went very wront while trying to send and error message")
+			c.Logger.Error("Something went very wrong while trying to send and error message")
 			c.Writer.Write([]byte("505"))
 		}
 	}
@@ -81,7 +81,6 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
 	x.posts = append(x.posts, HandleFunc{path, fn})
 }
 
-
 func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
 	inner_fn := func(c *Context) *Error {
         if !c.CheckAuthLevel(authLevel) {
@@ -97,6 +96,13 @@ func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
 }
 
 func (x *Handle) handleGets(context *Context) {
+    defer func() {
+        if r := recover(); r != nil { 
+            context.Logger.Error("Something went very wrong", "Error", r)
+	        handleError(&Error{500, "500"}, context)
+        }
+    }()
+
 	for _, s := range x.gets {
 		if s.path == context.R.URL.Path {
 			handleError(s.fn(context), context)
@@ -108,6 +114,13 @@ func (x *Handle) handleGets(context *Context) {
 }
 
 func (x *Handle) handlePosts(context *Context) {
+    defer func() {
+        if r := recover(); r != nil { 
+            context.Logger.Error("Something went very wrong", "Error", r)
+	        handleError(&Error{500, "500"}, context)
+        }
+    }()
+
 	for _, s := range x.posts {
 		if s.path == context.R.URL.Path {
 			handleError(s.fn(context), context)
@@ -119,6 +132,13 @@ func (x *Handle) handlePosts(context *Context) {
 }
 
 func (x *Handle) handleDeletes(context *Context) {
+    defer func() {
+        if r := recover(); r != nil { 
+            context.Logger.Error("Something went very wrong", "Error", r)
+	        handleError(&Error{500, "500"}, context)
+        }
+    }()
+
 	for _, s := range x.deletes {
 		if s.path == context.R.URL.Path {
 			handleError(s.fn(context), context)
@@ -155,6 +175,20 @@ type Context struct {
 	Handle      *Handle
 }
 
+
+func (c Context) GetDb() (*sql.DB) {
+    return c.Db
+}
+
+func (c Context) GetLogger() (*log.Logger) {
+    return c.Logger
+}
+
+func (c Context) Query(query string, args ...any) (*sql.Rows, error) {
+    return c.Db.Query(query, args...)
+}
+
+
 func (c Context) Prepare(str string) (*sql.Stmt, error) {
 	if c.Tx == nil {
 		return c.Db.Prepare(str)
@@ -199,19 +233,32 @@ func (c *Context) RollbackTx() error {
 	return nil
 }
 
-func (c Context) ToJSON(dat any) *Error {
+/**
+* Parse and vailidates the json
+*/
+func (c Context) ParseJson(dat any, str string) *Error {
+	decoder := json.NewDecoder(strings.NewReader(str))
 
+    return c.decodeAndValidade(decoder, dat)
+}
+
+func (c Context) ToJSON(dat any) *Error {
 	decoder := json.NewDecoder(c.R.Body)
     
+    return c.decodeAndValidade(decoder, dat)
+}
+
+func (c Context) decodeAndValidade(decoder *json.Decoder, dat any) *Error {
 	err := decoder.Decode(dat)
 	if err != nil {
-		return c.Error500(err)
+        c.Logger.Error("Failed to decode json", "dat", dat, "err", err)
+        return c.JsonBadRequest("Bad Request! Invalid json passed!");
 	}
 
     err = c.Handle.validate.Struct(dat)
     if err != nil {
         c.Logger.Error("Failed invalid json passed", "dat", dat, "err", err)
-        return c.JsonBadRequest("Bad Request! Invalid body passed!")
+        return c.JsonBadRequest("Bad Request! Invalid json passed!");
     }
 
 	return nil
@@ -246,7 +293,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
 	c.SetReportCaller(true)
 	c.Logger.Warn("Request failed with a bad request", "dat", dat)
 	c.SetReportCaller(false)
-	return c.SendJSONStatus(http.StatusBadRequest, dat)
+    return c.ErrorCode(nil, 404, dat)
 }
 
 func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
@@ -308,6 +355,10 @@ func (c Context) Error500(err error) *Error {
 	return c.ErrorCode(err, http.StatusInternalServerError, nil)
 }
 
+func (c Context) E500M(msg string, err error) *Error {
+	return c.ErrorCode(err, http.StatusInternalServerError, msg)
+}
+
 func (c *Context) requireAuth() bool {
 	if c.User == nil {
 		return true

logic/utils/utils.go

@@ -12,9 +12,29 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/charmbracelet/log"
 	"github.com/google/uuid"
 )
 
+
+type BasePack interface {
+	GetDb() *sql.DB
+	GetLogger() *log.Logger
+}
+
+type BasePackStruct struct {
+	Db     *sql.DB
+	Logger *log.Logger
+}
+
+func (b BasePackStruct) GetDb() (*sql.DB) {
+    return b.Db
+}
+
+func (b BasePackStruct) GetLogger() (*log.Logger) {
+    return b.Logger
+}
+
 func CheckEmpty(f url.Values, path string) bool {
 	return !f.Has(path) || f.Get(path) == ""
 }
@@ -184,7 +204,7 @@ func MyParseForm(r *http.Request) (vs url.Values, err error) {
 	return
 }
 
-type JustId struct { Id string }
+type JustId struct{ Id string }
 
 type Generic struct{ reflect.Type }
 
@@ -199,7 +219,7 @@ func generateQuery(t reflect.Type) (query string, nargs int) {
 		field := t.Field(i)
 		name, ok := field.Tag.Lookup("db")
 		if !ok {
-            name = field.Name;
+			name = field.Name
 		}
 
 		if name == "__nil__" {
@@ -214,7 +234,12 @@ func generateQuery(t reflect.Type) (query string, nargs int) {
 	return
 }
 
-func GetDbMultitple[T interface{}](c *Context, tablename string, args ...any) ([]*T, error) {
+type QueryInterface interface {
+	Prepare(str string) (*sql.Stmt, error)
+	Query(query string, args ...any) (*sql.Rows, error)
+}
+
+func GetDbMultitple[T interface{}](c QueryInterface, tablename string, args ...any) ([]*T, error) {
 	t := reflect.TypeFor[T]()
 
 	query, nargs := generateQuery(t)
@@ -248,7 +273,7 @@ func mapRow(store interface{}, rows *sql.Rows, nargs int) (err error) {
 	err = nil
 
 	val := reflect.Indirect(reflect.ValueOf(store))
-	scan_args := make([]interface{}, nargs);
+	scan_args := make([]interface{}, nargs)
 	for i := 0; i < nargs; i++ {
 		valueField := val.Field(i)
 		scan_args[i] = valueField.Addr().Interface()
@@ -269,13 +294,13 @@ func InsertReturnId(c *Context, store interface{}, tablename string, returnName
 
 	query2 := ""
 	for i := 0; i < nargs; i += 1 {
-        query2 += fmt.Sprintf("$%d,", i)
+		query2 += fmt.Sprintf("$%d,", i+1)
 	}
 	// Remove last quotation
 	query2 = query2[0 : len(query2)-1]
 
 	val := reflect.ValueOf(store).Elem()
-	scan_args := make([]interface{}, nargs);
+	scan_args := make([]interface{}, nargs)
 	for i := 0; i < nargs; i++ {
 		valueField := val.Field(i)
 		scan_args[i] = valueField.Interface()
@@ -299,12 +324,12 @@ func InsertReturnId(c *Context, store interface{}, tablename string, returnName
 	return
 }
 
-func GetDBOnce(c *Context, store interface{}, tablename string, args ...any) error {
+func GetDBOnce(db QueryInterface, store interface{}, tablename string, args ...any) error {
 	t := reflect.TypeOf(store).Elem()
 
 	query, nargs := generateQuery(t)
 
-	rows, err := c.Db.Query(fmt.Sprintf("select %s from %s", query, tablename), args...)
+	rows, err := db.Query(fmt.Sprintf("select %s from %s", query, tablename), args...)
 	if err != nil {
 		return err
 	}
@@ -317,7 +342,7 @@ func GetDBOnce(c *Context, store interface{}, tablename string, args ...any) err
 	err = nil
 
 	val := reflect.ValueOf(store).Elem()
-	scan_args := make([]interface{}, nargs);
+	scan_args := make([]interface{}, nargs)
 	for i := 0; i < nargs; i++ {
 		valueField := val.Field(i)
 		scan_args[i] = valueField.Addr().Interface()
@@ -331,3 +356,7 @@ func GetDBOnce(c *Context, store interface{}, tablename string, args ...any) err
 	return nil
 }
 
+func UpdateStatus(c *Context, table string, id string, status int) (err error) {
+	_, err = c.Db.Exec(fmt.Sprintf("update %s set status = $1 where id = $2", table), status, id)
+	return
+}

main.go

@@ -8,8 +8,10 @@ import (
 	_ "github.com/lib/pq"
 
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks"
 	models_utils "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/runner"
 )
 
 const (
@@ -36,6 +38,8 @@ func main() {
     config := LoadConfig()
     log.Info("Config loaded!", "config", config)
 
+    StartRunners(db, config)
+
 	//TODO check if file structure exists to save data
 	handle := NewHandler(db, config)
 
@@ -55,6 +59,7 @@ func main() {
 
 	usersEndpints(db, handle)
 	HandleModels(handle)
+    HandleTasks(handle)
 
 	handle.Startup()
 }

sql/tasks.sql

@@ -0,0 +1,33 @@
+-- drop table if exists tasks
+create table if not exists tasks (
+    id uuid primary key default gen_random_uuid(),
+    user_id uuid references users (id) not null,
+    model_id uuid references models (id) on delete cascade default null,
+
+    -- -2: Failed Running
+    -- -1: Failed Creation
+    --  0: Preparing
+    --  1: TODO
+    --  2: Picked up
+    --  3: Running
+    --  4: Failed
+    status integer default 1,
+    status_message text default '',
+
+    result text default '',
+    
+    -- -1: user said task is wrong
+    --  0: no user input
+    --  1: user said task is ok
+    user_confirmed integer default 0,
+    
+    -- Tells the user if the file has been already compacted into
+    --  embendings
+    compacted integer default 0,
+        
+    -- TODO move the training tasks to here
+    --  1: Classification
+    task_type integer,
+
+    created_on timestamp default current_timestamp
+)

webpage/src/routes/models/edit/+page.svelte

@@ -35,12 +35,14 @@
 
 	import ModelData from './ModelData.svelte';
 	import DeleteZip from './DeleteZip.svelte';
+	import RunModel from './RunModel.svelte';
 
+	import Tabs from 'src/lib/Tabs.svelte';
+	import TasksDataPage from './TasksDataPage.svelte';
 	import ModelDataPage from './ModelDataPage.svelte';
 
 	import 'src/styles/forms.css';
-	import RunModel from './RunModel.svelte';
+
-	import Tabs from 'src/lib/Tabs.svelte';
 	let model: Promise<Model> = $state(new Promise(() => {}));
     let _model: Model | undefined = $state(undefined);
 	let definitions: Promise<Definitions[]> = $state(new Promise(() => {}));
@@ -148,9 +150,19 @@
 					Model Data
 				</button>
 			{/if}
+			{#if _model && [5, 6, 7].includes(_model.status)}
+				<button
+					class="tab"
+					on:click|preventDefault={setActive('tasks')}
+					class:selected={isActive('tasks')}
+				>
+					Tasks
+				</button>
+			{/if}
 		</div>
         {#if _model}
 		    <ModelDataPage model={_model} on:reload={getModel} active={isActive('model-data')} />
+            <TasksDataPage model={_model} active={isActive('tasks')} />
         {/if}
 		<div class="content" class:selected={isActive('model')}>
 			{#await model}

webpage/src/routes/models/edit/RunModel.svelte

@@ -3,11 +3,14 @@
 	import type { Model } from "./+page.svelte";
     import FileUpload from "src/lib/FileUpload.svelte";
 	import MessageSimple from "src/lib/MessageSimple.svelte";
+	import { createEventDispatcher } from "svelte";
 
     let {model} = $props<{model: Model}>();
 
     let file: File | undefined = $state();
 
+    const dispatch = createEventDispatcher<{ upload: void }>();
+
     type Result = {
         class: string,
         confidence: number,
@@ -15,23 +18,24 @@
 
     let _result: Promise<Result | undefined> = $state(new Promise(() => {}));
     let run = $state(false);
+    let last_task: string | undefined = $state();
 
     let messages: MessageSimple;
 
     async function submit() {
-        console.log("here", file);
         if (!file) return;
         messages.clear();
 
         let form = new FormData();
-        form.append("id", model.id);
+        form.append("json_data", JSON.stringify({id: model.id}));
         form.append("file", file, "file");
 
         run = true;
         
         try {
-            _result = await postFormData('models/run', form);
+            const r = await postFormData('tasks/start/image', form);
-            console.log(await _result);
+            last_task = r.id
+            file = undefined;
         } catch (e) {
             if (e instanceof Response) {
                 messages.display(await e.json());
@@ -40,6 +44,7 @@
             }
         }
 
+        dispatch('upload');
     }
 </script>
     <form on:submit|preventDefault={submit}>
@@ -66,7 +71,11 @@
             Run
         </button>
         {#if run}
-            {#await _result then result}
+            {#await _result}
+                <h1>
+                    Processing Image {last_task}
+                </h1>
+            {:then result}
                 {#if !result}
                     <div class="result">
                         <h1>

webpage/src/routes/models/edit/TasksDataPage.svelte

@@ -0,0 +1,16 @@
+<script lang="ts">
+	import { post } from "src/lib/requests.svelte";
+	import type { Model } from "src/routes/models/edit/+page.svelte";
+	import RunModel from "./RunModel.svelte";
+	import TasksTable from "./TasksTable.svelte";
+
+    const { active, model } = $props<{ active?: boolean, model: Model }>();
+
+    let table: TasksTable;
+</script>
+
+
+<div class="content" class:selected={active}>
+    <RunModel model={model} on:upload={() => table.getList()} />
+    <TasksTable model={model} bind:this={table} />
+</div>

webpage/src/routes/models/edit/TasksTable.svelte

@@ -0,0 +1,185 @@
+<script lang="ts" context="module">
+	export type Task = {
+		id: string;
+		user_id: string;
+		model_id: string;
+		status: number;
+		status_message: string;
+		user_confirmed: number;
+		compacted: number;
+		type: number;
+        created: string;
+        result: string;
+	};
+</script>
+
+<script lang="ts">
+	import { post } from 'src/lib/requests.svelte';
+	import type { Model } from './+page.svelte';
+
+	let { model } = $props<{ model: Model, uploadCounter?: number }>();
+
+	let page = $state(0);
+	let showNext = $state(false);
+	let task_list = $state<Task[]>([]);
+
+    export async function getList() {
+		try {
+            const res = await post('tasks/list', {
+                id: model.id,
+                page: page,
+            });
+			showNext = res.show_next;
+			task_list = res.task_list;
+		} catch (e) {
+			console.error('TODO notify user', e);
+		}
+    }
+    
+    $effect(() => {
+        if (model) {
+            getList()
+        }
+    })
+</script>
+
+<div>
+    <h2>Tasks</h2>
+	<table>
+		<thead>
+			<tr>
+				<th> Task type </th>
+				<th>
+					<!-- Img -->
+				</th>
+				<th> User Confirmed </th>
+				<th> Result </th>
+				<th> Status </th>
+				<th> Status Message </th>
+				<th> Created </th>
+			</tr>
+		</thead>
+		<tbody>
+			{#each task_list as task}
+				<tr>
+					<td>
+                        {#if task.type == 1}
+                            Image Run
+                        {:else}
+                            {task.type}
+                        {/if}
+					</td>
+					<td class="text-center">
+						{#if task.type == 1}
+							<img
+								alt=""
+								src="/api/savedData/{model.id}/tasks/{task.id}.{model.format}"
+								height="30px"
+								width="30px"
+								style="object-fit: contain;"
+							/>
+						{:else}
+							TODO Show more information {task.status}
+						{/if}
+					</td>
+					<td>
+                        {#if task.type == 1}
+                            {#if task.status == 4}
+                                {#if task.user_confirmed == 0}
+                                    User has not agreed to the result of this task
+                                {:else if task.user_confirmed == -1}
+                                    User has disagred with the result of this task
+                                {:else if task.user_confirmed == 1}
+                                    User has aggred with the result of this task
+                                {:else} 
+                                    TODO {task.user_confirmed}
+                                {/if}
+                            {:else}
+                                -
+                            {/if}
+                        {:else}
+                            TODO Handle {task.type}
+                        {/if}
+					</td>
+                    <td>
+                        {#if task.status == 4}
+                            {#if task.type == 1}
+                                {@const temp = JSON.parse(task.result)}
+                                {temp.class}({temp.confidence * 100}%)
+                            {:else}
+                                {task.result}
+                            {/if}
+                        {/if}
+					</td>
+                    <td>
+                        {task.status}
+					</td>
+                    <td>
+                        {task.status_message}
+					</td>
+                    <td>
+                        {(new Date(task.created)).toLocaleString()}
+					</td>
+				</tr>
+			{/each}
+		</tbody>
+	</table>
+	<div class="flex justify-center align-center">
+		<div class="grow-1 flex justify-end align-center">
+			{#if page > 0}
+				<button on:click={() => (page -= 1)}> Prev </button>
+			{/if}
+		</div>
+
+		<div style="padding: 10px;">
+			{page}
+		</div>
+
+		<div class="grow-1 flex justify-start align-center">
+			{#if showNext}
+				<button on:click={() => (page += 1)}> Next </button>
+			{/if}
+		</div>
+	</div>
+</div>
+
+<style lang="scss">
+	.buttons {
+		width: 100%;
+		display: flex;
+		justify-content: space-between;
+
+		& > button {
+			margin: 3px 5px;
+		}
+	}
+
+	table {
+		width: 100%;
+		box-shadow: 0 2px 8px 1px #66666622;
+		border-radius: 10px;
+		border-collapse: collapse;
+		overflow: hidden;
+	}
+
+	table thead {
+		background: #60606022;
+	}
+
+	table tr td,
+	table tr th {
+		border-left: 1px solid #22222244;
+		padding: 15px;
+	}
+
+	table tr td:first-child,
+	table tr th:first-child {
+		border-left: none;
+	}
+
+	table tr td button,
+	table tr td .button {
+		padding: 5px 10px;
+		box-shadow: 0 2px 5px 1px #66666655;
+	}
+</style>