Improved classification performance

DockerfileServer

@@ -31,7 +31,10 @@ ADD  go.mod .
 ADD  go.sum .
 ADD  main.go .
 ADD  logic logic
+ADD entrypoint.sh .
 
 RUN go install || true
 
-CMD ["go", "run", "."]
+RUN go build .
+
+CMD ["./entrypoint.sh"]

config.toml

@@ -12,7 +12,7 @@ USER = "service"
 
 [Worker]
 PULLING_TIME = "500ms"
-NUMBER_OF_WORKERS = 1
+NUMBER_OF_WORKERS = 16
 
 [DB]
 MAX_CONNECTIONS = 600

docker-compose.yml

@@ -23,6 +23,13 @@ services:
             - db
         volumes:
             - "./config.toml:/app/config.toml"
+        deploy:
+            resources:
+                reservations:
+                    devices:
+                        - driver: nvidia
+                          count: 1
+                          capabilities: [gpu]
     proxy-server:
         image: andre-fyp-proxy
         networks:

entrypoint.sh

@@ -0,0 +1,4 @@
+#/bin/bash
+while true; do
+	./fyp
+done

logic/models/add.go

@@ -16,7 +16,6 @@ import (
 )
 
 func loadBaseImage(c *Context, id string) {
-	// TODO handle more types than png
 	infile, err := os.Open(path.Join("savedData", id, "baseimage.png"))
 	if err != nil {
 		c.Logger.Errorf("Failed to read image for model with id %s\n", id)

logic/models/run.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"os"
 	"path"
+	"runtime/debug"
 
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
@@ -37,11 +38,19 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
 	return image.Scale(0, 255)
 }
 
-func runModelNormal(model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
+func runModelNormal(model *BaseModel, def_id string, inputImage *tf.Tensor, data *RunnerModelData) (order int, confidence float32, err error) {
 	order = 0
 	err = nil
 
-	tf_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil)
+	var tf_model *tg.Model = nil
+
+	if data.Id != nil && *data.Id == def_id {
+		tf_model = data.Model
+	} else {
+		tf_model = tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil)
+		data.Model = tf_model
+		data.Id = &def_id
+	}
 
 	results := tf_model.Exec([]tf.Output{
 		tf_model.Op("StatefulPartitionedCall", 0),
@@ -125,10 +134,15 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
 	return
 }
 
-func ClassifyTask(base BasePack, task Task) (err error) {
+type RunnerModelData struct {
+	Id    *string
+	Model *tg.Model
+}
+
+func ClassifyTask(base BasePack, task Task, data *RunnerModelData) (err error) {
 	defer func() {
 		if r := recover(); r != nil {
-			base.GetLogger().Error("Task failed due to", "error", r)
+			base.GetLogger().Error("Task failed due to", "error", r, "stack", string(debug.Stack()))
 			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Task failed running")
 		}
 	}()
@@ -186,6 +200,8 @@ func ClassifyTask(base BasePack, task Task) (err error) {
 
 	if model.ModelType == 2 {
 		base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
+		data.Model = nil
+		data.Id = nil
 		vi, confidence, err = runModelExp(base, model, def_id, inputImage)
 		if err != nil {
 			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
@@ -193,7 +209,7 @@ func ClassifyTask(base BasePack, task Task) (err error) {
 		}
 	} else {
 		base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
-		vi, confidence, err = runModelNormal(model, def_id, inputImage)
+		vi, confidence, err = runModelNormal(model, def_id, inputImage, data)
 		if err != nil {
 			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
 			return

logic/models/train/train.go

@@ -1191,7 +1191,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
 	}
 	order++
 
-	loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
+	loop := max(1, int(math.Ceil((math.Log(float64(model.Width))/math.Log(float64(10)))))+1)
 	for i := 0; i < loop; i++ {
 		_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
 		order++
@@ -1299,7 +1299,7 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 	order++
 
 	// Create the blocks
-	loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
+	loop := int(math.Ceil((math.Log(float64(model.Width)) / math.Log(float64(10))))) + 1
 
 	/*if model.Width < 50 && model.Height < 50 {
 	loop = 0

logic/stats/tasks.go

@@ -68,7 +68,7 @@ func handleTasksStats(handle *Handle) {
 				} else if task.Status < 2 {
 					total.Classfication_pre += 1
 					hours[hour].Classfication_pre += 1
-				} else if task.Status < 4 {
+				} else if task.Status < 4 || task.Status == 5 {
 					total.Classfication_running += 1
 					hours[hour].Classfication_running += 1
 				}

logic/tasks/runner/runner.go

@@ -19,6 +19,8 @@ import (
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
 )
 
+var QUEUE_SIZE = 10
+
 /**
 * Actually runs the code
  */
@@ -47,17 +49,28 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 		Host:   config.Hostname,
 	}
 
+	loaded_model := RunnerModelData{
+		Id:    nil,
+		Model: nil,
+	}
+
+	count := 0
 	for task := range task_channel {
 		logger.Info("Got task", "task", task)
 		task.UpdateStatusLog(base, TASK_PICKED_UP, "Runner picked up task")
 
 		if task.TaskType == int(TASK_TYPE_CLASSIFICATION) {
 			logger.Info("Classification Task")
-			if err = ClassifyTask(base, task); err != nil {
+			if err = ClassifyTask(base, task, &loaded_model); err != nil {
 				logger.Error("Classification task failed", "error", err)
 			}
 
+			if count == QUEUE_SIZE {
 				back_channel <- index
+				count = 0
+			} else {
+				count += 1
+			}
 			continue
 		} else if task.TaskType == int(TASK_TYPE_TRAINING) {
 			logger.Info("Training Task")
@@ -65,7 +78,12 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 				logger.Error("Failed to tain the model", "error", err)
 			}
 
+			if count == QUEUE_SIZE {
 				back_channel <- index
+				count = 0
+			} else {
+				count += 1
+			}
 			continue
 		} else if task.TaskType == int(TASK_TYPE_RETRAINING) {
 			logger.Info("Retraining Task")
@@ -73,7 +91,12 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 				logger.Error("Failed to tain the model", "error", err)
 			}
 
+			if count == QUEUE_SIZE {
 				back_channel <- index
+				count = 0
+			} else {
+				count += 1
+			}
 			continue
 		} else if task.TaskType == int(TASK_TYPE_DELETE_USER) {
 			logger.Warn("User deleting Task")
@@ -81,13 +104,23 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
 				logger.Error("Failed to tain the model", "error", err)
 			}
 
+			if count == QUEUE_SIZE {
 				back_channel <- index
+				count = 0
+			} else {
+				count += 1
+			}
 			continue
 		}
 
 		logger.Error("Do not know how to route task", "task", task)
 		task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Do not know how to route task")
+		if count == QUEUE_SIZE {
 			back_channel <- index
+			count = 0
+		} else {
+			count += 1
+		}
 	}
 }
 
@@ -145,7 +178,7 @@ func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Tas
 /**
 * Tells the orcchestator to look at the task list from time to time
  */
-func attentionSeeker(config Config, back_channel chan int) {
+func attentionSeeker(config Config, db db.Db, back_channel chan int) {
 	logger := log.NewWithOptions(os.Stdout, log.Options{
 		ReportCaller:    true,
 		ReportTimestamp: true,
@@ -170,6 +203,20 @@ func attentionSeeker(config Config, back_channel chan int) {
 	for true {
 		back_channel <- 0
 
+		for {
+			var s struct {
+				Count int `json:"count(*)"`
+			}
+			err := GetDBOnce(db, &s, "tasks where stauts = 5 or status = 3")
+			if err != nil {
+				break
+			}
+			if s.Count == 0 {
+				break
+			}
+			time.Sleep(t)
+		}
+
 		time.Sleep(t)
 	}
 }
@@ -194,11 +241,16 @@ func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 	}
 
 	gpu_workers := config.GpuWorker.NumberOfWorkers
+	def_wait, err := time.ParseDuration(config.GpuWorker.Pulling)
+	if err != nil {
+		logger.Error("Failed to load", "error", err)
+		return
+	}
 
 	logger.Info("Starting runners")
 
 	task_runners := make([]chan Task, gpu_workers)
-	task_runners_used := make([]bool, gpu_workers)
+	task_runners_used := make([]int, gpu_workers)
 	// One more to accomudate the Attention Seeker channel
 	back_channel := make(chan int, gpu_workers+1)
 
@@ -213,12 +265,12 @@ func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 		}
 	}()
 
-	go attentionSeeker(config, back_channel)
+	// go attentionSeeker(config, db, back_channel)
 
 	// Start the runners
 	for i := 0; i < gpu_workers; i++ {
-		task_runners[i] = make(chan Task, 10)
+		task_runners[i] = make(chan Task, QUEUE_SIZE)
-		task_runners_used[i] = false
+		task_runners_used[i] = 0
 		AddLocalRunner(handler, LocalRunner{
 			RunnerNum: i + 1,
 			Task:      nil,
@@ -226,44 +278,55 @@ func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 		go runner(config, db, task_runners[i], i+1, back_channel)
 	}
 
-	var task_to_dispatch *Task = nil
+	used := 0
-
+	wait := time.Nanosecond * 100
-	for i := range back_channel {
+	for {
-
+		out := true
+		for out {
+			select {
+			case i := <-back_channel:
 				if i != 0 {
 					if i > 0 {
 						logger.Info("Runner freed", "runner", i)
-				task_runners_used[i-1] = false
+						task_runners_used[i-1] = 0
+						used = 0
 					} else if i < 0 {
 						logger.Error("Runner died! Restarting!", "runner", i)
 						i = int(math.Abs(float64(i)) - 1)
-				task_runners_used[i] = false
+						task_runners_used[i] = 0
+						used = 0
 						go runner(config, db, task_runners[i], i+1, back_channel)
 					}
 					AddLocalTask(handler, int(math.Abs(float64(i))), nil)
+				} else if used == len(task_runners_used) {
+					continue
+				}
+			case <-time.After(wait):
+				if wait == time.Nanosecond*100 {
+					wait = def_wait
+				}
+				out = false
+			}
 		}
 
-		if task_to_dispatch == nil {
+		for {
-			var task TaskT
+			tasks, err := GetDbMultitple[TaskT](db, "tasks as t "+
-			err := GetDBOnce(db, &task, "tasks as t "+
 				// Get depenencies
 				"left join tasks_dependencies as td on t.id=td.main_id "+
 				// Get the task that the depencey resolves to
 				"left join tasks as t2 on t2.id=td.dependent_id "+
 				"where t.status=1 "+
-				"group by t.id having count(td.id) filter (where t2.status in (0,1,2,3)) = 0;")
+				"group by t.id having count(td.id) filter (where t2.status in (0,1,2,3)) = 0 limit 20;")
 			if err != NotFoundError && err != nil {
 				log.Error("Failed to get tasks from db", "err", err)
 				continue
 			}
-			if err == NotFoundError {
+			if err == NotFoundError || len(tasks) == 0 {
-				task_to_dispatch = nil
+				break
-			} else {
-				temp := Task(task)
-				task_to_dispatch = &temp
-			}
 			}
 
+			for _, task_to_dispatch := range tasks {
+				ttd := Task(*task_to_dispatch)
 				if task_to_dispatch != nil && task_to_dispatch.TaskType != int(TASK_TYPE_DELETE_USER) {
 					// TODO split tasks into cpu tasks and GPU tasks
 					mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
@@ -282,7 +345,7 @@ func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 							continue
 						}
 
-				go handleRemoteTask(handler, base, k, *task_to_dispatch)
+						go handleRemoteTask(handler, base, k, ttd)
 						task_to_dispatch = nil
 						break
 					}
@@ -290,18 +353,32 @@ func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
 					mutex.Unlock()
 				}
 
+				used = 0
 				if task_to_dispatch != nil {
 					for i := 0; i < len(task_runners_used); i += 1 {
-				if !task_runners_used[i] {
+						if task_runners_used[i] <= QUEUE_SIZE {
-					task_runners[i] <- *task_to_dispatch
+							ttd.UpdateStatusLog(base, TASK_QUEUED, "Runner picked up task")
-					task_runners_used[i] = true
+							task_runners[i] <- ttd
-					AddLocalTask(handler, i+1, task_to_dispatch)
+							task_runners_used[i] += 1
+							AddLocalTask(handler, i+1, &ttd)
 							task_to_dispatch = nil
+							wait = time.Nanosecond * 100
 							break
+						} else {
+							used += 1
 						}
 					}
 				}
 
+				if used == len(task_runners_used) {
+					break
+				}
+			}
+
+			if used == len(task_runners_used) {
+				break
+			}
+		}
 	}
 }
 

logic/tasks/utils/utils.go

@@ -50,6 +50,7 @@ const (
 	TASK_PREPARING                  = 0
 	TASK_TODO                       = 1
 	TASK_PICKED_UP                  = 2
+	TASK_QUEUED                     = 5
 	TASK_RUNNING                    = 3
 	TASK_DONE                       = 4
 )

logic/utils/config.go

@@ -102,7 +102,7 @@ func (c *Config) Cleanup(db db.Db) {
 	failLog(err)
 	_, err = db.Exec("update models set status=$1 where status=$2", FAILED_PREPARING, PREPARING)
 	failLog(err)
-	_, err = db.Exec("update tasks set status=$1 where status=$2", TASK_TODO, TASK_PICKED_UP)
+	_, err = db.Exec("update tasks set status=$1 where status=$2 or status=$3", TASK_TODO, TASK_PICKED_UP, TASK_QUEUED)
 	failLog(err)
 
 	tasks, err := GetDbMultitple[Task](db, "tasks where status=$1", TASK_RUNNING)

nginx.proxy.conf

@@ -1,6 +1,6 @@
 
 events {
-    worker_connections 1024;
+    worker_connections 2024;
 }
 
 http {

run.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-podman run --rm --network host --gpus all --name fyp-server -it -v $(pwd):/app -e "TERM=xterm-256color" fyp-server bash
+podman run --network host --gpus all --replace --name fyp-server --ulimit=nofile=100000:100000 -it -v $(pwd):/app -e "TERM=xterm-256color" --restart=always andre-fyp-server

sql/models.sql

@@ -59,7 +59,6 @@ create table if not exists model_definition (
     accuracy real default 0,
     target_accuracy integer not null,
     epoch integer default 0,
-    -- TODO add max epoch
     -- 1: Pre Init
     -- 2: Init
     -- 3: Training
@@ -78,7 +77,7 @@ create table if not exists model_definition_layer (
     -- 1: input
     -- 2: dense
     -- 3: flatten
-    -- TODO add conv
+    -- 4: block
     layer_type integer not null,
     -- ei 28,28,1
     -- a 28x28 grayscale image
@@ -102,7 +101,6 @@ create table if not exists exp_model_head (
 
     accuracy real default 0,
 
-    -- TODO add max epoch
     -- 1: Pre Init
     -- 2: Init
     -- 3: Training

views/py/python_model_template.py

@@ -143,6 +143,15 @@ def addBlock(
         model.add(layers.Dropout(0.4))
     return model
 
+def resblock(x, kernelsize = 3, filters = 128):
+    fx = layers.Conv2D(filters, kernelsize, activation='relu', padding='same')(x)
+    fx = layers.BatchNormalization()(fx)
+    fx = layers.Conv2D(filters, kernelsize, padding='same')(fx)
+    out = layers.Add()([x,fx])
+    out = layers.ReLU()(out)
+    out = layers.BatchNormalization()(out)
+    return out
+
 
 {{ if .LoadPrev }}
 model = tf.keras.saving.load_model('{{.LastModelRunPath}}')

webpage/src/routes/models/edit/+page.svelte

@@ -42,6 +42,7 @@
 
 	import 'src/styles/forms.css';
 	import { notificationStore } from 'src/lib/NotificationsStore.svelte';
+	import Spinner from 'src/lib/Spinner.svelte';
 
 	let model: Promise<Model> = $state(new Promise(() => {}));
 	let _model: Model | undefined = $state(undefined);
@@ -188,7 +189,6 @@
 						<h1 class="text-center">
 							{m.name}
 						</h1>
-						<!-- TODO improve message -->
 						<h2 class="text-center">Failed to prepare model</h2>
 
 						<DeleteModel model={m} />
@@ -206,8 +206,7 @@
 				{:else if m.status == 3}
 					<BaseModelInfo model={m} />
 					<div class="card">
-						<!-- TODO improve this -->
+						Processing zip file... <Spinner />
-						Processing zip file...
 					</div>
 				{:else if m.status == -3 || m.status == -4}
 					<BaseModelInfo model={m} />

webpage/src/routes/models/edit/ModelData.svelte

@@ -70,16 +70,16 @@
 					<button class="tab" class:selected={isActive('upload')} onclick={setActive('upload')}>
 						Upload
 					</button>
-					<button
+					<!--button
 						class="tab"
 						class:selected={isActive('create-class')}
 						onclick={setActive('create-class')}
 					>
 						Create Class
-					</button>
+					</button-->
-					<button class="tab" class:selected={isActive('api')} onclick={setActive('api')}>
+					<!--button class="tab" class:selected={isActive('api')} onclick={setActive('api')}>
 						Api
-					</button>
+					</button-->
 				</div>
 				<div class="content" class:selected={isActive('upload')}>
 					<form onsubmit={preventDefault(uploadZip)}>
@@ -111,10 +111,10 @@
 						{/if}
 					</form>
 				</div>
-				<div class="content" class:selected={isActive('create-class')}>
+				<!--div class="content" class:selected={isActive('create-class')}>
 					<ModelTable {classes} {model} {onreload} />
-				</div>
+				</div-->
-				<div class="content" class:selected={isActive('api')}>TODO</div>
+				<!--div class="content" class:selected={isActive('api')}>TODO</div-->
 			</Tabs>
 			<div class="tabs"></div>
 		{:else}
@@ -122,7 +122,7 @@
 			{#if numberOfInvalidImages > 0}
 				<p class="danger">
 					There are images {numberOfInvalidImages} that were loaded that do not have the correct format.
-					These images will be delete when the model trains.
+					These images will be deleted when the model trains.
 				</p>
 			{/if}
 			<ModelTable {classes} {model} {onreload} />

webpage/src/routes/models/edit/TrainModel.svelte

@@ -54,7 +54,7 @@
 			{#if number_of_invalid_images > 0}
 				<p class="danger">
 					There are images {number_of_invalid_images} that were loaded that do not have the correct format.DeleteZip
-					These images will be delete when the model trains.
+					These images will be deleted when the model trains.
 				</p>
 			{/if}
 			<!-- TODO expading mode -->
@@ -101,14 +101,14 @@
 			<h2>To train the model please provide data to the model first</h2>
 		{/if}
 	</form>
-{:else}
+{:else if ![4, 6, 7].includes(model.status)}
 	<form class:submitted onsubmit={submitRetrain}>
 		{#if has_data}
 			<h2>This model has new classes and can be expanded</h2>
 			{#if number_of_invalid_images > 0}
 				<p class="danger">
 					There are images {number_of_invalid_images} that were loaded that do not have the correct format.DeleteZip
-					These images will be delete when the model trains.
+					These images will be deleted when the model trains.
 				</p>
 			{/if}
 			<button> Retrain </button>