added some nice graphs to the tasks

logic/models/run.go

@@ -118,6 +118,13 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
 }
 
 func ClassifyTask(base BasePack, task Task) (err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			base.GetLogger().Error("Task failed due to", "error", r)
+			task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Task failed running")
+		}
+	}()
+
 	task.UpdateStatusLog(base, TASK_RUNNING, "Runner running task")
 
 	model, err := GetBaseModel(base.GetDb(), *task.ModelId)

logic/models/train/train.go

@@ -384,7 +384,6 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
 		log.Error("Failed to get the exp head of the model")
 		return
 	} else if len(heads) != 1 {
-		log.Error("This training function can only train one model at the time")
 		err = errors.New("This training function can only train one model at the time")
 		return
 	}
@@ -1450,9 +1449,9 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
 	// Create the blocks
 	loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
 
-	if model.Width < 50 && model.Height < 50 {
+	/*if model.Width < 50 && model.Height < 50 {
 	loop = 0
-	}
+	}*/
 
 	log.Info("Size of the simple block", "loop", loop)
 
@@ -1541,7 +1540,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 		if model.Width > 100 && model.Height > 100 {
 			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		} else {
-			generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
+			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		}
 	} else if number_of_models == 3 {
 		for i := 0; i < number_of_models; i++ {
@@ -1550,7 +1549,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
 	} else {
 		// TODO handle incrisea the complexity
 		for i := 0; i < number_of_models; i++ {
-			generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
+			generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
 		}
 	}
 
@@ -1702,20 +1701,38 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
 
 	task.UpdateStatusLog(b, TASK_RUNNING, "Model retraining")
 
-	defId, err := GetDbVar[string](db, "md.id", "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
+	var defData struct {
+		Id             string  `db:"md.id"`
+		TargetAcuuracy float64 `db:"md.target_accuracy"`
+	}
+
+	err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
 	if err != nil {
 		failed()
 		return
 	}
 
+	var acc float64 = 0
+	var epocs = 0
+	// TODO make max epochs come from db
+	for acc*100 < defData.TargetAcuuracy && epocs < 20 {
 		// This is something I have to check
-	acc, err := trainDefinitionExpandExp(b, model, *defId, false)
+		acc, err = trainDefinitionExpandExp(b, model, defData.Id, epocs > 0)
 		if err != nil {
 			failed()
 			return
 		}
 
-	l.Info("Retrained model", "accuracy", acc)
+		l.Info("Retrained model", "accuracy", acc, "target", defData.TargetAcuuracy)
+
+		epocs += 1
+	}
+
+	if acc*100 < defData.TargetAcuuracy {
+		l.Error("Model never achived targetd accuracy", "acc", acc*100, "target", defData.TargetAcuuracy)
+		failed()
+		return
+	}
 
 	// TODO check accuracy
 	err = UpdateStatus(db, "models", model.Id, READY)

logic/stats/index.go

@@ -6,4 +6,5 @@ import (
 
 func HandleStats(handle *Handle) {
 	HandlePublicStats(handle)
+	handleTasksStats(handle)
 }

logic/stats/tasks.go

@@ -0,0 +1,95 @@
+package stats
+
+import (
+	"time"
+
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func handleTasksStats(handle *Handle) {
+	type ModelTasksStatsRequest struct {
+		ModelId string `json:"model_id" validate:"required"`
+	}
+	PostAuthJson(handle, "/stats/task/model/day", User_Normal, func(c *Context, dat *ModelTasksStatsRequest) *Error {
+		model, err := GetBaseModel(c, dat.ModelId)
+		if err == ModelNotFoundError {
+			return c.JsonBadRequest("Model not found!")
+		} else if err != nil {
+			return c.E500M("Failed to get model", err)
+		}
+
+		now := time.Now()
+		now = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
+
+		tasks, err := GetDbMultitple[Task](c, "tasks where model_id=$1 and created_on > $2 order by created_on asc;", model.Id, now)
+		if err != nil {
+			return c.E500M("Failed to get task informations", err)
+		}
+
+		type DataPoint struct {
+			Non_classfication_tasks_su  int `json:"nc_success"`
+			Non_classfication_tasks_err int `json:"nc_error"`
+
+			Classfication_su      int `json:"c_success"`
+			Classfication_fal     int `json:"c_failure"`
+			Classfication_un      int `json:"c_unknown"`
+			Classfication_err     int `json:"c_error"`
+			Classfication_pre     int `json:"c_pre_running"`
+			Classfication_running int `json:"c_running"`
+		}
+
+		total := DataPoint{}
+		hours := make([]DataPoint, 24)
+
+		for i := 0; i < 24; i++ {
+			hours[i] = DataPoint{}
+		}
+
+		for i := range tasks {
+			task := tasks[i]
+			hour := task.CreatedOn.Hour()
+			if task.TaskType == int(TASK_TYPE_CLASSIFICATION) {
+				if task.Status == 4 {
+					if task.UserConfirmed == 1 {
+						total.Classfication_su += 1
+						hours[hour].Classfication_su += 1
+					} else if task.UserConfirmed == -1 {
+						total.Classfication_fal += 1
+						hours[hour].Classfication_fal += 1
+					} else {
+						total.Classfication_un += 1
+						hours[hour].Classfication_un += 1
+					}
+				} else if task.Status < 0 {
+					total.Classfication_err += 1
+					hours[hour].Classfication_err += 1
+				} else if task.Status < 2 {
+					total.Classfication_pre += 1
+					hours[hour].Classfication_pre += 1
+				} else if task.Status < 4 {
+					total.Classfication_running += 1
+					hours[hour].Classfication_running += 1
+				}
+			} else {
+				if task.Status >= 0 {
+					total.Non_classfication_tasks_su += 1
+					hours[hour].Non_classfication_tasks_su += 1
+				} else {
+					total.Non_classfication_tasks_err += 1
+					hours[hour].Non_classfication_tasks_err += 1
+				}
+			}
+		}
+
+		data := struct {
+			Total DataPoint   `json:"total"`
+			Hours []DataPoint `json:"hours"`
+		}{
+			Total: total,
+			Hours: hours,
+		}
+		return c.SendJSON(data)
+	})
+}

webpage/src/lib/KeepPageSize.svelte

@@ -1,11 +0,0 @@
-<script lang="ts">
-	function updateSize() {
-		let s = document.body.getBoundingClientRect();
-		document.body.style.minHeight = `${Math.max(window.innerHeight, s.height)}px`;
-	}
-	$effect(() => {
-		updateSize();
-	});
-</script>
-
-<svelte:window on:scroll={updateSize} />

webpage/src/lib/Tooltip.svelte

@@ -28,7 +28,15 @@
 	}
 </script>
 
-<div bind:this={div} on:mouseover={mouseOver} on:mouseleave={mouseLeave} on:mousemove={mouseMove} on:focus={focus} role="tooltip">
+<div
+	bind:this={div}
+	on:mouseover={mouseOver}
+	on:mouseleave={mouseLeave}
+	on:mousemove={mouseMove}
+	on:focus={focus}
+	role="tooltip"
+	class="tooltipContainer"
+>
 	<slot />
 </div>
 
@@ -37,6 +45,9 @@
 {/if}
 
 <style>
+	.tooltipContainer {
+		display: inline-block;
+	}
 	.tooltip {
 		border: 1px solid #ddd;
 		box-shadow: 1px 1px 1px #ddd;

webpage/src/routes/models/edit/+page.svelte

@@ -42,7 +42,6 @@
 	import ModelDataPage from './ModelDataPage.svelte';
 
 	import 'src/styles/forms.css';
-	import KeepPageSize from 'src/lib/KeepPageSize.svelte';
 
 	let model: Promise<Model> = $state(new Promise(() => {}));
 	let _model: Model | undefined = $state(undefined);
@@ -137,8 +136,6 @@
 	{/await}
 </svelte:head>
 
-<KeepPageSize />
-
 <main>
 	{#if [4, 7].includes(_model?.status ?? 0)}
 		<div class="big-msg warning">The Model is currently training</div>

webpage/src/routes/models/edit/DeleteModel.svelte

@@ -27,7 +27,7 @@
 	}
 </script>
 
-<form on:submit|preventDefault={deleteModel} class:submmited>
+<form on:submit|preventDefault={deleteModel} class:submmited class="danger-bg">
 	<fieldset>
 		<label for="name">
 			To delete this model please type "{model.name}":

webpage/src/routes/models/edit/ModelDataPage.svelte

@@ -2,12 +2,12 @@
 	import { createEventDispatcher } from 'svelte';
 	import type { Model } from './+page.svelte';
 	import ModelData from './ModelData.svelte';
-	import { post } from 'src/lib/requests.svelte';
+	import { post, showMessage } from 'src/lib/requests.svelte';
 	import ModelDataPageStatsGraph from './ModelDataPageStatsGraph.svelte';
 	import type { ModelStats } from './types';
 	import DeleteZip from './DeleteZip.svelte';
 
-	let { model, active } = $props<{ model: Model; active?: boolean }>();
+	let { model, active }: { model: Model; active?: boolean } = $props();
 
 	const dispatch = createEventDispatcher<{ reload: void }>();
 
@@ -21,13 +21,14 @@
 		if (!model) return;
 		try {
 			stats = await post(`models/class/stats`, { id: model.id });
-		} catch {
-			return;
+		} catch (e) {
+			showMessage(e);
 		}
 	}
 </script>
 
-<div class="content" class:selected={active}>
+{#if active}
+	<div class="content selected">
 		{#if stats}
 			<ModelDataPageStatsGraph data={stats} />
 		{/if}
@@ -44,3 +45,4 @@
 			}}
 		/>
 	</div>
+{/if}

webpage/src/routes/models/edit/TasksDataPage.svelte

@@ -2,14 +2,18 @@
 	import { post } from 'src/lib/requests.svelte';
 	import type { Model } from 'src/routes/models/edit/+page.svelte';
 	import RunModel from './RunModel.svelte';
-	import TasksTable from './TasksTable.svelte';
+	import TasksTable from './tasks/TasksTable.svelte';
+	import Stats from './tasks/Stats.svelte';
 
-	const { active, model } = $props<{ active?: boolean; model: Model }>();
+	const { active, model }: { active?: boolean; model: Model } = $props();
 
 	let table: TasksTable;
 </script>
 
-<div class="content" class:selected={active}>
+{#if active}
+	<div class="content selected">
 		<RunModel {model} on:upload={() => table.getList()} on:taskReload={() => table.getList()} />
 		<TasksTable {model} bind:this={table} />
+		<Stats {model} />
 	</div>
+{/if}

webpage/src/routes/models/edit/tasks/Stats.svelte

@@ -0,0 +1,163 @@
+<script lang="ts">
+	import { onDestroy, onMount } from 'svelte';
+	import type { Model } from '../+page.svelte';
+	import { post, showMessage } from 'src/lib/requests.svelte';
+	import type { DataPoint, TasksStatsDay } from 'src/types/stats/task';
+
+	import {
+		Chart,
+		Title,
+		Tooltip,
+		Legend,
+		CategoryScale,
+		LinearScale,
+		PieController,
+		LineController,
+		Colors,
+		ArcElement,
+		PointElement,
+		LineElement
+	} from 'chart.js';
+	import ModelData from '../ModelData.svelte';
+
+	Chart.register(
+		Title,
+		Tooltip,
+		Legend,
+		ArcElement,
+		CategoryScale,
+		LinearScale,
+		PieController,
+		LineController,
+		Colors,
+		PointElement,
+		LineElement
+	);
+
+	let { model }: { model: Model } = $props();
+	// Load data
+	$effect(() => {
+		getStats();
+	});
+
+	async function getStats() {
+		if (pieChart && lineChart) {
+			return;
+		}
+		try {
+			const s = await post('stats/task/model/day', {
+				model_id: model.id
+			});
+			console.log(s);
+			createPie(s);
+			createLine(s);
+		} catch (e) {
+			showMessage(e);
+		}
+	}
+
+	let pie: HTMLCanvasElement;
+	let pieChart: Chart<'pie'> | undefined;
+	function createPie(s: TasksStatsDay) {
+		if (pieChart) {
+			pieChart.destroy();
+		}
+		let t = s.total;
+		pieChart = new Chart(pie, {
+			type: 'pie',
+			data: {
+				labels: [
+					'Classfication Error',
+					'Classfication Success',
+					'Classfication Failure',
+					'Classfication Preparing',
+					'Classfication Running',
+					'Classfication Unknown',
+					'Non Classfication Error',
+					'Non Classfication Success'
+				],
+				datasets: [
+					{
+						label: 'Total',
+						data: [
+							t.c_error,
+							t.c_success,
+							t.c_failure,
+							t.c_pre_running,
+							t.c_running,
+							t.c_unknown,
+							t.nc_error,
+							t.nc_success
+						]
+					}
+				]
+			},
+			options: {
+				animation: false
+			}
+		});
+	}
+
+	let line: HTMLCanvasElement;
+	let lineChart: Chart<'line'> | undefined;
+	function createLine(s: TasksStatsDay) {
+		if (lineChart) {
+			lineChart.destroy();
+		}
+		const keys: (keyof DataPoint)[] = Object.keys(s.total) as any;
+		let names: Record<keyof DataPoint, string> = {
+			c_error: 'Classfication Error',
+			c_success: 'Classfication Success',
+			c_failure: 'Classfication Failure',
+			c_pre_running: 'Classfication Preparing',
+			c_running: 'Classfication Running',
+			c_unknown: 'Classfication Unknown',
+			nc_error: 'Non Classfication Error',
+			nc_success: 'Non Classfication Success'
+		};
+		let t = s.total;
+		let labels = new Array(24).fill(0).map((_, i) => i);
+		lineChart = new Chart(line, {
+			type: 'line',
+			data: {
+				labels: labels,
+				datasets: keys.map((key) => ({
+					label: names[key],
+					data: s.hours.map((e) => e[key])
+				}))
+			},
+			options: {
+				animation: false
+			}
+		});
+	}
+
+	onDestroy(() => {
+		if (pieChart) {
+			pieChart.destroy();
+			pieChart = undefined;
+		}
+		if (lineChart) {
+			lineChart.destroy();
+			lineChart = undefined;
+		}
+	});
+</script>
+
+<h1>Statistics (Day)</h1>
+
+<h2>Total</h2>
+<div>
+	<canvas bind:this={pie}></canvas>
+</div>
+
+<h2>Hourly</h2>
+<div>
+	<canvas bind:this={line}></canvas>
+</div>
+
+<style lang="scss">
+	canvas {
+		width: 100%;
+	}
+</style>

webpage/src/routes/models/edit/tasks/TasksTable.svelte

@@ -29,12 +29,31 @@
 		3: 'Task running',
 		4: 'Task complete'
 	};
+	export const TaskTypeIcons: Record<number, string> = {
+		[-2]: 'bi-exclamation-octagon-fill',
+		[-1]: 'bi-exclamation-diamond-fill',
+		0: 'database-fill-gear',
+		1: 'database-fill-up',
+		2: 'database-fill-lock',
+		3: 'database-fill-gear',
+		4: 'database-fill-check'
+	};
+	export const TaskTypeColors: Record<number, string> = {
+		[-2]: 'red',
+		[-1]: 'red',
+		0: 'blue',
+		1: 'blue',
+		2: 'orange',
+		3: 'orange',
+		4: 'green'
+	};
 </script>
 
 <script lang="ts">
 	import { post, showMessage } from 'src/lib/requests.svelte';
-	import type { Model } from './+page.svelte';
+	import type { Model } from '../+page.svelte';
 	import MessageSimple from 'src/lib/MessageSimple.svelte';
+	import Tooltip from 'src/lib/Tooltip.svelte';
 
 	let { model }: { model: Model } = $props();
 
@@ -45,7 +64,7 @@
 	export async function getList() {
 		try {
 			const res = await post('tasks/list', {
-				id: model.id,
+				model_id: model.id,
 				page: page
 			});
 			showNext = res.show_next;
@@ -134,16 +153,22 @@
 								{:else}
 									TODO {task.user_confirmed}
 								{/if}
-								{#if task.user_confirmed == -1}
+								<div>
+									{#if task.user_confirmed != 1}
+										<Tooltip title="Agree with the result of the task">
 											<button type="button" on:click={userPreception(task.id, 1)}>
 												<span class="bi bi-check"></span>
 											</button>
+										</Tooltip>
 									{/if}
-								{#if task.user_confirmed == 1}
+									{#if task.user_confirmed != -1}
+										<Tooltip title="Disagree with the result">
 											<button class="danger" type="button" on:click={userPreception(task.id, -1)}>
 												<span class="bi bi-x-lg"></span>
 											</button>
+										</Tooltip>
 									{/if}
+								</div>
 							{:else}
 								-
 							{/if}
@@ -155,14 +180,20 @@
 						{#if task.status == 4}
 							{#if task.type == 1}
 								{@const temp = JSON.parse(task.result)}
-								{temp.class}({temp.confidence * 100}%)
+								{temp.class}({Math.round(temp.confidence * 100000) / 1000}%)
 							{:else}
 								{task.result}
 							{/if}
 						{/if}
 					</td>
-					<td>
-						{TaskTypeStrings[task.status]}
+					<td style="text-align: center;">
+						<Tooltip title={TaskTypeStrings[task.status]}>
+							<span
+								class="bi bi-{TaskTypeIcons[task.status]}"
+								style="color: {TaskTypeColors[task.status]}; font-size: 1.5em;"
+							>
+							</span>
+						</Tooltip>
 					</td>
 					<td>
 						{task.status_message}

webpage/src/types/stats/task.ts

@@ -0,0 +1,15 @@
+export type DataPoint = {
+	nc_success: number;
+	nc_error: number;
+	c_success: number;
+	c_failure: number;
+	c_unknown: number;
+	c_error: number;
+	c_pre_running: number;
+	c_running: number;
+};
+
+export type TasksStatsDay = {
+	total: DataPoint;
+	hours: DataPoint[];
+};