feat: improved visibility of dataset and improved speed for expand import zip file

This commit is contained in:
Andre Henriques 2024-04-09 17:28:59 +01:00
parent 00369640a8
commit 143ad3b02b
14 changed files with 727 additions and 464 deletions

11
go.mod
View File

@ -8,15 +8,20 @@ require (
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99
github.com/google/uuid v1.6.0
github.com/lib/pq v1.10.9
golang.org/x/crypto v0.18.0
golang.org/x/crypto v0.19.0
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.9.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.19.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
@ -24,6 +29,8 @@ require (
github.com/muesli/termenv v0.15.2 // indirect
github.com/rivo/uniseg v0.4.6 // indirect
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
golang.org/x/sys v0.16.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/protobuf v1.32.0 // indirect
)

18
go.sum
View File

@ -10,6 +10,8 @@ github.com/charmbracelet/log v0.2.5 h1:1yVvyKCKVV639RR4LIq1iy1Cs1AKxuNO+Hx2LJtk7
github.com/charmbracelet/log v0.2.5/go.mod h1:nQGK8tvc4pS9cvVEH/pWJiZ50eUq1aoXUOjGpXvdD0k=
github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw=
github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e h1:9+2AEFZymTi25FIIcDwuzcOPH04z9+fV6XeLiGORPDI=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe h1:7yELf1NFEwECpXMGowkoftcInMlVtLTCdwWLmxKgzNM=
@ -18,6 +20,12 @@ github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99 h1:8Bt1P/zy1gb37L4n8C
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99/go.mod h1:3YgYBeIX42t83uP27Bd4bSMxTnQhSbxl0pYSkCDB1tc=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@ -26,6 +34,8 @@ github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
@ -52,10 +62,14 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@ -63,6 +77,10 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=

View File

@ -309,58 +309,69 @@ func processZipFileExpand(c *Context, model *BaseModel) {
ids[name] = id
}
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
file_chans[i] = make(chan *zip.File, 2)
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
}
clean_up_channels := func() {
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
close(file_chans[i])
}
for i := 0; i < c.Handle.Config.NumberOfWorkers - 1; i++ {
_ = <- back_channel
}
close(back_channel)
}
first_round := true
channel_to_send := 0
// Parelalize this
for _, file := range reader.Reader.File {
// Skip if dir
if file.Name[len(file.Name)-1] == '/' {
continue
}
data, err := reader.Open(file.Name)
if err != nil {
failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name))
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name))
return
}
file_chans[channel_to_send] <- file
// TODO check if the file is a valid photo that matched the defined photo on the database
parts := strings.Split(file.Name, "/")
if first_round {
channel_to_send += 1
if c.Handle.Config.NumberOfWorkers == channel_to_send {
first_round = false
}
}
// Can not do else if because need to handle the case where the value changes in
// previous if
if !first_round {
new_id, ok := <- back_channel
if !ok {
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
}
mode := model_classes.DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
}
if new_id < 0 {
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
clean_up_channels()
failed("One of the workers failed due to db error")
return
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
if err != nil {
failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id))
return
}
channel_to_send = new_id
}
file_path := path.Join(base_path, data_point_id+"."+model.Format)
f, err := os.Create(file_path)
if err != nil {
failed(fmt.Sprintf("Could not create file %s\n", file_path))
return
}
defer f.Close()
f.Write(file_data)
if !testImgForModel(c, model, file_path) {
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
failed(fmt.Sprintf("Failed to update data point status"))
return
}
}
}
clean_up_channels()
c.Logger.Info("Added data to model", "id", model.Id)
ModelUpdateStatus(c, model.Id, READY)
}

View File

@ -5,6 +5,31 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
// Auth level set when path is definied as 1
func handleStats(c *Context) *Error {
var b struct {
Id string `json:"id" validate:"required"`
}
if _err := c.ToJSON(&b); _err != nil {
return _err;
}
type Row struct {
Name string `db:"mc.name" json:"name"`
Training string `db:"count(mdp.id) filter (where mdp.model_mode=1)" json:"training"`
Testing string `db:"count(mdp.id) filter (where mdp.model_mode=2)" json:"testing"`
}
rows, err := GetDbMultitple[Row](c, "model_data_point as mdp inner join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 group by mc.name order by mc.name asc;", b.Id)
if err != nil {
return c.Error500(err)
}
c.ShowMessage = false
return c.SendJSON(rows)
}
func handleList(handle *Handle) {
handle.Get("/models", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
@ -20,7 +45,10 @@ func handleList(handle *Handle) {
if err != nil {
return c.Error500(nil)
}
c.ShowMessage = true
return c.SendJSON(got)
})
handle.PostAuth("/models/class/stats", 1, handleStats)
}

View File

@ -15,6 +15,7 @@ import (
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
"github.com/charmbracelet/log"
"github.com/go-playground/validator/v10"
"github.com/goccy/go-json"
)
@ -43,6 +44,7 @@ type Handle struct {
posts []HandleFunc
deletes []HandleFunc
Config Config
validate *validator.Validate
}
func decodeBody(r *http.Request) (string, *Error) {
@ -79,6 +81,17 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
x.posts = append(x.posts, HandleFunc{path, fn})
}
func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
}
return fn(c)
}
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
x.deletes = append(x.deletes, HandleFunc{path, fn})
}
@ -195,6 +208,12 @@ func (c Context) ToJSON(dat any) *Error {
return c.Error500(err)
}
err = c.Handle.validate.Struct(dat)
if err != nil {
c.Logger.Error("Failed invalid json passed", "dat", dat, "err", err)
return c.JsonBadRequest("Bad Request! Invalid body passed!")
}
return nil
}
@ -476,7 +495,8 @@ func NewHandler(db *sql.DB, config Config) *Handle {
var gets []HandleFunc
var posts []HandleFunc
var deletes []HandleFunc
x := &Handle{db, gets, posts, deletes, config}
validate := validator.New()
x := &Handle{db, gets, posts, deletes, config, validate}
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {

Binary file not shown.

View File

@ -3,7 +3,8 @@
"version": "0.0.1",
"private": true,
"scripts": {
"dev": "vite dev",
"dev:raw": "vite dev",
"dev": "vite dev --port 5001 --host",
"build": "vite build",
"preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
@ -30,5 +31,9 @@
"typescript": "^5.0.0",
"vite": "^5.0.3"
},
"type": "module"
"type": "module",
"dependencies": {
"chart.js": "^4.4.2",
"svelte-chartjs": "^3.1.5"
}
}

View File

@ -1,5 +1,5 @@
<script lang="ts">
let {active} = $props<{active?: string}>();
let {active, nobox} = $props<{active?: string, nobox?: boolean}>();
function setActive(name: string) {
return () => active = name;
@ -10,7 +10,7 @@
}
</script>
<div class="tabs">
<div class="tabs" class:nobox>
<div class="tab-buttons">
<slot name="buttons" {setActive} {isActive} />
</div>
@ -58,5 +58,16 @@
display: block;
box-shadow: 0 2px 2px 1px #66666655;
}
&.nobox {
box-shadow: none;
overflow: visible;
:global(.content.selected) {
box-shadow: none;
}
}
}
</style>

View File

@ -6,8 +6,8 @@
width: number;
height: number;
status: number;
model_type: number;
format: string;
model_type: number;
format: string;
};
export type Layer = {
@ -25,85 +25,88 @@
</script>
<script lang="ts">
import { onMount } from "svelte";
import { onMount } from 'svelte';
import BaseModelInfo from "./BaseModelInfo.svelte";
import DeleteModel from "./DeleteModel.svelte";
import { goto } from "$app/navigation";
import { get, rdelete } from "src/lib/requests.svelte";
import MessageSimple from "$lib/MessageSimple.svelte"
import BaseModelInfo from './BaseModelInfo.svelte';
import DeleteModel from './DeleteModel.svelte';
import { goto } from '$app/navigation';
import { get, rdelete } from 'src/lib/requests.svelte';
import MessageSimple from '$lib/MessageSimple.svelte';
import ModelData from "./ModelData.svelte";
import DeleteZip from "./DeleteZip.svelte";
import ModelData from './ModelData.svelte';
import DeleteZip from './DeleteZip.svelte';
import 'src/styles/forms.css'
import RunModel from "./RunModel.svelte";
let model: Promise<Model> = $state(new Promise(() => {}));
let definitions: Promise<Definitions[]> = $state(new Promise(() => {}));
import ModelDataPage from './ModelDataPage.svelte';
let id: string | undefined = $state()
import 'src/styles/forms.css';
import RunModel from './RunModel.svelte';
import Tabs from 'src/lib/Tabs.svelte';
let model: Promise<Model> = $state(new Promise(() => {}));
let _model: Model | undefined = $state(undefined);
let definitions: Promise<Definitions[]> = $state(new Promise(() => {}));
async function getModel() {
try {
let temp_model: Model = await get(`models/edit?id=${id}`);
let id: string | undefined = $state();
console.log(temp_model)
async function getModel() {
try {
let temp_model: Model = await get(`models/edit?id=${id}`);
if (temp_model.status == 3) {
setTimeout(getModel, 2000);
}
if ([3, 6].includes(temp_model.status)) {
setTimeout(getModel, 2000);
}
if (temp_model.status == 4) {
setTimeout(getModel, 5000);
if (temp_model.status == 4) {
setTimeout(getModel, 5000);
definitions = await get(`models/edit/definitions?id=${id}`);
}
definitions = await get(`models/edit/definitions?id=${id}`);
}
model = Promise.resolve(temp_model);
} catch (e) {
if (e instanceof Response) {
model = Promise.reject(await e.json())
} else {
model = Promise.reject("Could not load model");
}
}
}
model = Promise.resolve(temp_model);
_model = temp_model;
} catch (e) {
if (e instanceof Response) {
model = Promise.reject(await e.json());
} else {
model = Promise.reject('Could not load model');
}
}
}
onMount(() => {
let url = new URLSearchParams(window.location.search);
const _id = url.get('id');
if (!_id) {
goto('/models')
return;
}
onMount(() => {
let url = new URLSearchParams(window.location.search);
const _id = url.get('id');
if (!_id) {
goto('/models');
return;
}
id = _id;
getModel();
});
id = _id;
getModel();
});
let resetMessages: MessageSimple;
let resetMessages: MessageSimple;
async function resetModel() {
resetMessages.display("");
let _model = await model;
async function resetModel() {
resetMessages.display('');
try {
await rdelete('models/train/reset', {
"id": _model.id
});
let _model = await model;
getModel();
} catch (e) {
if (e instanceof Response) {
resetMessages.display(await e.json())
} else {
resetMessages.display("Could not reset model!")
}
}
}
try {
await rdelete('models/train/reset', {
id: _model.id
});
// Auto reload after 2s when model.status 3,4
getModel();
} catch (e) {
if (e instanceof Response) {
resetMessages.display(await e.json());
} else {
resetMessages.display('Could not reset model!');
}
}
}
// Auto reload after 2s when model.status 3,4
</script>
<svelte:head>
@ -127,186 +130,208 @@
-->
<main>
{#await model}
Loading
{:then m}
{#if m.status == 1}
<div>
<h1 class="text-center">
{m.name}
</h1>
<!-- TODO add cool animation -->
<h2 class="text-center">Preparing the model</h2>
</div>
{:else if m.status == -1}
<div>
<h1 class="text-center">
{m.name}
</h1>
<!-- TODO improve message -->
<h2 class="text-center">Failed to prepare model</h2>
<Tabs active="model" let:isActive nobox>
<div slot="buttons" let:setActive let:isActive>
<button
class="tab"
on:click|preventDefault={setActive('model')}
class:selected={isActive('model')}
>
Model
</button>
{#if _model && [2, 3, 4, 5, 6, 7].includes(_model.status)}
<button
class="tab"
on:click|preventDefault={setActive('model-data')}
class:selected={isActive('model-data')}
>
Model Data
</button>
{/if}
</div>
{#if _model}
<ModelDataPage model={_model} on:reload={getModel} active={isActive('model-data')} />
{/if}
<div class="content" class:selected={isActive('model')}>
{#await model}
Loading
{:then m}
{#if m.status == 1}
<div>
<h1 class="text-center">
{m.name}
</h1>
<!-- TODO add cool animation -->
<h2 class="text-center">Preparing the model</h2>
</div>
{:else if m.status == -1}
<div>
<h1 class="text-center">
{m.name}
</h1>
<!-- TODO improve message -->
<h2 class="text-center">Failed to prepare model</h2>
<DeleteModel model={m} />
</div>
<!-- PRE TRAINING STATUS -->
{:else if m.status == 2}
<BaseModelInfo model={m} />
<ModelData model={m} on:reload={getModel} />
<!-- {{ template "train-model-card" . }} -->
<DeleteModel model={m} />
{:else if m.status == -2}
<BaseModelInfo model={m} />
<DeleteZip model={m} on:reload={getModel} />
<DeleteModel model={m} />
{:else if m.status == 3}
<BaseModelInfo model={m} />
<div class="card">
<!-- TODO improve this -->
Processing zip file...
</div>
{:else if m.status == -3 || m.status == -4}
<BaseModelInfo model={m} />
<form on:submit={resetModel}>
Failed Prepare for training.<br />
<div class="spacer"></div>
<MessageSimple bind:this={resetMessages} />
<button class="danger"> Try Again </button>
</form>
<DeleteModel model={m} />
{:else if m.status == 4}
<BaseModelInfo model={m} />
<!-- TODO request /models/edit?id={m.id} -->
<div class="card">
<!-- TODO improve this -->
Training the model...<br />
<!-- TODO Add progress status on definitions -->
{#await definitions}
Loading
{:then defs}
<table>
<thead>
<tr>
<th> Done Progress </th>
<th> Training Round Progress </th>
<th> Accuracy </th>
<th> Status </th>
</tr>
</thead>
<tbody>
{#each defs as def}
<tr>
<td>
{def.epoch}
</td>
<td>
{def.epoch_progress}/20
</td>
<td>
{def.accuracy}%
</td>
<td style="text-align: center;">
{#if def.status == 2}
<span class="bi bi-book" style="color: green;"></span>
{:else if [3, 6, -3].includes(def.status)}
<span
class="bi bi-book-half"
style="color: {{
'3': 'green',
'-3': 'red',
'6': 'orange'
}[String(def.status)]};"
></span>
{:else}
{def.status}
{/if}
</td>
</tr>
{#if def.status == 3 && def.layers}
<DeleteModel model={m} />
</div>
<!-- PRE TRAINING STATUS -->
{:else if m.status == 2}
<BaseModelInfo model={m} />
<ModelData model={m} on:reload={getModel} />
<!-- {{ template "train-model-card" . }} -->
<DeleteModel model={m} />
{:else if m.status == -2}
<BaseModelInfo model={m} />
<DeleteZip model={m} on:reload={getModel} />
<DeleteModel model={m} />
{:else if m.status == 3}
<BaseModelInfo model={m} />
<div class="card">
<!-- TODO improve this -->
Processing zip file...
</div>
{:else if m.status == -3 || m.status == -4}
<BaseModelInfo model={m} />
<form on:submit={resetModel}>
Failed Prepare for training.<br />
<div class="spacer"></div>
<MessageSimple bind:this={resetMessages} />
<button class="danger"> Try Again </button>
</form>
<DeleteModel model={m} />
{:else if m.status == 4}
<BaseModelInfo model={m} />
<!-- TODO request /models/edit?id={m.id} -->
<div class="card">
<!-- TODO improve this -->
Training the model...<br />
<!-- TODO Add progress status on definitions -->
{#await definitions}
Loading
{:then defs}
<table>
<thead>
<tr>
<td colspan="4">
<svg viewBox="0 200 1000 600">
{#each def.layers as layer, i}
{@const sep_mod =
def.layers.length > 8
? Math.max(10, 100 - (def.layers.length - 8) * 10)
: 100}
{#if layer.layer_type == 1}
<polygon
points="50,450 200,250 200,550 50,750"
stroke="black"
stroke-width="2"
fill="green"
></polygon>
{:else if layer.layer_type == 4}
<polygon
points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
i * sep_mod},550 {50 + i * sep_mod},750"
stroke="black"
stroke-width="2"
fill="orange"
>
</polygon>
{:else if layer.layer_type == 3}
<polygon
points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
i * sep_mod},550 {50 + i * sep_mod},750"
stroke="black"
stroke-width="2"
fill="red"
>
</polygon>
{:else if layer.layer_type == 2}
<polygon
points="{50 + i * sep_mod},550 {200 + i * sep_mod},350 {200 +
i * sep_mod},450 {50 + i * sep_mod},650"
stroke="black"
stroke-width="2"
fill="blue"
>
</polygon>
{:else}
<div>
{layer.layer_type}
{layer.shape}
</div>
{/if}
{/each}
</svg>
</td>
<th> Done Progress </th>
<th> Training Round Progress </th>
<th> Accuracy </th>
<th> Status </th>
</tr>
{/if}
{/each}
</tbody>
</table>
{/await}
<!-- TODO Add ability to stop training -->
</div>
{:else if [5, 6, -6, 7, -7].includes(m.status)}
<BaseModelInfo model={m} />
<RunModel model={m} />
{#if m.status == 6}
<div class="card">
Model expading... Processing ZIP file
</div>
{/if}
{#if m.status == -6}
<DeleteZip model={m} on:reload={getModel} expand />
{/if}
{#if m.status == -7}
<form>
<!-- TODO add more info about the failure -->
Failed to train the model!
Try to retrain
</form>
{/if}
{#if m.model_type == 2}
<ModelData model={m} on:reload={getModel} />
{/if}
<DeleteModel model={m} />
{:else}
<h1>Unknown Status of the model.</h1>
{/if}
{/await}
</thead>
<tbody>
{#each defs as def}
<tr>
<td>
{def.epoch}
</td>
<td>
{def.epoch_progress}/20
</td>
<td>
{def.accuracy}%
</td>
<td style="text-align: center;">
{#if def.status == 2}
<span class="bi bi-book" style="color: green;"></span>
{:else if [3, 6, -3].includes(def.status)}
<span
class="bi bi-book-half"
style="color: {{
'3': 'green',
'-3': 'red',
'6': 'orange'
}[String(def.status)]};"
></span>
{:else}
{def.status}
{/if}
</td>
</tr>
{#if def.status == 3 && def.layers}
<tr>
<td colspan="4">
<svg viewBox="0 200 1000 600">
{#each def.layers as layer, i}
{@const sep_mod =
def.layers.length > 8
? Math.max(10, 100 - (def.layers.length - 8) * 10)
: 100}
{#if layer.layer_type == 1}
<polygon
points="50,450 200,250 200,550 50,750"
stroke="black"
stroke-width="2"
fill="green"
></polygon>
{:else if layer.layer_type == 4}
<polygon
points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
i * sep_mod},550 {50 + i * sep_mod},750"
stroke="black"
stroke-width="2"
fill="orange"
>
</polygon>
{:else if layer.layer_type == 3}
<polygon
points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
i * sep_mod},550 {50 + i * sep_mod},750"
stroke="black"
stroke-width="2"
fill="red"
>
</polygon>
{:else if layer.layer_type == 2}
<polygon
points="{50 + i * sep_mod},550 {200 + i * sep_mod},350 {200 +
i * sep_mod},450 {50 + i * sep_mod},650"
stroke="black"
stroke-width="2"
fill="blue"
>
</polygon>
{:else}
<div>
{layer.layer_type}
{layer.shape}
</div>
{/if}
{/each}
</svg>
</td>
</tr>
{/if}
{/each}
</tbody>
</table>
{/await}
<!-- TODO Add ability to stop training -->
</div>
{:else if [5, 6, -6, 7, -7].includes(m.status)}
<BaseModelInfo model={m} />
<RunModel model={m} />
{#if m.status == 6}
<div class="card">Model expading... Processing ZIP file</div>
{/if}
{#if m.status == -6}
<DeleteZip model={m} on:reload={getModel} expand />
{/if}
{#if m.status == -7}
<form>
<!-- TODO add more info about the failure -->
Failed to train the model! Try to retrain
</form>
{/if}
{#if m.model_type == 2}
<ModelData simple model={m} on:reload={getModel} />
{/if}
<DeleteModel model={m} />
{:else}
<h1>Unknown Status of the model.</h1>
{/if}
{/await}
</div>
</Tabs>
</main>
<style lang="scss">

View File

@ -1,197 +1,195 @@
<script lang="ts" context="module">
export type Class = {
name: string;
id: string;
status: number;
}
export type Class = {
name: string;
id: string;
status: number;
};
</script>
<script lang="ts">
import FileUpload from "src/lib/FileUpload.svelte";
import Tabs from "src/lib/Tabs.svelte";
import type { Model } from "./+page.svelte";
import { postFormData, get } from "src/lib/requests.svelte";
import MessageSimple from "src/lib/MessageSimple.svelte";
import { createEventDispatcher } from "svelte";
import ModelTable from "./ModelTable.svelte";
import TrainModel from "./TrainModel.svelte";
import FileUpload from 'src/lib/FileUpload.svelte';
import Tabs from 'src/lib/Tabs.svelte';
import type { Model } from './+page.svelte';
import { postFormData, get } from 'src/lib/requests.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher } from 'svelte';
import ModelTable from './ModelTable.svelte';
import TrainModel from './TrainModel.svelte';
let { model } = $props<{model: Model}>();
let { model, simple } = $props<{ model: Model; simple?: boolean }>();
let classes: Class[] = $state([]);
let has_data: boolean = $state(false);
let classes: Class[] = $state([]);
let has_data: boolean = $state(false);
let file: File | undefined = $state();
let file: File | undefined = $state();
const dispatch = createEventDispatcher<{
reload: void
}>();
const dispatch = createEventDispatcher<{
reload: void;
}>();
let uploading: Promise<void> = $state(Promise.resolve())
let numberOfInvalidImages = $state(0);
let uploading: Promise<void> = $state(Promise.resolve());
let numberOfInvalidImages = $state(0);
let uploadImage: MessageSimple;
let uploadImage: MessageSimple;
async function uploadZip() {
if (!file) return;
async function uploadZip() {
if (!file) return;
uploading = new Promise(() => {});
uploading = new Promise(() => {});
let form = new FormData();
form.append('id', model.id);
form.append('file', file, 'upload.zip');
try {
await postFormData('models/data/upload', form);
dispatch('reload');
} catch (e) {
if (e instanceof Response) {
uploadImage.display(await e.json());
} else {
uploadImage.display('');
}
}
let form = new FormData();
form.append('id', model.id);
form.append('file', file, 'upload.zip');
uploading = Promise.resolve();
}
try {
await postFormData('models/data/upload', form);
dispatch('reload');
} catch (e) {
if (e instanceof Response) {
uploadImage.display(await e.json());
} else {
uploadImage.display('');
}
}
$effect(() => {
getData();
});
uploading = Promise.resolve();
}
async function getData() {
if (!model) return;
try {
let data = await get(`models/edit/classes?id=${model.id}`);
classes = data.classes
numberOfInvalidImages = data.number_of_invalid_images;
has_data = data.has_data;
} catch {
return;
}
}
$effect(() => {
getData();
});
async function getData() {
if (!model) return;
try {
let data = await get(`models/edit/classes?id=${model.id}`);
classes = data.classes;
numberOfInvalidImages = data.number_of_invalid_images;
has_data = data.has_data;
} catch {
return;
}
}
</script>
<div class="card">
<h3>
Training data
</h3>
{#if classes.length == 0}
<p>
You need to upload data so the model can train.
</p>
<Tabs active="upload" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive("upload")} on:click={setActive("upload")}>
Upload
</button>
<button class="tab" class:selected={isActive("create-class")} on:click={setActive("create-class")}>
Create Class
</button>
<button class="tab" class:selected={isActive("api")} on:click={setActive("api")}>
Api
</button>
</div>
<div class="content" class:selected={isActive("upload")}>
<form on:submit|preventDefault={uploadZip}>
<fieldset class="file-upload" >
<label for="file">Data file</label>
<div class="form-msg">
Please provide a file that has the training and testing data<br/>
The file must have 2 folders one with testing images and one with training images. <br/>
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
The class folders must have the images for the classes.
<pre>
training\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
{#if !simple}
<div class="card">
<h3>Training data</h3>
{#if classes.length == 0}
<p>You need to upload data so the model can train.</p>
<Tabs active="upload" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive('upload')} on:click={setActive('upload')}>
Upload
</button>
<button
class="tab"
class:selected={isActive('create-class')}
on:click={setActive('create-class')}
>
Create Class
</button>
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
Api
</button>
</div>
<div class="content" class:selected={isActive('upload')}>
<form on:submit|preventDefault={uploadZip}>
<fieldset class="file-upload">
<label for="file">Data file</label>
<div class="form-msg">
Please provide a file that has the training and testing data<br />
The file must have 2 folders one with testing images and one with training images.
<br />
Each of the folders will contain the classes of the model. The folders must be the same
in testing and training. The class folders must have the images for the classes.
<pre>
training\
class1\
img1.png
img2.png
img2.png
...
testing\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
testing\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
</pre>
</div>
<FileUpload replace_slot bind:file={file} accept="application/zip" notExpand >
<img src="/imgs/upload-icon.png" alt="" />
<span>
Upload Zip File
</span>
<div slot="replaced" style="display: inline;">
<img src="/imgs/upload-icon.png" alt="" />
<span>
File selected
</span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#if file}
{#await uploading}
<button disabled>
Uploading
</button>
{:then}
<button>
Add
</button>
{/await}
{/if}
</form>
</div>
<div class="content" class:selected={isActive("create-class")}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive("api")}>
TODO
</div>
</Tabs>
<div class="tabs">
</div>
{:else}
<p>
You need to upload data so the model can train.
</p>
{#if numberOfInvalidImages > 0}
<p class="danger">
There are images {numberOfInvalidImages} that were loaded that do not have the correct format. These images will be delete when the model trains.
</p>
{/if}
<Tabs active="create-class" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive("create-class")} on:click={setActive("create-class")}>
Create Class
</button>
<button class="tab" class:selected={isActive("api")} on:click={setActive("api")}>
Api
</button>
</div>
<div class="content" class:selected={isActive("create-class")}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive("api")}>
TODO
</div>
</Tabs>
{/if}
</div>
</div>
<FileUpload replace_slot bind:file accept="application/zip" notExpand>
<img src="/imgs/upload-icon.png" alt="" />
<span> Upload Zip File </span>
<div slot="replaced" style="display: inline;">
<img src="/imgs/upload-icon.png" alt="" />
<span> File selected </span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#if file}
{#await uploading}
<button disabled> Uploading </button>
{:then}
<button> Add </button>
{/await}
{/if}
</form>
</div>
<div class="content" class:selected={isActive('create-class')}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive('api')}>TODO</div>
</Tabs>
<div class="tabs"></div>
{:else}
<p>You need to upload data so the model can train.</p>
{#if numberOfInvalidImages > 0}
<p class="danger">
There are images {numberOfInvalidImages} that were loaded that do not have the correct format.
These images will be delete when the model trains.
</p>
{/if}
<Tabs active="create-class" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button
class="tab"
class:selected={isActive('create-class')}
on:click={setActive('create-class')}
>
Create Class
</button>
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
Api
</button>
</div>
<div class="content" class:selected={isActive('create-class')}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive('api')}>TODO</div>
</Tabs>
{/if}
</div>
{/if}
{#if classes.some((item) => item.status == 1) && ![-6, 6].includes(model.status)}
<TrainModel number_of_invalid_images={numberOfInvalidImages} {model} {has_data} on:reload={() => dispatch('reload')} />
<TrainModel
number_of_invalid_images={numberOfInvalidImages}
{model}
{has_data}
on:reload={() => dispatch('reload')}
/>
{/if}

View File

@ -0,0 +1,48 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
import type { Model } from "./+page.svelte";
import ModelData from "./ModelData.svelte";
import { post } from "src/lib/requests.svelte";
import ModelDataPageStatsGraph from "./ModelDataPageStatsGraph.svelte";
import type { ModelStats } from "./types";
import DeleteZip from "./DeleteZip.svelte";
let { model, active } = $props<{model: Model, active?: boolean}>()
const dispatch = createEventDispatcher<{ reload: void }>();
$effect(() => {
if (active)
getData();
});
let stats: ModelStats | undefined = $state();
async function getData() {
if (!model) return;
try {
stats = await post(`models/class/stats`, { id: model.id });
} catch {
return;
}
}
</script>
<div class="content" class:selected={active}>
{#if stats}
<ModelDataPageStatsGraph data={stats} />
{/if}
{#if [-6, -2].includes(model.status)}
<DeleteZip model={model} on:reload={() => dispatch('reload')} expand />
{/if}
<ModelData model={model} on:reload={() => {
getData();
dispatch('reload');
}} />
</div>

View File

@ -0,0 +1,89 @@
<script lang="ts">
import {
Chart,
Title,
Tooltip,
Legend,
BarElement,
CategoryScale,
LinearScale,
BarController,
Colors
} from 'chart.js';
import type { ModelStats } from './types';
Chart.register(
Title,
Tooltip,
Legend,
BarElement,
CategoryScale,
LinearScale,
BarController,
Colors
);
let { data } = $props<{ data: ModelStats }>();
let ctx: HTMLCanvasElement;
let chart: Chart;
$effect(() => {
if (data && ctx) {
if (chart) {
console.log("update")
chart.data = {
labels: data.map((a) => a.name),
datasets: [
{
label: 'Training',
data: data.map((a) => a.training)
},
{
label: 'Testing',
data: data.map((a) => a.testing)
}
]
};
chart.update('resize');
} else {
console.log("create")
chart = new Chart(ctx, {
type: 'bar',
data: {
labels: data.map((a) => a.name),
datasets: [
{
label: 'Training',
data: data.map((a) => a.training)
},
{
label: 'Testing',
data: data.map((a) => a.testing)
}
]
},
options: {
scales: {
x: {
stacked: true
},
y: {
stacked: true
}
}
}
});
}
}
});
</script>
<div><canvas bind:this={ctx} /></div>
<style lang="scss">
canvas {
width: 100%;
}
</style>

View File

@ -44,8 +44,6 @@
let res = await get('models/data/list?' + url.toString());
showNext = res.showNext;
image_list = res.image_list;
console.log(image_list);
} catch (e) {
console.error('TODO notify user', e);
}

View File

@ -0,0 +1,5 @@
export type ModelStats = Array<{
name: string,
training: number,
testing: number,
}>