feat: improved visibility of dataset and improved speed for expand import zip file
This commit is contained in:
parent
00369640a8
commit
143ad3b02b
11
go.mod
11
go.mod
@ -8,15 +8,20 @@ require (
|
||||
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/lib/pq v1.10.9
|
||||
golang.org/x/crypto v0.18.0
|
||||
golang.org/x/crypto v0.19.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/BurntSushi/toml v1.3.2 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/charmbracelet/lipgloss v0.9.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
||||
github.com/go-logfmt/logfmt v0.6.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.19.0 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
@ -24,6 +29,8 @@ require (
|
||||
github.com/muesli/termenv v0.15.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.6 // indirect
|
||||
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a // indirect
|
||||
golang.org/x/sys v0.16.0 // indirect
|
||||
golang.org/x/net v0.21.0 // indirect
|
||||
golang.org/x/sys v0.17.0 // indirect
|
||||
golang.org/x/text v0.14.0 // indirect
|
||||
google.golang.org/protobuf v1.32.0 // indirect
|
||||
)
|
||||
|
||||
18
go.sum
18
go.sum
@ -10,6 +10,8 @@ github.com/charmbracelet/log v0.2.5 h1:1yVvyKCKVV639RR4LIq1iy1Cs1AKxuNO+Hx2LJtk7
|
||||
github.com/charmbracelet/log v0.2.5/go.mod h1:nQGK8tvc4pS9cvVEH/pWJiZ50eUq1aoXUOjGpXvdD0k=
|
||||
github.com/charmbracelet/log v0.3.1 h1:TjuY4OBNbxmHWSwO3tosgqs5I3biyY8sQPny/eCMTYw=
|
||||
github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3zwHGWuuq8g=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e h1:9+2AEFZymTi25FIIcDwuzcOPH04z9+fV6XeLiGORPDI=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe h1:7yELf1NFEwECpXMGowkoftcInMlVtLTCdwWLmxKgzNM=
|
||||
@ -18,6 +20,12 @@ github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99 h1:8Bt1P/zy1gb37L4n8C
|
||||
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99/go.mod h1:3YgYBeIX42t83uP27Bd4bSMxTnQhSbxl0pYSkCDB1tc=
|
||||
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
|
||||
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4=
|
||||
github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
@ -26,6 +34,8 @@ github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
|
||||
github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
||||
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
|
||||
@ -52,10 +62,14 @@ golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg=
|
||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a h1:Q8/wZp0KX97QFTc2ywcOE0YRjZPVIx+MXInMzdvQqcA=
|
||||
golang.org/x/exp v0.0.0-20240119083558-1b970713d09a/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08=
|
||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
@ -63,6 +77,10 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
|
||||
golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
|
||||
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
|
||||
|
||||
@ -309,57 +309,68 @@ func processZipFileExpand(c *Context, model *BaseModel) {
|
||||
ids[name] = id
|
||||
}
|
||||
|
||||
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
|
||||
|
||||
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
|
||||
|
||||
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
|
||||
file_chans[i] = make(chan *zip.File, 2)
|
||||
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
|
||||
}
|
||||
|
||||
clean_up_channels := func() {
|
||||
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
|
||||
close(file_chans[i])
|
||||
}
|
||||
for i := 0; i < c.Handle.Config.NumberOfWorkers - 1; i++ {
|
||||
_ = <- back_channel
|
||||
}
|
||||
close(back_channel)
|
||||
}
|
||||
|
||||
first_round := true
|
||||
|
||||
channel_to_send := 0
|
||||
|
||||
// Parelalize this
|
||||
|
||||
for _, file := range reader.Reader.File {
|
||||
// Skip if dir
|
||||
if file.Name[len(file.Name)-1] == '/' {
|
||||
continue
|
||||
}
|
||||
|
||||
data, err := reader.Open(file.Name)
|
||||
if err != nil {
|
||||
failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name))
|
||||
return
|
||||
file_chans[channel_to_send] <- file
|
||||
|
||||
|
||||
if first_round {
|
||||
channel_to_send += 1
|
||||
if c.Handle.Config.NumberOfWorkers == channel_to_send {
|
||||
first_round = false
|
||||
}
|
||||
defer data.Close()
|
||||
file_data, err := io.ReadAll(data)
|
||||
if err != nil {
|
||||
failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name))
|
||||
}
|
||||
|
||||
// Can not do else if because need to handle the case where the value changes in
|
||||
// previous if
|
||||
if !first_round {
|
||||
new_id, ok := <- back_channel
|
||||
if !ok {
|
||||
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
|
||||
}
|
||||
|
||||
if new_id < 0 {
|
||||
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
|
||||
clean_up_channels()
|
||||
failed("One of the workers failed due to db error")
|
||||
return
|
||||
}
|
||||
|
||||
// TODO check if the file is a valid photo that matched the defined photo on the database
|
||||
|
||||
parts := strings.Split(file.Name, "/")
|
||||
|
||||
mode := model_classes.DATA_POINT_MODE_TRAINING
|
||||
if parts[0] == "testing" {
|
||||
mode = model_classes.DATA_POINT_MODE_TESTING
|
||||
channel_to_send = new_id
|
||||
}
|
||||
|
||||
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
|
||||
if err != nil {
|
||||
failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id))
|
||||
return
|
||||
}
|
||||
|
||||
file_path := path.Join(base_path, data_point_id+"."+model.Format)
|
||||
f, err := os.Create(file_path)
|
||||
if err != nil {
|
||||
failed(fmt.Sprintf("Could not create file %s\n", file_path))
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
f.Write(file_data)
|
||||
|
||||
if !testImgForModel(c, model, file_path) {
|
||||
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
|
||||
c.Logger.Warn("Not failling updating data point to status -1")
|
||||
message := "Image did not have valid format for the model"
|
||||
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
|
||||
failed(fmt.Sprintf("Failed to update data point status"))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
clean_up_channels()
|
||||
|
||||
c.Logger.Info("Added data to model", "id", model.Id)
|
||||
ModelUpdateStatus(c, model.Id, READY)
|
||||
|
||||
@ -5,6 +5,31 @@ import (
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||
)
|
||||
|
||||
// Auth level set when path is definied as 1
|
||||
func handleStats(c *Context) *Error {
|
||||
var b struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
}
|
||||
|
||||
if _err := c.ToJSON(&b); _err != nil {
|
||||
return _err;
|
||||
}
|
||||
|
||||
type Row struct {
|
||||
Name string `db:"mc.name" json:"name"`
|
||||
Training string `db:"count(mdp.id) filter (where mdp.model_mode=1)" json:"training"`
|
||||
Testing string `db:"count(mdp.id) filter (where mdp.model_mode=2)" json:"testing"`
|
||||
}
|
||||
|
||||
rows, err := GetDbMultitple[Row](c, "model_data_point as mdp inner join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 group by mc.name order by mc.name asc;", b.Id)
|
||||
if err != nil {
|
||||
return c.Error500(err)
|
||||
}
|
||||
|
||||
c.ShowMessage = false
|
||||
return c.SendJSON(rows)
|
||||
}
|
||||
|
||||
func handleList(handle *Handle) {
|
||||
handle.Get("/models", func(c *Context) *Error {
|
||||
if !c.CheckAuthLevel(1) {
|
||||
@ -21,6 +46,9 @@ func handleList(handle *Handle) {
|
||||
return c.Error500(nil)
|
||||
}
|
||||
|
||||
c.ShowMessage = true
|
||||
return c.SendJSON(got)
|
||||
})
|
||||
|
||||
handle.PostAuth("/models/class/stats", 1, handleStats)
|
||||
}
|
||||
|
||||
@ -15,6 +15,7 @@ import (
|
||||
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/go-playground/validator/v10"
|
||||
"github.com/goccy/go-json"
|
||||
)
|
||||
|
||||
@ -43,6 +44,7 @@ type Handle struct {
|
||||
posts []HandleFunc
|
||||
deletes []HandleFunc
|
||||
Config Config
|
||||
validate *validator.Validate
|
||||
}
|
||||
|
||||
func decodeBody(r *http.Request) (string, *Error) {
|
||||
@ -79,6 +81,17 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
|
||||
x.posts = append(x.posts, HandleFunc{path, fn})
|
||||
}
|
||||
|
||||
|
||||
func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
|
||||
inner_fn := func(c *Context) *Error {
|
||||
if !c.CheckAuthLevel(authLevel) {
|
||||
return nil
|
||||
}
|
||||
return fn(c)
|
||||
}
|
||||
x.posts = append(x.posts, HandleFunc{path, inner_fn})
|
||||
}
|
||||
|
||||
func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
|
||||
x.deletes = append(x.deletes, HandleFunc{path, fn})
|
||||
}
|
||||
@ -195,6 +208,12 @@ func (c Context) ToJSON(dat any) *Error {
|
||||
return c.Error500(err)
|
||||
}
|
||||
|
||||
err = c.Handle.validate.Struct(dat)
|
||||
if err != nil {
|
||||
c.Logger.Error("Failed invalid json passed", "dat", dat, "err", err)
|
||||
return c.JsonBadRequest("Bad Request! Invalid body passed!")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -476,7 +495,8 @@ func NewHandler(db *sql.DB, config Config) *Handle {
|
||||
var gets []HandleFunc
|
||||
var posts []HandleFunc
|
||||
var deletes []HandleFunc
|
||||
x := &Handle{db, gets, posts, deletes, config}
|
||||
validate := validator.New()
|
||||
x := &Handle{db, gets, posts, deletes, config, validate}
|
||||
|
||||
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
|
||||
Binary file not shown.
@ -3,7 +3,8 @@
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "vite dev",
|
||||
"dev:raw": "vite dev",
|
||||
"dev": "vite dev --port 5001 --host",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
|
||||
@ -30,5 +31,9 @@
|
||||
"typescript": "^5.0.0",
|
||||
"vite": "^5.0.3"
|
||||
},
|
||||
"type": "module"
|
||||
"type": "module",
|
||||
"dependencies": {
|
||||
"chart.js": "^4.4.2",
|
||||
"svelte-chartjs": "^3.1.5"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
<script lang="ts">
|
||||
let {active} = $props<{active?: string}>();
|
||||
let {active, nobox} = $props<{active?: string, nobox?: boolean}>();
|
||||
|
||||
function setActive(name: string) {
|
||||
return () => active = name;
|
||||
@ -10,7 +10,7 @@
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="tabs">
|
||||
<div class="tabs" class:nobox>
|
||||
<div class="tab-buttons">
|
||||
<slot name="buttons" {setActive} {isActive} />
|
||||
</div>
|
||||
@ -58,5 +58,16 @@
|
||||
display: block;
|
||||
box-shadow: 0 2px 2px 1px #66666655;
|
||||
}
|
||||
|
||||
&.nobox {
|
||||
box-shadow: none;
|
||||
overflow: visible;
|
||||
|
||||
|
||||
:global(.content.selected) {
|
||||
box-shadow: none;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
</style>
|
||||
|
||||
@ -25,31 +25,33 @@
|
||||
</script>
|
||||
|
||||
<script lang="ts">
|
||||
import { onMount } from "svelte";
|
||||
import { onMount } from 'svelte';
|
||||
|
||||
import BaseModelInfo from "./BaseModelInfo.svelte";
|
||||
import DeleteModel from "./DeleteModel.svelte";
|
||||
import { goto } from "$app/navigation";
|
||||
import { get, rdelete } from "src/lib/requests.svelte";
|
||||
import MessageSimple from "$lib/MessageSimple.svelte"
|
||||
import BaseModelInfo from './BaseModelInfo.svelte';
|
||||
import DeleteModel from './DeleteModel.svelte';
|
||||
import { goto } from '$app/navigation';
|
||||
import { get, rdelete } from 'src/lib/requests.svelte';
|
||||
import MessageSimple from '$lib/MessageSimple.svelte';
|
||||
|
||||
import ModelData from "./ModelData.svelte";
|
||||
import DeleteZip from "./DeleteZip.svelte";
|
||||
import ModelData from './ModelData.svelte';
|
||||
import DeleteZip from './DeleteZip.svelte';
|
||||
|
||||
import 'src/styles/forms.css'
|
||||
import RunModel from "./RunModel.svelte";
|
||||
import ModelDataPage from './ModelDataPage.svelte';
|
||||
|
||||
import 'src/styles/forms.css';
|
||||
import RunModel from './RunModel.svelte';
|
||||
import Tabs from 'src/lib/Tabs.svelte';
|
||||
let model: Promise<Model> = $state(new Promise(() => {}));
|
||||
let _model: Model | undefined = $state(undefined);
|
||||
let definitions: Promise<Definitions[]> = $state(new Promise(() => {}));
|
||||
|
||||
let id: string | undefined = $state()
|
||||
let id: string | undefined = $state();
|
||||
|
||||
async function getModel() {
|
||||
try {
|
||||
let temp_model: Model = await get(`models/edit?id=${id}`);
|
||||
|
||||
console.log(temp_model)
|
||||
|
||||
if (temp_model.status == 3) {
|
||||
if ([3, 6].includes(temp_model.status)) {
|
||||
setTimeout(getModel, 2000);
|
||||
}
|
||||
|
||||
@ -60,11 +62,12 @@
|
||||
}
|
||||
|
||||
model = Promise.resolve(temp_model);
|
||||
_model = temp_model;
|
||||
} catch (e) {
|
||||
if (e instanceof Response) {
|
||||
model = Promise.reject(await e.json())
|
||||
model = Promise.reject(await e.json());
|
||||
} else {
|
||||
model = Promise.reject("Could not load model");
|
||||
model = Promise.reject('Could not load model');
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -73,7 +76,7 @@
|
||||
let url = new URLSearchParams(window.location.search);
|
||||
const _id = url.get('id');
|
||||
if (!_id) {
|
||||
goto('/models')
|
||||
goto('/models');
|
||||
return;
|
||||
}
|
||||
|
||||
@ -84,21 +87,21 @@
|
||||
let resetMessages: MessageSimple;
|
||||
|
||||
async function resetModel() {
|
||||
resetMessages.display("");
|
||||
resetMessages.display('');
|
||||
|
||||
let _model = await model;
|
||||
|
||||
try {
|
||||
await rdelete('models/train/reset', {
|
||||
"id": _model.id
|
||||
id: _model.id
|
||||
});
|
||||
|
||||
getModel();
|
||||
} catch (e) {
|
||||
if (e instanceof Response) {
|
||||
resetMessages.display(await e.json())
|
||||
resetMessages.display(await e.json());
|
||||
} else {
|
||||
resetMessages.display("Could not reset model!")
|
||||
resetMessages.display('Could not reset model!');
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -127,6 +130,29 @@
|
||||
-->
|
||||
|
||||
<main>
|
||||
<Tabs active="model" let:isActive nobox>
|
||||
<div slot="buttons" let:setActive let:isActive>
|
||||
<button
|
||||
class="tab"
|
||||
on:click|preventDefault={setActive('model')}
|
||||
class:selected={isActive('model')}
|
||||
>
|
||||
Model
|
||||
</button>
|
||||
{#if _model && [2, 3, 4, 5, 6, 7].includes(_model.status)}
|
||||
<button
|
||||
class="tab"
|
||||
on:click|preventDefault={setActive('model-data')}
|
||||
class:selected={isActive('model-data')}
|
||||
>
|
||||
Model Data
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
{#if _model}
|
||||
<ModelDataPage model={_model} on:reload={getModel} active={isActive('model-data')} />
|
||||
{/if}
|
||||
<div class="content" class:selected={isActive('model')}>
|
||||
{#await model}
|
||||
Loading
|
||||
{:then m}
|
||||
@ -285,9 +311,7 @@
|
||||
<BaseModelInfo model={m} />
|
||||
<RunModel model={m} />
|
||||
{#if m.status == 6}
|
||||
<div class="card">
|
||||
Model expading... Processing ZIP file
|
||||
</div>
|
||||
<div class="card">Model expading... Processing ZIP file</div>
|
||||
{/if}
|
||||
{#if m.status == -6}
|
||||
<DeleteZip model={m} on:reload={getModel} expand />
|
||||
@ -295,18 +319,19 @@
|
||||
{#if m.status == -7}
|
||||
<form>
|
||||
<!-- TODO add more info about the failure -->
|
||||
Failed to train the model!
|
||||
Try to retrain
|
||||
Failed to train the model! Try to retrain
|
||||
</form>
|
||||
{/if}
|
||||
{#if m.model_type == 2}
|
||||
<ModelData model={m} on:reload={getModel} />
|
||||
<ModelData simple model={m} on:reload={getModel} />
|
||||
{/if}
|
||||
<DeleteModel model={m} />
|
||||
{:else}
|
||||
<h1>Unknown Status of the model.</h1>
|
||||
{/if}
|
||||
{/await}
|
||||
</div>
|
||||
</Tabs>
|
||||
</main>
|
||||
|
||||
<style lang="scss">
|
||||
|
||||
@ -3,19 +3,20 @@
|
||||
name: string;
|
||||
id: string;
|
||||
status: number;
|
||||
}
|
||||
};
|
||||
</script>
|
||||
<script lang="ts">
|
||||
import FileUpload from "src/lib/FileUpload.svelte";
|
||||
import Tabs from "src/lib/Tabs.svelte";
|
||||
import type { Model } from "./+page.svelte";
|
||||
import { postFormData, get } from "src/lib/requests.svelte";
|
||||
import MessageSimple from "src/lib/MessageSimple.svelte";
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import ModelTable from "./ModelTable.svelte";
|
||||
import TrainModel from "./TrainModel.svelte";
|
||||
|
||||
let { model } = $props<{model: Model}>();
|
||||
<script lang="ts">
|
||||
import FileUpload from 'src/lib/FileUpload.svelte';
|
||||
import Tabs from 'src/lib/Tabs.svelte';
|
||||
import type { Model } from './+page.svelte';
|
||||
import { postFormData, get } from 'src/lib/requests.svelte';
|
||||
import MessageSimple from 'src/lib/MessageSimple.svelte';
|
||||
import { createEventDispatcher } from 'svelte';
|
||||
import ModelTable from './ModelTable.svelte';
|
||||
import TrainModel from './TrainModel.svelte';
|
||||
|
||||
let { model, simple } = $props<{ model: Model; simple?: boolean }>();
|
||||
|
||||
let classes: Class[] = $state([]);
|
||||
let has_data: boolean = $state(false);
|
||||
@ -23,10 +24,10 @@
|
||||
let file: File | undefined = $state();
|
||||
|
||||
const dispatch = createEventDispatcher<{
|
||||
reload: void
|
||||
reload: void;
|
||||
}>();
|
||||
|
||||
let uploading: Promise<void> = $state(Promise.resolve())
|
||||
let uploading: Promise<void> = $state(Promise.resolve());
|
||||
let numberOfInvalidImages = $state(0);
|
||||
|
||||
let uploadImage: MessageSimple;
|
||||
@ -62,47 +63,48 @@
|
||||
if (!model) return;
|
||||
try {
|
||||
let data = await get(`models/edit/classes?id=${model.id}`);
|
||||
classes = data.classes
|
||||
classes = data.classes;
|
||||
numberOfInvalidImages = data.number_of_invalid_images;
|
||||
has_data = data.has_data;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<div class="card">
|
||||
<h3>
|
||||
Training data
|
||||
</h3>
|
||||
{#if !simple}
|
||||
<div class="card">
|
||||
<h3>Training data</h3>
|
||||
{#if classes.length == 0}
|
||||
<p>
|
||||
You need to upload data so the model can train.
|
||||
</p>
|
||||
<p>You need to upload data so the model can train.</p>
|
||||
<Tabs active="upload" let:isActive>
|
||||
<div slot="buttons" let:setActive let:isActive>
|
||||
<button class="tab" class:selected={isActive("upload")} on:click={setActive("upload")}>
|
||||
<button class="tab" class:selected={isActive('upload')} on:click={setActive('upload')}>
|
||||
Upload
|
||||
</button>
|
||||
<button class="tab" class:selected={isActive("create-class")} on:click={setActive("create-class")}>
|
||||
<button
|
||||
class="tab"
|
||||
class:selected={isActive('create-class')}
|
||||
on:click={setActive('create-class')}
|
||||
>
|
||||
Create Class
|
||||
</button>
|
||||
<button class="tab" class:selected={isActive("api")} on:click={setActive("api")}>
|
||||
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
|
||||
Api
|
||||
</button>
|
||||
</div>
|
||||
<div class="content" class:selected={isActive("upload")}>
|
||||
<div class="content" class:selected={isActive('upload')}>
|
||||
<form on:submit|preventDefault={uploadZip}>
|
||||
<fieldset class="file-upload" >
|
||||
<fieldset class="file-upload">
|
||||
<label for="file">Data file</label>
|
||||
<div class="form-msg">
|
||||
Please provide a file that has the training and testing data<br/>
|
||||
The file must have 2 folders one with testing images and one with training images. <br/>
|
||||
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
|
||||
The class folders must have the images for the classes.
|
||||
<pre>
|
||||
training\
|
||||
Please provide a file that has the training and testing data<br />
|
||||
The file must have 2 folders one with testing images and one with training images.
|
||||
<br />
|
||||
Each of the folders will contain the classes of the model. The folders must be the same
|
||||
in testing and training. The class folders must have the images for the classes.
|
||||
<pre>
|
||||
training\
|
||||
class1\
|
||||
img1.png
|
||||
img2.png
|
||||
@ -114,7 +116,7 @@
|
||||
img2.png
|
||||
...
|
||||
...
|
||||
testing\
|
||||
testing\
|
||||
class1\
|
||||
img1.png
|
||||
img2.png
|
||||
@ -128,70 +130,66 @@
|
||||
...
|
||||
</pre>
|
||||
</div>
|
||||
<FileUpload replace_slot bind:file={file} accept="application/zip" notExpand >
|
||||
<FileUpload replace_slot bind:file accept="application/zip" notExpand>
|
||||
<img src="/imgs/upload-icon.png" alt="" />
|
||||
<span>
|
||||
Upload Zip File
|
||||
</span>
|
||||
<span> Upload Zip File </span>
|
||||
<div slot="replaced" style="display: inline;">
|
||||
<img src="/imgs/upload-icon.png" alt="" />
|
||||
<span>
|
||||
File selected
|
||||
</span>
|
||||
<span> File selected </span>
|
||||
</div>
|
||||
</FileUpload>
|
||||
</fieldset>
|
||||
<MessageSimple bind:this={uploadImage} />
|
||||
{#if file}
|
||||
{#await uploading}
|
||||
<button disabled>
|
||||
Uploading
|
||||
</button>
|
||||
<button disabled> Uploading </button>
|
||||
{:then}
|
||||
<button>
|
||||
Add
|
||||
</button>
|
||||
<button> Add </button>
|
||||
{/await}
|
||||
{/if}
|
||||
</form>
|
||||
</div>
|
||||
<div class="content" class:selected={isActive("create-class")}>
|
||||
<div class="content" class:selected={isActive('create-class')}>
|
||||
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
|
||||
</div>
|
||||
<div class="content" class:selected={isActive("api")}>
|
||||
TODO
|
||||
</div>
|
||||
<div class="content" class:selected={isActive('api')}>TODO</div>
|
||||
</Tabs>
|
||||
<div class="tabs">
|
||||
</div>
|
||||
<div class="tabs"></div>
|
||||
{:else}
|
||||
<p>
|
||||
You need to upload data so the model can train.
|
||||
</p>
|
||||
<p>You need to upload data so the model can train.</p>
|
||||
{#if numberOfInvalidImages > 0}
|
||||
<p class="danger">
|
||||
There are images {numberOfInvalidImages} that were loaded that do not have the correct format. These images will be delete when the model trains.
|
||||
There are images {numberOfInvalidImages} that were loaded that do not have the correct format.
|
||||
These images will be delete when the model trains.
|
||||
</p>
|
||||
{/if}
|
||||
<Tabs active="create-class" let:isActive>
|
||||
<div slot="buttons" let:setActive let:isActive>
|
||||
<button class="tab" class:selected={isActive("create-class")} on:click={setActive("create-class")}>
|
||||
<button
|
||||
class="tab"
|
||||
class:selected={isActive('create-class')}
|
||||
on:click={setActive('create-class')}
|
||||
>
|
||||
Create Class
|
||||
</button>
|
||||
<button class="tab" class:selected={isActive("api")} on:click={setActive("api")}>
|
||||
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
|
||||
Api
|
||||
</button>
|
||||
</div>
|
||||
<div class="content" class:selected={isActive("create-class")}>
|
||||
<div class="content" class:selected={isActive('create-class')}>
|
||||
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
|
||||
</div>
|
||||
<div class="content" class:selected={isActive("api")}>
|
||||
TODO
|
||||
</div>
|
||||
<div class="content" class:selected={isActive('api')}>TODO</div>
|
||||
</Tabs>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
{#if classes.some((item) => item.status == 1) && ![-6, 6].includes(model.status)}
|
||||
<TrainModel number_of_invalid_images={numberOfInvalidImages} {model} {has_data} on:reload={() => dispatch('reload')} />
|
||||
<TrainModel
|
||||
number_of_invalid_images={numberOfInvalidImages}
|
||||
{model}
|
||||
{has_data}
|
||||
on:reload={() => dispatch('reload')}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
48
webpage/src/routes/models/edit/ModelDataPage.svelte
Normal file
48
webpage/src/routes/models/edit/ModelDataPage.svelte
Normal file
@ -0,0 +1,48 @@
|
||||
<script lang="ts">
|
||||
import { createEventDispatcher } from "svelte";
|
||||
import type { Model } from "./+page.svelte";
|
||||
import ModelData from "./ModelData.svelte";
|
||||
import { post } from "src/lib/requests.svelte";
|
||||
import ModelDataPageStatsGraph from "./ModelDataPageStatsGraph.svelte";
|
||||
import type { ModelStats } from "./types";
|
||||
import DeleteZip from "./DeleteZip.svelte";
|
||||
|
||||
let { model, active } = $props<{model: Model, active?: boolean}>()
|
||||
|
||||
const dispatch = createEventDispatcher<{ reload: void }>();
|
||||
|
||||
$effect(() => {
|
||||
if (active)
|
||||
getData();
|
||||
});
|
||||
|
||||
let stats: ModelStats | undefined = $state();
|
||||
|
||||
async function getData() {
|
||||
if (!model) return;
|
||||
try {
|
||||
stats = await post(`models/class/stats`, { id: model.id });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<div class="content" class:selected={active}>
|
||||
|
||||
{#if stats}
|
||||
<ModelDataPageStatsGraph data={stats} />
|
||||
{/if}
|
||||
|
||||
|
||||
{#if [-6, -2].includes(model.status)}
|
||||
<DeleteZip model={model} on:reload={() => dispatch('reload')} expand />
|
||||
{/if}
|
||||
|
||||
<ModelData model={model} on:reload={() => {
|
||||
getData();
|
||||
dispatch('reload');
|
||||
}} />
|
||||
</div>
|
||||
|
||||
@ -0,0 +1,89 @@
|
||||
<script lang="ts">
|
||||
import {
|
||||
Chart,
|
||||
Title,
|
||||
Tooltip,
|
||||
Legend,
|
||||
BarElement,
|
||||
CategoryScale,
|
||||
LinearScale,
|
||||
BarController,
|
||||
Colors
|
||||
} from 'chart.js';
|
||||
import type { ModelStats } from './types';
|
||||
|
||||
Chart.register(
|
||||
Title,
|
||||
Tooltip,
|
||||
Legend,
|
||||
BarElement,
|
||||
CategoryScale,
|
||||
LinearScale,
|
||||
BarController,
|
||||
Colors
|
||||
);
|
||||
|
||||
let { data } = $props<{ data: ModelStats }>();
|
||||
|
||||
let ctx: HTMLCanvasElement;
|
||||
|
||||
let chart: Chart;
|
||||
|
||||
$effect(() => {
|
||||
if (data && ctx) {
|
||||
if (chart) {
|
||||
console.log("update")
|
||||
chart.data = {
|
||||
labels: data.map((a) => a.name),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Training',
|
||||
data: data.map((a) => a.training)
|
||||
},
|
||||
{
|
||||
label: 'Testing',
|
||||
data: data.map((a) => a.testing)
|
||||
}
|
||||
]
|
||||
};
|
||||
chart.update('resize');
|
||||
} else {
|
||||
console.log("create")
|
||||
chart = new Chart(ctx, {
|
||||
type: 'bar',
|
||||
data: {
|
||||
labels: data.map((a) => a.name),
|
||||
datasets: [
|
||||
{
|
||||
label: 'Training',
|
||||
data: data.map((a) => a.training)
|
||||
},
|
||||
{
|
||||
label: 'Testing',
|
||||
data: data.map((a) => a.testing)
|
||||
}
|
||||
]
|
||||
},
|
||||
options: {
|
||||
scales: {
|
||||
x: {
|
||||
stacked: true
|
||||
},
|
||||
y: {
|
||||
stacked: true
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
||||
<div><canvas bind:this={ctx} /></div>
|
||||
|
||||
<style lang="scss">
|
||||
canvas {
|
||||
width: 100%;
|
||||
}
|
||||
</style>
|
||||
@ -44,8 +44,6 @@
|
||||
let res = await get('models/data/list?' + url.toString());
|
||||
showNext = res.showNext;
|
||||
image_list = res.image_list;
|
||||
|
||||
console.log(image_list);
|
||||
} catch (e) {
|
||||
console.error('TODO notify user', e);
|
||||
}
|
||||
|
||||
5
webpage/src/routes/models/edit/types.ts
Normal file
5
webpage/src/routes/models/edit/types.ts
Normal file
@ -0,0 +1,5 @@
|
||||
export type ModelStats = Array<{
|
||||
name: string,
|
||||
training: number,
|
||||
testing: number,
|
||||
}>
|
||||
Loading…
Reference in New Issue
Block a user