andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

Compare commits

base: andr3:torch
Branches Tags
andr3:main
andr3:runner
andr3:torch
..
compare: andr3:main
Branches Tags
andr3:main
andr3:runner
andr3:torch

7 Commits
torch ... main

Author SHA1 Message Date
Andre Henriques
652542d261 Improved classification performance 2024-05-15 05:32:49 +01:00
Andre Henriques
516d1d7634 added docker compose to run everything in one go 2024-05-12 15:29:36 +01:00
Andre Henriques
0c0d16c846 multiple fixes 2024-05-11 01:11:07 +01:00
Andre Henriques
0ac6ac8dce runner-go (#102) 
Reviewed-on: #102
Co-authored-by: Andre Henriques <andr3h3nriqu3s@gmail.com>
Co-committed-by: Andre Henriques <andr3h3nriqu3s@gmail.com>
 2024-05-10 02:13:02 +01:00
Andre Henriques
edd1e4c123 fix tensorflow version 2024-05-06 18:17:15 +01:00
Andre Henriques
7d346ba2ce closes #101 2024-05-02 16:58:01 +01:00
Andre Henriques
29b69deaf6 Fixed the model not training and running forever 2024-04-19 22:03:14 +01:00
100 changed files with 7411 additions and 4227 deletions
Show Stats Expand all files Collapse all files

5
.dockerignore
View File

@ -1,5 +0,0 @@
tmp/
testData/
savedData/
!savedData/.keep
fyp

2
DockerfileNginxDev → DockerfileProxy
View File

@ -1,6 +1,6 @@
# vi: ft=dockerfile
FROM docker.io/nginx
ADD nginx.dev.conf /nginx.conf
ADD nginx.proxy.conf /nginx.conf
CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"]

62
DockerfileServer
View File

@ -1,60 +1,40 @@
FROM docker.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
FROM docker.io/nvidia/cuda:12.3.2-devel-ubuntu22.04
ENV DEBIAN_FRONTEND=noninteractive
# Sometimes you have to get update twice because ?
RUN apt-get update
RUN apt-get update
RUN apt-get install -y wget sudo pkg-config libopencv-dev unzip python3-pip vim
RUN pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0
RUN mkdir /go
ENV GOPATH=/go
RUN apt-get install -y wget unzip python3-pip vim python3 python3-pip curl
RUN wget https://go.dev/dl/go1.22.2.linux-amd64.tar.gz
RUN tar -xvf go1.22.2.linux-amd64.tar.gz -C /usr/local
ENV PATH=$PATH:/usr/local/go/bin
ENV GOPATH=/go
RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.9.1.tar.gz" | tar -C /usr/local -xz'
RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.15.0.tar.gz" | tar -C /usr/local -xz'
RUN ldconfig
RUN ln -s /usr/bin/python3 /usr/bin/python
RUN python -m pip install nvidia-pyindex
ADD requirements.txt .
RUN python -m pip install -r requirements.txt
ENV CUDNN_PATH=/usr/local/lib/python3.10/dist-packages/nvidia/cudnn
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.10/dist-packages/nvidia/cudnn/lib
RUN mkdir /app
WORKDIR /app
ADD go.mod .
ADD go.sum .
ADD main.go .
ADD logic logic
ADD entrypoint.sh .
RUN go install || true
WORKDIR /root
RUN go build .
RUN wget https://git.andr3h3nriqu3s.com/andr3/gotch/raw/commit/22e75becf0432cda41a7c055a4d60ea435f76599/setup-libtorch.sh
RUN chmod +x setup-libtorch.sh
ENV CUDA_VER=11.8
ENV GOTCH_VER=v0.9.2
RUN bash setup-libtorch.sh
ENV GOTCH_LIBTORCH="/usr/local/lib/libtorch"
ENV REFRESH_SETUP=0
ENV LIBRARY_PATH="$LIBRARY_PATH:$GOTCH_LIBTORCH/lib"
ENV export CPATH="$CPATH:$GOTCH_LIBTORCH/lib:$GOTCH_LIBTORCH/include:$GOTCH_LIBTORCH/include/torch/csrc/api/include"
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GOTCH_LIBTORCH/lib:/usr/lib64-nvidia:/usr/local/cuda-${CUDA_VERSION}/lib64"
RUN wget https://git.andr3h3nriqu3s.com/andr3/gotch/raw/branch/master/setup-gotch.sh
RUN chmod +x setup-gotch.sh
RUN echo 'root ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
RUN bash setup-gotch.sh
RUN ln -s /usr/local/lib/libtorch/include/torch/csrc /usr/local/lib/libtorch/include/torch/csrc/api/include/torch
RUN mkdir -p /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch/csrc/api
RUN find /usr/local/lib/libtorch/include -maxdepth 1 -type d | tail -n +2 | grep -ve 'torch$' | xargs -I{} ln -s {} /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include
RUN ln -s /usr/local/lib/libtorch/include/torch/csrc/api/include /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch/csrc/api/include
RUN find /usr/local/lib/libtorch/include/torch -maxdepth 1 -type f | xargs -I{} ln -s {} /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch
RUN ln -s /usr/local/lib/libtorch/lib/libcudnn.so.8 /usr/local/lib/libcudnn.so
WORKDIR /app
ENV CGO_CXXFLAGS="-I/usr/local/lib/libtorch/include/torch/csrc/api/include/ -I/usr/local/lib/libtorch/include"
ENV CGO_CFLAGS="-I/usr/local/lib/libtorch/include/torch/csrc/api/include/ -I/usr/local/lib/libtorch/include"
ADD . .
RUN go build -x || true
CMD ["bash", "-c", "go run ."]
CMD ["./entrypoint.sh"]

42
README.md Normal file
View File

@ -0,0 +1,42 @@
# Configure the system
Go to the config.toml file and setup your hostname
# Build the containers
Running this commands on the root of the project will setup the nessesary.
Make sure that your docker/podman installation supports domain name resolution between containers
```bash
docker build -t andre-fyp-proxy -f DockerfileProxy
docker build -t andre-fyp-server -f DockerfileServer
cd webpage
docker build -t andre-fyp-web-server .
cd ..
```
# Run the docker compose
Running docker compose sets up the database server, the web page server, the proxy server and the main server
```bash
docker compose up
```
# Setup the Database
On another terminal instance create the database and tables.
Note: the password can be changed in the docker-compose file
```bash
PGPASSWORD=verysafepassword psql -h localhost -U postgres -f sql/base.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/user.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/models.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/tasks.sql
```
# Restart docker compose
Now restart docker compose and the system should be available under the domain name set up on the config.toml file

7
config.toml
View File

@ -12,7 +12,12 @@ USER = "service"
[Worker]
PULLING_TIME = "500ms"
NUMBER_OF_WORKERS = 1
NUMBER_OF_WORKERS = 16
[DB]
MAX_CONNECTIONS = 600
host = "db"
port = 5432
user = "postgres"
password = "verysafepassword"
dbname = "fyp"

53
docker-compose.yml
View File

@ -1,11 +1,44 @@
version: "3.1"
services:
db:
image: docker.andr3h3nriqu3s.com/services/postgres
command: -c 'max_connections=600'
restart: always
environment:
POSTGRES_PASSWORD: verysafepassword
ports:
- "5432:5432"
db:
image: docker.io/postgres:16.3
command: -c 'max_connections=600'
restart: always
networks:
- fyp-network
environment:
POSTGRES_PASSWORD: verysafepassword
ports:
- "5432:5432"
web-page:
image: andre-fyp-web-server
hostname: webpage
networks:
- fyp-network
server:
image: andre-fyp-server
hostname: server
networks:
- fyp-network
depends_on:
- db
volumes:
- "./config.toml:/app/config.toml"
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: 1
capabilities: [gpu]
proxy-server:
image: andre-fyp-proxy
networks:
- fyp-network
ports:
- "8000:8000"
depends_on:
- web-page
- server
networks:
fyp-network: {}

4
entrypoint.sh Executable file
View File

@ -0,0 +1,4 @@
#/bin/bash
while true; do
./fyp
done

7
go.mod
View File

@ -4,14 +4,16 @@ go 1.21
require (
github.com/charmbracelet/log v0.3.1
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99
github.com/google/uuid v1.6.0
github.com/lib/pq v1.10.9
golang.org/x/crypto v0.19.0
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2
github.com/BurntSushi/toml v1.3.2
github.com/goccy/go-json v0.10.2
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.9.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@ -19,7 +21,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.19.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx v3.6.2+incompatible // indirect

12
go.sum
View File

@ -1,7 +1,3 @@
git.andr3h3nriqu3s.com/andr3/gotch v0.9.1 h1:1q34JKV8cX80n7LXbJswlXCiRtNbzcvJ/vbgb6an1tA=
git.andr3h3nriqu3s.com/andr3/gotch v0.9.1/go.mod h1:FXusE3CHt8NLf5wynUGaHtIbToRuYifsZaC5EZH0pJY=
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2 h1:aZcsPgDVGVhrEFoer0upSkzPqJWNMxdUHRktP4s6MSc=
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2/go.mod h1:FXusE3CHt8NLf5wynUGaHtIbToRuYifsZaC5EZH0pJY=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
@ -17,6 +13,12 @@ github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3z
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e h1:9+2AEFZymTi25FIIcDwuzcOPH04z9+fV6XeLiGORPDI=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe h1:7yELf1NFEwECpXMGowkoftcInMlVtLTCdwWLmxKgzNM=
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99 h1:8Bt1P/zy1gb37L4n8CGgp1qmFwBV5729kxVfj0sqhJk=
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99/go.mod h1:3YgYBeIX42t83uP27Bd4bSMxTnQhSbxl0pYSkCDB1tc=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@ -72,9 +74,7 @@ github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg=
github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=

1
lib
View File

@ -1 +0,0 @@
/usr/local/lib

10
lib.go.back
View File

@ -1,10 +0,0 @@
package libtch
// #cgo LDFLAGS: -lstdc++ -ltorch -lc10 -ltorch_cpu -L${SRCDIR}/libtorch/lib
// #cgo LDFLAGS: -L/usr/local/cuda/lib64 -lcuda -lcudart -lcublas -lcudnn -lcaffe2_nvrtc -lnvrtc-builtins -lnvrtc -lnvToolsExt -lc10_cuda -ltorch_cuda
// #cgo CFLAGS: -I${SRCDIR} -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
// #cgo CFLAGS: -D_GLIBCXX_USE_CXX11_ABI=0
// #cgo CFLAGS: -I/usr/local/cuda/include
// #cgo CXXFLAGS: -std=c++17 -I${SRCDIR} -g -O3
// #cgo CXXFLAGS: -I${SRCDIR}/libtorch/lib -I${SRCDIR}/libtorch/include -I${SRCDIR}/libtorch/include/torch/csrc/api/include -I/opt/libtorch/include/torch/csrc/api/include
import "C"

10
logic/db_types/classes.go
View File

@ -16,9 +16,9 @@ const (
)
type ModelClass struct {
Id string `db:"mc.id"`
ModelId string `db:"mc.model_id"`
Name string `db:"mc.name"`
ClassOrder int `db:"mc.class_order"`
Status int `db:"mc.status"`
Id string `db:"mc.id" json:"id"`
ModelId string `db:"mc.model_id" json:"model_id"`
Name string `db:"mc.name" json:"name"`
ClassOrder int `db:"mc.class_order" json:"class_order"`
Status int `db:"mc.status" json:"status"`
}

20
logic/db_types/definitions.go
View File

@ -20,14 +20,14 @@ const (
)
type Definition struct {
Id string `db:"md.id"`
ModelId string `db:"md.model_id"`
Accuracy float64 `db:"md.accuracy"`
TargetAccuracy int `db:"md.target_accuracy"`
Epoch int `db:"md.epoch"`
Status int `db:"md.status"`
CreatedOn time.Time `db:"md.created_on"`
EpochProgress int `db:"md.epoch_progress"`
Id string `db:"md.id" json:"id"`
ModelId string `db:"md.model_id" json:"model_id"`
Accuracy float64 `db:"md.accuracy" json:"accuracy"`
TargetAccuracy int `db:"md.target_accuracy" json:"target_accuracy"`
Epoch int `db:"md.epoch" json:"epoch"`
Status int `db:"md.status" json:"status"`
CreatedOn time.Time `db:"md.created_on" json:"created"`
EpochProgress int `db:"md.epoch_progress" json:"epoch_progress"`
}
type SortByAccuracyDefinitions []*Definition
@ -87,9 +87,9 @@ func (d Definition) GetLayers(db db.Db, filter string, args ...any) (layer []*La
return GetDbMultitple[Layer](db, "model_definition_layer as mdl where mdl.def_id=$1 "+filter, args...)
}
func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64) (err error) {
func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64, epoch int) (err error) {
d.Accuracy = accuracy
d.Epoch += 1
d.Epoch += epoch
_, err = db.Exec("update model_definition set epoch=$1, accuracy=$2 where id=$3", d.Epoch, d.Accuracy, d.Id)
return
}

31
logic/db_types/layer.go
View File

@ -2,8 +2,10 @@ package dbtypes
import (
"encoding/json"
"fmt"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
"github.com/charmbracelet/log"
)
type LayerType int
@ -16,17 +18,30 @@ const (
)
type Layer struct {
Id string `db:"mdl.id"`
DefinitionId string `db:"mdl.def_id"`
LayerOrder string `db:"mdl.layer_order"`
LayerType LayerType `db:"mdl.layer_type"`
Shape string `db:"mdl.shape"`
ExpType string `db:"mdl.exp_type"`
Id string `db:"mdl.id" json:"id"`
DefinitionId string `db:"mdl.def_id" json:"definition_id"`
LayerOrder int `db:"mdl.layer_order" json:"layer_order"`
LayerType LayerType `db:"mdl.layer_type" json:"layer_type"`
Shape string `db:"mdl.shape" json:"shape"`
ExpType int `db:"mdl.exp_type" json:"exp_type"`
}
func (x *Layer) ShapeToSize() {
v := x.GetShape()
switch x.LayerType {
case LAYER_INPUT:
x.Shape = fmt.Sprintf("%d,%d", v[1], v[2])
case LAYER_DENSE:
x.Shape = fmt.Sprintf("(%d)", v[0])
default:
x.Shape = "ERROR"
}
}
func ShapeToString(args ...int) string {
text, err := json.Marshal(args)
if err != nil {
log.Error("json err!", "err", err)
panic("Could not generate Shape")
}
return string(text)
@ -35,12 +50,16 @@ func ShapeToString(args ...int) string {
func StringToShape(str string) (shape []int64) {
err := json.Unmarshal([]byte(str), &shape)
if err != nil {
log.Error("json err!", "err", err)
panic("Could not parse Shape")
}
return
}
func (l Layer) GetShape() []int64 {
if l.Shape == "" {
return []int64{}
}
return StringToShape(l.Shape)
}

131
logic/db_types/types.go
View File

@ -1,32 +1,44 @@
package dbtypes
import (
"errors"
"fmt"
"os"
"path"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
"github.com/jackc/pgx/v5"
)
const (
FAILED_TRAINING = -4
FAILED_PREPARING_TRAINING = -3
FAILED_PREPARING_ZIP_FILE = -2
FAILED_PREPARING = -1
type ModelStatus int
PREPARING = 1
CONFIRM_PRE_TRAINING = 2
PREPARING_ZIP_FILE = 3
TRAINING = 4
READY = 5
READY_ALTERATION = 6
READY_ALTERATION_FAILED = -6
const (
FAILED_TRAINING ModelStatus = -4
FAILED_PREPARING_TRAINING = -3
FAILED_PREPARING_ZIP_FILE = -2
FAILED_PREPARING = -1
PREPARING = 1
CONFIRM_PRE_TRAINING = 2
PREPARING_ZIP_FILE = 3
TRAINING = 4
READY = 5
READY_ALTERATION = 6
READY_ALTERATION_FAILED = -6
READY_RETRAIN = 7
READY_RETRAIN_FAILED = -7
)
type ModelDefinitionStatus int
const (
MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
MODEL_DEFINITION_STATUS_FAILED_TRAINING = -3
MODEL_DEFINITION_STATUS_PRE_INIT = 1
MODEL_DEFINITION_STATUS_INIT = 2
MODEL_DEFINITION_STATUS_TRAINING = 3
MODEL_DEFINITION_STATUS_PAUSED_TRAINING = 6
MODEL_DEFINITION_STATUS_TRANIED = 4
MODEL_DEFINITION_STATUS_READY = 5
)
type ModelHeadStatus int
const (
@ -38,19 +50,20 @@ const (
)
type BaseModel struct {
Name string
Status int
Id string
ModelType int `db:"model_type"`
ImageModeRaw string `db:"color_mode"`
ImageMode int `db:"0"`
Width int
Height int
Format string
CanTrain int `db:"can_train"`
Name string `json:"name"`
Status int `json:"status"`
Id string `json:"id"`
ModelType int `db:"model_type" json:"model_type"`
ImageModeRaw string `db:"color_mode" json:"image_more_raw"`
ImageMode int `db:"0" json:"image_mode"`
Width int `json:"width"`
Height int `json:"height"`
Format string `json:"format"`
CanTrain int `db:"can_train" json:"can_train"`
}
var ModelNotFoundError = errors.New("Model not found error")
func GetBaseModel(db db.Db, id string) (base *BaseModel, err error) {
var model BaseModel
err = GetDBOnce(db, &model, "models where id=$1", id)
@ -68,36 +81,6 @@ func (m BaseModel) CanEval() bool {
return true
}
func (m BaseModel) removeFailedDataPoints(c BasePack) (err error) {
rows, err := c.GetDb().Query("select mdp.id from model_data_point as mdp join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 and mdp.status=-1;", m.Id)
if err != nil {
return
}
defer rows.Close()
base_path := path.Join("savedData", m.Id, "data")
for rows.Next() {
var dataPointId string
err = rows.Scan(&dataPointId)
if err != nil {
return
}
p := path.Join(base_path, dataPointId+"."+m.Format)
c.GetLogger().Warn("Removing image", "path", p)
err = os.RemoveAll(p)
if err != nil {
return
}
}
_, err = c.GetDb().Exec("delete from model_data_point as mdp using model_classes as mc where mdp.class_id = mc.id and mc.model_id=$1 and mdp.status=-1;", m.Id)
return
}
// DO NOT Pass un filtered data on filters
func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
n_args := []any{m.Id}
@ -105,25 +88,21 @@ func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Def
return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
}
// DO NOT Pass un filtered data on filters
func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
n_args := []any{m.Id}
n_args = append(n_args, args...)
return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
}
type DataPointIterator struct {
rows pgx.Rows
Model BaseModel
func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
return
}
type DataPoint struct {
Class int
Path string
}
func (iter DataPointIterator) Close() {
iter.rows.Close()
Id string `json:"id"`
Class int `json:"class"`
Path string `json:"path"`
}
func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
@ -146,26 +125,20 @@ func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint,
if err = rows.Scan(&id, &class_order, &file_path); err != nil {
return
}
if file_path == "id://" {
data = append(data, DataPoint{
Path: path.Join("./savedData", m.Id, "data", id+"."+m.Format),
Class: class_order,
})
} else {
panic("TODO remote file path")
}
data = append(data, DataPoint{
Id: id,
Path: file_path,
Class: class_order,
})
}
return
}
const RGB string = "rgb"
const GRAY string = "greyscale"
func StringToImageMode(colorMode string) int {
switch colorMode {
case GRAY:
case "greyscale":
return 1
case RGB:
case "rgb":
return 3
default:
panic("unkown color mode")

17
logic/db_types/user.go
View File

@ -14,10 +14,19 @@ const (
)
type User struct {
Id string `db:"u.id"`
Username string `db:"u.username"`
Email string `db:"u.email"`
UserType int `db:"u.user_type"`
Id string `db:"u.id" json:"id"`
Username string `db:"u.username" json:"username"`
Email string `db:"u.email" json:"email"`
UserType int `db:"u.user_type" json:"user_type"`
}
func UserFromId(db db.Db, id string) (*User, error) {
var user User
err := GetDBOnce(db, &user, "users as u where u.id=$1", id)
if err != nil {
return nil, err
}
return &user, nil
}
func UserFromToken(db db.Db, token string) (*User, error) {

14
logic/db_types/utils.go
View File

@ -14,13 +14,11 @@ import (
"github.com/charmbracelet/log"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
db "git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
)
type BasePack interface {
db.Db
GetDb() db.Db
GetLogger() *log.Logger
GetHost() string
@ -44,18 +42,6 @@ func (b BasePackStruct) GetLogger() *log.Logger {
return b.Logger
}
func (c BasePackStruct) Query(query string, args ...any) (pgx.Rows, error) {
return c.Db.Query(query, args...)
}
func (c BasePackStruct) Exec(query string, args ...any) (pgconn.CommandTag, error) {
return c.Db.Exec(query, args...)
}
func (c BasePackStruct) Begin() (pgx.Tx, error) {
return c.Db.Begin()
}
func CheckEmpty(f url.Values, path string) bool {
return !f.Has(path) || f.Get(path) == ""
}

13
logic/models/add.go
View File

@ -16,7 +16,6 @@ import (
)
func loadBaseImage(c *Context, id string) {
// TODO handle more types than png
infile, err := os.Open(path.Join("savedData", id, "baseimage.png"))
if err != nil {
c.Logger.Errorf("Failed to read image for model with id %s\n", id)
@ -54,21 +53,29 @@ func loadBaseImage(c *Context, id string) {
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.RGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:
c.Logger.Error("Do not know how to handle this color model")
if src.ColorModel() == color.RGBA64Model {
c.Logger.Error("Color is rgb")
c.Logger.Error("Color is rgb 64")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Error("Color is nrgb 64")
} else if src.ColorModel() == color.AlphaModel {
c.Logger.Error("Color is alpha")
} else if src.ColorModel() == color.CMYKModel {
c.Logger.Error("Color is cmyk")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Error("Color is cmyk")
} else if src.ColorModel() == color.NYCbCrAModel {
c.Logger.Error("Color is cmyk a")
} else if src.ColorModel() == color.Alpha16Model {
c.Logger.Error("Color is cmyk a")
} else {
c.Logger.Error("Other so assuming color")
c.Logger.Error("Other so assuming color", "color mode", src.ColorModel())
}
ModelUpdateStatus(c, id, FAILED_PREPARING)

2
logic/models/classes/main.go
View File

@ -14,7 +14,7 @@ type ModelClassJSON struct {
Status int `json:"status"`
}
func ListClassesJSON(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
func ListClasses(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
return GetDbMultitple[ModelClassJSON](c.GetDb(), "model_classes where model_id=$1", model_id)
}

29
logic/models/data.go
View File

@ -136,17 +136,16 @@ func processZipFile(c *Context, model *BaseModel) {
return
}
if paths[0] != "training" {
if paths[0] == "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
} else if paths[0] == "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
c.Logger.Info("Diff", "testing", testing, "training", training)
failed("Testing and Training datesets are diferent")
return
c.Logger.Warn("Diff", "testing", testing, "training", training)
c.Logger.Warn("Testing and traing datasets differ")
}
base_path := path.Join("savedData", model.Id, "data")
@ -266,16 +265,15 @@ func processZipFileExpand(c *Context, model *BaseModel) {
return
}
if paths[0] != "training" {
if paths[0] == "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
} else if paths[0] == "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
failed("testing and training are diferent")
return
c.GetLogger().Warn("testing and training differ", "testing", testing, "training", training)
}
base_path := path.Join("savedData", model.Id, "data")
@ -435,7 +433,7 @@ func handleDataUpload(handle *Handle) {
}
model, err := GetBaseModel(handle.Db, id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
@ -468,7 +466,7 @@ func handleDataUpload(handle *Handle) {
}
PostAuthJson(handle, "/models/data/class/new", User_Normal, func(c *Context, obj *CreateNewEmptyClass) *Error {
model, err := GetBaseModel(c.Db, obj.Id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.E500M("Failed to get model information", err)
@ -518,7 +516,7 @@ func handleDataUpload(handle *Handle) {
c.Logger.Info("model", "model", *model_id)
model, err := GetBaseModel(c.Db, *model_id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.JsonBadRequest("Could not find the model")
} else if err != nil {
return c.E500M("Error getting model information", err)
@ -626,7 +624,7 @@ func handleDataUpload(handle *Handle) {
c.Logger.Info("Trying to expand model", "id", id)
model, err := GetBaseModel(handle.Db, id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
@ -636,7 +634,8 @@ func handleDataUpload(handle *Handle) {
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
c.GetLogger().Error("Model not in the ready status", "status", model.Status)
return c.JsonBadRequest("Model not in the correct state to add more classes")
}
// TODO mk this path configurable
@ -670,7 +669,7 @@ func handleDataUpload(handle *Handle) {
}
model, err := GetBaseModel(handle.Db, dat.Id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)

2
logic/models/delete.go
View File

@ -51,7 +51,7 @@ func handleDelete(handle *Handle) {
return c.E500M("Faield to get model", err)
}
switch model.Status {
switch ModelStatus(model.Status) {
case FAILED_TRAINING:
fallthrough
case FAILED_PREPARING_ZIP_FILE:

6
logic/models/edit.go
View File

@ -24,7 +24,7 @@ func handleEdit(handle *Handle) {
return c.Error500(err)
}
cls, err := model_classes.ListClassesJSON(c, model.Id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
return c.Error500(err)
}
@ -109,7 +109,7 @@ func handleEdit(handle *Handle) {
layers := []layerdef{}
for _, def := range defs {
if def.Status == DEFINITION_STATUS_TRAINING {
if def.Status == MODEL_DEFINITION_STATUS_TRAINING {
rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id)
if err != nil {
return c.Error500(err)
@ -166,7 +166,7 @@ func handleEdit(handle *Handle) {
for i, def := range defs {
var lay *[]layerdef = nil
if def.Status == DEFINITION_STATUS_TRAINING && !setLayers {
if def.Status == MODEL_DEFINITION_STATUS_TRAINING && !setLayers {
lay = &layers
setLayers = true
}

36
logic/models/run.go.notemp → logic/models/run.go
View File

@ -4,10 +4,12 @@ import (
"errors"
"os"
"path"
"runtime/debug"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
"github.com/charmbracelet/log"
tf "github.com/galeone/tensorflow/tensorflow/go"
"github.com/galeone/tensorflow/tensorflow/go/op"
tg "github.com/galeone/tfgo"
@ -19,6 +21,7 @@ func ReadPNG(scope *op.Scope, imagePath string, channels int64) *image.Image {
contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
output := op.DecodePng(scope.SubScope("DecodePng"), contents, op.DecodePngChannels(channels))
output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
image := &image.Image{
Tensor: tg.NewTensor(scope, output)}
return image.Scale(0, 255)
@ -29,16 +32,25 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
output := op.DecodePng(scope.SubScope("DecodeJpeg"), contents, op.DecodePngChannels(channels))
output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
image := &image.Image{
Tensor: tg.NewTensor(scope, output)}
return image.Scale(0, 255)
}
func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
func runModelNormal(model *BaseModel, def_id string, inputImage *tf.Tensor, data *RunnerModelData) (order int, confidence float32, err error) {
order = 0
err = nil
tf_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil)
var tf_model *tg.Model = nil
if data.Id != nil && *data.Id == def_id {
tf_model = data.Model
} else {
tf_model = tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil)
data.Model = tf_model
data.Id = &def_id
}
results := tf_model.Exec([]tf.Output{
tf_model.Op("StatefulPartitionedCall", 0),
@ -49,6 +61,8 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
var vmax float32 = 0.0
var predictions = results[0].Value().([][]float32)[0]
log.Info("preds", "preds", predictions)
for i, v := range predictions {
if v > vmax {
order = i
@ -62,10 +76,13 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
}
func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
log := base.GetLogger()
err = nil
order = 0
log.Info("Running base")
base_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "base", "model"), []string{"serve"}, nil)
//results := base_model.Exec([]tf.Output{
@ -86,7 +103,7 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
return
}
base.GetLogger().Info("test", "count", len(heads))
log.Info("Running heads", "heads", heads)
var vmax float32 = 0.0
@ -117,10 +134,15 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
return
}
func ClassifyTask(base BasePack, task Task) (err error) {
type RunnerModelData struct {
Id *string
Model *tg.Model
}
func ClassifyTask(base BasePack, task Task, data *RunnerModelData) (err error) {
defer func() {
if r := recover(); r != nil {
base.GetLogger().Error("Task failed due to", "error", r)
base.GetLogger().Error("Task failed due to", "error", r, "stack", string(debug.Stack()))
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Task failed running")
}
}()
@ -178,6 +200,8 @@ func ClassifyTask(base BasePack, task Task) (err error) {
if model.ModelType == 2 {
base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
data.Model = nil
data.Id = nil
vi, confidence, err = runModelExp(base, model, def_id, inputImage)
if err != nil {
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
@ -185,7 +209,7 @@ func ClassifyTask(base BasePack, task Task) (err error) {
}
} else {
base.GetLogger().Info("Running model normal", "model", model.Id, "def", def_id)
vi, confidence, err = runModelNormal(base, model, def_id, inputImage)
vi, confidence, err = runModelNormal(model, def_id, inputImage, data)
if err != nil {
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to run model")
return

2
logic/models/test.go
View File

@ -38,6 +38,8 @@ func TestImgForModel(c *Context, model *BaseModel, path string) (result bool) {
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.RGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:

118
logic/models/train/remote_train.go Normal file
View File

@ -0,0 +1,118 @@
package models_train
import (
"errors"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
"github.com/goccy/go-json"
)
func PrepareTraining(handler *Handle, b BasePack, task Task, runner_id string) (err error) {
l := b.GetLogger()
model, err := GetBaseModel(b.GetDb(), *task.ModelId)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
l.Error("Failed to get model information", "err", err)
return err
}
if model.Status != TRAINING {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
return errors.New("Model not in the right status")
}
// TODO do this when the runner says it's OK
//task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
// TODO move this to the runner part as well
var dat struct {
NumberOfModels int
Accuracy int
}
err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
}
if model.ModelType == 2 {
full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
l.Error("Failed to generate defintions", "err", full_error)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
} else {
error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if error != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
}
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
return
}
func CleanUpFailed(b BasePack, task *Task) {
db := b.GetDb()
l := b.GetLogger()
model, err := GetBaseModel(db, *task.ModelId)
if err != nil {
l.Error("Failed to get model", "err", err)
} else {
err = model.UpdateStatus(db, FAILED_TRAINING)
if err != nil {
l.Error("Failed to get status", err)
}
}
// Set the class status to trained
err = SetModelClassStatus(b, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to set class status")
return
}
}
func CleanUpFailedRetrain(b BasePack, task *Task) {
db := b.GetDb()
l := b.GetLogger()
model, err := GetBaseModel(db, *task.ModelId)
if err != nil {
l.Error("Failed to get model", "err", err)
} else {
err = model.UpdateStatus(db, FAILED_TRAINING)
if err != nil {
l.Error("Failed to get status", err)
}
}
ResetClasses(b, model)
ModelUpdateStatus(b, model.Id, READY_RETRAIN_FAILED)
var defData struct {
Id string `db:"md.id"`
TargetAcuuracy float64 `db:"md.target_accuracy"`
}
err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
if err != nil {
log.Error("failed to get def data", err)
return
}
_, err_ := db.Exec("delete from exp_model_head where def_id=$1 and status in (2,3)", defData.Id)
if err_ != nil {
panic(err_)
}
}

4
logic/models/train/reset.go
View File

@ -11,13 +11,13 @@ import (
func handleRest(handle *Handle) {
DeleteAuthJson(handle, "/models/train/reset", User_Normal, func(c *Context, dat *JustId) *Error {
model, err := GetBaseModel(c.Db, dat.Id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.E500M("Failed to get model", err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
if model.Status != FAILED_PREPARING_TRAINING && model.Status != int(FAILED_TRAINING) {
return c.JsonBadRequest("Model is not in status that be reset")
}

179
logic/models/train/torch/modelloader/modelloader.go
View File

@ -1,179 +0,0 @@
package imageloader
import (
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
types "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
"git.andr3h3nriqu3s.com/andr3/gotch"
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
"git.andr3h3nriqu3s.com/andr3/gotch/vision"
)
type Dataset struct {
TrainImages *torch.Tensor
TrainLabels *torch.Tensor
TestImages *torch.Tensor
TestLabels *torch.Tensor
TrainImagesSize int
TestImagesSize int
Device gotch.Device
}
func LoadImagesAndLables(db db.Db, m *types.BaseModel, mode types.DATA_POINT_MODE, classStart int, classEnd int) (imgs, labels *torch.Tensor, count int, err error) {
train_points, err := m.DataPoints(db, types.DATA_POINT_MODE_TRAINING)
if err != nil {
return
}
size := int64(classEnd - classStart + 1)
pimgs := []*torch.Tensor{}
plabels := []*torch.Tensor{}
for _, point := range train_points {
var img, label *torch.Tensor
img, err = vision.Load(point.Path)
if err != nil {
return
}
pimgs = append(pimgs, img)
t_label := make([]int, size)
if point.Class <= classEnd && point.Class >= classStart {
t_label[point.Class-classStart] = 1
}
label, err = torch.OfSlice(t_label)
if err != nil {
return
}
plabels = append(plabels, label)
}
imgs, err = torch.Concat(pimgs, 0)
if err != nil {
return
}
labels, err = torch.Stack(plabels, 0)
if err != nil {
return
}
count = len(pimgs)
imgs, err = torch.Stack(pimgs, 0)
if err != nil {
return
}
imgs, err = imgs.ToDtype(gotch.Float, false, false, true)
if err != nil {
return
}
labels, err = labels.ToDtype(gotch.Float, false, false, true)
if err != nil {
return
}
return
}
func NewDataset(db db.Db, m *types.BaseModel, classStart int, classEnd int) (ds *Dataset, err error) {
trainImages, trainLabels, train_count, err := LoadImagesAndLables(db, m, types.DATA_POINT_MODE_TRAINING, classStart, classEnd)
if err != nil {
return
}
testImages, testLabels, test_count, err := LoadImagesAndLables(db, m, types.DATA_POINT_MODE_TESTING, classStart, classEnd)
if err != nil {
return
}
ds = &Dataset{
TrainImages: trainImages,
TrainLabels: trainLabels,
TestImages: testImages,
TestLabels: testLabels,
TrainImagesSize: train_count,
TestImagesSize: test_count,
Device: gotch.CPU,
}
return
}
func (ds *Dataset) To(device gotch.Device) (err error) {
ds.TrainImages, err = ds.TrainImages.ToDevice(device, ds.TrainImages.DType(), device.IsCuda(), true, true)
if err != nil {
return
}
ds.TrainLabels, err = ds.TrainLabels.ToDevice(device, ds.TrainLabels.DType(), device.IsCuda(), true, true)
if err != nil {
return
}
ds.TestImages, err = ds.TestImages.ToDevice(device, ds.TestImages.DType(), device.IsCuda(), true, true)
if err != nil {
return
}
ds.TestLabels, err = ds.TestLabels.ToDevice(device, ds.TestLabels.DType(), device.IsCuda(), true, true)
if err != nil {
return
}
ds.Device = device
return
}
func (ds *Dataset) TestIter(batchSize int64) *torch.Iter2 {
return torch.MustNewIter2(ds.TestImages, ds.TestLabels, batchSize)
}
func (ds *Dataset) TrainIter(batchSize int64) (iter *torch.Iter2, err error) {
// Create a clone of the trainimages
size, err := ds.TrainImages.Size()
if err != nil {
return
}
train_images, err := torch.Zeros(size, gotch.Float, ds.Device)
if err != nil {
return
}
ds.TrainImages, err = ds.TrainImages.Clone(train_images, false)
if err != nil {
return
}
// Create a clone of the labels
size, err = ds.TrainLabels.Size()
if err != nil {
return
}
train_labels, err := torch.Zeros(size, gotch.Float, ds.Device)
if err != nil {
return
}
ds.TrainLabels, err = ds.TrainLabels.Clone(train_labels, false)
if err != nil {
return
}
iter, err = torch.NewIter2(train_images, train_labels, batchSize)
if err != nil {
return
}
return
}

174
logic/models/train/torch/nn/linear.go
View File

@ -1,174 +0,0 @@
package my_nn
// linear is a fully-connected layer
import (
"math"
"git.andr3h3nriqu3s.com/andr3/gotch/nn"
"git.andr3h3nriqu3s.com/andr3/gotch/ts"
"github.com/charmbracelet/log"
)
// LinearConfig is a configuration for a linear layer
type LinearConfig struct {
WsInit nn.Init // iniital weights
BsInit nn.Init // optional initial bias
Bias bool
}
// DefaultLinearConfig creates default LinearConfig with
// weights initiated using KaimingUniform and Bias is set to true
func DefaultLinearConfig() *LinearConfig {
negSlope := math.Sqrt(5)
return &LinearConfig{
// NOTE. KaimingUniform cause mem leak due to ts.Uniform()!!!
// Avoid using it now.
WsInit: nn.NewKaimingUniformInit(nn.WithKaimingNegativeSlope(negSlope)),
BsInit: nil,
Bias: true,
}
}
// Linear is a linear fully-connected layer
type Linear struct {
Ws *ts.Tensor
weight_name string
Bs *ts.Tensor
bias_name string
}
// NewLinear creates a new linear layer
// y = x*wT + b
// inDim - input dimension (x) [input features - columns]
// outDim - output dimension (y) [output features - columns]
// NOTE: w will have shape{outDim, inDim}; b will have shape{outDim}
func NewLinear(vs *Path, inDim, outDim int64, c *LinearConfig) *Linear {
var bias_name string
var bs *ts.Tensor
var err error
if c.Bias {
switch {
case c.BsInit == nil:
shape := []int64{inDim, outDim}
fanIn, _, err := nn.CalculateFans(shape)
or_panic(err)
bound := 0.0
if fanIn > 0 {
bound = 1 / math.Sqrt(float64(fanIn))
}
bsInit := nn.NewUniformInit(-bound, bound)
bs, bias_name, err = vs.NewVarNamed("bias", []int64{outDim}, bsInit)
or_panic(err)
// Find better way to do this
bs, err = bs.T(true)
or_panic(err)
bs, err = bs.T(true)
or_panic(err)
bs, err = bs.SetRequiresGrad(true, true)
or_panic(err)
err = bs.RetainGrad(false)
or_panic(err)
vs.varstore.UpdateVarTensor(bias_name, bs, true)
case c.BsInit != nil:
bs, bias_name, err = vs.NewVarNamed("bias", []int64{outDim}, c.BsInit)
or_panic(err)
}
}
ws, weight_name, err := vs.NewVarNamed("weight", []int64{outDim, inDim}, c.WsInit)
or_panic(err)
ws, err = ws.T(true)
or_panic(err)
ws, err = ws.SetRequiresGrad(true, true)
or_panic(err)
err = ws.RetainGrad(false)
or_panic(err)
vs.varstore.UpdateVarTensor(weight_name, ws, true)
return &Linear{
Ws: ws,
weight_name: weight_name,
Bs: bs,
bias_name: bias_name,
}
}
func (l *Linear) Debug() {
log.Info("Ws", "ws", l.Ws.MustGrad(false).MustMax(false).Float64Values())
log.Info("Bs", "bs", l.Bs.MustGrad(false).MustMax(false).Float64Values())
}
func (l *Linear) ExtractFromVarstore(vs *VarStore) {
l.Ws = vs.GetTensorOfVar(l.weight_name)
l.Bs = vs.GetTensorOfVar(l.bias_name)
}
// Implement `Module` for `Linear` struct:
// =======================================
// Forward proceeds input node through linear layer.
// NOTE:
// - It assumes that node has dimensions of 2 (matrix).
// To make it work for matrix multiplication, input node should
// has same number of **column** as number of **column** in
// `LinearLayer` `Ws` property as weights matrix will be
// transposed before multiplied to input node. (They are all used `inDim`)
// - Input node should have shape of `shape{batch size, input features}`.
// (shape{batchSize, inDim}). The input features is `inDim` while the
// output feature is `outDim` in `LinearConfig` struct.
//
// Example:
//
// inDim := 3
// outDim := 2
// batchSize := 4
// weights: 2x3
// [ 1 1 1
// 1 1 1 ]
//
// input node: 3x4
// [ 1 1 1
// 1 1 1
// 1 1 1
// 1 1 1 ]
func (l *Linear) Forward(xs *ts.Tensor) (retVal *ts.Tensor) {
mul, err := xs.Matmul(l.Ws, false)
or_panic(err)
if l.Bs != nil {
mul, err = mul.Add(l.Bs, false)
or_panic(err)
}
out, err := mul.Relu(false)
or_panic(err)
return out
}
// ForwardT implements ModuleT interface for Linear layer.
//
// NOTE: train param will not be used.
func (l *Linear) ForwardT(xs *ts.Tensor, train bool) (retVal *ts.Tensor) {
mul, err := xs.Matmul(l.Ws, true)
or_panic(err)
mul, err = mul.Add(l.Bs, true)
or_panic(err)
out, err := mul.Relu(true)
or_panic(err)
return out
}

603
logic/models/train/torch/nn/optimizer.go
View File

@ -1,603 +0,0 @@
package my_nn
// Optimizers to be used for gradient-descent based training.
import (
"fmt"
"math"
"github.com/charmbracelet/log"
"git.andr3h3nriqu3s.com/andr3/gotch/ts"
)
// Optimizer is a struct object to run gradient descent.
type Optimizer struct {
varstore *VarStore
opt *ts.COptimizer
// variablesInOptimizer uint8
variablesInOptimizer map[string]struct{}
config OptimizerConfig //interface{}
stepCount int
lr float64
}
func (o *Optimizer) Debug() {
for n, _ := range o.variablesInOptimizer {
v := o.varstore.GetVarOfName(n)
leaf, err := v.Tensor.IsLeaf(false)
or_panic(err)
retains, err := v.Tensor.RetainsGrad(false)
or_panic(err)
log.Info("[opt] var test", "n", n, "leaf", leaf, "retains", retains)
}
}
func (o *Optimizer) RefreshValues() (err error) {
opt, err := o.config.buildCOpt(o.lr)
if err != nil {
return
}
for name := range o.variablesInOptimizer {
v := o.varstore.GetVarOfName(name)
if v.Trainable {
if err = opt.AddParameter(v.Tensor, v.Group); err != nil {
err = fmt.Errorf("Optimizer defaultBuild - AddParameter failed: %w\n", err)
return
}
}
}
o.opt = opt
return
}
// OptimizerConfig defines Optimizer configurations. These configs can be used to build optimizer.
type OptimizerConfig interface {
buildCOpt(lr float64) (*ts.COptimizer, error)
// Build builds an optimizer with the specified learning rate handling variables stored in `vs`.
//
// NOTE: Build is a 'default' method. It can be called by wrapping
// 'DefaultBuild' function
// E.g. AdamOptimizerConfig struct have a method to fullfil `Build` method of
// OptimizerConfig by wrapping `DefaultBuild` like
// (config AdamOptimizerConfig) Build(vs VarStore, lr float64) (retVal Optimizer, err error){
// return defaultBuild(config, vs, lr)
// }
Build(vs *VarStore, lr float64) (*Optimizer, error)
}
// defaultBuild is `default` Build method for OptimizerConfig interface
func defaultBuild(config OptimizerConfig, vs *VarStore, lr float64) (*Optimizer, error) {
opt, err := config.buildCOpt(lr)
if err != nil {
return nil, err
}
names := make(map[string]struct{})
for name, v := range vs.vars {
if v.Trainable {
log.Info("Adding parameter", "name", name, "g", v.Group)
if err = opt.AddParameter(v.Tensor, v.Group); err != nil {
err = fmt.Errorf("Optimizer defaultBuild - AddParameter failed: %w\n", err)
return nil, err
}
}
names[name] = struct{}{}
}
return &Optimizer{
varstore: vs,
opt: opt,
variablesInOptimizer: names,
config: config,
stepCount: 0,
lr: 0,
}, nil
}
// SGD Optimizer:
//===============
// SGDConfig holds parameters for building the SGD (Stochastic Gradient Descent) optimizer.
type SGDConfig struct {
Momentum float64
Dampening float64
Wd float64
Nesterov bool
}
// DefaultSGDConfig creates SGDConfig with default values.
func DefaultSGDConfig() *SGDConfig {
return &SGDConfig{
Momentum: 0.0,
Dampening: 0.0,
Wd: 0.0,
Nesterov: false,
}
}
// NewSGD creates the configuration for a SGD optimizer with specified values
func NewSGDConfig(momentum, dampening, wd float64, nesterov bool) *SGDConfig {
return &SGDConfig{
Momentum: momentum,
Dampening: dampening,
Wd: wd,
Nesterov: nesterov,
}
}
// Implement OptimizerConfig interface for SGDConfig
func (c *SGDConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
return ts.Sgd(lr, c.Momentum, c.Dampening, c.Wd, c.Nesterov)
}
func (c *SGDConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
return defaultBuild(c, vs, lr)
}
// Adam optimizer:
// ===============
type AdamConfig struct {
Beta1 float64
Beta2 float64
Wd float64
}
// DefaultAdamConfig creates AdamConfig with default values
func DefaultAdamConfig() *AdamConfig {
return &AdamConfig{
Beta1: 0.9,
Beta2: 0.999,
Wd: 0.0,
}
}
// NewAdamConfig creates AdamConfig with specified values
func NewAdamConfig(beta1, beta2, wd float64) *AdamConfig {
return &AdamConfig{
Beta1: beta1,
Beta2: beta2,
Wd: wd,
}
}
// Implement OptimizerConfig interface for AdamConfig
func (c *AdamConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
return ts.Adam(lr, c.Beta1, c.Beta2, c.Wd)
}
func (c *AdamConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
return defaultBuild(c, vs, lr)
}
// AdamW optimizer:
// ===============
type AdamWConfig struct {
Beta1 float64
Beta2 float64
Wd float64
}
// DefaultAdamWConfig creates AdamWConfig with default values
func DefaultAdamWConfig() *AdamWConfig {
return &AdamWConfig{
Beta1: 0.9,
Beta2: 0.999,
Wd: 0.01,
}
}
// NewAdamWConfig creates AdamWConfig with specified values
func NewAdamWConfig(beta1, beta2, wd float64) *AdamWConfig {
return &AdamWConfig{
Beta1: beta1,
Beta2: beta2,
Wd: wd,
}
}
// Implement OptimizerConfig interface for AdamWConfig
func (c *AdamWConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
return ts.AdamW(lr, c.Beta1, c.Beta2, c.Wd)
}
// Build builds AdamW optimizer
func (c *AdamWConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
return defaultBuild(c, vs, lr)
}
// RMSProp optimizer:
// ===============
type RMSPropConfig struct {
Alpha float64
Eps float64
Wd float64
Momentum float64
Centered bool
}
// DefaultAdamConfig creates AdamConfig with default values
func DefaultRMSPropConfig() *RMSPropConfig {
return &RMSPropConfig{
Alpha: 0.99,
Eps: 1e-8,
Wd: 0.0,
Momentum: 0.0,
Centered: false,
}
}
// NewRMSPropConfig creates RMSPropConfig with specified values
func NewRMSPropConfig(alpha, eps, wd, momentum float64, centered bool) *RMSPropConfig {
return &RMSPropConfig{
Alpha: alpha,
Eps: eps,
Wd: wd,
Momentum: momentum,
Centered: centered,
}
}
// Implement OptimizerConfig interface for RMSPropConfig
func (c *RMSPropConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
return ts.RmsProp(lr, c.Alpha, c.Eps, c.Wd, c.Momentum, c.Centered)
}
func (c *RMSPropConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
return defaultBuild(c, vs, lr)
}
// Optimizer methods:
// ==================
func (opt *Optimizer) addMissingVariables() {
type param struct {
tensor *ts.Tensor
group uint
}
trainables := make(map[string]param)
for name, v := range opt.varstore.vars {
if v.Trainable {
trainables[name] = param{tensor: v.Tensor, group: v.Group}
}
}
missingVariables := len(trainables) - len(opt.variablesInOptimizer)
if missingVariables > 0 {
log.Info("INFO: Optimizer.addMissingVariables()...")
for name, x := range trainables {
if _, ok := opt.variablesInOptimizer[name]; !ok {
opt.opt.AddParameter(x.tensor, x.group)
opt.variablesInOptimizer[name] = struct{}{}
}
}
}
}
// ZeroGrad zeroes the gradient for the tensors tracked by this optimizer.
func (opt *Optimizer) ZeroGrad() error {
if err := opt.opt.ZeroGrad(); err != nil {
err = fmt.Errorf("Optimizer.ZeroGrad() failed: %w\n", err)
return err
}
return nil
}
// MustZeroGrad zeroes the gradient for the tensors tracked by this optimizer.
func (opt *Optimizer) MustZeroGrad() {
err := opt.ZeroGrad()
if err != nil {
log.Fatal(err)
}
}
// Clips gradient value at some specified maximum value.
func (opt *Optimizer) ClipGradValue(max float64) {
opt.varstore.Lock()
defer opt.varstore.Unlock()
for _, v := range opt.varstore.vars {
if v.Trainable {
// v.Tensor.MustGrad().Clamp_(ts.FloatScalar(-max), ts.FloatScalar(max))
gradTs := v.Tensor.MustGrad(false)
gradTs.Clamp_(ts.FloatScalar(-max), ts.FloatScalar(max))
}
}
}
// Step performs an optimization step, updating the tracked tensors based on their gradients.
func (opt *Optimizer) Step() error {
err := opt.opt.Step()
if err != nil {
err = fmt.Errorf("Optimizer.Step() failed: %w\n", err)
return err
}
opt.stepCount += 1
return nil
}
// MustStep performs an optimization step, updating the tracked tensors based on their gradients.
func (opt *Optimizer) MustStep() {
err := opt.Step()
if err != nil {
log.Fatal(err)
}
}
// ResetStepCount set step count to zero.
func (opt *Optimizer) ResetStepCount() {
opt.stepCount = 0
}
// StepCount get current step count.
func (opt *Optimizer) StepCount() int {
return opt.stepCount
}
// BackwardStep applies a backward step pass, update the gradients, and performs an optimization step.
func (opt *Optimizer) BackwardStep(loss *ts.Tensor) error {
err := opt.opt.ZeroGrad()
if err != nil {
err = fmt.Errorf("Optimizer.BackwardStep() failed: %w\n", err)
return err
}
loss.MustBackward()
err = opt.opt.Step()
if err != nil {
err = fmt.Errorf("Optimizer.BackwardStep() failed: %w\n", err)
return err
}
return nil
}
// MustBackwardStep applies a backward step pass, update the gradients, and performs an optimization step.
func (opt *Optimizer) MustBackwardStep(loss *ts.Tensor) {
err := opt.BackwardStep(loss)
if err != nil {
log.Fatal(err)
}
}
// BackwardStepClip applies a backward step pass, update the gradients, and performs an optimization step.
//
// The gradients are clipped based on `max` before being applied.
func (opt *Optimizer) BackwardStepClip(loss *ts.Tensor, max float64) error {
err := opt.opt.ZeroGrad()
if err != nil {
err = fmt.Errorf("Optimizer.BackwardStepClip() failed: %w\n", err)
return err
}
loss.MustBackward()
opt.ClipGradValue(max)
err = opt.opt.Step()
if err != nil {
err = fmt.Errorf("Optimizer.BackwardStepClip() failed: %w\n", err)
return err
}
return nil
}
// MustBackwardStepClip applies a backward step pass, update the gradients, and performs an optimization step.
//
// The gradients are clipped based on `max` before being applied.
func (opt *Optimizer) MustBackwardStepClip(loss *ts.Tensor, max float64) {
err := opt.BackwardStepClip(loss, max)
if err != nil {
log.Fatal(err)
}
}
type ClipOpts struct {
NormType float64
ErrorIfNonFinite bool
}
type ClipOpt func(*ClipOpts)
func defaultClipOpts() *ClipOpts {
return &ClipOpts{
NormType: 2.0,
ErrorIfNonFinite: false, // will switch to "true" in the future.
}
}
func WithNormType(v float64) ClipOpt {
return func(o *ClipOpts) {
o.NormType = v
}
}
func WithErrorIfNonFinite(v bool) ClipOpt {
return func(o *ClipOpts) {
o.ErrorIfNonFinite = v
}
}
// / Clips gradient L2 norm over all trainable parameters.
//
// The norm is computed over all gradients together, as if they were
// concatenated into a single vector.
//
// / Args:
// - max: max norm of the gradient
// - o.NormType. Type of the used p-norm, can be "inf" for infinity norm. Default= 2.0
// - o.ErrorIfNonFinite bool. If true, throw error if total norm of the gradients from paramters is "nan", "inf" or "-inf". Default=false
// Returns: total norm of the parameters (viewed as a single vector)
// ref. https://github.com/pytorch/pytorch/blob/cb4aeff7d8e4c70bb638cf159878c5204d0cc2da/torch/nn/utils/clip_grad.py#L59
func (opt *Optimizer) ClipGradNorm(max float64, opts ...ClipOpt) error {
o := defaultClipOpts()
for _, option := range opts {
option(o)
}
opt.varstore.Lock()
defer opt.varstore.Unlock()
parameters := opt.varstore.TrainableVariables()
if len(parameters) == 0 {
// return ts.MustOfSlice([]float64{0.0}), nil
return nil
}
var (
norms []*ts.Tensor
totalNorm *ts.Tensor
)
device := opt.varstore.device
// FIXME. What about mixed-precision?
dtype := parameters[0].DType()
if o.NormType == math.Inf(1) {
for _, v := range opt.varstore.vars {
n := v.Tensor.MustGrad(false).MustDetach(true).MustAbs(true).MustMax(true).MustTo(device, true)
norms = append(norms, n)
}
// total_norm = norms[0] if len(norms) == 1 else torch.max(torch.stack(norms))
totalNorm = ts.MustStack(norms, 0).MustMax(true)
} else {
for _, v := range opt.varstore.vars {
// x := v.Tensor.MustGrad(false).MustNorm(true)
// NOTE. tensor.Norm() is going to be deprecated. So use linalg_norm
// Ref. https://pytorch.org/docs/stable/generated/torch.linalg.norm.html#torch.linalg.norm
x := v.Tensor.MustGrad(false).MustDetach(true).MustLinalgNorm(ts.FloatScalar(o.NormType), nil, false, dtype, true)
norms = append(norms, x)
}
}
// totalNorm = ts.MustStack(norms, 0).MustNorm(true).MustAddScalar(ts.FloatScalar(1e-6), true)
// total_norm = torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type)
totalNorm = ts.MustStack(norms, 0).MustLinalgNorm(ts.FloatScalar(o.NormType), nil, false, dtype, true)
for _, x := range norms {
x.MustDrop()
}
totalNormVal := totalNorm.Float64Values(true)[0]
// if error_if_nonfinite and torch.logical_or(total_norm.isnan(), total_norm.isinf()):
if o.ErrorIfNonFinite && (math.IsNaN(totalNormVal) || math.IsInf(totalNormVal, 1)) {
err := fmt.Errorf("The total norm of order (%v) for gradients from 'parameters' is non-finite, so it cannot be clipped. To disable this error and scale the gradients by the non-finite norm anyway, set option.ErrorIfNonFinite= false", o.NormType)
return err
}
// clip_coef = max_norm / (total_norm + 1e-6)
// clipCoefTs := ts.TensorFrom([]float64{max}).MustDiv(totalNorm, true)
clipCoef := max / (totalNormVal + 1e-6)
// NOTE: multiplying by the clamped coef is redundant when the coef is clamped to 1, but doing so
// avoids a `if clip_coef < 1:` conditional which can require a CPU <=> device synchronization
// when the gradients do not reside in CPU memory.
// clip_coef_clamped = torch.clamp(clip_coef, max=1.0)
if clipCoef > 1.0 {
clipCoef = 1.0
}
for _, v := range opt.varstore.vars {
if v.Trainable {
// p.grad.detach().mul_(clip_coef_clamped.to(p.grad.device))
// v.Tensor.MustGrad(false).MustDetach(true).MustMulScalar_(ts.FloatScalar(clipCoef))
v.Tensor.MustGrad(false).MustMulScalar_(ts.FloatScalar(clipCoef))
}
}
return nil
}
// BackwardStepClipNorm applies a backward step pass, update the gradients, and performs an optimization step.
//
// The gradients L2 norm is clipped based on `max`.
func (opt *Optimizer) BackwardStepClipNorm(loss *ts.Tensor, max float64, opts ...ClipOpt) error {
err := opt.opt.ZeroGrad()
if err != nil {
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
return err
}
err = loss.Backward()
if err != nil {
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
return err
}
err = opt.ClipGradNorm(max, opts...)
if err != nil {
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
return err
}
err = opt.Step()
if err != nil {
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
return err
}
return nil
}
// MustBackwardStepClipNorm applies a backward step pass, update the gradients, and performs an optimization step.
//
// The gradients L2 norm is clipped based on `max`.
func (opt *Optimizer) MustBackwardStepClipNorm(loss *ts.Tensor, max float64, opts ...ClipOpt) {
err := opt.BackwardStepClipNorm(loss, max, opts...)
if err != nil {
log.Fatal(err)
}
}
// SetLR sets the optimizer learning rate.
//
// NOTE. it sets a SINGLE value of learning rate for all parameter groups.
// Most of the time, there's one parameter group.
func (opt *Optimizer) SetLR(lr float64) {
err := opt.opt.SetLearningRate(lr)
if err != nil {
log.Fatalf("Optimizer - SetLR method call error: %v\n", err)
}
}
func (opt *Optimizer) GetLRs() []float64 {
lrs, err := opt.opt.GetLearningRates()
if err != nil {
log.Fatalf("Optimizer - GetLRs method call error: %v\n", err)
}
return lrs
}
// SetLRs sets learning rates for ALL parameter groups respectively.
func (opt *Optimizer) SetLRs(lrs []float64) {
err := opt.opt.SetLearningRates(lrs)
if err != nil {
log.Fatalf("Optimizer - SetLRs method call error: %v\n", err)
}
}
// SetMomentum sets the optimizer momentum.
func (opt *Optimizer) SetMomentum(m float64) {
err := opt.opt.SetMomentum(m)
if err != nil {
log.Fatalf("Optimizer - SetMomentum method call error: %v\n", err)
}
}
func (opt *Optimizer) ParamGroupNum() int {
ngroup, err := opt.opt.ParamGroupNum()
if err != nil {
log.Fatalf("Optimizer - ParamGroupNum method call error: %v\n", err)
}
return int(ngroup)
}
func (opt *Optimizer) AddParamGroup(tensors []*ts.Tensor) {
err := opt.opt.AddParamGroup(tensors)
if err != nil {
log.Fatalf("Optimizer - ParamGroupNum method call error: %v\n", err)
}
}

18
logic/models/train/torch/nn/utils.go
View File

@ -1,18 +0,0 @@
package my_nn
import (
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
)
func or_panic(err error) {
if err != nil {
panic(err)
}
}
type MyLayer interface {
torch.ModuleT
ExtractFromVarstore(vs *VarStore)
Debug()
}

1359
logic/models/train/torch/nn/varstore.go
View File

File diff suppressed because it is too large Load Diff

120
logic/models/train/torch/torch.go
View File

@ -1,120 +0,0 @@
package train
import (
types "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
"git.andr3h3nriqu3s.com/andr3/gotch"
"github.com/charmbracelet/log"
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
)
type IForwardable interface {
Forward(xs *torch.Tensor) *torch.Tensor
}
// Container for a model
type ContainerModel struct {
Layers []my_nn.MyLayer
Vs *my_nn.VarStore
path *my_nn.Path
}
func (n *ContainerModel) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
if len(n.Layers) == 0 {
return x.MustShallowClone()
}
if len(n.Layers) == 1 {
log.Info("here")
return n.Layers[0].ForwardT(x, train)
}
// forward sequentially
outs := make([]*torch.Tensor, len(n.Layers))
for i := 0; i < len(n.Layers); i++ {
if i == 0 {
outs[0] = n.Layers[i].ForwardT(x, train)
//defer outs[0].MustDrop()
} else if i == len(n.Layers)-1 {
return n.Layers[i].ForwardT(outs[i-1], train)
} else {
outs[i] = n.Layers[i].ForwardT(outs[i-1], train)
//defer outs[i].MustDrop()
}
}
panic("Do not reach here")
}
func (n *ContainerModel) To(device gotch.Device) {
n.Vs.ToDevice(device)
for _, layer := range n.Layers {
layer.ExtractFromVarstore(n.Vs)
}
}
func (n *ContainerModel) Refresh() {
for _, layer := range n.Layers {
layer.ExtractFromVarstore(n.Vs)
}
}
func BuildModel(layers []*types.Layer, _lastLinearSize int64, addSigmoid bool) *ContainerModel {
base_vs := my_nn.NewVarStore(gotch.CPU)
vs := base_vs.Root()
m_layers := []my_nn.MyLayer{}
var lastLinearSize int64 = _lastLinearSize
lastLinearConv := []int64{}
for _, layer := range layers {
if layer.LayerType == types.LAYER_INPUT {
lastLinearConv = layer.GetShape()
log.Info("Input: ", "In:", lastLinearConv)
} else if layer.LayerType == types.LAYER_DENSE {
shape := layer.GetShape()
log.Info("New Dense: ", "In:", lastLinearSize, "out:", shape[0])
m_layers = append(m_layers, NewLinear(vs, lastLinearSize, shape[0]))
lastLinearSize = shape[0]
} else if layer.LayerType == types.LAYER_FLATTEN {
m_layers = append(m_layers, NewFlatten())
lastLinearSize = 1
for _, i := range lastLinearConv {
lastLinearSize *= i
}
log.Info("Flatten: ", "In:", lastLinearConv, "out:", lastLinearSize)
} else if layer.LayerType == types.LAYER_SIMPLE_BLOCK {
panic("TODO")
log.Info("New Block: ", "In:", lastLinearConv, "out:", []int64{lastLinearConv[1] / 2, lastLinearConv[2] / 2, 128})
//m_layers = append(m_layers, NewSimpleBlock(vs, lastLinearConv[0]))
lastLinearConv[0] = 128
lastLinearConv[1] /= 2
lastLinearConv[2] /= 2
}
}
if addSigmoid {
m_layers = append(m_layers, NewSigmoid())
}
b := &ContainerModel{
Layers: m_layers,
Vs: base_vs,
path: vs,
}
return b
}
func (model *ContainerModel) Debug() {
for _, v := range model.Layers {
v.Debug()
}
}
func SaveModel(model *ContainerModel, modelFn string) (err error) {
model.Vs.ToDevice(gotch.CPU)
return model.Vs.Save(modelFn)
}

152
logic/models/train/torch/utils.go
View File

@ -1,152 +0,0 @@
package train
import (
"unsafe"
my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
"github.com/charmbracelet/log"
"git.andr3h3nriqu3s.com/andr3/gotch/nn"
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
)
func or_panic(err error) {
if err != nil {
log.Fatal(err)
}
}
type SimpleBlock struct {
C1, C2 *nn.Conv2D
BN1 *nn.BatchNorm
}
// BasicBlock returns a BasicBlockModule instance
func NewSimpleBlock(_vs *my_nn.Path, inplanes int64) *SimpleBlock {
vs := (*nn.Path)(unsafe.Pointer(_vs))
conf1 := nn.DefaultConv2DConfig()
conf1.Stride = []int64{2, 2}
conf2 := nn.DefaultConv2DConfig()
conf2.Padding = []int64{2, 2}
b := &SimpleBlock{
C1: nn.NewConv2D(vs, inplanes, 128, 3, conf1),
C2: nn.NewConv2D(vs, 128, 128, 3, conf2),
BN1: nn.NewBatchNorm(vs, 2, 128, nn.DefaultBatchNormConfig()),
}
return b
}
// Forward method
func (b *SimpleBlock) Forward(x *torch.Tensor) *torch.Tensor {
identity := x
out := b.C1.Forward(x)
out = out.MustRelu(false)
out = b.C2.Forward(out)
out = out.MustRelu(false)
shape, err := out.Size()
or_panic(err)
out, err = out.AdaptiveAvgPool2d(shape, false)
or_panic(err)
out = b.BN1.Forward(out)
out, err = out.LeakyRelu(false)
or_panic(err)
out = out.MustAdd(identity, false)
out = out.MustRelu(false)
return out
}
func (b *SimpleBlock) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
identity := x
out := b.C1.ForwardT(x, train)
out = out.MustRelu(false)
out = b.C2.ForwardT(out, train)
out = out.MustRelu(false)
shape, err := out.Size()
or_panic(err)
out, err = out.AdaptiveAvgPool2d(shape, false)
or_panic(err)
out = b.BN1.ForwardT(out, train)
out, err = out.LeakyRelu(false)
or_panic(err)
out = out.MustAdd(identity, false)
out = out.MustRelu(false)
return out
}
// BasicBlock returns a BasicBlockModule instance
func NewLinear(vs *my_nn.Path, in, out int64) *my_nn.Linear {
config := my_nn.DefaultLinearConfig()
return my_nn.NewLinear(vs, in, out, config)
}
type Flatten struct{}
// BasicBlock returns a BasicBlockModule instance
func NewFlatten() *Flatten {
return &Flatten{}
}
// The flatten layer does not to move anything to the device
func (b *Flatten) ExtractFromVarstore(vs *my_nn.VarStore) {}
func (b *Flatten) Debug() {}
// Forward method
func (b *Flatten) Forward(x *torch.Tensor) *torch.Tensor {
out, err := x.Flatten(1, -1, false)
or_panic(err)
return out
}
func (b *Flatten) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
out, err := x.Flatten(1, -1, false)
or_panic(err)
return out
}
type Sigmoid struct{}
func NewSigmoid() *Sigmoid {
return &Sigmoid{}
}
// The sigmoid layer does not need to move anything to another device
func (b *Sigmoid) ExtractFromVarstore(vs *my_nn.VarStore) {}
func (b *Sigmoid) Debug() {}
func (b *Sigmoid) Forward(x *torch.Tensor) *torch.Tensor {
out, err := x.Sigmoid(false)
or_panic(err)
return out
}
func (b *Sigmoid) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
out, err := x.Sigmoid(false)
or_panic(err)
return out
}

982
logic/models/train/train_normal.go → logic/models/train/train.go
View File

File diff suppressed because it is too large Load Diff

4
logic/stats/tasks.go
View File

@ -14,7 +14,7 @@ func handleTasksStats(handle *Handle) {
}
PostAuthJson(handle, "/stats/task/model/day", User_Normal, func(c *Context, dat *ModelTasksStatsRequest) *Error {
model, err := GetBaseModel(c, dat.ModelId)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found!")
} else if err != nil {
return c.E500M("Failed to get model", err)
@ -68,7 +68,7 @@ func handleTasksStats(handle *Handle) {
} else if task.Status < 2 {
total.Classfication_pre += 1
hours[hour].Classfication_pre += 1
} else if task.Status < 4 {
} else if task.Status < 4 || task.Status == 5 {
total.Classfication_running += 1
hours[hour].Classfication_running += 1
}

2
logic/tasks/agreement.go
View File

@ -14,7 +14,7 @@ func handleRequests(x *Handle) {
PostAuthJson(x, "/task/agreement", User_Normal, func(c *Context, dat *AgreementRequest) *Error {
var task Task
err := GetDBOnce(c, &task, "tasks where id=$1", dat.Id)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.E500M("Failed to get task data", err)

2
logic/tasks/index.go
View File

@ -8,4 +8,6 @@ func HandleTasks(handle *Handle) {
handleUpload(handle)
handleList(handle)
handleRequests(handle)
handleRemoteRunner(handle)
handleRunnerData(handle)
}

2
logic/tasks/list.go
View File

@ -46,7 +46,7 @@ func handleList(handler *Handle) {
if requestData.ModelId != "" {
_, err := GetBaseModel(c.Db, requestData.ModelId)
if err == NotFoundError {
if err == ModelNotFoundError {
return c.SendJSONStatus(404, "Model not found!")
} else if err != nil {
return c.Error500(err)

960
logic/tasks/runner.go Normal file
View File

@ -0,0 +1,960 @@
package tasks
import (
"os"
"path"
"sync"
"time"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
)
func verifyRunner(c *Context, dat *JustId) (runner *Runner, e *Error) {
runner, err := GetRunner(c, dat.Id)
if err == NotFoundError {
e = c.JsonBadRequest("Could not find runner, please register runner first")
return
} else if err != nil {
e = c.E500M("Failed to get information about the runner", err)
return
}
if runner.Token != *c.Token {
return nil, c.SendJSONStatus(401, "Only runners can use this funcion")
}
return
}
type VerifyTask struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
}
type RunnerTrainDef struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
}
func verifyTask(x *Handle, c *Context, dat *VerifyTask) (task *Task, error *Error) {
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] == nil {
return nil, c.JsonBadRequest("Runner not active")
}
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
if runner_data["task"] == nil {
return nil, c.SendJSONStatus(404, "No active task")
}
return runner_data["task"].(*Task), nil
}
func clearRunnerTask(x *Handle, runner_id string) {
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[runner_id].(map[string]interface{})
runner_data["task"] = nil
runners[runner_id] = runner_data
x.DataMap["runners"] = runners
}
func handleRemoteRunner(x *Handle) {
type RegisterRunner struct {
Token string `json:"token" validate:"required"`
Type RunnerType `json:"type" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/register", User_Normal, func(c *Context, dat *RegisterRunner) *Error {
if *c.Token != dat.Token {
// TODO do admin
return c.E500M("Please make sure that the token is the same that is being registered", nil)
}
c.Logger.Info("test", "dat", dat)
var runner Runner
err := GetDBOnce(c, &runner, "remote_runner as ru where token=$1", dat.Token)
if err != NotFoundError && err != nil {
return c.E500M("Failed to get information remote runners", err)
}
if err != NotFoundError {
return c.JsonBadRequest("Token is already registered by a runner")
}
// TODO get id from token passed by when doing admin
var userId = c.User.Id
var new_runner = struct {
Type RunnerType
UserId string `db:"user_id"`
Token string
}{
Type: dat.Type,
Token: dat.Token,
UserId: userId,
}
id, err := InsertReturnId(c, &new_runner, "remote_runner", "id")
if err != nil {
return c.E500M("Failed to create remote runner", err)
}
return c.SendJSON(struct {
Id string `json:"id"`
}{
Id: id,
})
})
// TODO remove runner
PostAuthJson(x, "/tasks/runner/init", User_Normal, func(c *Context, dat *JustId) *Error {
runner, error := verifyRunner(c, dat)
if error != nil {
return error
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] != nil {
c.Logger.Info("Logger trying to register but already registerd")
c.ShowMessage = false
return c.SendJSON("Ok")
}
var new_runner = map[string]interface{}{}
new_runner["last_time_check"] = time.Now()
new_runner["runner_info"] = runner
runners[dat.Id] = new_runner
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/active", User_Normal, func(c *Context, dat *JustId) *Error {
_, error := verifyRunner(c, dat)
if error != nil {
return error
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] == nil {
return c.JsonBadRequest("Runner not active")
}
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
if runner_data["task"] == nil {
c.ShowMessage = false
return c.SendJSONStatus(404, "No active task")
}
c.ShowMessage = false
// This should be a task obj
return c.SendJSON(runner_data["task"])
})
PostAuthJson(x, "/tasks/runner/ready", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
err := task.UpdateStatus(c, TASK_RUNNING, "Task Running on Runner")
if err != nil {
return c.E500M("Failed to set task status", err)
}
return c.SendJSON("Ok")
})
type TaskFail struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
Reason string `json:"reason" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/fail", User_Normal, func(c *Context, dat *TaskFail) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
err := task.UpdateStatus(c, TASK_FAILED_RUNNING, dat.Reason)
if err != nil {
return c.E500M("Failed to set task status", err)
}
// Do extra clean up on tasks
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
CleanUpFailed(c, task)
case int(TASK_TYPE_RETRAINING):
CleanUpFailedRetrain(c, task)
case int(TASK_TYPE_CLASSIFICATION):
// DO nothing
default:
panic("Do not know how to handle this")
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/defs", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
var status DefinitionStatus
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
status = DEFINITION_STATUS_INIT
case int(TASK_TYPE_RETRAINING):
fallthrough
case int(TASK_TYPE_CLASSIFICATION):
status = DEFINITION_STATUS_READY
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
defs, err := model.GetDefinitions(c, "and md.status=$2", status)
if err != nil {
return c.E500M("Failed to get the model definitions", err)
}
return c.SendJSON(defs)
})
PostAuthJson(x, "/tasks/runner/classes", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
classes, err := model.GetClasses(c, "and status in ($2, $3) order by mc.class_order asc", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
case int(TASK_TYPE_RETRAINING):
classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
case int(TASK_TYPE_CLASSIFICATION):
classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TRAINED)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
})
type RunnerTrainDefStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Status DefinitionStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/def/status", User_Normal, func(c *Context, dat *RunnerTrainDefStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
err = def.UpdateStatus(c, dat.Status)
if err != nil {
return c.E500M("Failed to update model status", err)
}
return c.SendJSON("Ok")
})
type RunnerTrainDefHeadStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Status ModelHeadStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/def/head/status", User_Normal, func(c *Context, dat *RunnerTrainDefHeadStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
_, err = c.Exec("update exp_model_head set status=$1 where def_id=$2;", dat.Status, def.Id)
if err != nil {
log.Error("Failed to train definition!")
return c.E500M("Failed to train definition", err)
}
return c.SendJSON("Ok")
})
type RunnerRetrainDefHeadStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
HeadId string `json:"defId" validate:"required"`
Status ModelHeadStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/retrain/def/head/status", User_Normal, func(c *Context, dat *RunnerRetrainDefHeadStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_RETRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
if err := UpdateStatus(c.GetDb(), "exp_model_head", dat.HeadId, MODEL_DEFINITION_STATUS_TRAINING); err != nil {
return c.E500M("Failed to update head status", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/def/layers", User_Normal, func(c *Context, dat *RunnerTrainDef) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
// Do nothing
case int(TASK_TYPE_RETRAINING):
// Do nothing
default:
c.Logger.Error("Task not is not the right type to get the layers", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the layers")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
layers, err := def.GetLayers(c, " order by layer_order asc")
if err != nil {
return c.E500M("Failed to get layers", err)
}
return c.SendJSON(layers)
})
PostAuthJson(x, "/tasks/runner/train/datapoints", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
// DO nothing
case int(TASK_TYPE_RETRAINING):
// DO nothing
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
training_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
testing_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(struct {
Testing []DataPoint `json:"testing"`
Training []DataPoint `json:"training"`
}{
Testing: testing_points,
Training: training_points,
})
})
type RunnerTrainDefEpoch struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Epoch int `json:"epoch" validate:"required"`
Accuracy float64 `json:"accuracy" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/epoch", User_Normal, func(c *Context, dat *RunnerTrainDefEpoch) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
err = def.UpdateAfterEpoch(c, dat.Accuracy, dat.Epoch)
if err != nil {
return c.E500M("Failed to update model", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/mark-failed", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
_, err := c.Exec(
"update model_definition set status=$1 "+
"where model_id=$2 and status in ($3, $4)",
MODEL_DEFINITION_STATUS_CANCELD_TRAINING,
task.ModelId,
MODEL_DEFINITION_STATUS_TRAINING,
MODEL_DEFINITION_STATUS_PAUSED_TRAINING,
)
if err != nil {
return c.E500M("Failed to mark definition as failed", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/model", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
//DO NOTHING
case int(TASK_TYPE_RETRAINING):
//DO NOTHING
case int(TASK_TYPE_CLASSIFICATION):
//DO NOTHING
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
return c.SendJSON(model)
})
PostAuthJson(x, "/tasks/runner/heads", User_Normal, func(c *Context, dat *RunnerTrainDef) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
type ExpHead struct {
Id string `json:"id"`
Start int `db:"range_start" json:"start"`
End int `db:"range_end" json:"end"`
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
fallthrough
case int(TASK_TYPE_RETRAINING):
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](c, "exp_model_head where def_id=$1 and status in (2,3)", dat.DefId)
if err != nil {
return c.E500M("Failed getting active heads", err)
}
return c.SendJSON(heads)
case int(TASK_TYPE_CLASSIFICATION):
heads, err := GetDbMultitple[ExpHead](c, "exp_model_head where def_id=$1", dat.DefId)
if err != nil {
return c.E500M("Failed getting active heads", err)
}
return c.SendJSON(heads)
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
})
PostAuthJson(x, "/tasks/runner/train_exp/class/status/train", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model", err)
}
err = SetModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return c.E500M("Failed update status", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
var def Definition
err = GetDBOnce(c, &def, "model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
return c.SendJSON("Ok")
} else if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
return c.E500M("Failed to get model definition", err)
}
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
return c.E500M("Failed to update model definition", err)
}
to_delete, err := c.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if err = to_delete.Scan(&id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
// Set the class status to trained
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1;", model.Id)
if err != nil {
c.Logger.Error("Failed to set class status")
return c.E500M("Failed to set class status", err)
}
if err = model.UpdateStatus(c, READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to update status of model", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
})
type RunnerClassDone struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
Result string `json:"result" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/class/done", User_Normal, func(c *Context, dat *RunnerClassDone) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_CLASSIFICATION) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
err := task.SetResultText(c, dat.Result)
if err != nil {
return c.E500M("Failed to update the task", err)
}
err = task.UpdateStatus(c, TASK_DONE, "Task completed")
if err != nil {
return c.E500M("Failed to update task", err)
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train_exp/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
// TODO add check the to the model
var def Definition
err = GetDBOnce(c, &def, "model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
} else if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
return c.E500M("Failed to get model definition", err)
}
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
return c.E500M("Failed to update model definition", err)
}
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.GetLogger().Error("Failed to select model_definition to delete")
return c.E500M("Failed to select model definition to delete", err)
}
for _, d := range to_delete {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
// TODO Check if returning also works here
if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
if err = SplitModel(c, model); err != nil {
err = SetModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
c.Logger.Error("Failed to split the model! And Failed to set class status")
return c.E500M("Failed to split the model", err)
}
c.Logger.Error("Failed to split the model")
return c.E500M("Failed to split the model", err)
}
// Set the class status to trained
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
c.Logger.Error("Failed to set class status")
return c.E500M("Failed to set class status", err)
}
c.Logger.Warn("Removing base model for", "model", model.Id, "def", def.Id)
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model"))
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model.keras"))
if err = model.UpdateStatus(c, READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to update status of model", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/retrain/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_RETRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to set class status", err)
}
defs, err := model.GetDefinitions(c, "")
if err != nil {
return c.E500M("Failed to get definitions", err)
}
_, err = c.Exec("update exp_model_head set status=$1 where status=$2 and def_id=$3", MODEL_HEAD_STATUS_READY, MODEL_HEAD_STATUS_TRAINING, defs[0].Id)
if err != nil {
return c.E500M("Failed to set head status", err)
}
err = model.UpdateStatus(c, READY)
if err != nil {
return c.E500M("Failed to set class status", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
})
}

267
logic/tasks/runner/runner.go
View File

@ -5,20 +5,22 @@ import (
"math"
"os"
"runtime/debug"
"sync"
"time"
"github.com/charmbracelet/log"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/users"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
var QUEUE_SIZE = 10
/**
* Actually runs the code
*/
@ -47,18 +49,28 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
Host: config.Hostname,
}
loaded_model := RunnerModelData{
Id: nil,
Model: nil,
}
count := 0
for task := range task_channel {
logger.Info("Got task", "task", task)
task.UpdateStatusLog(base, TASK_PICKED_UP, "Runner picked up task")
if task.TaskType == int(TASK_TYPE_CLASSIFICATION) {
logger.Info("Classification Task")
/*if err = ClassifyTask(base, task); err != nil {
if err = ClassifyTask(base, task, &loaded_model); err != nil {
logger.Error("Classification task failed", "error", err)
}*/
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "TODO move tasks to pytorch")
}
back_channel <- index
if count == QUEUE_SIZE {
back_channel <- index
count = 0
} else {
count += 1
}
continue
} else if task.TaskType == int(TASK_TYPE_TRAINING) {
logger.Info("Training Task")
@ -66,7 +78,12 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
logger.Error("Failed to tain the model", "error", err)
}
back_channel <- index
if count == QUEUE_SIZE {
back_channel <- index
count = 0
} else {
count += 1
}
continue
} else if task.TaskType == int(TASK_TYPE_RETRAINING) {
logger.Info("Retraining Task")
@ -74,7 +91,12 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
logger.Error("Failed to tain the model", "error", err)
}
back_channel <- index
if count == QUEUE_SIZE {
back_channel <- index
count = 0
} else {
count += 1
}
continue
} else if task.TaskType == int(TASK_TYPE_DELETE_USER) {
logger.Warn("User deleting Task")
@ -82,20 +104,81 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
logger.Error("Failed to tain the model", "error", err)
}
back_channel <- index
if count == QUEUE_SIZE {
back_channel <- index
count = 0
} else {
count += 1
}
continue
}
logger.Error("Do not know how to route task", "task", task)
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Do not know how to route task")
back_channel <- index
if count == QUEUE_SIZE {
back_channel <- index
count = 0
} else {
count += 1
}
}
}
/**
* Handle remote runner
*/
func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Task) {
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
TimeFormat: time.Kitchen,
Prefix: fmt.Sprintf("Runner pre %s", runner_id),
})
defer func() {
if r := recover(); r != nil {
logger.Error("Runner failed to setup for runner", "due to", r, "stack", string(debug.Stack()))
// TODO maybe create better failed task
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to setup task for runner")
}
}()
err := task.UpdateStatus(base, TASK_PICKED_UP, "Failed to setup task for runner")
if err != nil {
logger.Error("Failed to mark task as PICK UP")
return
}
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
switch task.TaskType {
case int(TASK_TYPE_RETRAINING):
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
case int(TASK_TYPE_TRAINING):
if err := PrepareTraining(handler, base, task, runner_id); err != nil {
logger.Error("Failed to prepare for training", "err", err)
}
case int(TASK_TYPE_CLASSIFICATION):
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
default:
logger.Error("Not sure what to do panicing", "taskType", task.TaskType)
panic("not sure what to do")
}
}
/**
* Tells the orcchestator to look at the task list from time to time
*/
func attentionSeeker(config Config, back_channel chan int) {
func attentionSeeker(config Config, db db.Db, back_channel chan int) {
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
@ -120,6 +203,20 @@ func attentionSeeker(config Config, back_channel chan int) {
for true {
back_channel <- 0
for {
var s struct {
Count int `json:"count(*)"`
}
err := GetDBOnce(db, &s, "tasks where stauts = 5 or status = 3")
if err != nil {
break
}
if s.Count == 0 {
break
}
time.Sleep(t)
}
time.Sleep(t)
}
}
@ -127,7 +224,7 @@ func attentionSeeker(config Config, back_channel chan int) {
/**
* Manages what worker should to Work
*/
func RunnerOrchestrator(db db.Db, config Config) {
func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
@ -135,84 +232,156 @@ func RunnerOrchestrator(db db.Db, config Config) {
Prefix: "Runner Orchestrator Logger",
})
setupHandle(handler)
base := BasePackStruct{
Db: db,
Logger: logger,
Host: config.Hostname,
}
gpu_workers := config.GpuWorker.NumberOfWorkers
def_wait, err := time.ParseDuration(config.GpuWorker.Pulling)
if err != nil {
logger.Error("Failed to load", "error", err)
return
}
logger.Info("Starting runners")
task_runners := make([]chan Task, gpu_workers)
task_runners_used := make([]bool, gpu_workers)
task_runners_used := make([]int, gpu_workers)
// One more to accomudate the Attention Seeker channel
back_channel := make(chan int, gpu_workers+1)
defer func() {
if r := recover(); r != nil {
logger.Error("Recovered in Orchestrator restarting", "due to", r)
logger.Error("Recovered in Orchestrator restarting", "due to", r, "stack", string(debug.Stack()))
for x := range task_runners {
close(task_runners[x])
}
close(back_channel)
go RunnerOrchestrator(db, config)
go RunnerOrchestrator(db, config, handler)
}
}()
go attentionSeeker(config, back_channel)
// go attentionSeeker(config, db, back_channel)
// Start the runners
for i := 0; i < gpu_workers; i++ {
task_runners[i] = make(chan Task, 10)
task_runners_used[i] = false
task_runners[i] = make(chan Task, QUEUE_SIZE)
task_runners_used[i] = 0
AddLocalRunner(handler, LocalRunner{
RunnerNum: i + 1,
Task: nil,
})
go runner(config, db, task_runners[i], i+1, back_channel)
}
var task_to_dispatch *Task = nil
for i := range back_channel {
if i > 0 {
logger.Info("Runner freed", "runner", i)
task_runners_used[i-1] = false
} else if i < 0 {
logger.Error("Runner died! Restarting!", "runner", i)
i = int(math.Abs(float64(i)) - 1)
task_runners_used[i] = false
go runner(config, db, task_runners[i], i+1, back_channel)
used := 0
wait := time.Nanosecond * 100
for {
out := true
for out {
select {
case i := <-back_channel:
if i != 0 {
if i > 0 {
logger.Info("Runner freed", "runner", i)
task_runners_used[i-1] = 0
used = 0
} else if i < 0 {
logger.Error("Runner died! Restarting!", "runner", i)
i = int(math.Abs(float64(i)) - 1)
task_runners_used[i] = 0
used = 0
go runner(config, db, task_runners[i], i+1, back_channel)
}
AddLocalTask(handler, int(math.Abs(float64(i))), nil)
} else if used == len(task_runners_used) {
continue
}
case <-time.After(wait):
if wait == time.Nanosecond*100 {
wait = def_wait
}
out = false
}
}
if task_to_dispatch == nil {
var task TaskT
err := GetDBOnce(db, &task, "tasks as t "+
for {
tasks, err := GetDbMultitple[TaskT](db, "tasks as t "+
// Get depenencies
"left join tasks_dependencies as td on t.id=td.main_id "+
// Get the task that the depencey resolves to
"left join tasks as t2 on t2.id=td.dependent_id "+
"where t.status=1 "+
"group by t.id having count(td.id) filter (where t2.status in (0,1,2,3)) = 0;")
"group by t.id having count(td.id) filter (where t2.status in (0,1,2,3)) = 0 limit 20;")
if err != NotFoundError && err != nil {
log.Error("Failed to get tasks from db", "err", err)
continue
}
if err == NotFoundError {
task_to_dispatch = nil
} else {
temp := Task(task)
task_to_dispatch = &temp
if err == NotFoundError || len(tasks) == 0 {
break
}
}
if task_to_dispatch != nil {
for i := 0; i < len(task_runners_used); i += 1 {
if !task_runners_used[i] {
task_runners[i] <- *task_to_dispatch
task_runners_used[i] = true
task_to_dispatch = nil
for _, task_to_dispatch := range tasks {
ttd := Task(*task_to_dispatch)
if task_to_dispatch != nil && task_to_dispatch.TaskType != int(TASK_TYPE_DELETE_USER) {
// TODO split tasks into cpu tasks and GPU tasks
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
remote_runners := handler.DataMap["runners"].(map[string]interface{})
for k, v := range remote_runners {
runner_data := v.(map[string]interface{})
runner_info := runner_data["runner_info"].(*Runner)
if runner_data["task"] != nil {
continue
}
if runner_info.UserId != task_to_dispatch.UserId {
continue
}
go handleRemoteTask(handler, base, k, ttd)
task_to_dispatch = nil
break
}
mutex.Unlock()
}
used = 0
if task_to_dispatch != nil {
for i := 0; i < len(task_runners_used); i += 1 {
if task_runners_used[i] <= QUEUE_SIZE {
ttd.UpdateStatusLog(base, TASK_QUEUED, "Runner picked up task")
task_runners[i] <- ttd
task_runners_used[i] += 1
AddLocalTask(handler, i+1, &ttd)
task_to_dispatch = nil
wait = time.Nanosecond * 100
break
} else {
used += 1
}
}
}
if used == len(task_runners_used) {
break
}
}
}
if used == len(task_runners_used) {
break
}
}
}
}
func StartRunners(db db.Db, config Config) {
go RunnerOrchestrator(db, config)
func StartRunners(db db.Db, config Config, handler *Handle) {
go RunnerOrchestrator(db, config, handler)
}

51
logic/tasks/runner/utils.go Normal file
View File

@ -0,0 +1,51 @@
package task_runner
import (
"sync"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
type LocalRunner struct {
RunnerNum int `json:"id"`
Task *Task `json:"task"`
}
type LocalRunners map[int]*LocalRunner
func LockRunners(handler *Handle, t string) *sync.Mutex {
req := t + "_runners_mutex"
if t == "" {
req = "runners_mutex"
}
mutex := handler.DataMap[req].(*sync.Mutex)
mutex.Lock()
return mutex
}
func setupHandle(handler *Handle) {
// Setup Remote Runner data
handler.DataMap["runners"] = map[string]interface{}{}
handler.DataMap["runners_mutex"] = &sync.Mutex{}
// Setup Local Runner data
handler.DataMap["local_runners"] = &LocalRunners{}
handler.DataMap["local_runners_mutex"] = &sync.Mutex{}
}
func AddLocalRunner(handler *Handle, runner LocalRunner) {
mutex := LockRunners(handler, "local")
defer mutex.Unlock()
runners := handler.DataMap["local_runners"].(*LocalRunners)
(*runners)[runner.RunnerNum] = &runner
}
func AddLocalTask(handler *Handle, runner_id int, task *Task) {
mutex := LockRunners(handler, "local")
defer mutex.Unlock()
runners := handler.DataMap["local_runners"].(*LocalRunners)
(*(*runners)[runner_id]).Task = task
}

25
logic/tasks/runner_data.go Normal file
View File

@ -0,0 +1,25 @@
package tasks
import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/runner"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleRunnerData(x *Handle) {
type NonType struct{}
PostAuthJson(x, "/tasks/runner/info", User_Admin, func(c *Context, dat *NonType) *Error {
mutex_remote := LockRunners(x, "")
defer mutex_remote.Unlock()
mutex_local := LockRunners(x, "local")
defer mutex_local.Unlock()
return c.SendJSON(struct {
RemoteRunners map[string]interface{} `json:"remoteRunners"`
LocalRunner *LocalRunners `json:"localRunners"`
}{
RemoteRunners: x.DataMap["runners"].(map[string]interface{}),
LocalRunner: x.DataMap["local_runners"].(*LocalRunners),
})
})
}

29
logic/tasks/utils/runner.go Normal file
View File

@ -0,0 +1,29 @@
package tasks_utils
import (
"time"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
)
type RunnerType int64
const (
RUNNER_TYPE_GPU RunnerType = iota + 1
)
type Runner struct {
Id string `json:"id" db:"ru.id"`
UserId string `json:"user_id" db:"ru.user_id"`
Token string `json:"token" db:"ru.token"`
Type RunnerType `json:"type" db:"ru.type"`
CreateOn time.Time `json:"createOn" db:"ru.created_on"`
}
func GetRunner(db db.Db, id string) (ru *Runner, err error) {
var runner Runner
err = dbtypes.GetDBOnce(db, &runner, "remote_runner as ru where ru.id=$1", id)
ru = &runner
return
}

7
logic/tasks/utils/utils.go
View File

@ -50,6 +50,7 @@ const (
TASK_PREPARING = 0
TASK_TODO = 1
TASK_PICKED_UP = 2
TASK_QUEUED = 5
TASK_RUNNING = 3
TASK_DONE = 4
)
@ -101,7 +102,11 @@ func (t Task) SetResult(base BasePack, result any) (err error) {
if err != nil {
return
}
_, err = base.GetDb().Exec("update tasks set result=$1 where id=$2", text, t.Id)
return t.SetResultText(base, string(text))
}
func (t Task) SetResultText(base BasePack, text string) (err error) {
_, err = base.GetDb().Exec("update tasks set result=$1 where id=$2", []byte(text), t.Id)
return
}

23
logic/users/users.go
View File

@ -241,6 +241,17 @@ func UsersEndpints(db db.Db, handle *Handle) {
return c.SendJSON(userReturn)
})
PostAuthJson(handle, "/user/info/get", User_Admin, func(c *Context, dat *JustId) *Error {
var user *User
user, err := UserFromId(c, dat.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "User not found")
} else if err != nil {
return c.E500M("Could not get user", err)
}
return c.SendJSON(user)
})
// Handles updating users
type UpdateUserData struct {
Id string `json:"id"`
@ -412,6 +423,18 @@ func UsersEndpints(db db.Db, handle *Handle) {
return c.SendJSON("Ok")
})
handle.DeleteAuth("/user/token/logoff", User_Normal, func(c *Context) *Error {
if c.Token == nil {
return c.JsonBadRequest("Failed to get token")
}
_, err := c.Db.Exec("delete from tokens where token=$1;", c.Token)
if err != nil {
return c.E500M("Failed to delete token", err)
}
return c.SendJSON("OK")
})
type DeleteUser struct {
Id string `json:"id" validate:"required"`
Password string `json:"password" validate:"required"`

13
logic/utils/config.go
View File

@ -23,7 +23,12 @@ type ServiceUser struct {
}
type DbInfo struct {
MaxConnections int `toml:"max_connections"`
MaxConnections int `toml:"max_connections"`
Host string `toml:"host"`
Port int `toml:"port"`
User string `toml:"user"`
Password string `toml:"password"`
Dbname string `toml:"dbname"`
}
type Config struct {
@ -97,7 +102,7 @@ func (c *Config) Cleanup(db db.Db) {
failLog(err)
_, err = db.Exec("update models set status=$1 where status=$2", FAILED_PREPARING, PREPARING)
failLog(err)
_, err = db.Exec("update tasks set status=$1 where status=$2", TASK_TODO, TASK_PICKED_UP)
_, err = db.Exec("update tasks set status=$1 where status=$2 or status=$3", TASK_TODO, TASK_PICKED_UP, TASK_QUEUED)
failLog(err)
tasks, err := GetDbMultitple[Task](db, "tasks where status=$1", TASK_RUNNING)
@ -114,12 +119,16 @@ func (c *Config) Cleanup(db db.Db) {
tasks[i].UpdateStatus(base, TASK_FAILED_RUNNING, "Task inturupted by server restart please try again")
_, err = db.Exec("update models set status=$1 where id=$2", READY_RETRAIN_FAILED, tasks[i].ModelId)
failLog(err)
_, err = db.Exec("update model_classes set status=$1 where model_id=$2 and status=$3", CLASS_STATUS_TO_TRAIN, tasks[i].ModelId, CLASS_STATUS_TRAINING)
failLog(err)
continue
}
if tasks[i].TaskType == int(TASK_TYPE_TRAINING) {
tasks[i].UpdateStatus(base, TASK_FAILED_RUNNING, "Task inturupted by server restart please try again")
_, err = db.Exec("update models set status=$1 where id=$2", FAILED_TRAINING, tasks[i].ModelId)
failLog(err)
_, err = db.Exec("update model_classes set status=$1 where model_id=$2 and status=$3", CLASS_STATUS_TO_TRAIN, tasks[i].ModelId, CLASS_STATUS_TRAINING)
failLog(err)
continue
}
}

8
logic/utils/handler.go
View File

@ -175,7 +175,7 @@ func (x *Handle) DeleteAuth(path string, authLevel dbtypes.UserType, fn func(c *
}
return fn(c)
}
x.posts = append(x.posts, HandleFunc{path, inner_fn})
x.deletes = append(x.deletes, HandleFunc{path, inner_fn})
}
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
@ -374,7 +374,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
c.SetReportCaller(true)
c.Logger.Warn("Request failed with a bad request", "dat", dat)
c.SetReportCaller(false)
return c.ErrorCode(nil, 404, dat)
return c.SendJSONStatus(http.StatusBadRequest, dat)
}
func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
@ -392,7 +392,7 @@ func (c *Context) GetModelFromId(id_path string) (*dbtypes.BaseModel, *Error) {
}
model, err := dbtypes.GetBaseModel(c.Db, id)
if err == dbtypes.NotFoundError {
if err == dbtypes.ModelNotFoundError {
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return nil, c.Error500(err)
@ -449,7 +449,7 @@ func (x Handle) createContext(handler *Handle, r *http.Request, w http.ResponseW
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
TimeFormat: time.Kitchen,
TimeFormat: time.DateTime,
Prefix: r.URL.Path,
})

19
main.go
View File

@ -15,32 +15,25 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
const (
host = "localhost"
port = 5432
user = "postgres"
password = "verysafepassword"
dbname = "aistuff"
)
func main() {
func main_() {
config := LoadConfig()
log.Info("Config loaded!", "config", config)
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
"password=%s dbname=%s sslmode=disable",
host, port, user, password, dbname)
config.DbInfo.Host, config.DbInfo.Port, config.DbInfo.User, config.DbInfo.Password, config.DbInfo.Dbname)
db := db.StartUp(psqlInfo)
defer db.Close()
config := LoadConfig()
log.Info("Config loaded!", "config", config)
config.GenerateToken(db)
StartRunners(db, config)
//TODO check if file structure exists to save data
handle := NewHandler(db, config)
StartRunners(db, config, handle)
config.Cleanup(db)
// TODO Handle this in other way

8
nginx.dev.conf → nginx.proxy.conf
View File

@ -1,6 +1,6 @@
events {
worker_connections 1024;
worker_connections 2024;
}
http {
@ -13,11 +13,11 @@ http {
server {
listen 8000;
client_max_body_size 1G;
client_max_body_size 5G;
location / {
proxy_http_version 1.1;
proxy_pass http://127.0.0.1:5001;
proxy_pass http://webpage:5001;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
@ -25,7 +25,7 @@ http {
location /api {
proxy_pass http://127.0.0.1:5002;
proxy_pass http://server:5002;
}
}
}

5
requirements.txt Normal file
View File

@ -0,0 +1,5 @@
tensorflow[and-cuda] == 2.15.1
pandas
# Make sure to install the nvidia pyindex first
# nvidia-pyindex
nvidia-cudnn

5
run.sh
View File

@ -1,3 +1,2 @@
#!/bin/fish
podman run --rm --network host --gpus all -ti -v (pwd):/app -e "TERM=xterm-256color" fyp-server bash
#!/bin/bash
podman run --network host --gpus all --replace --name fyp-server --ulimit=nofile=100000:100000 -it -v $(pwd):/app -e "TERM=xterm-256color" --restart=always andre-fyp-server

2
sql/base.sql
View File

@ -1 +1 @@
CREATE DATABASE aistuff;
CREATE DATABASE fyp;

6
sql/models.sql
View File

@ -35,7 +35,7 @@ create table if not exists model_classes (
-- 1: to_train
-- 2: training
-- 3: trained
status integer default 1,
status integer default 1
);
-- drop table if exists model_data_point;
@ -59,7 +59,6 @@ create table if not exists model_definition (
accuracy real default 0,
target_accuracy integer not null,
epoch integer default 0,
-- TODO add max epoch
-- 1: Pre Init
-- 2: Init
-- 3: Training
@ -78,7 +77,7 @@ create table if not exists model_definition_layer (
-- 1: input
-- 2: dense
-- 3: flatten
-- TODO add conv
-- 4: block
layer_type integer not null,
-- ei 28,28,1
-- a 28x28 grayscale image
@ -102,7 +101,6 @@ create table if not exists exp_model_head (
accuracy real default 0,
-- TODO add max epoch
-- 1: Pre Init
-- 2: Init
-- 3: Training

11
sql/tasks.sql
View File

@ -38,3 +38,14 @@ create table if not exists tasks_dependencies (
main_id uuid references tasks (id) on delete cascade not null,
dependent_id uuid references tasks (id) on delete cascade not null
);
create table if not exists remote_runner (
id uuid primary key default gen_random_uuid(),
user_id uuid references users (id) on delete cascade not null,
token text not null,
-- 1: GPU
type integer,
created_on timestamp default current_timestamp
);

120
test.go
View File

@ -1,120 +0,0 @@
package main
import (
"git.andr3h3nriqu3s.com/andr3/gotch"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch"
//my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
"github.com/charmbracelet/log"
)
func main() {
log.Info("Hello world")
m := train.BuildModel([]*dbtypes.Layer{
&dbtypes.Layer{
LayerType: dbtypes.LAYER_INPUT,
Shape: "[ 3, 28, 28 ]",
},
&dbtypes.Layer{
LayerType: dbtypes.LAYER_FLATTEN,
},
&dbtypes.Layer{
LayerType: dbtypes.LAYER_DENSE,
Shape: "[ 27 ]",
},
&dbtypes.Layer{
LayerType: dbtypes.LAYER_DENSE,
Shape: "[ 18 ]",
},
// &dbtypes.Layer{
// LayerType: dbtypes.LAYER_DENSE,
// Shape: "[ 9 ]",
// },
}, 0, true)
//var err error
d := gotch.CudaIfAvailable()
log.Info("device", "d", d)
m.To(d)
var count = 0
// vars1 := m.Vs.Variables()
//
// for k, v := range vars1 {
// ones := torch.MustOnes(v.MustSize(), gotch.Float, d)
// v := ones.MustSetRequiresGrad(true, false)
// v.MustDrop()
// ones.RetainGrad(false)
//
// m.Vs.UpdateVarTensor(k, ones, true)
// m.Refresh()
// }
//
// opt, err := my_nn.DefaultAdamConfig().Build(m.Vs, 0.001)
// if err != nil {
// return
// }
log.Info("start")
for count < 100 {
ones := torch.MustOnes([]int64{1, 3, 28, 28}, gotch.Float, d)
// ones = ones.MustSetRequiresGrad(true, true)
// ones.RetainGrad(false)
res := m.ForwardT(ones, true)
//res = res.MustSetRequiresGrad(true, true)
//res.RetainGrad(false)
outs := torch.MustZeros([]int64{1, 18}, gotch.Float, d)
loss, err := res.BinaryCrossEntropyWithLogits(outs, &torch.Tensor{}, &torch.Tensor{}, 2, false)
if err != nil {
log.Fatal(err)
}
// loss = loss.MustSetRequiresGrad(true, true)
//opt.ZeroGrad()
log.Info("loss", "loss", loss.Float64Values())
loss.MustBackward()
//opt.Step()
// log.Info(mean.MustGrad(false).Float64Values())
//ones_grad = ones.MustGrad(true).MustMax(true).Float64Values()[0]
// log.Info(res.MustGrad(true).MustMax(true).Float64Values())
// log.Info(ones_grad)
vars := m.Vs.Variables()
for k, v := range vars {
log.Info("[grad check]", "k", k, "grad", v.MustGrad(false).MustMax(true).Float64Values())
}
m.Debug()
outs.MustDrop()
count += 1
log.Fatal("grad zero")
}
log.Warn("out")
}

23
views/py/python_model_template.py
View File

@ -9,9 +9,9 @@ import requests
class NotifyServerCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, log, *args, **kwargs):
{{ if .HeadId }}
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
{{ else }}
requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}')
requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["val_accuracy"]}&definition={{.DefId}}')
{{end}}
@ -82,7 +82,7 @@ def prepare_dataset(ds: tf.data.Dataset, size: int) -> tf.data.Dataset:
def filterDataset(path):
path = tf.strings.regex_replace(path, DATA_DIR_PREPARE, "")
{{ if eq .Model.Format "png" }}
path = tf.strings.regex_replace(path, ".png", "")
{{ else if eq .Model.Format "jpeg" }}
@ -90,7 +90,7 @@ def filterDataset(path):
{{ else }}
ERROR
{{ end }}
return tf.reshape(table.lookup(tf.strings.as_string([path])), []) != -1
seed = random.randint(0, 100000000)
@ -135,14 +135,23 @@ def addBlock(
model.add(layers.ReLU())
if top:
if pooling_same:
model.add(pool_func(padding="same", strides=(1, 1)))
model.add(pool_func(pool_size=(2,2), padding="same", strides=(1, 1)))
else:
model.add(pool_func())
model.add(pool_func(pool_size=(2,2)))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU())
model.add(layers.Dropout(0.4))
return model
def resblock(x, kernelsize = 3, filters = 128):
fx = layers.Conv2D(filters, kernelsize, activation='relu', padding='same')(x)
fx = layers.BatchNormalization()(fx)
fx = layers.Conv2D(filters, kernelsize, padding='same')(fx)
out = layers.Add()([x,fx])
out = layers.ReLU()(out)
out = layers.BatchNormalization()(out)
return out
{{ if .LoadPrev }}
model = tf.keras.saving.load_model('{{.LastModelRunPath}}')
@ -172,7 +181,7 @@ model.compile(
his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
NotifyServerCallback(),
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)])
acc = his.history["accuracy"]

2
views/py/python_model_template_expand.py
View File

@ -10,7 +10,7 @@ import numpy as np
class NotifyServerCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, log, *args, **kwargs):
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
DATA_DIR = "{{ .DataDir }}"

1
webpage/.dockerignore Symbolic link
View File

@ -0,0 +1 @@
.gitignore

8
webpage/.eslintrc.cjs
View File

@ -27,5 +27,11 @@ module.exports = {
parser: '@typescript-eslint/parser'
}
}
]
],
rules: {
'svelte/no-at-html-tags': 'off',
// TODO remove this
'@typescript-eslint/no-explicit-any': 'off'
}
};

9
webpage/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM docker.io/node:22
ADD . .
RUN npm install
RUN npm run build
CMD ["npm", "run", "preview"]

BIN
webpage/bun.lockb
View File

Binary file not shown.

4125
webpage/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

75
webpage/package.json
View File

@ -1,38 +1,41 @@
{
"name": "webpage",
"version": "0.0.1",
"private": true,
"scripts": {
"dev:raw": "vite dev",
"dev": "vite dev --port 5001 --host",
"build": "vite build",
"preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "3.0.0",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.37.0",
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"sass": "^1.75.0",
"svelte": "5.0.0-next.104",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.8"
},
"type": "module",
"dependencies": {
"chart.js": "^4.4.2"
}
"name": "webpage",
"version": "0.0.1",
"private": true,
"scripts": {
"dev:raw": "vite dev",
"dev": "vite dev --port 5001 --host",
"build": "vite build",
"preview": "vite preview --port 5001 --host",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/d3": "^7.4.3",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.37.0",
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"sass": "^1.75.0",
"svelte": "^5.0.0-next.104",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.8"
},
"type": "module",
"dependencies": {
"chart.js": "^4.4.2",
"d3": "^7.9.0",
"highlight.js": "^11.9.0"
}
}

10
webpage/src/NavBar.svelte
View File

@ -15,8 +15,18 @@
{/if}
<li class="expand"></li>
{#if userStore.user}
{#if userStore.user.user_type == 2}
<li>
<a href="/admin/runners">
<span class="bi bi-cpu-fill"></span>
Runner
</a>
</li>
{/if}
<li>
<a href="/user/info"> <span class="bi bi-person-fill"></span> {userStore.user.username} </a>
</li>
<li>
<a href="/logout"> <span class="bi bi-box-arrow-right"></span> Logout </a>
</li>
{:else}

2
webpage/src/app.html
View File

@ -9,6 +9,8 @@
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
<link href="https://fonts.googleapis.com/css2?family=Andada+Pro:ital,wght@0,400..840;1,400..840&family=Bebas+Neue&family=Fira+Code:wght@300..700&display=swap" rel="stylesheet">
<link
href="https://fonts.googleapis.com/css2?family=Andada+Pro:ital,wght@0,400..840;1,400..840&family=Bebas+Neue&display=swap"
rel="stylesheet"

4
webpage/src/lib/FileUpload.svelte
View File

@ -36,7 +36,7 @@
class="icon"
class:adapt={replace_slot && file && !notExpand}
type="button"
on:click={() => fileInput.click()}
onclick={() => fileInput.click()}
>
{#if replace_slot && file}
<slot name="replaced" {file}>
@ -54,6 +54,6 @@
required
{accept}
bind:this={fileInput}
on:change={onChange}
onchange={onChange}
/>
</div>

57
webpage/src/lib/MessageSimple.svelte
View File

@ -1,57 +0,0 @@
<script context="module" lang="ts">
export type DisplayFn = (
msg: string,
options?: {
type?: 'error' | 'success';
timeToShow?: number;
}
) => void;
</script>
<script lang="ts">
let message = $state<string | undefined>(undefined);
let type = $state<'error' | 'success'>('error');
let timeout: number | undefined = undefined;
export function clear() {
if (timeout) clearTimeout(timeout);
message = undefined;
}
export function display(
msg: string,
options?: {
type?: 'error' | 'success';
timeToShow?: number;
}
) {
if (timeout) clearTimeout(timeout);
if (!msg) {
message = undefined;
return;
}
let { type: l_type, timeToShow } = options ?? { type: 'error', timeToShow: undefined };
if (l_type) {
type = l_type;
}
message = msg;
if (timeToShow) {
timeout = setTimeout(() => {
message = undefined;
timeout = undefined;
}, timeToShow);
}
}
</script>
{#if message}
<div class="form-msg {type}">
{message}
</div>
{/if}

10
webpage/src/lib/Tooltip.svelte
View File

@ -1,5 +1,5 @@
<script lang="ts">
let { title } = $props<{ title: string }>();
let { title }: { title: string } = $props();
let isHovered = $state(false);
let x = $state(0);
@ -30,10 +30,10 @@
<div
bind:this={div}
on:mouseover={mouseOver}
on:mouseleave={mouseLeave}
on:mousemove={mouseMove}
on:focus={focus}
onmouseover={mouseOver}
onmouseleave={mouseLeave}
onmousemove={mouseMove}
onfocus={focus}
role="tooltip"
class="tooltipContainer"
>

6
webpage/src/lib/utils.ts Normal file
View File

@ -0,0 +1,6 @@
export function preventDefault(fn: any) {
return function (event: Event) {
event.preventDefault();
fn.call(this, event);
};
}

7
webpage/src/routes/UserStore.svelte.ts
View File

@ -1,6 +1,7 @@
import { goto } from '$app/navigation';
import { rdelete } from '$lib/requests.svelte';
type User = {
export type User = {
token: string;
id: string;
user_type: number;
@ -33,6 +34,10 @@ export function createUserStore() {
if (value) {
localStorage.setItem('user', JSON.stringify(value));
} else {
if (user) {
// Request the deletion of the token
rdelete('/user/token/logoff', {});
}
localStorage.removeItem('user');
}
user = value;

350
webpage/src/routes/admin/runners/+page.svelte Normal file
View File

@ -0,0 +1,350 @@
<script lang="ts">
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
import { userStore } from 'src/routes/UserStore.svelte';
import { onMount } from 'svelte';
import * as d3 from 'd3';
import type { Base } from './types';
import CardInfo from './CardInfo.svelte';
let width = $state(0);
let height = $state(0);
function drag(simulation: d3.Simulation<d3.HierarchyNode<Base>, undefined>) {
function dragstarted(event: any, d: any) {
if (!event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
selected = d.data;
}
function dragged(event: any, d: any) {
d.fx = event.x;
d.fy = event.y;
}
function dragended(event: any, d: any) {
if (!event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
return d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended);
}
let graph: HTMLDivElement;
let selected: Base | undefined = $state();
async function getData() {
const dataObj: Base = {
name: 'API',
type: 'api',
children: []
};
if (!dataObj.children) throw new Error();
const localRunners: Base[] = [];
const remotePairs: Record<string, Base[]> = {};
try {
let data = await post('tasks/runner/info', {});
if (Object.keys(data.localRunners).length > 0) {
for (const objId of Object.keys(data.localRunners)) {
localRunners.push({ name: objId, type: 'local_runner', task: data.localRunners[objId] });
}
dataObj.children.push({
name: 'local runners',
type: 'runner_group',
children: localRunners
});
}
if (Object.keys(data.remoteRunners).length > 0) {
for (const objId of Object.keys(data.remoteRunners)) {
let obj = data.remoteRunners[objId];
if (remotePairs[obj.runner_info.user_id as string]) {
remotePairs[obj.runner_info.user_id as string].push({
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
});
} else {
remotePairs[data.remoteRunners[objId].runner_info.user_id] = [
{
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
}
];
}
}
}
dataObj.children.push({
name: 'remote runners',
type: 'runner_group',
task: undefined,
children: Object.keys(remotePairs).map(
(name) =>
({
name,
type: 'user_group',
task: undefined,
children: remotePairs[name]
}) as Base
)
});
} catch (ex) {
showMessage(ex, notificationStore, 'Failed to get Runner information');
return;
}
const root = d3.hierarchy(dataObj);
const links = root.links();
const nodes = root.descendants();
console.log(root, links, nodes);
const simulation = d3
.forceSimulation(nodes)
.force(
'link',
d3
.forceLink(links)
.id((d: any) => d.id)
.distance((d: any) => {
let data = d.source.data as Base;
switch (data.type) {
case 'api':
return 150;
case 'runner_group':
return 90;
case 'user_group':
return 80;
case 'runner':
case 'local_runner':
return 20;
default:
throw new Error();
}
})
.strength(1)
)
.force('charge', d3.forceManyBody().strength(-1000))
.force('x', d3.forceX())
.force('y', d3.forceY());
const svg = d3
.create('svg')
.attr('width', width)
.attr('height', height - 62)
.attr('viewBox', [-width / 2, -height / 2, width, height])
.attr('style', 'max-width: 100%; height: auto;');
// Append links.
const link = svg
.append('g')
.attr('stroke', '#999')
.attr('stroke-opacity', 0.6)
.selectAll('line')
.data(links)
.join('line');
const database_svg = `
<svg xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-database" viewBox="0 0 32 32">
<path transform="scale(2)" d="M4.318 2.687C5.234 2.271 6.536 2 8 2s2.766.27 3.682.687C12.644 3.125 13 3.627 13 4c0 .374-.356.875-1.318 1.313C10.766 5.729 9.464 6 8 6s-2.766-.27-3.682-.687C3.356 4.875 3 4.373 3 4c0-.374.356-.875 1.318-1.313M13 5.698V7c0 .374-.356.875-1.318 1.313C10.766 8.729 9.464 9 8 9s-2.766-.27-3.682-.687C3.356 7.875 3 7.373 3 7V5.698c.271.202.58.378.904.525C4.978 6.711 6.427 7 8 7s3.022-.289 4.096-.777A5 5 0 0 0 13 5.698M14 4c0-1.007-.875-1.755-1.904-2.223C11.022 1.289 9.573 1 8 1s-3.022.289-4.096.777C2.875 2.245 2 2.993 2 4v9c0 1.007.875 1.755 1.904 2.223C4.978 15.71 6.427 16 8 16s3.022-.289 4.096-.777C13.125 14.755 14 14.007 14 13zm-1 4.698V10c0 .374-.356.875-1.318 1.313C10.766 11.729 9.464 12 8 12s-2.766-.27-3.682-.687C3.356 10.875 3 10.373 3 10V8.698c.271.202.58.378.904.525C4.978 9.71 6.427 10 8 10s3.022-.289 4.096-.777A5 5 0 0 0 13 8.698m0 3V13c0 .374-.356.875-1.318 1.313C10.766 14.729 9.464 15 8 15s-2.766-.27-3.682-.687C3.356 13.875 3 13.373 3 13v-1.302c.271.202.58.378.904.525C4.978 12.71 6.427 13 8 13s3.022-.289 4.096-.777c.324-.147.633-.323.904-.525"/>
</svg>
`;
const cpu_svg = `
<svg stroke="white" fill="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-cpu-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M6.5 6a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5z"/>
<path transform="scale(2)" d="M5.5.5a.5.5 0 0 0-1 0V2A2.5 2.5 0 0 0 2 4.5H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2A2.5 2.5 0 0 0 4.5 14v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14a2.5 2.5 0 0 0 2.5-2.5h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14A2.5 2.5 0 0 0 11.5 2V.5a.5.5 0 0 0-1 0V2h-1V.5a.5.5 0 0 0-1 0V2h-1V.5a.5.5 0 0 0-1 0V2h-1zm1 4.5h3A1.5 1.5 0 0 1 11 6.5v3A1.5 1.5 0 0 1 9.5 11h-3A1.5 1.5 0 0 1 5 9.5v-3A1.5 1.5 0 0 1 6.5 5"/>
</svg>
`;
const user_svg = `
<svg fill="white" stroke="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-person-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M3 14s-1 0-1-1 1-4 6-4 6 3 6 4-1 1-1 1zm5-6a3 3 0 1 0 0-6 3 3 0 0 0 0 6"/>
</svg>
`;
const inbox_fill = `
<svg fill="white" stroke="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-inbox-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M4.98 4a.5.5 0 0 0-.39.188L1.54 8H6a.5.5 0 0 1 .5.5 1.5 1.5 0 1 0 3 0A.5.5 0 0 1 10 8h4.46l-3.05-3.812A.5.5 0 0 0 11.02 4zm-1.17-.437A1.5 1.5 0 0 1 4.98 3h6.04a1.5 1.5 0 0 1 1.17.563l3.7 4.625a.5.5 0 0 1 .106.374l-.39 3.124A1.5 1.5 0 0 1 14.117 13H1.883a1.5 1.5 0 0 1-1.489-1.314l-.39-3.124a.5.5 0 0 1 .106-.374z"/>
</svg>
`;
const node = svg
.append('g')
.attr('fill', '#fff')
.attr('stroke', '#000')
.attr('stroke-width', 1.5)
.selectAll('g')
.data(nodes)
.join('g')
.attr('style', 'cursor: pointer;')
.call(drag(simulation) as any)
.on('click', (e) => {
console.log('test');
function findData(obj: HTMLElement) {
if ((obj as any).__data__) {
return (obj as any).__data__;
}
if (!obj.parentElement) {
throw new Error();
}
return findData(obj.parentElement);
}
let obj = findData(e.srcElement);
console.log(obj);
selected = obj.data;
});
node
.append('circle')
.attr('fill', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
return '#caf0f8';
case 'runner_group':
return '#00b4d8';
case 'user_group':
return '#0000ff';
case 'runner':
case 'local_runner':
return '#03045e';
default:
throw new Error();
}
})
.attr('stroke', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
case 'user_group':
case 'runner_group':
return '#fff';
case 'runner':
case 'local_runner':
// TODO make this relient on the stauts
return '#000';
default:
throw new Error();
}
})
.attr('r', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
return 30;
case 'runner_group':
return 20;
case 'user_group':
return 25;
case 'runner':
case 'local_runner':
return 30;
default:
throw new Error();
}
})
.append('title')
.text((d: any) => d.data.name);
node
.filter((d) => {
return ['api', 'local_runner', 'runner', 'user_group', 'runner_group'].includes(
d.data.type
);
})
.append('g')
.html((d) => {
switch (d.data.type) {
case 'api':
return database_svg;
case 'user_group':
return user_svg;
case 'runner_group':
return inbox_fill;
case 'local_runner':
case 'runner':
return cpu_svg;
default:
throw new Error();
}
});
simulation.on('tick', () => {
link
.attr('x1', (d: any) => d.source.x)
.attr('y1', (d: any) => d.source.y)
.attr('x2', (d: any) => d.target.x)
.attr('y2', (d: any) => d.target.y);
node
.select('circle')
.attr('cx', (d: any) => d.x)
.attr('cy', (d: any) => d.y);
node
.select('svg')
.attr('x', (d: any) => d.x - 16)
.attr('y', (d: any) => d.y - 16);
});
//invalidation.then(() => simulation.stop());
graph.appendChild(svg.node() as any);
}
$effect(() => {
console.log(selected);
});
onMount(() => {
// Check if logged in and admin
if (!userStore.user || userStore.user.user_type != 2) {
goto('/');
return;
}
getData();
});
</script>
<svelte:window bind:innerWidth={width} bind:innerHeight={height} />
<svelte:head>
<title>Runners</title>
</svelte:head>
<div class="graph-container">
<div class="graph" bind:this={graph}></div>
{#if selected}
<div class="selected">
<CardInfo item={selected} />
</div>
{/if}
</div>
<style lang="css">
.graph-container {
position: relative;
.selected {
position: absolute;
right: 40px;
top: 40px;
width: 20%;
height: auto;
padding: 20px;
background: white;
border-radius: 20px;
box-shadow: 1px 1px 8px 2px #22222244;
}
}
</style>

90
webpage/src/routes/admin/runners/CardInfo.svelte Normal file
View File

@ -0,0 +1,90 @@
<script lang="ts">
import { post, showMessage } from 'src/lib/requests.svelte';
import type { Base } from './types';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import type { User } from 'src/routes/UserStore.svelte';
import Spinner from 'src/lib/Spinner.svelte';
import Tooltip from 'src/lib/Tooltip.svelte';
let { item }: { item: Base } = $props();
let user_data: User | undefined = $state();
async function getUserData(id: string) {
try {
user_data = await post('user/info/get', { id });
console.log(user_data);
} catch (ex) {
showMessage(ex, notificationStore, 'Could not get user information');
}
}
$effect(() => {
user_data = undefined;
if (item.type == 'user_group') {
getUserData(item.name);
} else if (item.type == 'runner') {
getUserData(item.parent ?? '');
}
});
</script>
{#if item.type == 'api'}
<h3>API</h3>
{:else if item.type == 'runner_group'}
<h3>Runner Group</h3>
This reprents a the group of {item.name}.
{:else if item.type == 'user_group'}
<h3>User</h3>
{#if user_data}
All Runners connected to this node bellong to <span class="accent">{user_data.username}</span>
{:else}
<div style="text-align: center;">
<Spinner />
</div>
{/if}
{:else if item.type == 'local_runner'}
<h3>Local Runner</h3>
This is a local runner
<div>
{#if item.task}
This runner is runing a <Tooltip title={item.task.id}>task</Tooltip>
{:else}
Not running any task
{/if}
</div>
{:else if item.type == 'runner'}
<h3>Runner</h3>
{#if user_data}
<p>
This is a remote runner. This runner is owned by<span class="accent"
>{user_data?.username}</span
>
</p>
<div>
{#if item.task}
This runner is runing a <Tooltip title={item.task.id}>task</Tooltip>
{:else}
Not running any task
{/if}
</div>
{:else}
<div style="text-align: center;">
<Spinner />
</div>
{/if}
{:else}
{item.type}
{/if}
<style lang="scss">
h3 {
text-align: center;
margin: 0;
}
.accent {
background: #22222222;
padding: 1px;
border-radius: 5px;
}
</style>

10
webpage/src/routes/admin/runners/types.ts Normal file
View File

@ -0,0 +1,10 @@
import type { Task } from 'src/routes/models/edit/tasks/types';
export type BaseType = 'api' | 'runner_group' | 'user_group' | 'runner' | 'local_runner';
export type Base = {
name: string;
type: BaseType;
children?: Base[];
task?: Task;
parent?: string;
};

3
webpage/src/routes/login/+page.svelte
View File

@ -3,6 +3,7 @@
import 'src/styles/forms.css';
import { userStore } from '../UserStore.svelte';
import { goto } from '$app/navigation';
import { preventDefault } from 'src/lib/utils';
let submitted = $state(false);
@ -39,7 +40,7 @@
<div class="login-page">
<div>
<h1>Login</h1>
<form on:submit|preventDefault={onSubmit} class:submitted>
<form onsubmit={preventDefault(onSubmit)} class:submitted>
<fieldset>
<label for="email">Email</label>
<input type="email" required name="email" bind:value={loginData.email} />

96
webpage/src/routes/models/+page.svelte
View File

@ -1,26 +1,22 @@
<script lang="ts">
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { onMount } from 'svelte';
import { get } from '$lib/requests.svelte';
import { get, showMessage } from '$lib/requests.svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import Spinner from 'src/lib/Spinner.svelte';
let list = $state<
{
name: string;
id: string;
}[]
>([]);
let message: MessageSimple;
| {
name: string;
id: string;
}[]
| undefined
>(undefined);
onMount(async () => {
try {
list = await get('models');
} catch (e) {
if (e instanceof Response) {
message.display(await e.json());
} else {
message.display('Could not request list of models');
}
showMessage(e, notificationStore, 'Could not request list of models');
}
});
</script>
@ -30,39 +26,44 @@
</svelte:head>
<main>
<MessageSimple bind:this={message} />
{#if list.length > 0}
<div class="list-header">
<h2>My Models</h2>
<div class="expand"></div>
<a class="button" href="/models/add"> New </a>
</div>
<table class="table">
<thead>
<tr>
<th> Name </th>
<th>
<!-- Open Button -->
</th>
</tr>
</thead>
<tbody>
{#each list as item}
{#if list}
{#if list.length > 0}
<div class="list-header">
<h2>My Models</h2>
<div class="expand"></div>
<a class="button" href="/models/add"> New </a>
</div>
<table class="table">
<thead>
<tr>
<td>
{item.name}
</td>
<td class="text-center">
<a class="button simple" href="/models/edit?id={item.id}"> Edit </a>
</td>
<th> Name </th>
<th>
<!-- Open Button -->
</th>
</tr>
{/each}
</tbody>
</table>
</thead>
<tbody>
{#each list as item}
<tr>
<td>
{item.name}
</td>
<td class="text-center">
<a class="button simple" href="/models/edit?id={item.id}"> Edit </a>
</td>
</tr>
{/each}
</tbody>
</table>
{:else}
<h2 class="text-center">You don't have any models</h2>
<div class="text-center">
<a class="button padded" href="/models/add"> Create a new model </a>
</div>
{/if}
{:else}
<h2 class="text-center">You don't have any models</h2>
<div class="text-center">
<a class="button padded" href="/models/add"> Create a new model </a>
<div style="text-align: center;">
<Spinner />
</div>
{/if}
</main>
@ -84,11 +85,4 @@
.list-header .expand {
flex-grow: 1;
}
.list-header .button,
.list-header button {
padding: 10px 10px;
height: calc(100% - 20px);
margin-top: 5px;
}
</style>

17
webpage/src/routes/models/add/+page.svelte
View File

@ -1,15 +1,14 @@
<script lang="ts">
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { postFormData } from 'src/lib/requests.svelte';
import { postFormData, showMessage } from 'src/lib/requests.svelte';
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import 'src/styles/forms.css';
import { preventDefault } from 'src/lib/utils';
let submitted = $state(false);
let message: MessageSimple;
let buttonClicked: Promise<void> = $state(Promise.resolve());
let data = $state<{
@ -21,7 +20,6 @@
});
async function onSubmit() {
message.display('');
buttonClicked = new Promise<void>(() => {});
if (!data.file || !data.name) return;
@ -34,11 +32,7 @@
let id = await postFormData('models/add', formData);
goto(`/models/edit?id=${id}`);
} catch (e) {
if (e instanceof Response) {
message.display(await e.json());
} else {
message.display('Was not able to create model');
}
showMessage(e, notificationStore, 'Was not able to create model');
}
buttonClicked = Promise.resolve();
@ -51,7 +45,7 @@
<main>
<h1>Create new Model</h1>
<form class:submitted on:submit|preventDefault={onSubmit}>
<form class:submitted onsubmit={preventDefault(onSubmit)}>
<fieldset>
<label for="name">Name</label>
<input id="name" name="name" required bind:value={data.name} />
@ -75,7 +69,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={message} />
{#await buttonClicked}
<div class="text-center">File Uploading</div>
{:then}

48
webpage/src/routes/models/edit/+page.svelte
View File

@ -30,18 +30,19 @@
import BaseModelInfo from './BaseModelInfo.svelte';
import DeleteModel from './DeleteModel.svelte';
import { goto } from '$app/navigation';
import { get, rdelete } from 'src/lib/requests.svelte';
import MessageSimple from '$lib/MessageSimple.svelte';
import { get, rdelete, showMessage } from 'src/lib/requests.svelte';
import { preventDefault } from 'src/lib/utils';
import ModelData from './ModelData.svelte';
import DeleteZip from './DeleteZip.svelte';
import RunModel from './RunModel.svelte';
import Tabs from 'src/lib/Tabs.svelte';
import TasksDataPage from './TasksDataPage.svelte';
import ModelDataPage from './ModelDataPage.svelte';
import 'src/styles/forms.css';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import Spinner from 'src/lib/Spinner.svelte';
let model: Promise<Model> = $state(new Promise(() => {}));
let _model: Model | undefined = $state(undefined);
@ -92,10 +93,7 @@
getModel();
});
let resetMessages: MessageSimple;
async function resetModel() {
resetMessages.display('');
let _model = await model;
try {
@ -105,11 +103,7 @@
getModel();
} catch (e) {
if (e instanceof Response) {
resetMessages.display(await e.json());
} else {
resetMessages.display('Could not reset model!');
}
showMessage(e, notificationStore, 'Could not reset model!');
}
}
@ -147,7 +141,8 @@
<div slot="buttons" let:setActive let:isActive>
<button
class="tab"
on:click|preventDefault={setActive('model')}
type="button"
onclick={setActive('model')}
class:selected={isActive('model')}
>
Model
@ -155,7 +150,8 @@
{#if _model && [2, 3, 4, 5, 6, 7, -6, -7].includes(_model.status)}
<button
class="tab"
on:click|preventDefault={setActive('model-data')}
type="button"
onclick={setActive('model-data')}
class:selected={isActive('model-data')}
>
Model Data
@ -164,7 +160,8 @@
{#if _model && [5, 6, 7, -6, -7].includes(_model.status)}
<button
class="tab"
on:click|preventDefault={setActive('tasks')}
type="button"
onclick={setActive('tasks')}
class:selected={isActive('tasks')}
>
Tasks
@ -172,7 +169,7 @@
{/if}
</div>
{#if _model}
<ModelDataPage model={_model} on:reload={getModel} active={isActive('model-data')} />
<ModelDataPage model={_model} onreload={getModel} active={isActive('model-data')} />
<TasksDataPage model={_model} active={isActive('tasks')} />
{/if}
<div class="content" class:selected={isActive('model')}>
@ -192,7 +189,6 @@
<h1 class="text-center">
{m.name}
</h1>
<!-- TODO improve message -->
<h2 class="text-center">Failed to prepare model</h2>
<DeleteModel model={m} />
@ -200,25 +196,23 @@
<!-- PRE TRAINING STATUS -->
{:else if m.status == 2}
<BaseModelInfo model={m} />
<ModelData model={m} on:reload={getModel} />
<ModelData model={m} onreload={getModel} />
<!-- {{ template "train-model-card" . }} -->
<DeleteModel model={m} />
{:else if m.status == -2}
<BaseModelInfo model={m} />
<DeleteZip model={m} on:reload={getModel} />
<DeleteZip model={m} onreload={getModel} />
<DeleteModel model={m} />
{:else if m.status == 3}
<BaseModelInfo model={m} />
<div class="card">
<!-- TODO improve this -->
Processing zip file...
Processing zip file... <Spinner />
</div>
{:else if m.status == -3 || m.status == -4}
<BaseModelInfo model={m} />
<form on:submit|preventDefault={resetModel}>
<form onsubmit={preventDefault(resetModel)}>
Failed Prepare for training.<br />
<div class="spacer"></div>
<MessageSimple bind:this={resetMessages} />
<button class="danger"> Try Again </button>
</form>
<DeleteModel model={m} />
@ -337,7 +331,7 @@
<div class="card">Model expading... Processing ZIP file</div>
{/if}
{#if m.status == -6}
<DeleteZip model={m} on:reload={getModel} expand />
<DeleteZip model={m} onreload={getModel} expand />
{/if}
{#if m.status == -7}
<form>
@ -346,7 +340,7 @@
</form>
{/if}
{#if m.model_type == 2}
<ModelData simple model={m} on:reload={getModel} />
<ModelData simple model={m} onreload={getModel} />
{/if}
<DeleteModel model={m} />
{:else}
@ -383,10 +377,4 @@
table tr th:first-child {
border-left: none;
}
table tr td button,
table tr td .button {
padding: 5px 10px;
box-shadow: 0 2px 5px 1px #66666655;
}
</style>

2
webpage/src/routes/models/edit/BaseModelInfo.svelte
View File

@ -1,6 +1,6 @@
<script lang="ts">
import type { Model } from './+page.svelte';
let { model } = $props<{ model: Model }>();
let { model }: { model: Model } = $props();
</script>
<div class="card model-card">

18
webpage/src/routes/models/edit/DeleteModel.svelte
View File

@ -1,39 +1,31 @@
<script lang="ts">
import MessageSimple from 'src/lib/MessageSimple.svelte';
import type { Model } from './+page.svelte';
import { rdelete } from '$lib/requests.svelte';
import { rdelete, showMessage } from '$lib/requests.svelte';
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
let { model } = $props<{ model: Model }>();
let { model }: { model: Model } = $props();
let name: string = $state('');
let submmited: boolean = $state(false);
let messageSimple: MessageSimple;
async function deleteModel() {
submmited = true;
messageSimple.display('');
try {
await rdelete('models/delete', { id: model.id, name });
goto('/models');
} catch (e) {
if (e instanceof Response) {
messageSimple.display(await e.json());
} else {
messageSimple.display('Could not delete the model');
}
showMessage(e, notificationStore, 'Could not delete the model');
}
}
</script>
<form on:submit|preventDefault={deleteModel} class:submmited class="danger-bg">
<form onsubmit={deleteModel} class:submmited class="danger-bg">
<fieldset>
<label for="name">
To delete this model please type "{model.name}":
</label>
<input name="name" id="name" required bind:value={name} />
</fieldset>
<MessageSimple bind:this={messageSimple} />
<button class="danger"> Delete </button>
</form>

33
webpage/src/routes/models/edit/DeleteZip.svelte
View File

@ -1,32 +1,30 @@
<script lang="ts">
import { rdelete } from 'src/lib/requests.svelte';
import { rdelete, showMessage } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher } from 'svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { preventDefault } from 'src/lib/utils';
let message: MessageSimple;
let { model, expand } = $props<{ model: Model; expand?: boolean }>();
const dispatch = createEventDispatcher<{ reload: void }>();
let {
model,
expand,
onreload = () => {}
}: {
model: Model;
expand?: boolean;
onreload?: () => void;
} = $props();
async function deleteZip() {
message.clear();
try {
await rdelete('models/data/delete-zip-file', { id: model.id });
dispatch('reload');
onreload();
} catch (e) {
if (e instanceof Response) {
message.display(await e.json());
} else {
message.display('Could not delete the zip file');
}
showMessage(e, notificationStore, 'Could not delete the zip file');
}
}
</script>
<form on:submit|preventDefault={deleteZip}>
<form onsubmit={preventDefault(deleteZip)}>
{#if expand}
Failed to proccess the zip file.<br />
Delete file and upload a correct version do add more classes.<br />
@ -37,6 +35,5 @@
<br />
{/if}
<div class="spacer"></div>
<MessageSimple bind:this={message} />
<button class="danger"> Delete Zip File </button>
</form>

90
webpage/src/routes/models/edit/ModelData.svelte
View File

@ -1,38 +1,29 @@
<script lang="ts" context="module">
export type Class = {
name: string;
id: string;
status: number;
};
</script>
<script lang="ts">
import FileUpload from 'src/lib/FileUpload.svelte';
import Tabs from 'src/lib/Tabs.svelte';
import type { Model } from './+page.svelte';
import { postFormData, get } from 'src/lib/requests.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher } from 'svelte';
import type { Class } from './types';
import { postFormData, get, showMessage } from 'src/lib/requests.svelte';
import ModelTable from './ModelTable.svelte';
import TrainModel from './TrainModel.svelte';
import ZipStructure from './ZipStructure.svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { preventDefault } from 'src/lib/utils';
let { model, simple } = $props<{ model: Model; simple?: boolean }>();
let {
model,
simple,
onreload = () => {}
}: { model: Model; simple?: boolean; onreload?: () => void } = $props();
let classes: Class[] = $state([]);
let has_data: boolean = $state(false);
let file: File | undefined = $state();
const dispatch = createEventDispatcher<{
reload: void;
}>();
let uploading: Promise<void> = $state(Promise.resolve());
let numberOfInvalidImages = $state(0);
let uploadImage: MessageSimple;
async function uploadZip() {
if (!file) return;
@ -44,13 +35,9 @@
try {
await postFormData('models/data/upload', form);
dispatch('reload');
onreload();
} catch (e) {
if (e instanceof Response) {
uploadImage.display(await e.json());
} else {
uploadImage.display('');
}
showMessage(e, notificationStore, 'Could not upload data');
}
uploading = Promise.resolve();
@ -67,8 +54,8 @@
classes = data.classes;
numberOfInvalidImages = data.number_of_invalid_images;
has_data = data.has_data;
} catch {
return;
} catch (e) {
showMessage(e, notificationStore, 'Could not get information on classes');
}
}
</script>
@ -80,22 +67,22 @@
<p>You need to upload data so the model can train.</p>
<Tabs active="upload" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive('upload')} on:click={setActive('upload')}>
<button class="tab" class:selected={isActive('upload')} onclick={setActive('upload')}>
Upload
</button>
<button
<!--button
class="tab"
class:selected={isActive('create-class')}
on:click={setActive('create-class')}
onclick={setActive('create-class')}
>
Create Class
</button>
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
</button-->
<!--button class="tab" class:selected={isActive('api')} onclick={setActive('api')}>
Api
</button>
</button-->
</div>
<div class="content" class:selected={isActive('upload')}>
<form on:submit|preventDefault={uploadZip}>
<form onsubmit={preventDefault(uploadZip)}>
<fieldset class="file-upload">
<label for="file">Data file</label>
<div class="form-msg">
@ -115,7 +102,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#if file}
{#await uploading}
<button disabled> Uploading </button>
@ -125,10 +111,10 @@
{/if}
</form>
</div>
<div class="content" class:selected={isActive('create-class')}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive('api')}>TODO</div>
<!--div class="content" class:selected={isActive('create-class')}>
<ModelTable {classes} {model} {onreload} />
</div-->
<!--div class="content" class:selected={isActive('api')}>TODO</div-->
</Tabs>
<div class="tabs"></div>
{:else}
@ -136,36 +122,14 @@
{#if numberOfInvalidImages > 0}
<p class="danger">
There are images {numberOfInvalidImages} that were loaded that do not have the correct format.
These images will be delete when the model trains.
These images will be deleted when the model trains.
</p>
{/if}
<Tabs active="create-class" let:isActive>
<div slot="buttons" let:setActive let:isActive>
<button
class="tab"
class:selected={isActive('create-class')}
on:click={setActive('create-class')}
>
Create Class
</button>
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}>
Api
</button>
</div>
<div class="content" class:selected={isActive('create-class')}>
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive('api')}>TODO</div>
</Tabs>
<ModelTable {classes} {model} {onreload} />
{/if}
</div>
{/if}
{#if classes.some((item) => item.status == 1) && ![-6, 6].includes(model.status)}
<TrainModel
number_of_invalid_images={numberOfInvalidImages}
{model}
{has_data}
on:reload={() => dispatch('reload')}
/>
<TrainModel number_of_invalid_images={numberOfInvalidImages} {model} {has_data} {onreload} />
{/if}

15
webpage/src/routes/models/edit/ModelDataPage.svelte
View File

@ -1,5 +1,4 @@
<script lang="ts">
import { createEventDispatcher } from 'svelte';
import type { Model } from './+page.svelte';
import ModelData from './ModelData.svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
@ -7,9 +6,11 @@
import type { ModelStats } from './types';
import DeleteZip from './DeleteZip.svelte';
let { model, active }: { model: Model; active?: boolean } = $props();
const dispatch = createEventDispatcher<{ reload: void }>();
let {
model,
active,
onreload = () => {}
}: { model: Model; active?: boolean; onreload?: () => void } = $props();
$effect(() => {
if (active) getData();
@ -34,14 +35,14 @@
{/if}
{#if [-6, -2].includes(model.status)}
<DeleteZip {model} on:reload={() => dispatch('reload')} expand />
<DeleteZip {model} {onreload} expand />
{/if}
<ModelData
{model}
on:reload={() => {
onreload={() => {
getData();
dispatch('reload');
onreload();
}}
/>
</div>

2
webpage/src/routes/models/edit/ModelDataPageStatsGraph.svelte
View File

@ -97,7 +97,7 @@
});
</script>
<div><canvas bind:this={ctx} /></div>
<div><canvas bind:this={ctx}></canvas></div>
<style lang="scss">
canvas {

101
webpage/src/routes/models/edit/ModelTable.svelte
View File

@ -1,33 +1,24 @@
<script lang="ts" context="module">
export type Image = {
file_path: string;
mode: number;
status: number;
id: string;
};
</script>
<script lang="ts">
import Tabs from 'src/lib/Tabs.svelte';
import type { Class } from './ModelData.svelte';
import type { Class, Image } from './types';
import { post, postFormData, rdelete, showMessage } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher } from 'svelte';
import ZipStructure from './ZipStructure.svelte';
const dispatch = createEventDispatcher<{ reload: void }>();
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { preventDefault } from 'src/lib/utils.js';
import CreateNewClass from './api/CreateNewClass.svelte';
let selected_class: Class | undefined = $state();
let { classes, model }: { classes: Class[]; model: Model } = $props();
let { classes, model, onreload }: { classes: Class[]; model: Model; onreload?: () => void } =
$props();
let createClass: { className: string } = $state({
className: ''
});
let page = $state(0);
let page = $state(-1);
let showNext = $state(false);
let image_list = $state<Image[]>([]);
@ -41,9 +32,10 @@
});
async function getList() {
if (!selected_class) return;
try {
let res = await post('models/data/list', {
id: selected_class?.id ?? '',
id: selected_class.id,
page: page
});
showNext = res.showNext;
@ -53,19 +45,20 @@
}
}
$effect(() => {
getList();
});
$effect(() => {
if (selected_class) {
page = 0;
getList();
}
});
let file: File | undefined = $state();
let uploadImage: MessageSimple;
let uploading = $state(Promise.resolve());
async function uploadZip() {
uploadImage.clear();
if (!file) return;
uploading = new Promise(() => {});
@ -76,19 +69,14 @@
try {
await postFormData('models/data/class/upload', form);
dispatch('reload');
if (onreload) onreload();
} catch (e) {
if (e instanceof Response) {
uploadImage.display(await e.json());
} else {
uploadImage.display('');
}
showMessage(e, notificationStore, 'Failed to upload');
}
uploading = Promise.resolve();
}
let createNewClassMessages: MessageSimple;
async function createNewClass() {
try {
const r = await post('models/data/class/new', {
@ -100,7 +88,7 @@
classes = classes;
getList();
} catch (e) {
showMessage(e, createNewClassMessages);
showMessage(e, notificationStore);
}
}
@ -109,12 +97,11 @@
rdelete('models/data/point', { id });
getList();
} catch (e) {
console.error('TODO notify user', e);
showMessage(e, notificationStore);
}
}
let addFile: File | undefined = $state();
let addImageMessages: MessageSimple;
let adding = $state(Promise.resolve());
let uploadImageDialog: HTMLDialogElement;
async function addImage() {
@ -136,7 +123,7 @@
addFile = undefined;
getList();
} catch (e) {
showMessage(e, addImageMessages);
showMessage(e, notificationStore);
}
}
</script>
@ -151,30 +138,30 @@
{#each classes as item}
<button
style="width: auto; white-space: nowrap;"
on:click={() => setActiveClass(item, setActive)}
onclick={() => setActiveClass(item, setActive)}
class="tab"
class:selected={isActive(item.name)}
>
{item.name}
{#if model.model_type == 2}
{#if item.status == 1}
<span class="bi bi-book" style="color: orange;" />
<span class="bi bi-book" style="color: orange;"></span>
{:else if item.status == 2}
<span class="bi bi-book" style="color: green;" />
<span class="bi bi-book" style="color: green;"></span>
{:else if item.status == 3}
<span class="bi bi-check" style="color: green;" />
<span class="bi bi-check" style="color: green;"></span>
{/if}
{/if}
</button>
{/each}
</div>
<button
on:click={() => {
onclick={() => {
setActive('-----New Class-----')();
selected_class = undefined;
}}
>
<span class="bi bi-plus" />
<span class="bi bi-plus"></span>
</button>
</div>
{#if selected_class == undefined && isActive('-----New Class-----')}
@ -184,21 +171,31 @@
<div slot="buttons" let:setActive let:isActive>
<button
class="tab"
on:click|preventDefault={setActive('zip')}
type="button"
onclick={setActive('zip')}
class:selected={isActive('zip')}
>
Zip
</button>
<button
class="tab"
on:click|preventDefault={setActive('empty')}
type="button"
onclick={setActive('empty')}
class:selected={isActive('empty')}
>
Empty Class
</button>
<button
class="tab"
type="button"
onclick={setActive('api')}
class:selected={isActive('api')}
>
API
</button>
</div>
<div class="content" class:selected={isActive('zip')}>
<form on:submit|preventDefault={uploadZip}>
<form onsubmit={preventDefault(uploadZip)}>
<fieldset class="file-upload">
<label for="file">Data file</label>
<div class="form-msg">
@ -218,7 +215,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#if file}
{#await uploading}
<button disabled> Uploading </button>
@ -229,7 +225,7 @@
</form>
</div>
<div class="content" class:selected={isActive('empty')}>
<form on:submit|preventDefault={createNewClass}>
<form onsubmit={preventDefault(createNewClass)}>
<div class="form-msg">
This Creates an empty class that allows images to be added after
</div>
@ -237,10 +233,12 @@
<label for="className">Class Name</label>
<input required name="className" bind:value={createClass.className} />
</fieldset>
<MessageSimple bind:this={createNewClassMessages} />
<button> Create New Class </button>
</form>
</div>
<div class="content" class:selected={isActive('api')}>
<CreateNewClass {model} />
</div>
</Tabs>
</div>
{/if}
@ -258,7 +256,7 @@
{:else}
Class to train
{/if}
<button on:click={() => uploadImageDialog.showModal()}> Upload Image </button>
<button onclick={() => uploadImageDialog.showModal()}> Upload Image </button>
</h2>
<table>
<thead>
@ -314,7 +312,7 @@
{/if}
</td>
<td style="width: 3ch">
<button class="danger" on:click={() => deleteDataPoint(image.id)}>
<button class="danger" onclick={() => deleteDataPoint(image.id)}>
<span class="bi bi-trash"></span>
</button>
</td>
@ -325,7 +323,7 @@
<div class="flex justify-center align-center">
<div class="grow-1 flex justify-end align-center">
{#if page > 0}
<button on:click={() => (page -= 1)}> Prev </button>
<button onclick={() => (page -= 1)}> Prev </button>
{/if}
</div>
@ -335,7 +333,7 @@
<div class="grow-1 flex justify-start align-center">
{#if showNext}
<button on:click={() => (page += 1)}> Next </button>
<button onclick={() => (page += 1)}> Next </button>
{/if}
</div>
</div>
@ -345,7 +343,7 @@
{/if}
<dialog class="newImageDialog" bind:this={uploadImageDialog}>
<form on:submit|preventDefault={addImage}>
<form onsubmit={preventDefault(addImage)}>
<fieldset class="file-upload">
<label for="file">Data file</label>
<div class="form-msg">
@ -365,7 +363,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={addImageMessages} />
{#if addFile}
{#await adding}
<button disabled> Uploading </button>
@ -415,10 +412,4 @@
table tr th:first-child {
border-left: none;
}
table tr td button,
table tr td .button {
padding: 5px 10px;
box-shadow: 0 2px 5px 1px #66666655;
}
</style>

172
webpage/src/routes/models/edit/RunModel.svelte
View File

@ -1,26 +1,29 @@
<script lang="ts">
import { post, postFormData } from 'src/lib/requests.svelte';
import { post, postFormData, showMessage } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher, onDestroy } from 'svelte';
import { onDestroy } from 'svelte';
import Spinner from 'src/lib/Spinner.svelte';
import type { Task } from './TasksTable.svelte';
import type { Task } from './tasks/types';
import Tabs from 'src/lib/Tabs.svelte';
import hljs from 'highlight.js';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { preventDefault } from 'src/lib/utils';
let { model } = $props<{ model: Model }>();
let {
model,
onupload = () => {},
ontaskReload = () => {}
}: { model: Model; onupload?: () => void; ontaskReload?: () => void } = $props();
let file: File | undefined = $state();
const dispatch = createEventDispatcher<{ upload: void; taskReload: void }>();
let _result: Promise<Task> = $state(new Promise(() => {}));
let run = $state(false);
let last_task: string | undefined = $state();
let last_task_timeout: number | null = null;
let messages: MessageSimple;
async function reloadLastTimeout() {
if (!last_task) {
return;
@ -31,7 +34,7 @@
const r = await post('tasks/task', { id: last_task });
if ([0, 1, 2, 3].includes(r.status)) {
setTimeout(reloadLastTimeout, 500);
setTimeout(() => dispatch('taskReload'), 500);
setTimeout(ontaskReload, 500);
} else {
_result = Promise.resolve(r);
}
@ -42,7 +45,6 @@
async function submit() {
if (!file) return;
messages.clear();
let form = new FormData();
form.append('json_data', JSON.stringify({ id: model.id }));
@ -56,14 +58,10 @@
file = undefined;
last_task_timeout = setTimeout(() => reloadLastTimeout());
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display('Could not run the model');
}
showMessage(e, notificationStore, 'Could not run the model');
}
dispatch('upload');
onupload();
}
onDestroy(() => {
@ -73,39 +71,111 @@
});
</script>
<form on:submit|preventDefault={submit}>
<fieldset class="file-upload">
<label for="file">Image</label>
<div class="form-msg">Run image through them model and get the result</div>
<Tabs active="upload" let:isActive>
<div class="buttons" slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive('upload')} onclick={setActive('upload')}>
Upload
</button>
<button class="tab" class:selected={isActive('api')} onclick={setActive('api')}> Api </button>
</div>
<div class="content" class:selected={isActive('api')}>
<div class="codeinfo">
To perform an image classfication please follow the example bellow:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`let form = new FormData();
form.append('json_data', JSON.stringify({ id: '${model.id}' }));
form.append('file', file, 'file');
<FileUpload replace_slot bind:file accept="image/*">
<img src="/imgs/upload-icon.png" alt="" />
<span> Upload image </span>
<div slot="replaced-name">
<span> Image selected </span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={messages} />
<button> Run </button>
{#if run}
{#await _result}
<h1>
Processing Image! <Spinner />
</h1>
{:then result}
{#if result.status == 4}
{@const res = JSON.parse(result.result)}
<div>
<h1>Result</h1>
The image was classified as {res.class} with confidence: {res.confidence}
</div>
{:else}
<div class="result">
<h1>There was a problem running the task:</h1>
{result?.status_message}
</div>
const headers = new Headers();
headers.append('response-type', 'application/json');
headers.append('token', token);
const r = await fetch('${window.location.protocol}//${window.location.hostname}/api/tasks/start/image', {
method: 'POST',
headers: headers,
body: form
});`,
{ language: 'javascript' }
).value}</pre>
On Success the request will return a json with this format:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`{ id "00000000-0000-0000-0000-000000000000" }`,
{ language: 'json' }
).value}</pre>
This id can be used to query the API for the result of the task:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`const headers = new Headers();
headers.append('content-type', 'application/json');
headers.append('token', token);
const r = await fetch('${window.location.protocol}//${window.location.hostname}/api/tasks/task', {
method: 'POST',
headers: headers,
body: JSON.stringify({ id: '00000000-0000-0000-0000-000000000000' })
});`,
{ language: 'javascript' }
).value}</pre>
Once the task shows the status as 4 then the data can be obatined in the result field: The successful
return value has this type:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`{
"id": string,
"user_id": string,
"model_id": string,
"status": number,
"status_message": string,
"user_confirmed": number,
"compacted": number,
"type": number,
"extra_task_info": string,
"result": string,
"created": string
}`,
{ language: 'javascript' }
).value}</pre>
</div>
</div>
<div class="content" class:selected={isActive('upload')}>
<form onsubmit={preventDefault(submit)} style="box-shadow: none;">
<fieldset class="file-upload">
<label for="file">Image</label>
<div class="form-msg">Run image through them model and get the result</div>
<FileUpload replace_slot bind:file accept="image/*">
<img src="/imgs/upload-icon.png" alt="" />
<span> Upload image </span>
<div slot="replaced-name">
<span> Image selected </span>
</div>
</FileUpload>
</fieldset>
<button> Run </button>
{#if run}
{#await _result}
<h1>
Processing Image! <Spinner />
</h1>
{:then result}
{#if result.status == 4}
{@const res = JSON.parse(result.result)}
<div>
<h1>Result</h1>
The image was classified as {res.class} with confidence: {res.confidence}
</div>
{:else}
<div class="result">
<h1>There was a problem running the task:</h1>
{result?.status_message}
</div>
{/if}
{/await}
{/if}
{/await}
{/if}
</form>
</form>
</div>
</Tabs>
<style lang="scss">
.codeinfo {
padding: 20px;
}
</style>

3
webpage/src/routes/models/edit/TasksDataPage.svelte
View File

@ -1,5 +1,4 @@
<script lang="ts">
import { post } from 'src/lib/requests.svelte';
import type { Model } from 'src/routes/models/edit/+page.svelte';
import RunModel from './RunModel.svelte';
import TasksTable from './tasks/TasksTable.svelte';
@ -12,7 +11,7 @@
{#if active}
<div class="content selected">
<RunModel {model} on:upload={() => table.getList()} on:taskReload={() => table.getList()} />
<RunModel {model} onupload={() => table.getList()} ontaskReload={() => table.getList()} />
<TasksTable {model} bind:this={table} />
<Stats {model} />
</div>

50
webpage/src/routes/models/edit/TrainModel.svelte
View File

@ -1,14 +1,20 @@
<script lang="ts">
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import type { Model } from './+page.svelte';
import { post } from 'src/lib/requests.svelte';
import { createEventDispatcher } from 'svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
import { preventDefault } from 'src/lib/utils';
let { number_of_invalid_images, has_data, model } = $props<{
let {
number_of_invalid_images,
has_data,
model,
onreload = () => {}
}: {
number_of_invalid_images: number;
has_data: boolean;
model: Model;
}>();
onreload?: () => void;
} = $props();
let data = $state({
model_type: 'simple',
@ -18,54 +24,39 @@
let submitted = $state(false);
let dispatch = createEventDispatcher<{ reload: void }>();
let messages: MessageSimple;
async function submit() {
messages.clear();
submitted = true;
try {
await post('models/train', {
id: model.id,
...data
});
dispatch('reload');
onreload();
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display('Could not start the training of the model');
}
showMessage(e, notificationStore, 'Could not start the training of the model');
}
}
async function submitRetrain() {
messages.clear();
submitted = true;
try {
await post('model/train/retrain', { id: model.id });
dispatch('reload');
onreload();
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display('Could not start the training of the model');
}
showMessage(e, notificationStore, 'Could not start the training of the model');
}
}
</script>
{#if model.status == 2}
<form class:submitted on:submit|preventDefault={submit}>
<form class:submitted onsubmit={preventDefault(submit)}>
{#if has_data}
{#if number_of_invalid_images > 0}
<p class="danger">
There are images {number_of_invalid_images} that were loaded that do not have the correct format.DeleteZip
These images will be delete when the model trains.
These images will be deleted when the model trains.
</p>
{/if}
<MessageSimple bind:this={messages} />
<!-- TODO expading mode -->
<fieldset>
<legend> Model Type </legend>
@ -110,17 +101,16 @@
<h2>To train the model please provide data to the model first</h2>
{/if}
</form>
{:else}
<form class:submitted on:submit|preventDefault={submitRetrain}>
{:else if ![4, 6, 7].includes(model.status)}
<form class:submitted onsubmit={submitRetrain}>
{#if has_data}
<h2>This model has new classes and can be expanded</h2>
{#if number_of_invalid_images > 0}
<p class="danger">
There are images {number_of_invalid_images} that were loaded that do not have the correct format.DeleteZip
These images will be delete when the model trains.
These images will be deleted when the model trains.
</p>
{/if}
<MessageSimple bind:this={messages} />
<button> Retrain </button>
{:else}
<h2>To train the model please provide data to the model first</h2>

27
webpage/src/routes/models/edit/api/CreateNewClass.svelte Normal file
View File

@ -0,0 +1,27 @@
<script lang="ts">
import hljs from 'highlight.js';
import type { Model } from '../+page.svelte';
let { model }: { model: Model } = $props();
</script>
To create a new class via the API you can:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`let form = new FormData();
form.append('json_data', JSON.stringify({
id: '${model.id}',
name: 'New class name'
}));
form.append('file', file, 'file');
const headers = new Headers();
headers.append('response-type', 'application/json');
headers.append('token', token);
const r = await fetch('${window.location.protocol}//${window.location.hostname}/models/data/class/new', {
method: 'POST',
headers: headers,
body: form
});`,
{ language: 'javascript' }
).value}</pre>

57
webpage/src/routes/models/edit/tasks/Stats.svelte
View File

@ -1,5 +1,5 @@
<script lang="ts">
import { onDestroy, onMount } from 'svelte';
import { onDestroy } from 'svelte';
import type { Model } from '../+page.svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
import type { DataPoint, TasksStatsDay } from 'src/types/stats/task';
@ -18,7 +18,6 @@
PointElement,
LineElement
} from 'chart.js';
import ModelData from '../ModelData.svelte';
Chart.register(
Title,
@ -57,7 +56,9 @@
}
let pie: HTMLCanvasElement;
let pie2: HTMLCanvasElement;
let pieChart: Chart<'pie'> | undefined;
let pie2Chart: Chart<'pie'> | undefined;
function createPie(s: TasksStatsDay) {
if (pieChart) {
pieChart.destroy();
@ -72,23 +73,31 @@
'Classfication Failure',
'Classfication Preparing',
'Classfication Running',
'Classfication Unknown',
'Non Classfication Error',
'Non Classfication Success'
'Classfication Unknown'
],
datasets: [
{
label: 'Total',
data: [
t.c_error,
t.c_success,
t.c_failure,
t.c_pre_running,
t.c_running,
t.c_unknown,
t.nc_error,
t.nc_success
]
data: [t.c_error, t.c_success, t.c_failure, t.c_pre_running, t.c_running, t.c_unknown]
}
]
},
options: {
animation: false
}
});
if (pie2Chart) {
pieChart.destroy();
}
pie2Chart = new Chart(pie2, {
type: 'pie',
data: {
labels: ['Non Classfication Error', 'Non Classfication Success'],
datasets: [
{
label: 'Total',
data: [t.nc_error, t.nc_success]
}
]
},
@ -115,7 +124,6 @@
nc_error: 'Non Classfication Error',
nc_success: 'Non Classfication Success'
};
let t = s.total;
let labels = new Array(24).fill(0).map((_, i) => i);
lineChart = new Chart(line, {
type: 'line',
@ -147,8 +155,13 @@
<h1>Statistics (Day)</h1>
<h2>Total</h2>
<div>
<canvas bind:this={pie}></canvas>
<div class="pies">
<div>
<canvas bind:this={pie}></canvas>
</div>
<div>
<canvas bind:this={pie2}></canvas>
</div>
</div>
<h2>Hourly</h2>
@ -160,4 +173,12 @@
canvas {
width: 100%;
}
.pies {
display: flex;
align-content: stretch;
div {
width: 50%;
}
}
</style>

29
webpage/src/routes/models/edit/tasks/TasksTable.svelte
View File

@ -1,16 +1,4 @@
<script lang="ts" context="module">
export type Task = {
id: string;
user_id: string;
model_id: string;
status: number;
status_message: string;
user_confirmed: number;
compacted: number;
type: number;
created: string;
result: string;
};
export const TaskType = {
TASK_FAILED_RUNNING: -2,
TASK_FAILED_CREATION: -1,
@ -52,9 +40,10 @@
<script lang="ts">
import { post, showMessage } from 'src/lib/requests.svelte';
import type { Model } from '../+page.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import Tooltip from 'src/lib/Tooltip.svelte';
import type { Task } from './types';
let { model }: { model: Model } = $props();
let page = $state(0);
@ -80,7 +69,6 @@
}
});
let userPreceptionMessages: MessageSimple;
// This returns a function that performs the call and does not do the call it self
function userPreception(task: string, agree: number) {
return async function () {
@ -99,7 +87,6 @@
<div>
<h2>Tasks</h2>
<MessageSimple bind:this={userPreceptionMessages} />
<table>
<thead>
<tr>
@ -156,14 +143,14 @@
<div>
{#if task.user_confirmed != 1}
<Tooltip title="Agree with the result of the task">
<button type="button" on:click={userPreception(task.id, 1)}>
<button type="button" onclick={userPreception(task.id, 1)}>
<span class="bi bi-check"></span>
</button>
</Tooltip>
{/if}
{#if task.user_confirmed != -1}
<Tooltip title="Disagree with the result">
<button class="danger" type="button" on:click={userPreception(task.id, -1)}>
<button class="danger" type="button" onclick={userPreception(task.id, -1)}>
<span class="bi bi-x-lg"></span>
</button>
</Tooltip>
@ -208,7 +195,7 @@
<div class="flex justify-center align-center">
<div class="grow-1 flex justify-end align-center">
{#if page > 0}
<button on:click={() => (page -= 1)}> Prev </button>
<button onclick={() => (page -= 1)}> Prev </button>
{/if}
</div>
@ -218,7 +205,7 @@
<div class="grow-1 flex justify-start align-center">
{#if showNext}
<button on:click={() => (page += 1)}> Next </button>
<button onclick={() => (page += 1)}> Next </button>
{/if}
</div>
</div>
@ -229,10 +216,6 @@
width: 100%;
display: flex;
justify-content: space-between;
& > button {
margin: 3px 5px;
}
}
table {

12
webpage/src/routes/models/edit/tasks/types.ts Normal file
View File

@ -0,0 +1,12 @@
export type Task = {
id: string;
user_id: string;
model_id: string;
status: number;
status_message: string;
user_confirmed: number;
compacted: number;
type: number;
created: string;
result: string;
};

13
webpage/src/routes/models/edit/types.ts
View File

@ -3,3 +3,16 @@ export type ModelStats = Array<{
training: number;
testing: number;
}>;
export type Class = {
name: string;
id: string;
status: number;
};
export type Image = {
file_path: string;
mode: number;
status: number;
id: string;
};

3
webpage/src/routes/register/+page.svelte
View File

@ -3,6 +3,7 @@
import 'src/styles/forms.css';
import { userStore } from '../UserStore.svelte';
import { goto } from '$app/navigation';
import { preventDefault } from 'src/lib/utils';
let submitted = $state(false);
@ -39,7 +40,7 @@
<div class="login-page">
<div>
<h1>Register</h1>
<form on:submit|preventDefault={onSubmit} class:submitted>
<form onsubmit={preventDefault(onSubmit)} class:submitted>
<fieldset>
<label for="username">Username</label>
<input required name="username" bind:value={loginData.username} />

41
webpage/src/routes/user/info/+page.svelte
View File

@ -4,10 +4,11 @@
import { onMount } from 'svelte';
import 'src/styles/forms.css';
import { post } from 'src/lib/requests.svelte';
import MessageSimple, { type DisplayFn } from 'src/lib/MessageSimple.svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
import TokenTable from './TokenTable.svelte';
import DeleteUser from './DeleteUser.svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { preventDefault } from 'src/lib/utils';
onMount(() => {
if (!userStore.isLogin()) {
@ -26,12 +27,8 @@
let submiitedEmail = $state(false);
let submiitedPassword = $state(false);
let msgEmail: MessageSimple;
let msgPassword: MessageSimple;
async function onSubmitEmail() {
submiitedEmail = true;
msgEmail.display('');
if (!userStore.user) return;
@ -44,31 +41,31 @@
...userStore.user,
...req
};
msgEmail.display('User updated successufly!', { type: 'success', timeToShow: 10000 });
notificationStore.add({
message: 'User updated successufly!',
type: 'success',
timeToLive: 10000
});
} catch (e) {
if (e instanceof Response) {
msgEmail.display(await e.json());
} else {
msgEmail.display('Could not update email');
}
showMessage(e, notificationStore, 'Could not update email');
}
}
async function onSubmitPassword() {
submiitedPassword = true;
msgPassword.display('');
if (!userStore.user) return;
try {
await post('user/info/password', passwordData);
passwordData = { old_password: '', password: '', password2: '' };
msgPassword.display('Password updated successufly!', { type: 'success', timeToShow: 10000 });
notificationStore.add({
message: 'Password updated successufly!',
type: 'success',
timeToLive: 10000
});
} catch (e) {
if (e instanceof Response) {
msgPassword.display(await e.json());
} else {
msgPassword.display('Could not update password');
}
showMessage(e, notificationStore, 'Could not update password');
}
}
</script>
@ -80,15 +77,14 @@
<div class="login-page">
<div>
<h1>User Infomation</h1>
<form on:submit|preventDefault={onSubmitEmail} class:submiitedEmail>
<form onsubmit={onSubmitEmail} class:submiitedEmail>
<fieldset>
<label for="email">Email</label>
<input type="email" required name="email" bind:value={email} />
</fieldset>
<MessageSimple bind:this={msgEmail} />
<button> Update </button>
</form>
<form on:submit|preventDefault={onSubmitPassword} class:submiitedPassword>
<form onsubmit={preventDefault(onSubmitPassword)} class:submiitedPassword>
<fieldset>
<label for="old_password">Old Password</label>
<input
@ -106,7 +102,6 @@
<label for="password2">Repeat New Password</label>
<input required bind:value={passwordData.password2} name="password2" type="password" />
</fieldset>
<MessageSimple bind:this={msgPassword} />
<div>
<button> Update </button>
</div>

167
webpage/src/routes/user/info/AddToken.svelte
View File

@ -1,36 +1,32 @@
<script lang="ts">
import {createEventDispatcher} from 'svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import Tooltip from 'src/lib/Tooltip.svelte';
import 'src/styles/forms.css';
import {post} from 'src/lib/requests.svelte';
import Spinner from 'src/lib/Spinner.svelte';
import { post } from 'src/lib/requests.svelte';
import Spinner from 'src/lib/Spinner.svelte';
import { preventDefault } from 'src/lib/utils';
const dispatch = createEventDispatcher<{reload: void}>();
let { onreload = () => {} }: { onreload?: () => void } = $props();
let addNewToken = $state(false);
let messages: MessageSimple;
let expiry_date: HTMLInputElement = $state(undefined as any);
type NewToken = {
name: string,
expiry: number,
token: string,
}
let token: Promise<NewToken> | undefined = $state();
type NewToken = {
name: string;
expiry: number;
token: string;
};
let token: Promise<NewToken> | undefined = $state();
let newToken: {
name: string;
expiry: string;
password: string;
password: string;
} = $state({
name: '',
expiry: '',
password: '',
password: ''
});
async function createToken(e: SubmitEvent & { currentTarget: HTMLFormElement }) {
@ -47,86 +43,85 @@
}
}
try {
const r = await post("user/token/add", {
name: newToken.name,
expiry: expiry,
password: newToken.password,
});
token = Promise.resolve(r)
setTimeout(() => dispatch('reload'), 500)
} catch (e) {
token = undefined;
console.error("Notify user", e)
}
try {
const r = await post('user/token/add', {
name: newToken.name,
expiry: expiry,
password: newToken.password
});
token = Promise.resolve(r);
setTimeout(onreload, 500);
} catch (e) {
token = undefined;
console.error('Notify user', e);
}
}
$effect(() => {
if (expiry_date) {
if (isNaN(Number(newToken.expiry))) {
expiry_date.setCustomValidity('Invalid Number');
} else {
expiry_date.setCustomValidity('');
}
}
if (expiry_date) {
if (isNaN(Number(newToken.expiry))) {
expiry_date.setCustomValidity('Invalid Number');
} else {
expiry_date.setCustomValidity('');
}
}
});
</script>
{#if addNewToken}
<div>
<h2>Add New Token</h2>
{#if !token}
<form on:submit|preventDefault={createToken}>
<fieldset>
<label for="name">Name</label>
<input required bind:value={newToken.name} name="name" />
</fieldset>
<fieldset>
<label for="expiry_date">Expiry Date</label>
<div class="flex">
<input bind:this={expiry_date} bind:value={newToken.expiry} name="expiry_date" />
<Tooltip title="Time in seconds. Leave empty to last forever">
<span class="center-question bi bi-question-circle-fill" />
</Tooltip>
</div>
</fieldset>
<fieldset>
<label for="password">Password</label>
<input required bind:value={newToken.password} name="password" />
</fieldset>
<MessageSimple bind:this={messages} />
<div>
<button> Update </button>
</div>
</form>
{:else}
{#await token}
<Spinner /> Generating
{:then t}
<h3> Token generated </h3>
<form on:submit|preventDefault={() => {}}>
<fieldset>
<label for="token">Token</label>
<div class="flex">
<input value={t.token} on:input={(e) => e.preventDefault() } name="token" />
<div style="width: 5em;">
<button on:click={() => navigator.clipboard.writeText(t.token)} >
<span class="bi bi-clipboard" />
</button>
</div>
</div>
</fieldset>
<div>
<button on:click={() => token = undefined}> Generate new token </button>
</div>
</form>
{:catch e}
{e}
{/await}
{/if}
{#if !token}
<form onsubmit={preventDefault(createToken)}>
<fieldset>
<label for="name">Name</label>
<input required bind:value={newToken.name} name="name" />
</fieldset>
<fieldset>
<label for="expiry_date">Expiry Date</label>
<div class="flex">
<input bind:this={expiry_date} bind:value={newToken.expiry} name="expiry_date" />
<Tooltip title="Time in seconds. Leave empty to last forever">
<span class="center-question bi bi-question-circle-fill"></span>
</Tooltip>
</div>
</fieldset>
<fieldset>
<label for="password">Password</label>
<input required bind:value={newToken.password} name="password" />
</fieldset>
<div>
<button> Update </button>
</div>
</form>
{:else}
{#await token}
<Spinner /> Generating
{:then t}
<h3>Token generated</h3>
<form onsubmit={preventDefault(() => {})}>
<fieldset>
<label for="token">Token</label>
<div class="flex">
<input value={t.token} oninput={(e) => e.preventDefault()} name="token" />
<div style="width: 5em;">
<button onclick={() => navigator.clipboard.writeText(t.token)}>
<span class="bi bi-clipboard"></span>
</button>
</div>
</div>
</fieldset>
<div>
<button onclick={() => (token = undefined)}> Generate new token </button>
</div>
</form>
{:catch e}
{e}
{/await}
{/if}
</div>
{:else}
<div>
<button class="expander" on:click={() => (addNewToken = true)}> Add New Token </button>
<button class="expander" onclick={() => (addNewToken = true)}> Add New Token </button>
</div>
{/if}

3
webpage/src/routes/user/info/DeleteUser.svelte
View File

@ -2,6 +2,7 @@
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { rdelete, showMessage } from 'src/lib/requests.svelte';
import { preventDefault } from 'src/lib/utils';
import { userStore } from 'src/routes/UserStore.svelte';
let data = $state({ password: '' });
@ -23,7 +24,7 @@
}
</script>
<form class="danger-bg" on:submit|preventDefault={deleteUser}>
<form class="danger-bg" onsubmit={preventDefault(deleteUser)}>
<h2 class="no-top-margin">Delete user</h2>
Deleting the user will delete all your data stored in the service including the images.
<fieldset>

26
webpage/src/routes/user/info/TokenTable.svelte
View File

@ -10,7 +10,7 @@
import { post, rdelete } from 'src/lib/requests.svelte';
import { userStore } from 'src/routes/UserStore.svelte';
import AddToken from './AddToken.svelte';
import AddToken from './AddToken.svelte';
let page = $state(0);
let showNext = $state(false);
@ -70,7 +70,7 @@
{new Date(token.create_date).toLocaleString()}
</td>
<td>
<button class="danger" on:click={() => removeToken(token)}>
<button class="danger" onclick={() => removeToken(token)}>
<span class="bi bi-trash"></span>
</button>
</td>
@ -81,7 +81,7 @@
<div class="flex justify-center align-center">
<div class="grow-1 flex justify-end align-center">
{#if page > 0}
<button on:click={() => (page -= 1)}> Prev </button>
<button onclick={() => (page -= 1)}> Prev </button>
{/if}
</div>
@ -91,25 +91,15 @@
<div class="grow-1 flex justify-start align-center">
{#if showNext}
<button on:click={() => (page += 1)}> Next </button>
<button onclick={() => (page += 1)}> Next </button>
{/if}
</div>
</div>
</div>
<AddToken on:reload={getList} />
<AddToken onreload={getList} />
<style lang="scss">
.buttons {
width: 100%;
display: flex;
justify-content: space-between;
& > button {
margin: 3px 5px;
}
}
table {
width: 100%;
box-shadow: 0 2px 8px 1px #66666622;
@ -132,10 +122,4 @@
table tr th:first-child {
border-left: none;
}
table tr td button,
table tr td .button {
padding: 5px 10px;
box-shadow: 0 2px 5px 1px #66666655;
}
</style>

10
webpage/src/styles/app.css
View File

@ -65,6 +65,7 @@ button.expander::after {
a.button {
text-decoration: none;
height: 1.6em;
}
.flex {
@ -190,3 +191,12 @@ form.danger-bg {
background-color: var(--danger-transparent);
border: 1px solid var(--danger);
}
pre {
font-family: 'Fira Code';
background-color: #f6f8fa;
word-break: break-word;
white-space: pre-wrap;
border-radius: 10px;
padding: 10px;
}

8
webpage/svelte.config.js
View File

@ -12,10 +12,10 @@ const config = {
// If your environment is not supported or you settled on a specific environment, switch out the adapter.
// See https://kit.svelte.dev/docs/adapters for more information about adapters.
adapter: adapter(),
alias: {
src: "src",
routes: "src/routes",
}
alias: {
src: 'src',
routes: 'src/routes'
}
}
};

7
webpage/vite.config.ts
View File

@ -2,5 +2,10 @@ import { sveltekit } from '@sveltejs/kit/vite';
import { defineConfig } from 'vite';
export default defineConfig({
plugins: [sveltekit()]
plugins: [sveltekit()],
build: {
commonjsOptions: {
esmExternals: true
}
}
});