Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b1e4211e6a | |||
| e22df8adc9 | |||
| 7d346ba2ce | |||
| 29b69deaf6 |
@@ -1,5 +0,0 @@
|
||||
tmp/
|
||||
testData/
|
||||
savedData/
|
||||
!savedData/.keep
|
||||
fyp
|
||||
@@ -1,19 +1,30 @@
|
||||
FROM docker.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
|
||||
FROM docker.io/nvidia/cuda:12.3.2-devel-ubuntu22.04
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Sometimes you have to get update twice because ?
|
||||
RUN apt-get update
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y wget sudo pkg-config libopencv-dev unzip python3-pip vim
|
||||
|
||||
RUN pip install torch==2.1.0 torchvision==0.16.0 torchaudio==2.1.0
|
||||
|
||||
RUN mkdir /go
|
||||
ENV GOPATH=/go
|
||||
RUN apt-get install -y wget unzip python3-pip vim python3 python3-pip curl
|
||||
|
||||
RUN wget https://go.dev/dl/go1.22.2.linux-amd64.tar.gz
|
||||
RUN tar -xvf go1.22.2.linux-amd64.tar.gz -C /usr/local
|
||||
ENV PATH=$PATH:/usr/local/go/bin
|
||||
ENV GOPATH=/go
|
||||
|
||||
RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.9.1.tar.gz" | tar -C /usr/local -xz'
|
||||
# RUN bash -c 'curl -L "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-cpu-linux-x86_64-2.13.1.tar.gz" | tar -C /usr/local -xz'
|
||||
RUN ldconfig
|
||||
|
||||
RUN ln -s /usr/bin/python3 /usr/bin/python
|
||||
RUN python -m pip install nvidia-pyindex
|
||||
ADD requirements.txt .
|
||||
RUN python -m pip install -r requirements.txt
|
||||
|
||||
ENV CUDNN_PATH=/usr/local/lib/python3.10/dist-packages/nvidia/cudnn
|
||||
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/python3.10/dist-packages/nvidia/cudnn/lib
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
|
||||
ADD go.mod .
|
||||
@@ -23,38 +34,4 @@ ADD logic logic
|
||||
|
||||
RUN go install || true
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN wget https://git.andr3h3nriqu3s.com/andr3/gotch/raw/commit/22e75becf0432cda41a7c055a4d60ea435f76599/setup-libtorch.sh
|
||||
RUN chmod +x setup-libtorch.sh
|
||||
ENV CUDA_VER=11.8
|
||||
ENV GOTCH_VER=v0.9.2
|
||||
RUN bash setup-libtorch.sh
|
||||
ENV GOTCH_LIBTORCH="/usr/local/lib/libtorch"
|
||||
|
||||
ENV REFRESH_SETUP=0
|
||||
|
||||
ENV LIBRARY_PATH="$LIBRARY_PATH:$GOTCH_LIBTORCH/lib"
|
||||
ENV export CPATH="$CPATH:$GOTCH_LIBTORCH/lib:$GOTCH_LIBTORCH/include:$GOTCH_LIBTORCH/include/torch/csrc/api/include"
|
||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GOTCH_LIBTORCH/lib:/usr/lib64-nvidia:/usr/local/cuda-${CUDA_VERSION}/lib64"
|
||||
RUN wget https://git.andr3h3nriqu3s.com/andr3/gotch/raw/branch/master/setup-gotch.sh
|
||||
RUN chmod +x setup-gotch.sh
|
||||
RUN echo 'root ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
|
||||
RUN bash setup-gotch.sh
|
||||
|
||||
RUN ln -s /usr/local/lib/libtorch/include/torch/csrc /usr/local/lib/libtorch/include/torch/csrc/api/include/torch
|
||||
RUN mkdir -p /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch/csrc/api
|
||||
RUN find /usr/local/lib/libtorch/include -maxdepth 1 -type d | tail -n +2 | grep -ve 'torch$' | xargs -I{} ln -s {} /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include
|
||||
RUN ln -s /usr/local/lib/libtorch/include/torch/csrc/api/include /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch/csrc/api/include
|
||||
RUN find /usr/local/lib/libtorch/include/torch -maxdepth 1 -type f | xargs -I{} ln -s {} /go/pkg/mod/git.andr3h3nriqu3s.com/andr3/gotch@v0.9.2/libtch/libtorch/include/torch
|
||||
RUN ln -s /usr/local/lib/libtorch/lib/libcudnn.so.8 /usr/local/lib/libcudnn.so
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV CGO_CXXFLAGS="-I/usr/local/lib/libtorch/include/torch/csrc/api/include/ -I/usr/local/lib/libtorch/include"
|
||||
ENV CGO_CFLAGS="-I/usr/local/lib/libtorch/include/torch/csrc/api/include/ -I/usr/local/lib/libtorch/include"
|
||||
|
||||
ADD . .
|
||||
RUN go build -x || true
|
||||
|
||||
CMD ["bash", "-c", "go run ."]
|
||||
CMD ["go", "run", "."]
|
||||
|
||||
3
go.mod
3
go.mod
@@ -4,10 +4,11 @@ go 1.21
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/log v0.3.1
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe
|
||||
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/lib/pq v1.10.9
|
||||
golang.org/x/crypto v0.19.0
|
||||
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2
|
||||
)
|
||||
|
||||
require (
|
||||
|
||||
12
go.sum
12
go.sum
@@ -1,7 +1,3 @@
|
||||
git.andr3h3nriqu3s.com/andr3/gotch v0.9.1 h1:1q34JKV8cX80n7LXbJswlXCiRtNbzcvJ/vbgb6an1tA=
|
||||
git.andr3h3nriqu3s.com/andr3/gotch v0.9.1/go.mod h1:FXusE3CHt8NLf5wynUGaHtIbToRuYifsZaC5EZH0pJY=
|
||||
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2 h1:aZcsPgDVGVhrEFoer0upSkzPqJWNMxdUHRktP4s6MSc=
|
||||
git.andr3h3nriqu3s.com/andr3/gotch v0.9.2/go.mod h1:FXusE3CHt8NLf5wynUGaHtIbToRuYifsZaC5EZH0pJY=
|
||||
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
|
||||
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
@@ -17,6 +13,12 @@ github.com/charmbracelet/log v0.3.1/go.mod h1:OR4E1hutLsax3ZKpXbgUqPtTjQfrh1pG3z
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e h1:9+2AEFZymTi25FIIcDwuzcOPH04z9+fV6XeLiGORPDI=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20221023090153-6b7fa0680c3e/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe h1:7yELf1NFEwECpXMGowkoftcInMlVtLTCdwWLmxKgzNM=
|
||||
github.com/galeone/tensorflow/tensorflow/go v0.0.0-20240119075110-6ad3cf65adfe/go.mod h1:TelZuq26kz2jysARBwOrTv16629hyUsHmIoj54QqyFo=
|
||||
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99 h1:8Bt1P/zy1gb37L4n8CGgp1qmFwBV5729kxVfj0sqhJk=
|
||||
github.com/galeone/tfgo v0.0.0-20230715013254-16113111dc99/go.mod h1:3YgYBeIX42t83uP27Bd4bSMxTnQhSbxl0pYSkCDB1tc=
|
||||
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
|
||||
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -72,9 +74,7 @@ github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg=
|
||||
github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc=
|
||||
|
||||
10
lib.go.back
10
lib.go.back
@@ -1,10 +0,0 @@
|
||||
package libtch
|
||||
|
||||
// #cgo LDFLAGS: -lstdc++ -ltorch -lc10 -ltorch_cpu -L${SRCDIR}/libtorch/lib
|
||||
// #cgo LDFLAGS: -L/usr/local/cuda/lib64 -lcuda -lcudart -lcublas -lcudnn -lcaffe2_nvrtc -lnvrtc-builtins -lnvrtc -lnvToolsExt -lc10_cuda -ltorch_cuda
|
||||
// #cgo CFLAGS: -I${SRCDIR} -O3 -Wall -Wno-unused-variable -Wno-deprecated-declarations -Wno-c++11-narrowing -g -Wno-sign-compare -Wno-unused-function
|
||||
// #cgo CFLAGS: -D_GLIBCXX_USE_CXX11_ABI=0
|
||||
// #cgo CFLAGS: -I/usr/local/cuda/include
|
||||
// #cgo CXXFLAGS: -std=c++17 -I${SRCDIR} -g -O3
|
||||
// #cgo CXXFLAGS: -I${SRCDIR}/libtorch/lib -I${SRCDIR}/libtorch/include -I${SRCDIR}/libtorch/include/torch/csrc/api/include -I/opt/libtorch/include/torch/csrc/api/include
|
||||
import "C"
|
||||
@@ -16,9 +16,9 @@ const (
|
||||
)
|
||||
|
||||
type ModelClass struct {
|
||||
Id string `db:"mc.id"`
|
||||
ModelId string `db:"mc.model_id"`
|
||||
Name string `db:"mc.name"`
|
||||
ClassOrder int `db:"mc.class_order"`
|
||||
Status int `db:"mc.status"`
|
||||
Id string `db:"mc.id" json:"id"`
|
||||
ModelId string `db:"mc.model_id" json:"model_id"`
|
||||
Name string `db:"mc.name" json:"name"`
|
||||
ClassOrder int `db:"mc.class_order" json:"class_order"`
|
||||
Status int `db:"mc.status" json:"status"`
|
||||
}
|
||||
|
||||
@@ -20,14 +20,14 @@ const (
|
||||
)
|
||||
|
||||
type Definition struct {
|
||||
Id string `db:"md.id"`
|
||||
ModelId string `db:"md.model_id"`
|
||||
Accuracy float64 `db:"md.accuracy"`
|
||||
TargetAccuracy int `db:"md.target_accuracy"`
|
||||
Epoch int `db:"md.epoch"`
|
||||
Status int `db:"md.status"`
|
||||
CreatedOn time.Time `db:"md.created_on"`
|
||||
EpochProgress int `db:"md.epoch_progress"`
|
||||
Id string `db:"md.id" json:"id"`
|
||||
ModelId string `db:"md.model_id" json:"model_id"`
|
||||
Accuracy float64 `db:"md.accuracy" json:"accuracy"`
|
||||
TargetAccuracy int `db:"md.target_accuracy" json:"target_accuracy"`
|
||||
Epoch int `db:"md.epoch" json:"epoch"`
|
||||
Status int `db:"md.status" json:"status"`
|
||||
CreatedOn time.Time `db:"md.created_on" json:"created"`
|
||||
EpochProgress int `db:"md.epoch_progress" json:"epoch_progress"`
|
||||
}
|
||||
|
||||
type SortByAccuracyDefinitions []*Definition
|
||||
|
||||
@@ -16,12 +16,12 @@ const (
|
||||
)
|
||||
|
||||
type Layer struct {
|
||||
Id string `db:"mdl.id"`
|
||||
DefinitionId string `db:"mdl.def_id"`
|
||||
LayerOrder string `db:"mdl.layer_order"`
|
||||
LayerType LayerType `db:"mdl.layer_type"`
|
||||
Shape string `db:"mdl.shape"`
|
||||
ExpType string `db:"mdl.exp_type"`
|
||||
Id string `db:"mdl.id" json:"id"`
|
||||
DefinitionId string `db:"mdl.def_id" json:"definition_id"`
|
||||
LayerOrder string `db:"mdl.layer_order" json:"layer_order"`
|
||||
LayerType LayerType `db:"mdl.layer_type" json:"layer_type"`
|
||||
Shape string `db:"mdl.shape" json:"shape"`
|
||||
ExpType string `db:"mdl.exp_type" json:"exp_type"`
|
||||
}
|
||||
|
||||
func ShapeToString(args ...int) string {
|
||||
|
||||
@@ -1,32 +1,45 @@
|
||||
package dbtypes
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
const (
|
||||
FAILED_TRAINING = -4
|
||||
FAILED_PREPARING_TRAINING = -3
|
||||
FAILED_PREPARING_ZIP_FILE = -2
|
||||
FAILED_PREPARING = -1
|
||||
type ModelStatus int
|
||||
|
||||
PREPARING = 1
|
||||
CONFIRM_PRE_TRAINING = 2
|
||||
PREPARING_ZIP_FILE = 3
|
||||
TRAINING = 4
|
||||
READY = 5
|
||||
READY_ALTERATION = 6
|
||||
READY_ALTERATION_FAILED = -6
|
||||
const (
|
||||
FAILED_TRAINING ModelStatus = -4
|
||||
FAILED_PREPARING_TRAINING = -3
|
||||
FAILED_PREPARING_ZIP_FILE = -2
|
||||
FAILED_PREPARING = -1
|
||||
PREPARING = 1
|
||||
CONFIRM_PRE_TRAINING = 2
|
||||
PREPARING_ZIP_FILE = 3
|
||||
TRAINING = 4
|
||||
READY = 5
|
||||
READY_ALTERATION = 6
|
||||
READY_ALTERATION_FAILED = -6
|
||||
|
||||
READY_RETRAIN = 7
|
||||
READY_RETRAIN_FAILED = -7
|
||||
)
|
||||
|
||||
type ModelDefinitionStatus int
|
||||
|
||||
const (
|
||||
MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
|
||||
MODEL_DEFINITION_STATUS_FAILED_TRAINING = -3
|
||||
MODEL_DEFINITION_STATUS_PRE_INIT = 1
|
||||
MODEL_DEFINITION_STATUS_INIT = 2
|
||||
MODEL_DEFINITION_STATUS_TRAINING = 3
|
||||
MODEL_DEFINITION_STATUS_PAUSED_TRAINING = 6
|
||||
MODEL_DEFINITION_STATUS_TRANIED = 4
|
||||
MODEL_DEFINITION_STATUS_READY = 5
|
||||
)
|
||||
|
||||
type ModelHeadStatus int
|
||||
|
||||
const (
|
||||
@@ -51,6 +64,8 @@ type BaseModel struct {
|
||||
CanTrain int `db:"can_train"`
|
||||
}
|
||||
|
||||
var ModelNotFoundError = errors.New("Model not found error")
|
||||
|
||||
func GetBaseModel(db db.Db, id string) (base *BaseModel, err error) {
|
||||
var model BaseModel
|
||||
err = GetDBOnce(db, &model, "models where id=$1", id)
|
||||
@@ -68,36 +83,6 @@ func (m BaseModel) CanEval() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (m BaseModel) removeFailedDataPoints(c BasePack) (err error) {
|
||||
rows, err := c.GetDb().Query("select mdp.id from model_data_point as mdp join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 and mdp.status=-1;", m.Id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
base_path := path.Join("savedData", m.Id, "data")
|
||||
|
||||
for rows.Next() {
|
||||
var dataPointId string
|
||||
err = rows.Scan(&dataPointId)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
p := path.Join(base_path, dataPointId+"."+m.Format)
|
||||
|
||||
c.GetLogger().Warn("Removing image", "path", p)
|
||||
|
||||
err = os.RemoveAll(p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
_, err = c.GetDb().Exec("delete from model_data_point as mdp using model_classes as mc where mdp.class_id = mc.id and mc.model_id=$1 and mdp.status=-1;", m.Id)
|
||||
return
|
||||
}
|
||||
|
||||
// DO NOT Pass un filtered data on filters
|
||||
func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
|
||||
n_args := []any{m.Id}
|
||||
@@ -105,25 +90,20 @@ func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Def
|
||||
return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
|
||||
}
|
||||
|
||||
// DO NOT Pass un filtered data on filters
|
||||
func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
|
||||
n_args := []any{m.Id}
|
||||
n_args = append(n_args, args...)
|
||||
return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
|
||||
}
|
||||
|
||||
type DataPointIterator struct {
|
||||
rows pgx.Rows
|
||||
Model BaseModel
|
||||
func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
|
||||
_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
|
||||
return
|
||||
}
|
||||
|
||||
type DataPoint struct {
|
||||
Class int
|
||||
Path string
|
||||
}
|
||||
|
||||
func (iter DataPointIterator) Close() {
|
||||
iter.rows.Close()
|
||||
Class int `json:"class"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
|
||||
@@ -158,14 +138,11 @@ func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint,
|
||||
return
|
||||
}
|
||||
|
||||
const RGB string = "rgb"
|
||||
const GRAY string = "greyscale"
|
||||
|
||||
func StringToImageMode(colorMode string) int {
|
||||
switch colorMode {
|
||||
case GRAY:
|
||||
case "greyscale":
|
||||
return 1
|
||||
case RGB:
|
||||
case "rgb":
|
||||
return 3
|
||||
default:
|
||||
panic("unkown color mode")
|
||||
|
||||
@@ -14,13 +14,11 @@ import (
|
||||
"github.com/charmbracelet/log"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
|
||||
db "git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
)
|
||||
|
||||
type BasePack interface {
|
||||
db.Db
|
||||
GetDb() db.Db
|
||||
GetLogger() *log.Logger
|
||||
GetHost() string
|
||||
@@ -44,18 +42,6 @@ func (b BasePackStruct) GetLogger() *log.Logger {
|
||||
return b.Logger
|
||||
}
|
||||
|
||||
func (c BasePackStruct) Query(query string, args ...any) (pgx.Rows, error) {
|
||||
return c.Db.Query(query, args...)
|
||||
}
|
||||
|
||||
func (c BasePackStruct) Exec(query string, args ...any) (pgconn.CommandTag, error) {
|
||||
return c.Db.Exec(query, args...)
|
||||
}
|
||||
|
||||
func (c BasePackStruct) Begin() (pgx.Tx, error) {
|
||||
return c.Db.Begin()
|
||||
}
|
||||
|
||||
func CheckEmpty(f url.Values, path string) bool {
|
||||
return !f.Has(path) || f.Get(path) == ""
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ type ModelClassJSON struct {
|
||||
Status int `json:"status"`
|
||||
}
|
||||
|
||||
func ListClassesJSON(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
|
||||
func ListClasses(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
|
||||
return GetDbMultitple[ModelClassJSON](c.GetDb(), "model_classes where model_id=$1", model_id)
|
||||
}
|
||||
|
||||
|
||||
@@ -435,7 +435,7 @@ func handleDataUpload(handle *Handle) {
|
||||
}
|
||||
|
||||
model, err := GetBaseModel(handle.Db, id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||
} else if err != nil {
|
||||
return c.Error500(err)
|
||||
@@ -468,7 +468,7 @@ func handleDataUpload(handle *Handle) {
|
||||
}
|
||||
PostAuthJson(handle, "/models/data/class/new", User_Normal, func(c *Context, obj *CreateNewEmptyClass) *Error {
|
||||
model, err := GetBaseModel(c.Db, obj.Id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.JsonBadRequest("Model not found")
|
||||
} else if err != nil {
|
||||
return c.E500M("Failed to get model information", err)
|
||||
@@ -518,7 +518,7 @@ func handleDataUpload(handle *Handle) {
|
||||
c.Logger.Info("model", "model", *model_id)
|
||||
|
||||
model, err := GetBaseModel(c.Db, *model_id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.JsonBadRequest("Could not find the model")
|
||||
} else if err != nil {
|
||||
return c.E500M("Error getting model information", err)
|
||||
@@ -626,7 +626,7 @@ func handleDataUpload(handle *Handle) {
|
||||
c.Logger.Info("Trying to expand model", "id", id)
|
||||
|
||||
model, err := GetBaseModel(handle.Db, id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||
} else if err != nil {
|
||||
return c.Error500(err)
|
||||
@@ -670,7 +670,7 @@ func handleDataUpload(handle *Handle) {
|
||||
}
|
||||
|
||||
model, err := GetBaseModel(handle.Db, dat.Id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||
} else if err != nil {
|
||||
return c.Error500(err)
|
||||
|
||||
@@ -51,7 +51,7 @@ func handleDelete(handle *Handle) {
|
||||
return c.E500M("Faield to get model", err)
|
||||
}
|
||||
|
||||
switch model.Status {
|
||||
switch ModelStatus(model.Status) {
|
||||
case FAILED_TRAINING:
|
||||
fallthrough
|
||||
case FAILED_PREPARING_ZIP_FILE:
|
||||
|
||||
@@ -24,7 +24,7 @@ func handleEdit(handle *Handle) {
|
||||
return c.Error500(err)
|
||||
}
|
||||
|
||||
cls, err := model_classes.ListClassesJSON(c, model.Id)
|
||||
cls, err := model_classes.ListClasses(c, model.Id)
|
||||
if err != nil {
|
||||
return c.Error500(err)
|
||||
}
|
||||
@@ -109,7 +109,7 @@ func handleEdit(handle *Handle) {
|
||||
layers := []layerdef{}
|
||||
|
||||
for _, def := range defs {
|
||||
if def.Status == DEFINITION_STATUS_TRAINING {
|
||||
if def.Status == MODEL_DEFINITION_STATUS_TRAINING {
|
||||
rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id)
|
||||
if err != nil {
|
||||
return c.Error500(err)
|
||||
@@ -166,7 +166,7 @@ func handleEdit(handle *Handle) {
|
||||
|
||||
for i, def := range defs {
|
||||
var lay *[]layerdef = nil
|
||||
if def.Status == DEFINITION_STATUS_TRAINING && !setLayers {
|
||||
if def.Status == MODEL_DEFINITION_STATUS_TRAINING && !setLayers {
|
||||
lay = &layers
|
||||
setLayers = true
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
tf "github.com/galeone/tensorflow/tensorflow/go"
|
||||
"github.com/galeone/tensorflow/tensorflow/go/op"
|
||||
tg "github.com/galeone/tfgo"
|
||||
@@ -19,6 +20,7 @@ func ReadPNG(scope *op.Scope, imagePath string, channels int64) *image.Image {
|
||||
contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
|
||||
output := op.DecodePng(scope.SubScope("DecodePng"), contents, op.DecodePngChannels(channels))
|
||||
output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
|
||||
output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
|
||||
image := &image.Image{
|
||||
Tensor: tg.NewTensor(scope, output)}
|
||||
return image.Scale(0, 255)
|
||||
@@ -29,6 +31,7 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
|
||||
contents := op.ReadFile(scope.SubScope("ReadFile"), op.Const(scope.SubScope("filename"), imagePath))
|
||||
output := op.DecodePng(scope.SubScope("DecodeJpeg"), contents, op.DecodePngChannels(channels))
|
||||
output = op.ExpandDims(scope.SubScope("ExpandDims"), output, op.Const(scope.SubScope("axis"), []int32{0}))
|
||||
output = op.ExpandDims(scope.SubScope("Stack"), output, op.Const(scope.SubScope("axis"), []int32{1}))
|
||||
image := &image.Image{
|
||||
Tensor: tg.NewTensor(scope, output)}
|
||||
return image.Scale(0, 255)
|
||||
@@ -49,6 +52,8 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
|
||||
var vmax float32 = 0.0
|
||||
var predictions = results[0].Value().([][]float32)[0]
|
||||
|
||||
log.Info("preds", "preds", predictions)
|
||||
|
||||
for i, v := range predictions {
|
||||
if v > vmax {
|
||||
order = i
|
||||
@@ -62,10 +67,13 @@ func runModelNormal(base BasePack, model *BaseModel, def_id string, inputImage *
|
||||
}
|
||||
|
||||
func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) {
|
||||
log := base.GetLogger()
|
||||
|
||||
err = nil
|
||||
order = 0
|
||||
|
||||
log.Info("Running base")
|
||||
|
||||
base_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "base", "model"), []string{"serve"}, nil)
|
||||
|
||||
//results := base_model.Exec([]tf.Output{
|
||||
@@ -86,7 +94,7 @@ func runModelExp(base BasePack, model *BaseModel, def_id string, inputImage *tf.
|
||||
return
|
||||
}
|
||||
|
||||
base.GetLogger().Info("test", "count", len(heads))
|
||||
log.Info("Running heads", "heads", heads)
|
||||
|
||||
var vmax float32 = 0.0
|
||||
|
||||
79
logic/models/train/remote_train.go
Normal file
79
logic/models/train/remote_train.go
Normal file
@@ -0,0 +1,79 @@
|
||||
package models_train
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||
"github.com/goccy/go-json"
|
||||
)
|
||||
|
||||
func PrepareTraining(handler *Handle, b BasePack, task Task, runner_id string) (err error) {
|
||||
l := b.GetLogger()
|
||||
|
||||
model, err := GetBaseModel(b.GetDb(), *task.ModelId)
|
||||
if err != nil {
|
||||
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
|
||||
l.Error("Failed to get model information", "err", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if model.Status != TRAINING {
|
||||
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
|
||||
return errors.New("Model not in the right status")
|
||||
}
|
||||
|
||||
// TODO do this when the runner says it's OK
|
||||
//task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
|
||||
|
||||
// TODO move this to the runner part as well
|
||||
var dat struct {
|
||||
NumberOfModels int
|
||||
Accuracy int
|
||||
}
|
||||
|
||||
err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
|
||||
if err != nil {
|
||||
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
|
||||
}
|
||||
|
||||
if model.ModelType == 2 {
|
||||
panic("TODO")
|
||||
full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
|
||||
if full_error != nil {
|
||||
l.Error("Failed to generate defintions", "err", full_error)
|
||||
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
|
||||
return errors.New("Failed to generate definitions")
|
||||
}
|
||||
} else {
|
||||
error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
|
||||
if error != nil {
|
||||
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
|
||||
return errors.New("Failed to generate definitions")
|
||||
}
|
||||
}
|
||||
|
||||
runners := handler.DataMap["runners"].(map[string]interface{})
|
||||
runner := runners[runner_id].(map[string]interface{})
|
||||
runner["task"] = &task
|
||||
|
||||
runners[runner_id] = runner
|
||||
handler.DataMap["runners"] = runners
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func CleanUpFailed(b BasePack, task *Task) {
|
||||
db := b.GetDb()
|
||||
l := b.GetLogger()
|
||||
model, err := GetBaseModel(db, *task.ModelId)
|
||||
if err != nil {
|
||||
l.Error("Failed to get model", "err", err)
|
||||
} else {
|
||||
err = model.UpdateStatus(db, FAILED_TRAINING)
|
||||
if err != nil {
|
||||
l.Error("Failed to get status", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
func handleRest(handle *Handle) {
|
||||
DeleteAuthJson(handle, "/models/train/reset", User_Normal, func(c *Context, dat *JustId) *Error {
|
||||
model, err := GetBaseModel(c.Db, dat.Id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.JsonBadRequest("Model not found")
|
||||
} else if err != nil {
|
||||
return c.E500M("Failed to get model", err)
|
||||
}
|
||||
|
||||
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
|
||||
if model.Status != FAILED_PREPARING_TRAINING && model.Status != int(FAILED_TRAINING) {
|
||||
return c.JsonBadRequest("Model is not in status that be reset")
|
||||
}
|
||||
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
package imageloader
|
||||
|
||||
import (
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
types "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch"
|
||||
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch/vision"
|
||||
)
|
||||
|
||||
type Dataset struct {
|
||||
TrainImages *torch.Tensor
|
||||
TrainLabels *torch.Tensor
|
||||
TestImages *torch.Tensor
|
||||
TestLabels *torch.Tensor
|
||||
TrainImagesSize int
|
||||
TestImagesSize int
|
||||
Device gotch.Device
|
||||
}
|
||||
|
||||
func LoadImagesAndLables(db db.Db, m *types.BaseModel, mode types.DATA_POINT_MODE, classStart int, classEnd int) (imgs, labels *torch.Tensor, count int, err error) {
|
||||
train_points, err := m.DataPoints(db, types.DATA_POINT_MODE_TRAINING)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
size := int64(classEnd - classStart + 1)
|
||||
|
||||
pimgs := []*torch.Tensor{}
|
||||
plabels := []*torch.Tensor{}
|
||||
|
||||
for _, point := range train_points {
|
||||
var img, label *torch.Tensor
|
||||
img, err = vision.Load(point.Path)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
pimgs = append(pimgs, img)
|
||||
|
||||
t_label := make([]int, size)
|
||||
if point.Class <= classEnd && point.Class >= classStart {
|
||||
t_label[point.Class-classStart] = 1
|
||||
}
|
||||
|
||||
label, err = torch.OfSlice(t_label)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
plabels = append(plabels, label)
|
||||
}
|
||||
|
||||
imgs, err = torch.Concat(pimgs, 0)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
labels, err = torch.Stack(plabels, 0)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
count = len(pimgs)
|
||||
|
||||
imgs, err = torch.Stack(pimgs, 0)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
imgs, err = imgs.ToDtype(gotch.Float, false, false, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
labels, err = labels.ToDtype(gotch.Float, false, false, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func NewDataset(db db.Db, m *types.BaseModel, classStart int, classEnd int) (ds *Dataset, err error) {
|
||||
trainImages, trainLabels, train_count, err := LoadImagesAndLables(db, m, types.DATA_POINT_MODE_TRAINING, classStart, classEnd)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
testImages, testLabels, test_count, err := LoadImagesAndLables(db, m, types.DATA_POINT_MODE_TESTING, classStart, classEnd)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds = &Dataset{
|
||||
TrainImages: trainImages,
|
||||
TrainLabels: trainLabels,
|
||||
TestImages: testImages,
|
||||
TestLabels: testLabels,
|
||||
TrainImagesSize: train_count,
|
||||
TestImagesSize: test_count,
|
||||
Device: gotch.CPU,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (ds *Dataset) To(device gotch.Device) (err error) {
|
||||
ds.TrainImages, err = ds.TrainImages.ToDevice(device, ds.TrainImages.DType(), device.IsCuda(), true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.TrainLabels, err = ds.TrainLabels.ToDevice(device, ds.TrainLabels.DType(), device.IsCuda(), true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.TestImages, err = ds.TestImages.ToDevice(device, ds.TestImages.DType(), device.IsCuda(), true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.TestLabels, err = ds.TestLabels.ToDevice(device, ds.TestLabels.DType(), device.IsCuda(), true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.Device = device
|
||||
return
|
||||
}
|
||||
|
||||
func (ds *Dataset) TestIter(batchSize int64) *torch.Iter2 {
|
||||
return torch.MustNewIter2(ds.TestImages, ds.TestLabels, batchSize)
|
||||
}
|
||||
|
||||
func (ds *Dataset) TrainIter(batchSize int64) (iter *torch.Iter2, err error) {
|
||||
|
||||
// Create a clone of the trainimages
|
||||
size, err := ds.TrainImages.Size()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
train_images, err := torch.Zeros(size, gotch.Float, ds.Device)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.TrainImages, err = ds.TrainImages.Clone(train_images, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
|
||||
// Create a clone of the labels
|
||||
size, err = ds.TrainLabels.Size()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
train_labels, err := torch.Zeros(size, gotch.Float, ds.Device)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
ds.TrainLabels, err = ds.TrainLabels.Clone(train_labels, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
iter, err = torch.NewIter2(train_images, train_labels, batchSize)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
@@ -1,174 +0,0 @@
|
||||
package my_nn
|
||||
|
||||
// linear is a fully-connected layer
|
||||
|
||||
import (
|
||||
"math"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch/nn"
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
// LinearConfig is a configuration for a linear layer
|
||||
type LinearConfig struct {
|
||||
WsInit nn.Init // iniital weights
|
||||
BsInit nn.Init // optional initial bias
|
||||
Bias bool
|
||||
}
|
||||
|
||||
// DefaultLinearConfig creates default LinearConfig with
|
||||
// weights initiated using KaimingUniform and Bias is set to true
|
||||
func DefaultLinearConfig() *LinearConfig {
|
||||
negSlope := math.Sqrt(5)
|
||||
return &LinearConfig{
|
||||
// NOTE. KaimingUniform cause mem leak due to ts.Uniform()!!!
|
||||
// Avoid using it now.
|
||||
WsInit: nn.NewKaimingUniformInit(nn.WithKaimingNegativeSlope(negSlope)),
|
||||
BsInit: nil,
|
||||
Bias: true,
|
||||
}
|
||||
}
|
||||
|
||||
// Linear is a linear fully-connected layer
|
||||
type Linear struct {
|
||||
Ws *ts.Tensor
|
||||
weight_name string
|
||||
Bs *ts.Tensor
|
||||
bias_name string
|
||||
}
|
||||
|
||||
// NewLinear creates a new linear layer
|
||||
// y = x*wT + b
|
||||
// inDim - input dimension (x) [input features - columns]
|
||||
// outDim - output dimension (y) [output features - columns]
|
||||
// NOTE: w will have shape{outDim, inDim}; b will have shape{outDim}
|
||||
func NewLinear(vs *Path, inDim, outDim int64, c *LinearConfig) *Linear {
|
||||
var bias_name string
|
||||
var bs *ts.Tensor
|
||||
var err error
|
||||
if c.Bias {
|
||||
switch {
|
||||
case c.BsInit == nil:
|
||||
shape := []int64{inDim, outDim}
|
||||
fanIn, _, err := nn.CalculateFans(shape)
|
||||
or_panic(err)
|
||||
bound := 0.0
|
||||
if fanIn > 0 {
|
||||
bound = 1 / math.Sqrt(float64(fanIn))
|
||||
}
|
||||
bsInit := nn.NewUniformInit(-bound, bound)
|
||||
bs, bias_name, err = vs.NewVarNamed("bias", []int64{outDim}, bsInit)
|
||||
or_panic(err)
|
||||
|
||||
// Find better way to do this
|
||||
bs, err = bs.T(true)
|
||||
or_panic(err)
|
||||
bs, err = bs.T(true)
|
||||
or_panic(err)
|
||||
|
||||
bs, err = bs.SetRequiresGrad(true, true)
|
||||
or_panic(err)
|
||||
|
||||
err = bs.RetainGrad(false)
|
||||
or_panic(err)
|
||||
|
||||
vs.varstore.UpdateVarTensor(bias_name, bs, true)
|
||||
|
||||
case c.BsInit != nil:
|
||||
bs, bias_name, err = vs.NewVarNamed("bias", []int64{outDim}, c.BsInit)
|
||||
or_panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
ws, weight_name, err := vs.NewVarNamed("weight", []int64{outDim, inDim}, c.WsInit)
|
||||
or_panic(err)
|
||||
|
||||
ws, err = ws.T(true)
|
||||
or_panic(err)
|
||||
|
||||
ws, err = ws.SetRequiresGrad(true, true)
|
||||
or_panic(err)
|
||||
|
||||
err = ws.RetainGrad(false)
|
||||
or_panic(err)
|
||||
|
||||
|
||||
vs.varstore.UpdateVarTensor(weight_name, ws, true)
|
||||
|
||||
|
||||
return &Linear{
|
||||
Ws: ws,
|
||||
weight_name: weight_name,
|
||||
Bs: bs,
|
||||
bias_name: bias_name,
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Linear) Debug() {
|
||||
log.Info("Ws", "ws", l.Ws.MustGrad(false).MustMax(false).Float64Values())
|
||||
log.Info("Bs", "bs", l.Bs.MustGrad(false).MustMax(false).Float64Values())
|
||||
}
|
||||
|
||||
func (l *Linear) ExtractFromVarstore(vs *VarStore) {
|
||||
l.Ws = vs.GetTensorOfVar(l.weight_name)
|
||||
l.Bs = vs.GetTensorOfVar(l.bias_name)
|
||||
}
|
||||
|
||||
// Implement `Module` for `Linear` struct:
|
||||
// =======================================
|
||||
|
||||
// Forward proceeds input node through linear layer.
|
||||
// NOTE:
|
||||
// - It assumes that node has dimensions of 2 (matrix).
|
||||
// To make it work for matrix multiplication, input node should
|
||||
// has same number of **column** as number of **column** in
|
||||
// `LinearLayer` `Ws` property as weights matrix will be
|
||||
// transposed before multiplied to input node. (They are all used `inDim`)
|
||||
// - Input node should have shape of `shape{batch size, input features}`.
|
||||
// (shape{batchSize, inDim}). The input features is `inDim` while the
|
||||
// output feature is `outDim` in `LinearConfig` struct.
|
||||
//
|
||||
// Example:
|
||||
//
|
||||
// inDim := 3
|
||||
// outDim := 2
|
||||
// batchSize := 4
|
||||
// weights: 2x3
|
||||
// [ 1 1 1
|
||||
// 1 1 1 ]
|
||||
//
|
||||
// input node: 3x4
|
||||
// [ 1 1 1
|
||||
// 1 1 1
|
||||
// 1 1 1
|
||||
// 1 1 1 ]
|
||||
func (l *Linear) Forward(xs *ts.Tensor) (retVal *ts.Tensor) {
|
||||
mul, err := xs.Matmul(l.Ws, false)
|
||||
or_panic(err)
|
||||
if l.Bs != nil {
|
||||
mul, err = mul.Add(l.Bs, false)
|
||||
or_panic(err)
|
||||
}
|
||||
|
||||
out, err := mul.Relu(false)
|
||||
or_panic(err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// ForwardT implements ModuleT interface for Linear layer.
|
||||
//
|
||||
// NOTE: train param will not be used.
|
||||
func (l *Linear) ForwardT(xs *ts.Tensor, train bool) (retVal *ts.Tensor) {
|
||||
mul, err := xs.Matmul(l.Ws, true)
|
||||
or_panic(err)
|
||||
|
||||
|
||||
mul, err = mul.Add(l.Bs, true)
|
||||
or_panic(err)
|
||||
|
||||
out, err := mul.Relu(true)
|
||||
or_panic(err)
|
||||
return out
|
||||
}
|
||||
@@ -1,603 +0,0 @@
|
||||
package my_nn
|
||||
|
||||
// Optimizers to be used for gradient-descent based training.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
)
|
||||
|
||||
// Optimizer is a struct object to run gradient descent.
|
||||
type Optimizer struct {
|
||||
varstore *VarStore
|
||||
opt *ts.COptimizer
|
||||
// variablesInOptimizer uint8
|
||||
variablesInOptimizer map[string]struct{}
|
||||
config OptimizerConfig //interface{}
|
||||
stepCount int
|
||||
lr float64
|
||||
}
|
||||
|
||||
func (o *Optimizer) Debug() {
|
||||
for n, _ := range o.variablesInOptimizer {
|
||||
v := o.varstore.GetVarOfName(n)
|
||||
leaf, err := v.Tensor.IsLeaf(false)
|
||||
or_panic(err)
|
||||
|
||||
retains, err := v.Tensor.RetainsGrad(false)
|
||||
or_panic(err)
|
||||
|
||||
log.Info("[opt] var test", "n", n, "leaf", leaf, "retains", retains)
|
||||
}
|
||||
}
|
||||
|
||||
func (o *Optimizer) RefreshValues() (err error) {
|
||||
opt, err := o.config.buildCOpt(o.lr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for name := range o.variablesInOptimizer {
|
||||
v := o.varstore.GetVarOfName(name)
|
||||
if v.Trainable {
|
||||
if err = opt.AddParameter(v.Tensor, v.Group); err != nil {
|
||||
err = fmt.Errorf("Optimizer defaultBuild - AddParameter failed: %w\n", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
o.opt = opt
|
||||
return
|
||||
}
|
||||
|
||||
// OptimizerConfig defines Optimizer configurations. These configs can be used to build optimizer.
|
||||
type OptimizerConfig interface {
|
||||
buildCOpt(lr float64) (*ts.COptimizer, error)
|
||||
|
||||
// Build builds an optimizer with the specified learning rate handling variables stored in `vs`.
|
||||
//
|
||||
// NOTE: Build is a 'default' method. It can be called by wrapping
|
||||
// 'DefaultBuild' function
|
||||
// E.g. AdamOptimizerConfig struct have a method to fullfil `Build` method of
|
||||
// OptimizerConfig by wrapping `DefaultBuild` like
|
||||
// (config AdamOptimizerConfig) Build(vs VarStore, lr float64) (retVal Optimizer, err error){
|
||||
// return defaultBuild(config, vs, lr)
|
||||
// }
|
||||
Build(vs *VarStore, lr float64) (*Optimizer, error)
|
||||
}
|
||||
|
||||
// defaultBuild is `default` Build method for OptimizerConfig interface
|
||||
func defaultBuild(config OptimizerConfig, vs *VarStore, lr float64) (*Optimizer, error) {
|
||||
opt, err := config.buildCOpt(lr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
names := make(map[string]struct{})
|
||||
for name, v := range vs.vars {
|
||||
if v.Trainable {
|
||||
log.Info("Adding parameter", "name", name, "g", v.Group)
|
||||
if err = opt.AddParameter(v.Tensor, v.Group); err != nil {
|
||||
err = fmt.Errorf("Optimizer defaultBuild - AddParameter failed: %w\n", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
names[name] = struct{}{}
|
||||
}
|
||||
|
||||
return &Optimizer{
|
||||
varstore: vs,
|
||||
opt: opt,
|
||||
variablesInOptimizer: names,
|
||||
config: config,
|
||||
stepCount: 0,
|
||||
lr: 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// SGD Optimizer:
|
||||
//===============
|
||||
|
||||
// SGDConfig holds parameters for building the SGD (Stochastic Gradient Descent) optimizer.
|
||||
type SGDConfig struct {
|
||||
Momentum float64
|
||||
Dampening float64
|
||||
Wd float64
|
||||
Nesterov bool
|
||||
}
|
||||
|
||||
// DefaultSGDConfig creates SGDConfig with default values.
|
||||
func DefaultSGDConfig() *SGDConfig {
|
||||
return &SGDConfig{
|
||||
Momentum: 0.0,
|
||||
Dampening: 0.0,
|
||||
Wd: 0.0,
|
||||
Nesterov: false,
|
||||
}
|
||||
}
|
||||
|
||||
// NewSGD creates the configuration for a SGD optimizer with specified values
|
||||
func NewSGDConfig(momentum, dampening, wd float64, nesterov bool) *SGDConfig {
|
||||
return &SGDConfig{
|
||||
Momentum: momentum,
|
||||
Dampening: dampening,
|
||||
Wd: wd,
|
||||
Nesterov: nesterov,
|
||||
}
|
||||
}
|
||||
|
||||
// Implement OptimizerConfig interface for SGDConfig
|
||||
func (c *SGDConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
|
||||
return ts.Sgd(lr, c.Momentum, c.Dampening, c.Wd, c.Nesterov)
|
||||
}
|
||||
|
||||
func (c *SGDConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
|
||||
return defaultBuild(c, vs, lr)
|
||||
}
|
||||
|
||||
// Adam optimizer:
|
||||
// ===============
|
||||
|
||||
type AdamConfig struct {
|
||||
Beta1 float64
|
||||
Beta2 float64
|
||||
Wd float64
|
||||
}
|
||||
|
||||
// DefaultAdamConfig creates AdamConfig with default values
|
||||
func DefaultAdamConfig() *AdamConfig {
|
||||
return &AdamConfig{
|
||||
Beta1: 0.9,
|
||||
Beta2: 0.999,
|
||||
Wd: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
// NewAdamConfig creates AdamConfig with specified values
|
||||
func NewAdamConfig(beta1, beta2, wd float64) *AdamConfig {
|
||||
return &AdamConfig{
|
||||
Beta1: beta1,
|
||||
Beta2: beta2,
|
||||
Wd: wd,
|
||||
}
|
||||
}
|
||||
|
||||
// Implement OptimizerConfig interface for AdamConfig
|
||||
func (c *AdamConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
|
||||
return ts.Adam(lr, c.Beta1, c.Beta2, c.Wd)
|
||||
}
|
||||
|
||||
func (c *AdamConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
|
||||
return defaultBuild(c, vs, lr)
|
||||
}
|
||||
|
||||
// AdamW optimizer:
|
||||
// ===============
|
||||
|
||||
type AdamWConfig struct {
|
||||
Beta1 float64
|
||||
Beta2 float64
|
||||
Wd float64
|
||||
}
|
||||
|
||||
// DefaultAdamWConfig creates AdamWConfig with default values
|
||||
func DefaultAdamWConfig() *AdamWConfig {
|
||||
return &AdamWConfig{
|
||||
Beta1: 0.9,
|
||||
Beta2: 0.999,
|
||||
Wd: 0.01,
|
||||
}
|
||||
}
|
||||
|
||||
// NewAdamWConfig creates AdamWConfig with specified values
|
||||
func NewAdamWConfig(beta1, beta2, wd float64) *AdamWConfig {
|
||||
return &AdamWConfig{
|
||||
Beta1: beta1,
|
||||
Beta2: beta2,
|
||||
Wd: wd,
|
||||
}
|
||||
}
|
||||
|
||||
// Implement OptimizerConfig interface for AdamWConfig
|
||||
func (c *AdamWConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
|
||||
return ts.AdamW(lr, c.Beta1, c.Beta2, c.Wd)
|
||||
}
|
||||
|
||||
// Build builds AdamW optimizer
|
||||
func (c *AdamWConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
|
||||
return defaultBuild(c, vs, lr)
|
||||
}
|
||||
|
||||
// RMSProp optimizer:
|
||||
// ===============
|
||||
|
||||
type RMSPropConfig struct {
|
||||
Alpha float64
|
||||
Eps float64
|
||||
Wd float64
|
||||
Momentum float64
|
||||
Centered bool
|
||||
}
|
||||
|
||||
// DefaultAdamConfig creates AdamConfig with default values
|
||||
func DefaultRMSPropConfig() *RMSPropConfig {
|
||||
return &RMSPropConfig{
|
||||
Alpha: 0.99,
|
||||
Eps: 1e-8,
|
||||
Wd: 0.0,
|
||||
Momentum: 0.0,
|
||||
Centered: false,
|
||||
}
|
||||
}
|
||||
|
||||
// NewRMSPropConfig creates RMSPropConfig with specified values
|
||||
func NewRMSPropConfig(alpha, eps, wd, momentum float64, centered bool) *RMSPropConfig {
|
||||
return &RMSPropConfig{
|
||||
Alpha: alpha,
|
||||
Eps: eps,
|
||||
Wd: wd,
|
||||
Momentum: momentum,
|
||||
Centered: centered,
|
||||
}
|
||||
}
|
||||
|
||||
// Implement OptimizerConfig interface for RMSPropConfig
|
||||
func (c *RMSPropConfig) buildCOpt(lr float64) (*ts.COptimizer, error) {
|
||||
return ts.RmsProp(lr, c.Alpha, c.Eps, c.Wd, c.Momentum, c.Centered)
|
||||
}
|
||||
|
||||
func (c *RMSPropConfig) Build(vs *VarStore, lr float64) (*Optimizer, error) {
|
||||
return defaultBuild(c, vs, lr)
|
||||
}
|
||||
|
||||
// Optimizer methods:
|
||||
// ==================
|
||||
|
||||
func (opt *Optimizer) addMissingVariables() {
|
||||
type param struct {
|
||||
tensor *ts.Tensor
|
||||
group uint
|
||||
}
|
||||
trainables := make(map[string]param)
|
||||
for name, v := range opt.varstore.vars {
|
||||
if v.Trainable {
|
||||
trainables[name] = param{tensor: v.Tensor, group: v.Group}
|
||||
}
|
||||
}
|
||||
missingVariables := len(trainables) - len(opt.variablesInOptimizer)
|
||||
if missingVariables > 0 {
|
||||
log.Info("INFO: Optimizer.addMissingVariables()...")
|
||||
for name, x := range trainables {
|
||||
if _, ok := opt.variablesInOptimizer[name]; !ok {
|
||||
opt.opt.AddParameter(x.tensor, x.group)
|
||||
opt.variablesInOptimizer[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ZeroGrad zeroes the gradient for the tensors tracked by this optimizer.
|
||||
func (opt *Optimizer) ZeroGrad() error {
|
||||
if err := opt.opt.ZeroGrad(); err != nil {
|
||||
err = fmt.Errorf("Optimizer.ZeroGrad() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustZeroGrad zeroes the gradient for the tensors tracked by this optimizer.
|
||||
func (opt *Optimizer) MustZeroGrad() {
|
||||
err := opt.ZeroGrad()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Clips gradient value at some specified maximum value.
|
||||
func (opt *Optimizer) ClipGradValue(max float64) {
|
||||
opt.varstore.Lock()
|
||||
defer opt.varstore.Unlock()
|
||||
|
||||
for _, v := range opt.varstore.vars {
|
||||
if v.Trainable {
|
||||
// v.Tensor.MustGrad().Clamp_(ts.FloatScalar(-max), ts.FloatScalar(max))
|
||||
gradTs := v.Tensor.MustGrad(false)
|
||||
gradTs.Clamp_(ts.FloatScalar(-max), ts.FloatScalar(max))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step performs an optimization step, updating the tracked tensors based on their gradients.
|
||||
func (opt *Optimizer) Step() error {
|
||||
err := opt.opt.Step()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Optimizer.Step() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
opt.stepCount += 1
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustStep performs an optimization step, updating the tracked tensors based on their gradients.
|
||||
func (opt *Optimizer) MustStep() {
|
||||
err := opt.Step()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// ResetStepCount set step count to zero.
|
||||
func (opt *Optimizer) ResetStepCount() {
|
||||
opt.stepCount = 0
|
||||
}
|
||||
|
||||
// StepCount get current step count.
|
||||
func (opt *Optimizer) StepCount() int {
|
||||
return opt.stepCount
|
||||
}
|
||||
|
||||
// BackwardStep applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
func (opt *Optimizer) BackwardStep(loss *ts.Tensor) error {
|
||||
err := opt.opt.ZeroGrad()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Optimizer.BackwardStep() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
loss.MustBackward()
|
||||
err = opt.opt.Step()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Optimizer.BackwardStep() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustBackwardStep applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
func (opt *Optimizer) MustBackwardStep(loss *ts.Tensor) {
|
||||
err := opt.BackwardStep(loss)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// BackwardStepClip applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
//
|
||||
// The gradients are clipped based on `max` before being applied.
|
||||
func (opt *Optimizer) BackwardStepClip(loss *ts.Tensor, max float64) error {
|
||||
err := opt.opt.ZeroGrad()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Optimizer.BackwardStepClip() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
loss.MustBackward()
|
||||
opt.ClipGradValue(max)
|
||||
err = opt.opt.Step()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Optimizer.BackwardStepClip() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustBackwardStepClip applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
//
|
||||
// The gradients are clipped based on `max` before being applied.
|
||||
func (opt *Optimizer) MustBackwardStepClip(loss *ts.Tensor, max float64) {
|
||||
err := opt.BackwardStepClip(loss, max)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type ClipOpts struct {
|
||||
NormType float64
|
||||
ErrorIfNonFinite bool
|
||||
}
|
||||
|
||||
type ClipOpt func(*ClipOpts)
|
||||
|
||||
func defaultClipOpts() *ClipOpts {
|
||||
return &ClipOpts{
|
||||
NormType: 2.0,
|
||||
ErrorIfNonFinite: false, // will switch to "true" in the future.
|
||||
}
|
||||
}
|
||||
|
||||
func WithNormType(v float64) ClipOpt {
|
||||
return func(o *ClipOpts) {
|
||||
o.NormType = v
|
||||
}
|
||||
}
|
||||
|
||||
func WithErrorIfNonFinite(v bool) ClipOpt {
|
||||
return func(o *ClipOpts) {
|
||||
o.ErrorIfNonFinite = v
|
||||
}
|
||||
}
|
||||
|
||||
// / Clips gradient L2 norm over all trainable parameters.
|
||||
//
|
||||
// The norm is computed over all gradients together, as if they were
|
||||
// concatenated into a single vector.
|
||||
//
|
||||
// / Args:
|
||||
// - max: max norm of the gradient
|
||||
// - o.NormType. Type of the used p-norm, can be "inf" for infinity norm. Default= 2.0
|
||||
// - o.ErrorIfNonFinite bool. If true, throw error if total norm of the gradients from paramters is "nan", "inf" or "-inf". Default=false
|
||||
// Returns: total norm of the parameters (viewed as a single vector)
|
||||
// ref. https://github.com/pytorch/pytorch/blob/cb4aeff7d8e4c70bb638cf159878c5204d0cc2da/torch/nn/utils/clip_grad.py#L59
|
||||
func (opt *Optimizer) ClipGradNorm(max float64, opts ...ClipOpt) error {
|
||||
o := defaultClipOpts()
|
||||
for _, option := range opts {
|
||||
option(o)
|
||||
}
|
||||
|
||||
opt.varstore.Lock()
|
||||
defer opt.varstore.Unlock()
|
||||
parameters := opt.varstore.TrainableVariables()
|
||||
if len(parameters) == 0 {
|
||||
// return ts.MustOfSlice([]float64{0.0}), nil
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
norms []*ts.Tensor
|
||||
totalNorm *ts.Tensor
|
||||
)
|
||||
|
||||
device := opt.varstore.device
|
||||
|
||||
// FIXME. What about mixed-precision?
|
||||
dtype := parameters[0].DType()
|
||||
|
||||
if o.NormType == math.Inf(1) {
|
||||
for _, v := range opt.varstore.vars {
|
||||
n := v.Tensor.MustGrad(false).MustDetach(true).MustAbs(true).MustMax(true).MustTo(device, true)
|
||||
norms = append(norms, n)
|
||||
}
|
||||
// total_norm = norms[0] if len(norms) == 1 else torch.max(torch.stack(norms))
|
||||
totalNorm = ts.MustStack(norms, 0).MustMax(true)
|
||||
} else {
|
||||
for _, v := range opt.varstore.vars {
|
||||
// x := v.Tensor.MustGrad(false).MustNorm(true)
|
||||
|
||||
// NOTE. tensor.Norm() is going to be deprecated. So use linalg_norm
|
||||
// Ref. https://pytorch.org/docs/stable/generated/torch.linalg.norm.html#torch.linalg.norm
|
||||
x := v.Tensor.MustGrad(false).MustDetach(true).MustLinalgNorm(ts.FloatScalar(o.NormType), nil, false, dtype, true)
|
||||
norms = append(norms, x)
|
||||
}
|
||||
}
|
||||
|
||||
// totalNorm = ts.MustStack(norms, 0).MustNorm(true).MustAddScalar(ts.FloatScalar(1e-6), true)
|
||||
// total_norm = torch.norm(torch.stack([torch.norm(p.grad.detach(), norm_type).to(device) for p in parameters]), norm_type)
|
||||
totalNorm = ts.MustStack(norms, 0).MustLinalgNorm(ts.FloatScalar(o.NormType), nil, false, dtype, true)
|
||||
for _, x := range norms {
|
||||
x.MustDrop()
|
||||
}
|
||||
|
||||
totalNormVal := totalNorm.Float64Values(true)[0]
|
||||
// if error_if_nonfinite and torch.logical_or(total_norm.isnan(), total_norm.isinf()):
|
||||
if o.ErrorIfNonFinite && (math.IsNaN(totalNormVal) || math.IsInf(totalNormVal, 1)) {
|
||||
err := fmt.Errorf("The total norm of order (%v) for gradients from 'parameters' is non-finite, so it cannot be clipped. To disable this error and scale the gradients by the non-finite norm anyway, set option.ErrorIfNonFinite= false", o.NormType)
|
||||
return err
|
||||
}
|
||||
|
||||
// clip_coef = max_norm / (total_norm + 1e-6)
|
||||
// clipCoefTs := ts.TensorFrom([]float64{max}).MustDiv(totalNorm, true)
|
||||
clipCoef := max / (totalNormVal + 1e-6)
|
||||
// NOTE: multiplying by the clamped coef is redundant when the coef is clamped to 1, but doing so
|
||||
// avoids a `if clip_coef < 1:` conditional which can require a CPU <=> device synchronization
|
||||
// when the gradients do not reside in CPU memory.
|
||||
// clip_coef_clamped = torch.clamp(clip_coef, max=1.0)
|
||||
if clipCoef > 1.0 {
|
||||
clipCoef = 1.0
|
||||
}
|
||||
for _, v := range opt.varstore.vars {
|
||||
if v.Trainable {
|
||||
// p.grad.detach().mul_(clip_coef_clamped.to(p.grad.device))
|
||||
// v.Tensor.MustGrad(false).MustDetach(true).MustMulScalar_(ts.FloatScalar(clipCoef))
|
||||
v.Tensor.MustGrad(false).MustMulScalar_(ts.FloatScalar(clipCoef))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BackwardStepClipNorm applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
//
|
||||
// The gradients L2 norm is clipped based on `max`.
|
||||
func (opt *Optimizer) BackwardStepClipNorm(loss *ts.Tensor, max float64, opts ...ClipOpt) error {
|
||||
err := opt.opt.ZeroGrad()
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
err = loss.Backward()
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = opt.ClipGradNorm(max, opts...)
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = opt.Step()
|
||||
if err != nil {
|
||||
err := fmt.Errorf("Optimizer.BackwardStepClipNorm() failed: %w\n", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustBackwardStepClipNorm applies a backward step pass, update the gradients, and performs an optimization step.
|
||||
//
|
||||
// The gradients L2 norm is clipped based on `max`.
|
||||
func (opt *Optimizer) MustBackwardStepClipNorm(loss *ts.Tensor, max float64, opts ...ClipOpt) {
|
||||
err := opt.BackwardStepClipNorm(loss, max, opts...)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SetLR sets the optimizer learning rate.
|
||||
//
|
||||
// NOTE. it sets a SINGLE value of learning rate for all parameter groups.
|
||||
// Most of the time, there's one parameter group.
|
||||
func (opt *Optimizer) SetLR(lr float64) {
|
||||
err := opt.opt.SetLearningRate(lr)
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - SetLR method call error: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (opt *Optimizer) GetLRs() []float64 {
|
||||
lrs, err := opt.opt.GetLearningRates()
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - GetLRs method call error: %v\n", err)
|
||||
}
|
||||
|
||||
return lrs
|
||||
}
|
||||
|
||||
// SetLRs sets learning rates for ALL parameter groups respectively.
|
||||
func (opt *Optimizer) SetLRs(lrs []float64) {
|
||||
err := opt.opt.SetLearningRates(lrs)
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - SetLRs method call error: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// SetMomentum sets the optimizer momentum.
|
||||
func (opt *Optimizer) SetMomentum(m float64) {
|
||||
err := opt.opt.SetMomentum(m)
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - SetMomentum method call error: %v\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (opt *Optimizer) ParamGroupNum() int {
|
||||
ngroup, err := opt.opt.ParamGroupNum()
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - ParamGroupNum method call error: %v\n", err)
|
||||
}
|
||||
|
||||
return int(ngroup)
|
||||
}
|
||||
|
||||
func (opt *Optimizer) AddParamGroup(tensors []*ts.Tensor) {
|
||||
err := opt.opt.AddParamGroup(tensors)
|
||||
if err != nil {
|
||||
log.Fatalf("Optimizer - ParamGroupNum method call error: %v\n", err)
|
||||
}
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
package my_nn
|
||||
|
||||
import (
|
||||
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
)
|
||||
|
||||
func or_panic(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
type MyLayer interface {
|
||||
torch.ModuleT
|
||||
|
||||
ExtractFromVarstore(vs *VarStore)
|
||||
Debug()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,120 +0,0 @@
|
||||
package train
|
||||
|
||||
import (
|
||||
types "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch"
|
||||
"github.com/charmbracelet/log"
|
||||
|
||||
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
)
|
||||
|
||||
type IForwardable interface {
|
||||
Forward(xs *torch.Tensor) *torch.Tensor
|
||||
}
|
||||
|
||||
// Container for a model
|
||||
type ContainerModel struct {
|
||||
Layers []my_nn.MyLayer
|
||||
Vs *my_nn.VarStore
|
||||
path *my_nn.Path
|
||||
}
|
||||
|
||||
func (n *ContainerModel) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
|
||||
if len(n.Layers) == 0 {
|
||||
return x.MustShallowClone()
|
||||
}
|
||||
|
||||
if len(n.Layers) == 1 {
|
||||
log.Info("here")
|
||||
return n.Layers[0].ForwardT(x, train)
|
||||
}
|
||||
|
||||
// forward sequentially
|
||||
outs := make([]*torch.Tensor, len(n.Layers))
|
||||
for i := 0; i < len(n.Layers); i++ {
|
||||
if i == 0 {
|
||||
outs[0] = n.Layers[i].ForwardT(x, train)
|
||||
//defer outs[0].MustDrop()
|
||||
} else if i == len(n.Layers)-1 {
|
||||
return n.Layers[i].ForwardT(outs[i-1], train)
|
||||
} else {
|
||||
outs[i] = n.Layers[i].ForwardT(outs[i-1], train)
|
||||
//defer outs[i].MustDrop()
|
||||
}
|
||||
}
|
||||
panic("Do not reach here")
|
||||
}
|
||||
|
||||
func (n *ContainerModel) To(device gotch.Device) {
|
||||
n.Vs.ToDevice(device)
|
||||
for _, layer := range n.Layers {
|
||||
layer.ExtractFromVarstore(n.Vs)
|
||||
}
|
||||
}
|
||||
|
||||
func (n *ContainerModel) Refresh() {
|
||||
for _, layer := range n.Layers {
|
||||
layer.ExtractFromVarstore(n.Vs)
|
||||
}
|
||||
}
|
||||
|
||||
func BuildModel(layers []*types.Layer, _lastLinearSize int64, addSigmoid bool) *ContainerModel {
|
||||
|
||||
base_vs := my_nn.NewVarStore(gotch.CPU)
|
||||
vs := base_vs.Root()
|
||||
|
||||
m_layers := []my_nn.MyLayer{}
|
||||
|
||||
var lastLinearSize int64 = _lastLinearSize
|
||||
lastLinearConv := []int64{}
|
||||
|
||||
for _, layer := range layers {
|
||||
if layer.LayerType == types.LAYER_INPUT {
|
||||
lastLinearConv = layer.GetShape()
|
||||
log.Info("Input: ", "In:", lastLinearConv)
|
||||
} else if layer.LayerType == types.LAYER_DENSE {
|
||||
shape := layer.GetShape()
|
||||
log.Info("New Dense: ", "In:", lastLinearSize, "out:", shape[0])
|
||||
m_layers = append(m_layers, NewLinear(vs, lastLinearSize, shape[0]))
|
||||
lastLinearSize = shape[0]
|
||||
} else if layer.LayerType == types.LAYER_FLATTEN {
|
||||
m_layers = append(m_layers, NewFlatten())
|
||||
lastLinearSize = 1
|
||||
for _, i := range lastLinearConv {
|
||||
lastLinearSize *= i
|
||||
}
|
||||
log.Info("Flatten: ", "In:", lastLinearConv, "out:", lastLinearSize)
|
||||
} else if layer.LayerType == types.LAYER_SIMPLE_BLOCK {
|
||||
panic("TODO")
|
||||
log.Info("New Block: ", "In:", lastLinearConv, "out:", []int64{lastLinearConv[1] / 2, lastLinearConv[2] / 2, 128})
|
||||
//m_layers = append(m_layers, NewSimpleBlock(vs, lastLinearConv[0]))
|
||||
lastLinearConv[0] = 128
|
||||
lastLinearConv[1] /= 2
|
||||
lastLinearConv[2] /= 2
|
||||
}
|
||||
}
|
||||
|
||||
if addSigmoid {
|
||||
m_layers = append(m_layers, NewSigmoid())
|
||||
}
|
||||
|
||||
b := &ContainerModel{
|
||||
Layers: m_layers,
|
||||
Vs: base_vs,
|
||||
path: vs,
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (model *ContainerModel) Debug() {
|
||||
for _, v := range model.Layers {
|
||||
v.Debug()
|
||||
}
|
||||
}
|
||||
|
||||
func SaveModel(model *ContainerModel, modelFn string) (err error) {
|
||||
model.Vs.ToDevice(gotch.CPU)
|
||||
return model.Vs.Save(modelFn)
|
||||
}
|
||||
@@ -1,152 +0,0 @@
|
||||
package train
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
|
||||
my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch/nn"
|
||||
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
)
|
||||
|
||||
func or_panic(err error) {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type SimpleBlock struct {
|
||||
C1, C2 *nn.Conv2D
|
||||
BN1 *nn.BatchNorm
|
||||
}
|
||||
|
||||
// BasicBlock returns a BasicBlockModule instance
|
||||
func NewSimpleBlock(_vs *my_nn.Path, inplanes int64) *SimpleBlock {
|
||||
vs := (*nn.Path)(unsafe.Pointer(_vs))
|
||||
|
||||
conf1 := nn.DefaultConv2DConfig()
|
||||
conf1.Stride = []int64{2, 2}
|
||||
|
||||
conf2 := nn.DefaultConv2DConfig()
|
||||
conf2.Padding = []int64{2, 2}
|
||||
|
||||
b := &SimpleBlock{
|
||||
C1: nn.NewConv2D(vs, inplanes, 128, 3, conf1),
|
||||
C2: nn.NewConv2D(vs, 128, 128, 3, conf2),
|
||||
BN1: nn.NewBatchNorm(vs, 2, 128, nn.DefaultBatchNormConfig()),
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// Forward method
|
||||
func (b *SimpleBlock) Forward(x *torch.Tensor) *torch.Tensor {
|
||||
identity := x
|
||||
|
||||
out := b.C1.Forward(x)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
out = b.C2.Forward(out)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
shape, err := out.Size()
|
||||
or_panic(err)
|
||||
|
||||
out, err = out.AdaptiveAvgPool2d(shape, false)
|
||||
or_panic(err)
|
||||
|
||||
out = b.BN1.Forward(out)
|
||||
out, err = out.LeakyRelu(false)
|
||||
or_panic(err)
|
||||
|
||||
out = out.MustAdd(identity, false)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *SimpleBlock) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
|
||||
identity := x
|
||||
|
||||
out := b.C1.ForwardT(x, train)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
out = b.C2.ForwardT(out, train)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
shape, err := out.Size()
|
||||
or_panic(err)
|
||||
|
||||
out, err = out.AdaptiveAvgPool2d(shape, false)
|
||||
or_panic(err)
|
||||
|
||||
out = b.BN1.ForwardT(out, train)
|
||||
out, err = out.LeakyRelu(false)
|
||||
or_panic(err)
|
||||
|
||||
out = out.MustAdd(identity, false)
|
||||
out = out.MustRelu(false)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
// BasicBlock returns a BasicBlockModule instance
|
||||
func NewLinear(vs *my_nn.Path, in, out int64) *my_nn.Linear {
|
||||
config := my_nn.DefaultLinearConfig()
|
||||
return my_nn.NewLinear(vs, in, out, config)
|
||||
}
|
||||
|
||||
type Flatten struct{}
|
||||
|
||||
// BasicBlock returns a BasicBlockModule instance
|
||||
func NewFlatten() *Flatten {
|
||||
return &Flatten{}
|
||||
}
|
||||
|
||||
// The flatten layer does not to move anything to the device
|
||||
func (b *Flatten) ExtractFromVarstore(vs *my_nn.VarStore) {}
|
||||
func (b *Flatten) Debug() {}
|
||||
|
||||
// Forward method
|
||||
func (b *Flatten) Forward(x *torch.Tensor) *torch.Tensor {
|
||||
|
||||
out, err := x.Flatten(1, -1, false)
|
||||
or_panic(err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Flatten) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
|
||||
|
||||
out, err := x.Flatten(1, -1, false)
|
||||
or_panic(err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
type Sigmoid struct{}
|
||||
|
||||
func NewSigmoid() *Sigmoid {
|
||||
return &Sigmoid{}
|
||||
}
|
||||
|
||||
// The sigmoid layer does not need to move anything to another device
|
||||
func (b *Sigmoid) ExtractFromVarstore(vs *my_nn.VarStore) {}
|
||||
func (b *Sigmoid) Debug() {}
|
||||
|
||||
func (b *Sigmoid) Forward(x *torch.Tensor) *torch.Tensor {
|
||||
out, err := x.Sigmoid(false)
|
||||
or_panic(err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func (b *Sigmoid) ForwardT(x *torch.Tensor, train bool) *torch.Tensor {
|
||||
out, err := x.Sigmoid(false)
|
||||
or_panic(err)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -14,7 +14,7 @@ func handleTasksStats(handle *Handle) {
|
||||
}
|
||||
PostAuthJson(handle, "/stats/task/model/day", User_Normal, func(c *Context, dat *ModelTasksStatsRequest) *Error {
|
||||
model, err := GetBaseModel(c, dat.ModelId)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.JsonBadRequest("Model not found!")
|
||||
} else if err != nil {
|
||||
return c.E500M("Failed to get model", err)
|
||||
|
||||
7
logic/tasks/README.md
Normal file
7
logic/tasks/README.md
Normal file
@@ -0,0 +1,7 @@
|
||||
# Runner Protocol
|
||||
|
||||
```
|
||||
/----\
|
||||
\/ |
|
||||
Register -> Init -> Active ---> Ready -> Info
|
||||
```
|
||||
@@ -14,7 +14,7 @@ func handleRequests(x *Handle) {
|
||||
PostAuthJson(x, "/task/agreement", User_Normal, func(c *Context, dat *AgreementRequest) *Error {
|
||||
var task Task
|
||||
err := GetDBOnce(c, &task, "tasks where id=$1", dat.Id)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.JsonBadRequest("Model not found")
|
||||
} else if err != nil {
|
||||
return c.E500M("Failed to get task data", err)
|
||||
|
||||
@@ -8,4 +8,5 @@ func HandleTasks(handle *Handle) {
|
||||
handleUpload(handle)
|
||||
handleList(handle)
|
||||
handleRequests(handle)
|
||||
handleRemoteRunner(handle)
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func handleList(handler *Handle) {
|
||||
|
||||
if requestData.ModelId != "" {
|
||||
_, err := GetBaseModel(c.Db, requestData.ModelId)
|
||||
if err == NotFoundError {
|
||||
if err == ModelNotFoundError {
|
||||
return c.SendJSONStatus(404, "Model not found!")
|
||||
} else if err != nil {
|
||||
return c.Error500(err)
|
||||
|
||||
386
logic/tasks/runner.go
Normal file
386
logic/tasks/runner.go
Normal file
@@ -0,0 +1,386 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||
)
|
||||
|
||||
func verifyRunner(c *Context, dat *JustId) (runner *Runner, e *Error) {
|
||||
runner, err := GetRunner(c, dat.Id)
|
||||
if err == NotFoundError {
|
||||
e = c.JsonBadRequest("Could not find runner, please register runner first")
|
||||
return
|
||||
} else if err != nil {
|
||||
e = c.E500M("Failed to get information about the runner", err)
|
||||
return
|
||||
}
|
||||
|
||||
if runner.Token != *c.Token {
|
||||
return nil, c.SendJSONStatus(401, "Only runners can use this funcion")
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
type VerifyTask struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
TaskId string `json:"taskId" validate:"required"`
|
||||
}
|
||||
|
||||
func verifyTask(x *Handle, c *Context, dat *VerifyTask) (task *Task, error *Error) {
|
||||
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
|
||||
if runners[dat.Id] == nil {
|
||||
return nil, c.JsonBadRequest("Runner not active")
|
||||
}
|
||||
|
||||
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
|
||||
|
||||
if runner_data["task"] == nil {
|
||||
return nil, c.SendJSONStatus(404, "No active task")
|
||||
}
|
||||
|
||||
return runner_data["task"].(*Task), nil
|
||||
}
|
||||
|
||||
func handleRemoteRunner(x *Handle) {
|
||||
|
||||
type RegisterRunner struct {
|
||||
Token string `json:"token" validate:"required"`
|
||||
Type RunnerType `json:"type" validate:"required"`
|
||||
}
|
||||
PostAuthJson(x, "/tasks/runner/register", User_Normal, func(c *Context, dat *RegisterRunner) *Error {
|
||||
if *c.Token != dat.Token {
|
||||
// TODO do admin
|
||||
return c.E500M("Please make sure that the token is the same that is being registered", nil)
|
||||
}
|
||||
|
||||
c.Logger.Info("test", "dat", dat)
|
||||
|
||||
var runner Runner
|
||||
err := GetDBOnce(c, &runner, "remote_runner as ru where token=$1", dat.Token)
|
||||
if err != NotFoundError && err != nil {
|
||||
return c.E500M("Failed to get information remote runners", err)
|
||||
}
|
||||
if err != NotFoundError {
|
||||
return c.JsonBadRequest("Token is already registered by a runner")
|
||||
}
|
||||
|
||||
// TODO get id from token passed by when doing admin
|
||||
var userId = c.User.Id
|
||||
|
||||
var new_runner = struct {
|
||||
Type RunnerType
|
||||
UserId string `db:"user_id"`
|
||||
Token string
|
||||
}{
|
||||
Type: dat.Type,
|
||||
Token: dat.Token,
|
||||
UserId: userId,
|
||||
}
|
||||
|
||||
id, err := InsertReturnId(c, &new_runner, "remote_runner", "id")
|
||||
if err != nil {
|
||||
return c.E500M("Failed to create remote runner", err)
|
||||
}
|
||||
|
||||
return c.SendJSON(struct {
|
||||
Id string `json:"id"`
|
||||
}{
|
||||
Id: id,
|
||||
})
|
||||
})
|
||||
|
||||
// TODO remove runner
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/init", User_Normal, func(c *Context, dat *JustId) *Error {
|
||||
runner, error := verifyRunner(c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
|
||||
if runners[dat.Id] != nil {
|
||||
c.Logger.Info("Logger trying to register but already registerd")
|
||||
c.ShowMessage = false
|
||||
return c.SendJSON("Ok")
|
||||
}
|
||||
|
||||
var new_runner = map[string]interface{}{}
|
||||
new_runner["last_time_check"] = time.Now()
|
||||
new_runner["runner_info"] = runner
|
||||
|
||||
runners[dat.Id] = new_runner
|
||||
|
||||
x.DataMap["runners"] = runners
|
||||
|
||||
return c.SendJSON("Ok")
|
||||
})
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/active", User_Normal, func(c *Context, dat *JustId) *Error {
|
||||
_, error := verifyRunner(c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
|
||||
if runners[dat.Id] == nil {
|
||||
return c.JsonBadRequest("Runner not active")
|
||||
}
|
||||
|
||||
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
|
||||
|
||||
if runner_data["task"] == nil {
|
||||
c.ShowMessage = false
|
||||
return c.SendJSONStatus(404, "No active task")
|
||||
}
|
||||
|
||||
c.ShowMessage = false
|
||||
// This should be a task obj
|
||||
return c.SendJSON(runner_data["task"])
|
||||
})
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/ready", User_Normal, func(c *Context, dat *VerifyTask) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
err := task.UpdateStatus(c, TASK_RUNNING, "Task Running on Runner")
|
||||
if err != nil {
|
||||
return c.E500M("Failed to set task status", err)
|
||||
}
|
||||
|
||||
return c.SendJSON("Ok")
|
||||
})
|
||||
|
||||
type TaskFail struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
TaskId string `json:"taskId" validate:"required"`
|
||||
Reason string `json:"reason" validate:"required"`
|
||||
}
|
||||
PostAuthJson(x, "/tasks/runner/fail", User_Normal, func(c *Context, dat *TaskFail) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
err := task.UpdateStatus(c, TASK_FAILED_RUNNING, dat.Reason)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to set task status", err)
|
||||
}
|
||||
|
||||
// Do extra clean up on tasks
|
||||
switch task.TaskType {
|
||||
case int(TASK_TYPE_TRAINING):
|
||||
CleanUpFailed(c, task)
|
||||
default:
|
||||
panic("Do not know how to handle this")
|
||||
}
|
||||
|
||||
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
|
||||
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
|
||||
runner_data["task"] = nil
|
||||
|
||||
runners[dat.Id] = runner_data
|
||||
x.DataMap["runners"] = runners
|
||||
|
||||
return c.SendJSON("Ok")
|
||||
})
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/train/defs", User_Normal, func(c *Context, dat *VerifyTask) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
if task.TaskType != int(TASK_TYPE_TRAINING) {
|
||||
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
|
||||
return c.JsonBadRequest("Task is not the right type go get the definitions")
|
||||
}
|
||||
|
||||
model, err := GetBaseModel(c, *task.ModelId)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get model information", err)
|
||||
}
|
||||
|
||||
defs, err := model.GetDefinitions(c, "and md.status=$2", DEFINITION_STATUS_INIT)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get the model definitions", err)
|
||||
}
|
||||
|
||||
return c.SendJSON(defs)
|
||||
})
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/train/classes", User_Normal, func(c *Context, dat *VerifyTask) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
if task.TaskType != int(TASK_TYPE_TRAINING) {
|
||||
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
|
||||
return c.JsonBadRequest("Task is not the right type go get the definitions")
|
||||
}
|
||||
|
||||
model, err := GetBaseModel(c, *task.ModelId)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get model information", err)
|
||||
}
|
||||
|
||||
classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TO_TRAIN)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get the model classes", err)
|
||||
}
|
||||
|
||||
return c.SendJSON(classes)
|
||||
})
|
||||
|
||||
type RunnerTrainDefStatus struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
TaskId string `json:"taskId" validate:"required"`
|
||||
DefId string `json:"defId" validate:"required"`
|
||||
Status DefinitionStatus `json:"status" validate:"required"`
|
||||
}
|
||||
PostAuthJson(x, "/tasks/runner/train/def/status", User_Normal, func(c *Context, dat *RunnerTrainDefStatus) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
if task.TaskType != int(TASK_TYPE_TRAINING) {
|
||||
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
|
||||
return c.JsonBadRequest("Task is not the right type go get the definitions")
|
||||
}
|
||||
|
||||
def, err := GetDefinition(c, dat.DefId)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get definition information", err)
|
||||
}
|
||||
|
||||
err = def.UpdateStatus(c, dat.Status)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to update model status", err)
|
||||
}
|
||||
|
||||
return c.SendJSON("Ok")
|
||||
})
|
||||
|
||||
type RunnerTrainDefLayers struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
TaskId string `json:"taskId" validate:"required"`
|
||||
DefId string `json:"defId" validate:"required"`
|
||||
}
|
||||
PostAuthJson(x, "/tasks/runner/train/def/layers", User_Normal, func(c *Context, dat *RunnerTrainDefLayers) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
if task.TaskType != int(TASK_TYPE_TRAINING) {
|
||||
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
|
||||
return c.JsonBadRequest("Task is not the right type go get the definitions")
|
||||
}
|
||||
|
||||
def, err := GetDefinition(c, dat.DefId)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get definition information", err)
|
||||
}
|
||||
|
||||
layers, err := def.GetLayers(c, " order by layer_order asc")
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get layers", err)
|
||||
}
|
||||
|
||||
return c.SendJSON(layers)
|
||||
})
|
||||
|
||||
PostAuthJson(x, "/tasks/runner/train/datapoints", User_Normal, func(c *Context, dat *VerifyTask) *Error {
|
||||
_, error := verifyRunner(c, &JustId{Id: dat.Id})
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
task, error := verifyTask(x, c, dat)
|
||||
if error != nil {
|
||||
return error
|
||||
}
|
||||
|
||||
if task.TaskType != int(TASK_TYPE_TRAINING) {
|
||||
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
|
||||
return c.JsonBadRequest("Task is not the right type go get the definitions")
|
||||
}
|
||||
|
||||
model, err := GetBaseModel(c, *task.ModelId)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get model information", err)
|
||||
}
|
||||
|
||||
training_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get the model classes", err)
|
||||
}
|
||||
testing_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to get the model classes", err)
|
||||
}
|
||||
|
||||
return c.SendJSON(struct {
|
||||
Testing []DataPoint `json:"testing"`
|
||||
Training []DataPoint `json:"training"`
|
||||
}{
|
||||
Testing: testing_points,
|
||||
Training: training_points,
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -5,14 +5,14 @@ import (
|
||||
"math"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/charmbracelet/log"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
|
||||
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/users"
|
||||
@@ -53,10 +53,9 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
|
||||
|
||||
if task.TaskType == int(TASK_TYPE_CLASSIFICATION) {
|
||||
logger.Info("Classification Task")
|
||||
/*if err = ClassifyTask(base, task); err != nil {
|
||||
if err = ClassifyTask(base, task); err != nil {
|
||||
logger.Error("Classification task failed", "error", err)
|
||||
}*/
|
||||
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "TODO move tasks to pytorch")
|
||||
}
|
||||
|
||||
back_channel <- index
|
||||
continue
|
||||
@@ -92,6 +91,45 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle remote runner
|
||||
*/
|
||||
func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Task) {
|
||||
logger := log.NewWithOptions(os.Stdout, log.Options{
|
||||
ReportCaller: true,
|
||||
ReportTimestamp: true,
|
||||
TimeFormat: time.Kitchen,
|
||||
Prefix: fmt.Sprintf("Runner pre %s", runner_id),
|
||||
})
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
logger.Error("Runner failed to setup for runner", "due to", r, "stack", string(debug.Stack()))
|
||||
// TODO maybe create better failed task
|
||||
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to setup task for runner")
|
||||
}
|
||||
}()
|
||||
|
||||
err := task.UpdateStatus(base, TASK_PICKED_UP, "Failed to setup task for runner")
|
||||
if err != nil {
|
||||
logger.Error("Failed to mark task as PICK UP")
|
||||
return
|
||||
}
|
||||
|
||||
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
defer mutex.Unlock()
|
||||
|
||||
switch task.TaskType {
|
||||
case int(TASK_TYPE_TRAINING):
|
||||
if err := PrepareTraining(handler, base, task, runner_id); err != nil {
|
||||
logger.Error("Failed to prepare for training", "err", err)
|
||||
}
|
||||
default:
|
||||
logger.Error("Not sure what to do panicing", "taskType", task.TaskType)
|
||||
panic("not sure what to do")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the orcchestator to look at the task list from time to time
|
||||
*/
|
||||
@@ -127,7 +165,7 @@ func attentionSeeker(config Config, back_channel chan int) {
|
||||
/**
|
||||
* Manages what worker should to Work
|
||||
*/
|
||||
func RunnerOrchestrator(db db.Db, config Config) {
|
||||
func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
|
||||
logger := log.NewWithOptions(os.Stdout, log.Options{
|
||||
ReportCaller: true,
|
||||
ReportTimestamp: true,
|
||||
@@ -135,6 +173,16 @@ func RunnerOrchestrator(db db.Db, config Config) {
|
||||
Prefix: "Runner Orchestrator Logger",
|
||||
})
|
||||
|
||||
// Setup vars
|
||||
handler.DataMap["runners"] = map[string]interface{}{}
|
||||
handler.DataMap["runners_mutex"] = &sync.Mutex{}
|
||||
|
||||
base := BasePackStruct{
|
||||
Db: db,
|
||||
Logger: logger,
|
||||
Host: config.Hostname,
|
||||
}
|
||||
|
||||
gpu_workers := config.GpuWorker.NumberOfWorkers
|
||||
|
||||
logger.Info("Starting runners")
|
||||
@@ -151,7 +199,7 @@ func RunnerOrchestrator(db db.Db, config Config) {
|
||||
close(task_runners[x])
|
||||
}
|
||||
close(back_channel)
|
||||
go RunnerOrchestrator(db, config)
|
||||
go RunnerOrchestrator(db, config, handler)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -200,19 +248,45 @@ func RunnerOrchestrator(db db.Db, config Config) {
|
||||
}
|
||||
|
||||
if task_to_dispatch != nil {
|
||||
for i := 0; i < len(task_runners_used); i += 1 {
|
||||
if !task_runners_used[i] {
|
||||
task_runners[i] <- *task_to_dispatch
|
||||
task_runners_used[i] = true
|
||||
|
||||
// Only let CPU tasks be done by the local users
|
||||
if task_to_dispatch.TaskType == int(TASK_TYPE_DELETE_USER) {
|
||||
for i := 0; i < len(task_runners_used); i += 1 {
|
||||
if !task_runners_used[i] {
|
||||
task_runners[i] <- *task_to_dispatch
|
||||
task_runners_used[i] = true
|
||||
task_to_dispatch = nil
|
||||
break
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
|
||||
mutex.Lock()
|
||||
remote_runners := handler.DataMap["runners"].(map[string]interface{})
|
||||
|
||||
for k, v := range remote_runners {
|
||||
runner_data := v.(map[string]interface{})
|
||||
runner_info := runner_data["runner_info"].(*Runner)
|
||||
|
||||
if runner_data["task"] != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if runner_info.UserId == task_to_dispatch.UserId {
|
||||
go handleRemoteTask(handler, base, k, *task_to_dispatch)
|
||||
task_to_dispatch = nil
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
mutex.Unlock()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func StartRunners(db db.Db, config Config) {
|
||||
go RunnerOrchestrator(db, config)
|
||||
func StartRunners(db db.Db, config Config, handler *Handle) {
|
||||
go RunnerOrchestrator(db, config, handler)
|
||||
}
|
||||
|
||||
29
logic/tasks/utils/runner.go
Normal file
29
logic/tasks/utils/runner.go
Normal file
@@ -0,0 +1,29 @@
|
||||
package tasks_utils
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
)
|
||||
|
||||
type RunnerType int64
|
||||
|
||||
const (
|
||||
RUNNER_TYPE_GPU RunnerType = iota + 1
|
||||
)
|
||||
|
||||
type Runner struct {
|
||||
Id string `json:"id" db:"ru.id"`
|
||||
UserId string `json:"user_id" db:"ru.user_id"`
|
||||
Token string `json:"token" db:"ru.token"`
|
||||
Type RunnerType `json:"type" db:"ru.type"`
|
||||
CreateOn time.Time `json:"createOn" db:"ru.created_on"`
|
||||
}
|
||||
|
||||
func GetRunner(db db.Db, id string) (ru *Runner, err error) {
|
||||
var runner Runner
|
||||
err = dbtypes.GetDBOnce(db, &runner, "remote_runner as ru where ru.id=$1", id)
|
||||
ru = &runner
|
||||
return
|
||||
}
|
||||
@@ -412,6 +412,18 @@ func UsersEndpints(db db.Db, handle *Handle) {
|
||||
return c.SendJSON("Ok")
|
||||
})
|
||||
|
||||
handle.DeleteAuth("/user/token/logoff", User_Normal, func(c *Context) *Error {
|
||||
if c.Token == nil {
|
||||
return c.JsonBadRequest("Failed to get token")
|
||||
}
|
||||
_, err := c.Db.Exec("delete from tokens where token=$1;", c.Token)
|
||||
if err != nil {
|
||||
return c.E500M("Failed to delete token", err)
|
||||
}
|
||||
|
||||
return c.SendJSON("OK")
|
||||
})
|
||||
|
||||
type DeleteUser struct {
|
||||
Id string `json:"id" validate:"required"`
|
||||
Password string `json:"password" validate:"required"`
|
||||
|
||||
@@ -175,7 +175,7 @@ func (x *Handle) DeleteAuth(path string, authLevel dbtypes.UserType, fn func(c *
|
||||
}
|
||||
return fn(c)
|
||||
}
|
||||
x.posts = append(x.posts, HandleFunc{path, inner_fn})
|
||||
x.deletes = append(x.deletes, HandleFunc{path, inner_fn})
|
||||
}
|
||||
|
||||
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
|
||||
@@ -374,7 +374,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
|
||||
c.SetReportCaller(true)
|
||||
c.Logger.Warn("Request failed with a bad request", "dat", dat)
|
||||
c.SetReportCaller(false)
|
||||
return c.ErrorCode(nil, 404, dat)
|
||||
return c.SendJSONStatus(http.StatusBadRequest, dat)
|
||||
}
|
||||
|
||||
func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
|
||||
@@ -392,7 +392,7 @@ func (c *Context) GetModelFromId(id_path string) (*dbtypes.BaseModel, *Error) {
|
||||
}
|
||||
|
||||
model, err := dbtypes.GetBaseModel(c.Db, id)
|
||||
if err == dbtypes.NotFoundError {
|
||||
if err == dbtypes.ModelNotFoundError {
|
||||
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||
} else if err != nil {
|
||||
return nil, c.Error500(err)
|
||||
|
||||
6
main.go
6
main.go
@@ -23,7 +23,7 @@ const (
|
||||
dbname = "aistuff"
|
||||
)
|
||||
|
||||
func main_() {
|
||||
func main() {
|
||||
|
||||
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
|
||||
"password=%s dbname=%s sslmode=disable",
|
||||
@@ -36,11 +36,11 @@ func main_() {
|
||||
log.Info("Config loaded!", "config", config)
|
||||
config.GenerateToken(db)
|
||||
|
||||
StartRunners(db, config)
|
||||
|
||||
//TODO check if file structure exists to save data
|
||||
handle := NewHandler(db, config)
|
||||
|
||||
StartRunners(db, config, handle)
|
||||
|
||||
config.Cleanup(db)
|
||||
|
||||
// TODO Handle this in other way
|
||||
|
||||
@@ -13,7 +13,7 @@ http {
|
||||
server {
|
||||
listen 8000;
|
||||
|
||||
client_max_body_size 1G;
|
||||
client_max_body_size 5G;
|
||||
|
||||
location / {
|
||||
proxy_http_version 1.1;
|
||||
|
||||
6
requirements.txt
Normal file
6
requirements.txt
Normal file
@@ -0,0 +1,6 @@
|
||||
# tensorflow[and-cuda] == 2.15.1
|
||||
tensorflow[and-cuda] == 2.9.1
|
||||
pandas
|
||||
# Make sure to install the nvidia pyindex first
|
||||
# nvidia-pyindex
|
||||
nvidia-cudnn
|
||||
5
run.sh
5
run.sh
@@ -1,3 +1,2 @@
|
||||
#!/bin/fish
|
||||
podman run --rm --network host --gpus all -ti -v (pwd):/app -e "TERM=xterm-256color" fyp-server bash
|
||||
|
||||
#!/bin/bash
|
||||
podman run --rm --network host --gpus all -ti -v $(pwd):/app -e "TERM=xterm-256color" fyp-server bash
|
||||
|
||||
1
runner/.gitignore
vendored
Normal file
1
runner/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
target/
|
||||
1936
runner/Cargo.lock
generated
Normal file
1936
runner/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
17
runner/Cargo.toml
Normal file
17
runner/Cargo.toml
Normal file
@@ -0,0 +1,17 @@
|
||||
[package]
|
||||
name = "runner"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.82"
|
||||
serde = { version = "1.0.200", features = ["derive"] }
|
||||
toml = "0.8.12"
|
||||
reqwest = { version = "0.12", features = ["json"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
serde_json = "1.0.116"
|
||||
serde_repr = "0.1"
|
||||
tch = { version = "0.16.0", features = ["download-libtorch"] }
|
||||
rand = "0.8.5"
|
||||
12
runner/Dockerfile
Normal file
12
runner/Dockerfile
Normal file
@@ -0,0 +1,12 @@
|
||||
FROM docker.io/nvidia/cuda:11.7.1-devel-ubuntu22.04
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y curl
|
||||
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
|
||||
ENV PATH="$PATH:/root/.cargo/bin"
|
||||
RUN rustup toolchain install stable
|
||||
|
||||
RUN apt-get install -y pkg-config libssl-dev
|
||||
|
||||
WORKDIR /app
|
||||
3
runner/config.toml
Normal file
3
runner/config.toml
Normal file
@@ -0,0 +1,3 @@
|
||||
hostname = "https://testing.andr3h3nriqu3s.com/api"
|
||||
token = "d2bc41e8293937bcd9397870c98f97acc9603f742924b518e193cd1013e45d57897aa302b364001c72b458afcfb34239dfaf38a66b318e5cbc973eea"
|
||||
data_path = "/home/andr3/Documents/my-repos/fyp"
|
||||
1
runner/data.toml
Normal file
1
runner/data.toml
Normal file
@@ -0,0 +1 @@
|
||||
id = "a7cec9e9-1d05-4633-8bc5-6faabe4fd5a3"
|
||||
2
runner/run.sh
Executable file
2
runner/run.sh
Executable file
@@ -0,0 +1,2 @@
|
||||
#!/bin/bash
|
||||
podman run --rm --network host --gpus all -ti -v $(pwd):/app -e "TERM=xterm-256color" fyp-runner bash
|
||||
115
runner/src/dataloader.rs
Normal file
115
runner/src/dataloader.rs
Normal file
@@ -0,0 +1,115 @@
|
||||
use crate::{model::DataPoint, settings::ConfigFile};
|
||||
use std::{path::Path, sync::Arc};
|
||||
use tch::Tensor;
|
||||
|
||||
pub struct DataLoader {
|
||||
pub batch_size: i64,
|
||||
pub len: usize,
|
||||
pub inputs: Vec<Tensor>,
|
||||
pub labels: Vec<Tensor>,
|
||||
pub pos: usize,
|
||||
}
|
||||
|
||||
fn import_image(
|
||||
item: &DataPoint,
|
||||
base_path: &Path,
|
||||
classes_len: i64,
|
||||
inputs: &mut Vec<Tensor>,
|
||||
labels: &mut Vec<Tensor>,
|
||||
) {
|
||||
inputs.push(
|
||||
tch::vision::image::load(base_path.join(&item.path))
|
||||
.ok()
|
||||
.unwrap()
|
||||
.unsqueeze(0),
|
||||
);
|
||||
|
||||
if item.class >= 0 {
|
||||
let t = tch::Tensor::from_slice(&[item.class]).onehot(classes_len as i64);
|
||||
labels.push(t);
|
||||
} else {
|
||||
labels.push(tch::Tensor::zeros(
|
||||
[1, classes_len as i64],
|
||||
(tch::Kind::Float, tch::Device::Cpu),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl DataLoader {
|
||||
pub fn new(
|
||||
config: Arc<ConfigFile>,
|
||||
data: Vec<DataPoint>,
|
||||
classes_len: i64,
|
||||
batch_size: i64,
|
||||
) -> DataLoader {
|
||||
let len: f64 = (data.len() as f64) / (batch_size as f64);
|
||||
let min_len: i64 = len.floor() as i64;
|
||||
let max_len: i64 = len.ceil() as i64;
|
||||
|
||||
println!(
|
||||
"Creating dataloader data len: {} len: {} min_len: {} max_len:{}",
|
||||
data.len(),
|
||||
len,
|
||||
min_len,
|
||||
max_len
|
||||
);
|
||||
|
||||
let base_path = Path::new(&config.data_path);
|
||||
|
||||
let mut inputs: Vec<Tensor> = Vec::new();
|
||||
let mut all_labels: Vec<Tensor> = Vec::new();
|
||||
|
||||
for batch in 0..min_len {
|
||||
let mut batch_acc: Vec<Tensor> = Vec::new();
|
||||
let mut labels: Vec<Tensor> = Vec::new();
|
||||
for image in 0..batch_size {
|
||||
let i: usize = (batch * batch_size + image).try_into().unwrap();
|
||||
let item = &data[i];
|
||||
import_image(item, base_path, classes_len, &mut batch_acc, &mut labels)
|
||||
}
|
||||
inputs.push(tch::Tensor::cat(&batch_acc[0..], 0));
|
||||
all_labels.push(tch::Tensor::cat(&labels[0..], 0));
|
||||
}
|
||||
|
||||
// Import the last batch that has irregular sizing
|
||||
if min_len != max_len {
|
||||
let mut batch_acc: Vec<Tensor> = Vec::new();
|
||||
let mut labels: Vec<Tensor> = Vec::new();
|
||||
for image in 0..(data.len() - (batch_size * min_len) as usize) {
|
||||
let i: usize = (min_len * batch_size + (image as i64)) as usize;
|
||||
let item = &data[i];
|
||||
import_image(item, base_path, classes_len, &mut batch_acc, &mut labels);
|
||||
}
|
||||
inputs.push(tch::Tensor::cat(&batch_acc[0..], 0));
|
||||
all_labels.push(tch::Tensor::cat(&labels[0..], 0));
|
||||
}
|
||||
|
||||
println!("ins shape: {:?}", inputs[0].size());
|
||||
|
||||
return DataLoader {
|
||||
batch_size,
|
||||
inputs,
|
||||
labels: all_labels,
|
||||
len: max_len as usize,
|
||||
pos: 0,
|
||||
};
|
||||
}
|
||||
|
||||
pub fn restart(self: &mut DataLoader) {
|
||||
self.pos = 0;
|
||||
}
|
||||
|
||||
pub fn next(self: &mut DataLoader) -> Option<(Tensor, Tensor)> {
|
||||
if self.pos >= self.len {
|
||||
return None;
|
||||
}
|
||||
let input = self.inputs[self.pos].empty_like();
|
||||
self.inputs[self.pos] = self.inputs[self.pos].clone(&input);
|
||||
let label = self.labels[self.pos].empty_like();
|
||||
self.labels[self.pos] = self.labels[self.pos].clone(&label);
|
||||
|
||||
self.pos += 1;
|
||||
|
||||
return Some((input, label));
|
||||
}
|
||||
}
|
||||
206
runner/src/main.rs
Normal file
206
runner/src/main.rs
Normal file
@@ -0,0 +1,206 @@
|
||||
mod dataloader;
|
||||
mod model;
|
||||
mod settings;
|
||||
mod tasks;
|
||||
mod training;
|
||||
mod types;
|
||||
|
||||
use crate::settings::*;
|
||||
use crate::tasks::{fail_task, Task, TaskType};
|
||||
use crate::training::handle_train;
|
||||
use anyhow::{bail, Result};
|
||||
use reqwest::StatusCode;
|
||||
use serde_json::json;
|
||||
use std::{fs, process::exit, sync::Arc, time::Duration};
|
||||
|
||||
enum ResultAlive {
|
||||
Ok,
|
||||
Error,
|
||||
NotInit,
|
||||
}
|
||||
|
||||
async fn send_keep_alive_message(
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
) -> ResultAlive {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let to_send = json!({
|
||||
"id": runner_data.id,
|
||||
});
|
||||
|
||||
let resp = client
|
||||
.post(format!("{}/tasks/runner/beat", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await;
|
||||
|
||||
if resp.is_err() {
|
||||
return ResultAlive::Error;
|
||||
}
|
||||
|
||||
let resp = resp.ok();
|
||||
|
||||
if resp.is_none() {
|
||||
return ResultAlive::Error;
|
||||
}
|
||||
|
||||
let resp = resp.unwrap();
|
||||
|
||||
// TODO see if the message is related to not being inited
|
||||
if resp.status() != 200 {
|
||||
println!("Could not connect with the status");
|
||||
return ResultAlive::Error;
|
||||
}
|
||||
|
||||
ResultAlive::Ok
|
||||
}
|
||||
|
||||
async fn keep_alive(config: Arc<ConfigFile>, runner_data: Arc<RunnerData>) -> Result<()> {
|
||||
let mut failed = 0;
|
||||
loop {
|
||||
match send_keep_alive_message(config.clone(), runner_data.clone()).await {
|
||||
ResultAlive::Error => failed += 1,
|
||||
ResultAlive::NotInit => {
|
||||
println!("Runner not inited! Restarting!");
|
||||
exit(1)
|
||||
}
|
||||
ResultAlive::Ok => failed = 0,
|
||||
}
|
||||
|
||||
// TODO move to config
|
||||
if failed > 20 {
|
||||
println!("Failed to connect to API! More than 20 times in a row stoping");
|
||||
exit(1)
|
||||
}
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_task(
|
||||
task: Task,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
) -> Result<()> {
|
||||
let res = match task.task_type {
|
||||
TaskType::Training => handle_train(&task, config.clone(), runner_data.clone()).await,
|
||||
_ => {
|
||||
println!("Do not know how to handle this task #{:?}", task);
|
||||
bail!("Failed")
|
||||
}
|
||||
};
|
||||
|
||||
if res.is_err() {
|
||||
println!("task failed #{:?}", res);
|
||||
fail_task(
|
||||
&task,
|
||||
config,
|
||||
runner_data,
|
||||
"Do not know how to handle this kind of task",
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
// Load config file
|
||||
let config_data = fs::read_to_string("./config.toml")?;
|
||||
let mut config: ConfigFile = toml::from_str(&config_data)?;
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
if config.config_path == None {
|
||||
config.config_path = Some(String::from("./data.toml"))
|
||||
}
|
||||
|
||||
let runner_data: RunnerData = load_runner_data(&config).await?;
|
||||
|
||||
let to_send = json!({
|
||||
"id": runner_data.id,
|
||||
});
|
||||
|
||||
// Inform the server that the runner is available
|
||||
let resp = client
|
||||
.post(format!("{}/tasks/runner/init", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if resp.status() != 200 {
|
||||
println!(
|
||||
"Could not connect with the api: status {} body {}",
|
||||
resp.status(),
|
||||
resp.text().await?
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let res = resp.json::<String>().await?;
|
||||
if res != "Ok" {
|
||||
print!("Unexpected problem: {}", res);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let config = Arc::new(config);
|
||||
let runner_data = Arc::new(runner_data);
|
||||
|
||||
let config_alive = config.clone();
|
||||
let runner_data_alive = runner_data.clone();
|
||||
std::thread::spawn(move || keep_alive(config_alive.clone(), runner_data_alive.clone()));
|
||||
|
||||
println!("Started main loop");
|
||||
loop {
|
||||
//TODO move time to config
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
|
||||
let to_send = json!({ "id": runner_data.id });
|
||||
|
||||
let resp = client
|
||||
.post(format!("{}/tasks/runner/active", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await;
|
||||
|
||||
if resp.is_err() || resp.as_ref().ok().is_none() {
|
||||
println!("Failed to get info from server {:?}", resp);
|
||||
continue;
|
||||
}
|
||||
|
||||
let resp = resp?;
|
||||
|
||||
match resp.status() {
|
||||
// No active task
|
||||
StatusCode::NOT_FOUND => (),
|
||||
StatusCode::OK => {
|
||||
println!("Found task!");
|
||||
|
||||
let task: Result<Task, reqwest::Error> = resp.json().await;
|
||||
if task.is_err() || task.as_ref().ok().is_none() {
|
||||
println!("Failed to resolve the json {:?}", task);
|
||||
continue;
|
||||
}
|
||||
|
||||
let task = task?;
|
||||
|
||||
let res = handle_task(task, config.clone(), runner_data.clone()).await;
|
||||
|
||||
if res.is_err() || res.as_ref().ok().is_none() {
|
||||
println!("Failed to run the task");
|
||||
}
|
||||
|
||||
_ = res;
|
||||
()
|
||||
}
|
||||
_ => {
|
||||
println!("Unexpected error #{:?}", resp);
|
||||
exit(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
117
runner/src/model/mod.rs
Normal file
117
runner/src/model/mod.rs
Normal file
@@ -0,0 +1,117 @@
|
||||
use anyhow::bail;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use tch::{
|
||||
nn::{self, Module},
|
||||
Device,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Model {
|
||||
pub vs: nn::VarStore,
|
||||
pub seq: nn::Sequential,
|
||||
pub layers: Vec<Layer>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Serialize_repr, Deserialize_repr)]
|
||||
#[repr(i8)]
|
||||
pub enum LayerType {
|
||||
Input = 1,
|
||||
Dense = 2,
|
||||
Flatten = 3,
|
||||
SimpleBlock = 4,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct Layer {
|
||||
pub id: String,
|
||||
pub definition_id: String,
|
||||
pub layer_order: String,
|
||||
pub layer_type: LayerType,
|
||||
pub shape: String,
|
||||
pub exp_type: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct DataPoint {
|
||||
pub class: i64,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
pub fn build_model(layers: Vec<Layer>, last_linear_size: i64, add_sigmoid: bool) -> Model {
|
||||
let vs = nn::VarStore::new(Device::Cuda(0));
|
||||
|
||||
let mut seq = nn::seq();
|
||||
|
||||
let mut last_linear_size = last_linear_size;
|
||||
let mut last_linear_conv: Vec<i64> = Vec::new();
|
||||
|
||||
for layer in layers.iter() {
|
||||
match layer.layer_type {
|
||||
LayerType::Input => {
|
||||
last_linear_conv = serde_json::from_str(&layer.shape).ok().unwrap();
|
||||
println!("Layer: Input, In: {:?}", last_linear_conv);
|
||||
}
|
||||
LayerType::Dense => {
|
||||
let shape: Vec<i64> = serde_json::from_str(&layer.shape).ok().unwrap();
|
||||
println!("Layer: Dense, In: {}, Out: {}", last_linear_size, shape[0]);
|
||||
seq = seq
|
||||
.add(nn::linear(
|
||||
&vs.root(),
|
||||
last_linear_size,
|
||||
shape[0],
|
||||
Default::default(),
|
||||
))
|
||||
.add_fn(|xs| xs.relu());
|
||||
last_linear_size = shape[0];
|
||||
}
|
||||
LayerType::Flatten => {
|
||||
seq = seq.add_fn(|xs| xs.flatten(1, -1));
|
||||
last_linear_size = 1;
|
||||
for i in &last_linear_conv {
|
||||
last_linear_size *= i;
|
||||
}
|
||||
println!(
|
||||
"Layer: flatten, In: {:?}, Out: {}",
|
||||
last_linear_conv, last_linear_size
|
||||
)
|
||||
}
|
||||
LayerType::SimpleBlock => {
|
||||
let new_last_linear_conv =
|
||||
vec![128, last_linear_conv[1] / 2, last_linear_conv[2] / 2];
|
||||
println!(
|
||||
"Layer: block, In: {:?}, Put: {:?}",
|
||||
last_linear_conv, new_last_linear_conv,
|
||||
);
|
||||
let out_size = vec![new_last_linear_conv[1], new_last_linear_conv[2]];
|
||||
seq = seq
|
||||
.add(nn::conv2d(
|
||||
&vs.root(),
|
||||
last_linear_conv[0],
|
||||
128,
|
||||
3,
|
||||
nn::ConvConfig::default(),
|
||||
))
|
||||
.add_fn(|xs| xs.relu())
|
||||
.add(nn::conv2d(
|
||||
&vs.root(),
|
||||
128,
|
||||
128,
|
||||
3,
|
||||
nn::ConvConfig::default(),
|
||||
))
|
||||
.add_fn(|xs| xs.relu())
|
||||
.add_fn(move |xs| xs.adaptive_avg_pool2d([out_size[1], out_size[1]]))
|
||||
.add_fn(|xs| xs.leaky_relu());
|
||||
//m_layers = append(m_layers, NewSimpleBlock(vs, lastLinearConv[0]))
|
||||
last_linear_conv = new_last_linear_conv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if add_sigmoid {
|
||||
seq = seq.add_fn(|xs| xs.sigmoid());
|
||||
}
|
||||
|
||||
return Model { vs, layers, seq };
|
||||
}
|
||||
57
runner/src/settings.rs
Normal file
57
runner/src/settings.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use anyhow::{bail, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::json;
|
||||
use std::{fs, path};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ConfigFile {
|
||||
// Hostname to connect with the api
|
||||
pub hostname: String,
|
||||
// Token used in the api to authenticate
|
||||
pub token: String,
|
||||
// Path to where to store some generated configuration values
|
||||
// defaults to ./data.toml
|
||||
pub config_path: Option<String>,
|
||||
// Data Path
|
||||
// Path to where the data is mounted
|
||||
pub data_path: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
pub struct RunnerData {
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
pub async fn load_runner_data(config: &ConfigFile) -> Result<RunnerData> {
|
||||
let data_path = config.config_path.as_ref().unwrap();
|
||||
let data_path = path::Path::new(&*data_path);
|
||||
|
||||
if data_path.exists() {
|
||||
let runner_data = fs::read_to_string(data_path)?;
|
||||
Ok(toml::from_str(&runner_data)?)
|
||||
} else {
|
||||
let client = reqwest::Client::new();
|
||||
let to_send = json!({
|
||||
"token": config.token,
|
||||
"type": 1,
|
||||
});
|
||||
|
||||
let register_resp = client
|
||||
.post(format!("{}/tasks/runner/register", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if register_resp.status() != 200 {
|
||||
bail!(format!("Could not create runner {:#?}", register_resp));
|
||||
}
|
||||
|
||||
let runner_data: RunnerData = register_resp.json().await?;
|
||||
|
||||
fs::write(data_path, toml::to_string(&runner_data)?)
|
||||
.expect("Faield to save data for runner");
|
||||
|
||||
Ok(runner_data)
|
||||
}
|
||||
}
|
||||
90
runner/src/tasks.rs
Normal file
90
runner/src/tasks.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{bail, Result};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use serde_repr::Deserialize_repr;
|
||||
|
||||
use crate::{ConfigFile, RunnerData};
|
||||
|
||||
#[derive(Clone, Copy, Deserialize_repr, Debug)]
|
||||
#[repr(i8)]
|
||||
pub enum TaskStatus {
|
||||
FailedRunning = -2,
|
||||
FailedCreation = -1,
|
||||
Preparing = 0,
|
||||
Todo = 1,
|
||||
PickedUp = 2,
|
||||
Running = 3,
|
||||
Done = 4,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize_repr, Debug)]
|
||||
#[repr(i8)]
|
||||
pub enum TaskType {
|
||||
Classification = 1,
|
||||
Training = 2,
|
||||
Retraining = 3,
|
||||
DeleteUser = 4,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct Task {
|
||||
pub id: String,
|
||||
pub user_id: String,
|
||||
pub model_id: String,
|
||||
pub status: TaskStatus,
|
||||
pub status_message: String,
|
||||
pub user_confirmed: i8,
|
||||
pub compacted: i8,
|
||||
#[serde(alias = "type")]
|
||||
pub task_type: TaskType,
|
||||
pub extra_task_info: String,
|
||||
pub result: String,
|
||||
pub created: String,
|
||||
}
|
||||
|
||||
pub async fn fail_task(
|
||||
task: &Task,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
reason: &str,
|
||||
) -> Result<()> {
|
||||
println!("Marking Task as failed");
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let to_send = json!({
|
||||
"id": runner_data.id,
|
||||
"taskId": task.id,
|
||||
"reason": reason
|
||||
});
|
||||
|
||||
let resp = client
|
||||
.post(format!("{}/tasks/runner/fail", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if resp.status() != 200 {
|
||||
println!("Failed to update status of task");
|
||||
bail!("Failed to update status of task");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl Task {
|
||||
pub async fn fail(
|
||||
self: &mut Task,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
reason: &str,
|
||||
) -> Result<()> {
|
||||
fail_task(self, config, runner_data, reason).await?;
|
||||
self.status = TaskStatus::FailedRunning;
|
||||
self.status_message = reason.to_string();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
599
runner/src/training.rs
Normal file
599
runner/src/training.rs
Normal file
@@ -0,0 +1,599 @@
|
||||
use crate::{
|
||||
dataloader::DataLoader,
|
||||
model::{self, build_model},
|
||||
settings::{ConfigFile, RunnerData},
|
||||
tasks::{fail_task, Task},
|
||||
types::{DataPointRequest, Definition, ModelClass},
|
||||
};
|
||||
use std::{
|
||||
io::{self, Write},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use anyhow::Result;
|
||||
use rand::{seq::SliceRandom, thread_rng};
|
||||
use serde_json::json;
|
||||
use tch::{
|
||||
nn::{self, Module, OptimizerConfig},
|
||||
Cuda, Tensor,
|
||||
};
|
||||
|
||||
pub async fn handle_train(
|
||||
task: &Task,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
) -> Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
println!("Preparing to train a model");
|
||||
|
||||
let to_send = json!({
|
||||
"id": runner_data.id,
|
||||
"taskId": task.id,
|
||||
});
|
||||
|
||||
let mut defs: Vec<Definition> = client
|
||||
.post(format!("{}/tasks/runner/train/defs", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
if defs.len() == 0 {
|
||||
println!("No defs found");
|
||||
fail_task(task, config, runner_data, "No definitions found").await?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let classes: Vec<ModelClass> = client
|
||||
.post(format!("{}/tasks/runner/train/classes", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
let data: DataPointRequest = client
|
||||
.post(format!("{}/tasks/runner/train/datapoints", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
let mut testing = data.testing;
|
||||
|
||||
testing.shuffle(&mut thread_rng());
|
||||
|
||||
let mut data_loader = DataLoader::new(config.clone(), testing, classes.len() as i64, 64);
|
||||
|
||||
// TODO make this a vec
|
||||
let mut model: Option<model::Model> = None;
|
||||
|
||||
loop {
|
||||
let config = config.clone();
|
||||
let runner_data = runner_data.clone();
|
||||
let mut to_remove: Vec<usize> = Vec::new();
|
||||
|
||||
let mut def_iter = defs.iter_mut();
|
||||
|
||||
let mut i: usize = 0;
|
||||
while let Some(def) = def_iter.next() {
|
||||
def.updateStatus(
|
||||
task,
|
||||
config.clone(),
|
||||
runner_data.clone(),
|
||||
crate::types::DefinitionStatus::Training,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let model_err = train_definition(
|
||||
def,
|
||||
&mut data_loader,
|
||||
model,
|
||||
config.clone(),
|
||||
runner_data.clone(),
|
||||
&task,
|
||||
)
|
||||
.await;
|
||||
|
||||
if model_err.is_err() {
|
||||
println!("Failed to create model {:?}", model_err);
|
||||
model = None;
|
||||
to_remove.push(i);
|
||||
continue;
|
||||
}
|
||||
|
||||
model = model_err?;
|
||||
|
||||
i += 1;
|
||||
}
|
||||
|
||||
defs = defs
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.filter(|&(i, _)| to_remove.iter().any(|b| *b == i))
|
||||
.map(|(_, e)| e)
|
||||
.collect();
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
fail_task(task, config, runner_data, "TODO").await?;
|
||||
Ok(())
|
||||
|
||||
/*
|
||||
for {
|
||||
// Keep track of definitions that did not train fast enough
|
||||
var toRemove ToRemoveList = []int{}
|
||||
|
||||
for i, def := range definitions {
|
||||
|
||||
accuracy, ml_model, err := trainDefinition(c, model, def, models[def.Id], classes)
|
||||
if err != nil {
|
||||
log.Error("Failed to train definition!Err:", "err", err)
|
||||
def.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
|
||||
toRemove = append(toRemove, i)
|
||||
continue
|
||||
}
|
||||
models[def.Id] = ml_model
|
||||
|
||||
if accuracy >= float64(def.TargetAccuracy) {
|
||||
log.Info("Found a definition that reaches target_accuracy!")
|
||||
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, DEFINITION_STATUS_TRANIED, def.Epoch, def.Id)
|
||||
if err != nil {
|
||||
log.Error("Failed to train definition!Err:\n", "err", err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", DEFINITION_STATUS_CANCELD_TRAINING, def.Id, model.Id, DEFINITION_STATUS_FAILED_TRAINING)
|
||||
if err != nil {
|
||||
log.Error("Failed to train definition!Err:\n", "err", err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return err
|
||||
}
|
||||
|
||||
finished = true
|
||||
break
|
||||
}
|
||||
|
||||
if def.Epoch > MAX_EPOCH {
|
||||
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.TargetAccuracy)
|
||||
def.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
|
||||
toRemove = append(toRemove, i)
|
||||
continue
|
||||
}
|
||||
|
||||
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.Epoch, DEFINITION_STATUS_PAUSED_TRAINING, def.Id)
|
||||
if err != nil {
|
||||
log.Error("Failed to train definition!Err:\n", "err", err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if finished {
|
||||
break
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(toRemove))
|
||||
|
||||
log.Info("Round done", "toRemove", toRemove)
|
||||
|
||||
for _, n := range toRemove {
|
||||
// Clean up unsed models
|
||||
models[definitions[n].Id] = nil
|
||||
definitions = remove(definitions, n)
|
||||
}
|
||||
|
||||
len_def := len(definitions)
|
||||
|
||||
if len_def == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
if len_def == 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
sort.Sort(sort.Reverse(definitions))
|
||||
|
||||
acc := definitions[0].Accuracy - 20.0
|
||||
|
||||
log.Info("Training models, Highest acc", "acc", definitions[0].Accuracy, "mod_acc", acc)
|
||||
|
||||
toRemove = []int{}
|
||||
for i, def := range definitions {
|
||||
if def.Accuracy < acc {
|
||||
toRemove = append(toRemove, i)
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("Removing due to accuracy", "toRemove", toRemove)
|
||||
|
||||
sort.Sort(sort.Reverse(toRemove))
|
||||
for _, n := range toRemove {
|
||||
log.Warn("Removing definition not fast enough learning", "n", n)
|
||||
definitions[n].UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
|
||||
models[definitions[n].Id] = nil
|
||||
definitions = remove(definitions, n)
|
||||
}
|
||||
}
|
||||
|
||||
var def Definition
|
||||
err = GetDBOnce(c, &def, "model_definition as md where md.model_id=$1 and md.status=$2 order by md.accuracy desc limit 1;", model.Id, DEFINITION_STATUS_TRANIED)
|
||||
if err != nil {
|
||||
if err == NotFoundError {
|
||||
log.Error("All definitions failed to train!")
|
||||
} else {
|
||||
log.Error("DB: failed to read definition", "err", err)
|
||||
}
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return
|
||||
}
|
||||
|
||||
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
|
||||
log.Error("Failed to update model definition", "err", err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return
|
||||
}
|
||||
|
||||
to_delete, err := db.Query("select id from model_definition where status != $1 and model_id=$2", DEFINITION_STATUS_READY, model.Id)
|
||||
if err != nil {
|
||||
log.Error("Failed to select model_definition to delete")
|
||||
log.Error(err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return
|
||||
}
|
||||
defer to_delete.Close()
|
||||
|
||||
for to_delete.Next() {
|
||||
var id string
|
||||
if err = to_delete.Scan(&id); err != nil {
|
||||
log.Error("Failed to scan the id of a model_definition to delete", "err", err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return
|
||||
}
|
||||
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
|
||||
}
|
||||
|
||||
// TODO Check if returning also works here
|
||||
if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", DEFINITION_STATUS_READY, model.Id); err != nil {
|
||||
log.Error("Failed to delete model_definition")
|
||||
log.Error(err)
|
||||
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
|
||||
return
|
||||
}
|
||||
|
||||
ModelUpdateStatus(c, model.Id, READY)
|
||||
|
||||
return
|
||||
*/
|
||||
}
|
||||
|
||||
async fn train_definition(
|
||||
def: &Definition,
|
||||
data_loader: &mut DataLoader,
|
||||
model: Option<model::Model>,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
task: &Task,
|
||||
) -> Result<Option<model::Model>> {
|
||||
let client = reqwest::Client::new();
|
||||
println!("About to start training definition");
|
||||
|
||||
let mut accuracy = 0;
|
||||
|
||||
let model = model.unwrap_or({
|
||||
let layers: Vec<model::Layer> = client
|
||||
.post(format!("{}/tasks/runner/train/def/layers", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(
|
||||
json!({
|
||||
"id": runner_data.id,
|
||||
"taskId": task.id,
|
||||
"defId": def.id,
|
||||
})
|
||||
.to_string(),
|
||||
)
|
||||
.send()
|
||||
.await?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
build_model(layers, 0, true)
|
||||
});
|
||||
|
||||
// TODO CUDA
|
||||
// get device
|
||||
// Move model to cuda
|
||||
|
||||
let mut opt = nn::Adam::default().build(&model.vs, 1e-3)?;
|
||||
|
||||
let mut last_acc = 0.0;
|
||||
|
||||
for epoch in 1..40 {
|
||||
data_loader.restart();
|
||||
let mut mean_loss: f64 = 0.0;
|
||||
let mut mean_acc: f64 = 0.0;
|
||||
while let Some((inputs, labels)) = data_loader.next() {
|
||||
let inputs = inputs
|
||||
.to_kind(tch::Kind::Float)
|
||||
.to_device(tch::Device::Cuda(0));
|
||||
let labels = labels
|
||||
.to_kind(tch::Kind::Float)
|
||||
.to_device(tch::Device::Cuda(0));
|
||||
let out = model.seq.forward(&inputs);
|
||||
let weight: Option<Tensor> = None;
|
||||
let loss = out.binary_cross_entropy(&labels, weight, tch::Reduction::Mean);
|
||||
opt.backward_step(&loss);
|
||||
mean_loss += loss
|
||||
.to_device(tch::Device::Cpu)
|
||||
.unsqueeze(0)
|
||||
.double_value(&[0]);
|
||||
|
||||
let out = out.to_device(tch::Device::Cpu);
|
||||
|
||||
let test = out.empty_like();
|
||||
_ = out.clone(&test);
|
||||
|
||||
let out = test.argmax(1, true);
|
||||
|
||||
let mut labels = labels.to_device(tch::Device::Cpu);
|
||||
|
||||
labels = labels.unsqueeze(-1);
|
||||
|
||||
let size = out.size()[0];
|
||||
|
||||
let mut acc = 0;
|
||||
for i in 0..size {
|
||||
let res = out.double_value(&[i]);
|
||||
let exp = labels.double_value(&[i, res as i64]);
|
||||
if exp == 1.0 {
|
||||
acc += 1;
|
||||
}
|
||||
}
|
||||
|
||||
mean_acc += acc as f64 / size as f64;
|
||||
last_acc = acc as f64 / size as f64;
|
||||
}
|
||||
print!(
|
||||
"\repoch: {} loss: {} acc: {} l acc: {} ",
|
||||
epoch,
|
||||
mean_loss / data_loader.len as f64,
|
||||
mean_acc / data_loader.len as f64,
|
||||
last_acc
|
||||
);
|
||||
io::stdout().flush().expect("Unable to flush stdout");
|
||||
}
|
||||
|
||||
println!("\nlast acc: {}", last_acc);
|
||||
|
||||
return Ok(Some(model));
|
||||
/*
|
||||
|
||||
opt, err := my_nn.DefaultAdamConfig().Build(model.Vs, 0.001)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for epoch := 0; epoch < EPOCH_PER_RUN; epoch++ {
|
||||
var trainIter *torch.Iter2
|
||||
trainIter, err = ds.TrainIter(32)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// trainIter.ToDevice(device)
|
||||
|
||||
log.Info("epoch", "epoch", epoch)
|
||||
|
||||
var trainLoss float64 = 0
|
||||
var trainCorrect float64 = 0
|
||||
ok := true
|
||||
for ok {
|
||||
var item torch.Iter2Item
|
||||
var loss *torch.Tensor
|
||||
item, ok = trainIter.Next()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
data := item.Data
|
||||
data, err = data.ToDevice(device, gotch.Float, false, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var size []int64
|
||||
size, err = data.Size()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
var zeros *torch.Tensor
|
||||
zeros, err = torch.Zeros(size, gotch.Float, device)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
data, err = zeros.Add(data, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("\n\nhere 1, data\n\n", "retains", data.MustRetainsGrad(false), "requires", data.MustRequiresGrad())
|
||||
|
||||
data, err = data.SetRequiresGrad(true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("\n\nhere 2, data\n\n", "retains", data.MustRetainsGrad(false), "requires", data.MustRequiresGrad())
|
||||
|
||||
err = data.RetainGrad(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("\n\nhere 2, data\n\n", "retains", data.MustRetainsGrad(false), "requires", data.MustRequiresGrad())
|
||||
|
||||
pred := model.ForwardT(data, true)
|
||||
pred, err = pred.SetRequiresGrad(true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = pred.RetainGrad(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
label := item.Label
|
||||
label, err = label.ToDevice(device, gotch.Float, false, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
label, err = label.SetRequiresGrad(true, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = label.RetainGrad(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate loss
|
||||
loss, err = pred.BinaryCrossEntropyWithLogits(label, &torch.Tensor{}, &torch.Tensor{}, 2, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
loss, err = loss.SetRequiresGrad(true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = loss.RetainGrad(false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = opt.ZeroGrad()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = loss.Backward()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("pred grad", "pred", pred.MustGrad(false).MustMax(false).Float64Values())
|
||||
log.Info("pred grad", "outs", label.MustGrad(false).MustMax(false).Float64Values())
|
||||
log.Info("pred grad", "data", data.MustGrad(false).MustMax(false).Float64Values(), "lol", data.MustRetainsGrad(false))
|
||||
|
||||
vars := model.Vs.Variables()
|
||||
|
||||
for k, v := range vars {
|
||||
log.Info("[grad check]", "k", k, "grad", v.MustGrad(false).MustMax(false).Float64Values(), "lol", v.MustRetainsGrad(false))
|
||||
}
|
||||
|
||||
model.Debug()
|
||||
|
||||
err = opt.Step()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
trainLoss = loss.Float64Values()[0]
|
||||
|
||||
// Calculate accuracy
|
||||
/ *var p_pred, p_labels *torch.Tensor
|
||||
p_pred, err = pred.Argmax([]int64{1}, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
p_labels, err = item.Label.Argmax([]int64{1}, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
floats := p_pred.Float64Values()
|
||||
floats_labels := p_labels.Float64Values()
|
||||
|
||||
for i := range floats {
|
||||
if floats[i] == floats_labels[i] {
|
||||
trainCorrect += 1
|
||||
}
|
||||
} * /
|
||||
|
||||
panic("fornow")
|
||||
}
|
||||
|
||||
//v := []float64{}
|
||||
|
||||
log.Info("model training epoch done loss", "loss", trainLoss, "correct", trainCorrect, "out", ds.TrainImagesSize, "accuracy", trainCorrect/float64(ds.TrainImagesSize))
|
||||
|
||||
/ *correct := int64(0)
|
||||
//torch.NoGrad(func() {
|
||||
ok = true
|
||||
testIter := ds.TestIter(64)
|
||||
for ok {
|
||||
var item torch.Iter2Item
|
||||
item, ok = testIter.Next()
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
output := model.Forward(item.Data)
|
||||
|
||||
var pred, labels *torch.Tensor
|
||||
pred, err = output.Argmax([]int64{1}, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
labels, err = item.Label.Argmax([]int64{1}, true, false)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
floats := pred.Float64Values()
|
||||
floats_labels := labels.Float64Values()
|
||||
|
||||
for i := range floats {
|
||||
if floats[i] == floats_labels[i] {
|
||||
correct += 1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
accuracy = float64(correct) / float64(ds.TestImagesSize)
|
||||
|
||||
log.Info("Eval accuracy", "accuracy", accuracy)
|
||||
|
||||
err = def.UpdateAfterEpoch(db, accuracy*100)
|
||||
if err != nil {
|
||||
return
|
||||
}* /
|
||||
//})
|
||||
}
|
||||
|
||||
result_path := path.Join(getDir(), "savedData", m.Id, "defs", def.Id)
|
||||
err = os.MkdirAll(result_path, os.ModePerm)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
err = my_torch.SaveModel(model, path.Join(result_path, "model.dat"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Info("Model finished training!", "accuracy", accuracy)
|
||||
return
|
||||
*/
|
||||
}
|
||||
89
runner/src/types.rs
Normal file
89
runner/src/types.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
use crate::{model, tasks::Task, ConfigFile, RunnerData};
|
||||
use anyhow::{bail, Result};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use serde_repr::{Deserialize_repr, Serialize_repr};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone, Copy, Deserialize_repr, Serialize_repr, Debug)]
|
||||
#[repr(i8)]
|
||||
pub enum DefinitionStatus {
|
||||
CanceldTraining = -4,
|
||||
FailedTraining = -3,
|
||||
PreInit = 1,
|
||||
Init = 2,
|
||||
Training = 3,
|
||||
PausedTraining = 6,
|
||||
Tranied = 4,
|
||||
Ready = 5,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct Definition {
|
||||
pub id: String,
|
||||
pub model_id: String,
|
||||
pub accuracy: f64,
|
||||
pub target_accuracy: i64,
|
||||
pub epoch: i64,
|
||||
pub status: i64,
|
||||
pub created: String,
|
||||
pub epoch_progress: i64,
|
||||
}
|
||||
|
||||
impl Definition {
|
||||
pub async fn updateStatus(
|
||||
self: &mut Definition,
|
||||
task: &Task,
|
||||
config: Arc<ConfigFile>,
|
||||
runner_data: Arc<RunnerData>,
|
||||
status: DefinitionStatus,
|
||||
) -> Result<()> {
|
||||
println!("Marking Task as faield");
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let to_send = json!({
|
||||
"id": runner_data.id,
|
||||
"taskId": task.id,
|
||||
"defId": self.id,
|
||||
"status": status,
|
||||
});
|
||||
|
||||
let resp = client
|
||||
.post(format!("{}/tasks/runner/train/def/status", config.hostname))
|
||||
.header("token", &config.token)
|
||||
.body(to_send.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
if resp.status() != 200 {
|
||||
println!("Failed to update status of task");
|
||||
bail!("Failed to update status of task");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Deserialize_repr, Debug)]
|
||||
#[repr(i8)]
|
||||
pub enum ModelClassStatus {
|
||||
ToTrain = 1,
|
||||
Training = 2,
|
||||
Trained = 3,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct ModelClass {
|
||||
pub id: String,
|
||||
pub model_id: String,
|
||||
pub name: String,
|
||||
pub class_order: i64,
|
||||
pub status: ModelClassStatus,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct DataPointRequest {
|
||||
pub testing: Vec<model::DataPoint>,
|
||||
pub training: Vec<model::DataPoint>,
|
||||
}
|
||||
@@ -38,3 +38,14 @@ create table if not exists tasks_dependencies (
|
||||
main_id uuid references tasks (id) on delete cascade not null,
|
||||
dependent_id uuid references tasks (id) on delete cascade not null
|
||||
);
|
||||
|
||||
create table if not exists remote_runner (
|
||||
id uuid primary key default gen_random_uuid(),
|
||||
user_id uuid references users (id) on delete cascade not null,
|
||||
token text not null,
|
||||
|
||||
-- 1: GPU
|
||||
type integer,
|
||||
|
||||
created_on timestamp default current_timestamp
|
||||
);
|
||||
|
||||
120
test.go
120
test.go
@@ -1,120 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"git.andr3h3nriqu3s.com/andr3/gotch"
|
||||
|
||||
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch"
|
||||
|
||||
//my_nn "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/nn"
|
||||
|
||||
torch "git.andr3h3nriqu3s.com/andr3/gotch/ts"
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
log.Info("Hello world")
|
||||
|
||||
m := train.BuildModel([]*dbtypes.Layer{
|
||||
&dbtypes.Layer{
|
||||
LayerType: dbtypes.LAYER_INPUT,
|
||||
Shape: "[ 3, 28, 28 ]",
|
||||
},
|
||||
&dbtypes.Layer{
|
||||
LayerType: dbtypes.LAYER_FLATTEN,
|
||||
},
|
||||
&dbtypes.Layer{
|
||||
LayerType: dbtypes.LAYER_DENSE,
|
||||
Shape: "[ 27 ]",
|
||||
},
|
||||
&dbtypes.Layer{
|
||||
LayerType: dbtypes.LAYER_DENSE,
|
||||
Shape: "[ 18 ]",
|
||||
},
|
||||
// &dbtypes.Layer{
|
||||
// LayerType: dbtypes.LAYER_DENSE,
|
||||
// Shape: "[ 9 ]",
|
||||
// },
|
||||
}, 0, true)
|
||||
|
||||
//var err error
|
||||
|
||||
d := gotch.CudaIfAvailable()
|
||||
|
||||
log.Info("device", "d", d)
|
||||
|
||||
m.To(d)
|
||||
|
||||
var count = 0
|
||||
|
||||
// vars1 := m.Vs.Variables()
|
||||
//
|
||||
// for k, v := range vars1 {
|
||||
// ones := torch.MustOnes(v.MustSize(), gotch.Float, d)
|
||||
// v := ones.MustSetRequiresGrad(true, false)
|
||||
// v.MustDrop()
|
||||
// ones.RetainGrad(false)
|
||||
//
|
||||
// m.Vs.UpdateVarTensor(k, ones, true)
|
||||
// m.Refresh()
|
||||
// }
|
||||
//
|
||||
// opt, err := my_nn.DefaultAdamConfig().Build(m.Vs, 0.001)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
|
||||
log.Info("start")
|
||||
|
||||
for count < 100 {
|
||||
|
||||
ones := torch.MustOnes([]int64{1, 3, 28, 28}, gotch.Float, d)
|
||||
// ones = ones.MustSetRequiresGrad(true, true)
|
||||
// ones.RetainGrad(false)
|
||||
|
||||
res := m.ForwardT(ones, true)
|
||||
//res = res.MustSetRequiresGrad(true, true)
|
||||
//res.RetainGrad(false)
|
||||
|
||||
outs := torch.MustZeros([]int64{1, 18}, gotch.Float, d)
|
||||
|
||||
loss, err := res.BinaryCrossEntropyWithLogits(outs, &torch.Tensor{}, &torch.Tensor{}, 2, false)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
// loss = loss.MustSetRequiresGrad(true, true)
|
||||
|
||||
//opt.ZeroGrad()
|
||||
|
||||
log.Info("loss", "loss", loss.Float64Values())
|
||||
|
||||
loss.MustBackward()
|
||||
|
||||
//opt.Step()
|
||||
|
||||
// log.Info(mean.MustGrad(false).Float64Values())
|
||||
//ones_grad = ones.MustGrad(true).MustMax(true).Float64Values()[0]
|
||||
// log.Info(res.MustGrad(true).MustMax(true).Float64Values())
|
||||
|
||||
// log.Info(ones_grad)
|
||||
|
||||
vars := m.Vs.Variables()
|
||||
|
||||
for k, v := range vars {
|
||||
log.Info("[grad check]", "k", k, "grad", v.MustGrad(false).MustMax(true).Float64Values())
|
||||
}
|
||||
|
||||
m.Debug()
|
||||
|
||||
outs.MustDrop()
|
||||
|
||||
count += 1
|
||||
|
||||
log.Fatal("grad zero")
|
||||
|
||||
}
|
||||
|
||||
log.Warn("out")
|
||||
|
||||
}
|
||||
@@ -9,9 +9,9 @@ import requests
|
||||
class NotifyServerCallback(tf.keras.callbacks.Callback):
|
||||
def on_epoch_end(self, epoch, log, *args, **kwargs):
|
||||
{{ if .HeadId }}
|
||||
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
|
||||
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
|
||||
{{ else }}
|
||||
requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}')
|
||||
requests.get(f'{{ .Host }}/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["val_accuracy"]}&definition={{.DefId}}')
|
||||
{{end}}
|
||||
|
||||
|
||||
@@ -82,7 +82,7 @@ def prepare_dataset(ds: tf.data.Dataset, size: int) -> tf.data.Dataset:
|
||||
|
||||
def filterDataset(path):
|
||||
path = tf.strings.regex_replace(path, DATA_DIR_PREPARE, "")
|
||||
|
||||
|
||||
{{ if eq .Model.Format "png" }}
|
||||
path = tf.strings.regex_replace(path, ".png", "")
|
||||
{{ else if eq .Model.Format "jpeg" }}
|
||||
@@ -90,7 +90,7 @@ def filterDataset(path):
|
||||
{{ else }}
|
||||
ERROR
|
||||
{{ end }}
|
||||
|
||||
|
||||
return tf.reshape(table.lookup(tf.strings.as_string([path])), []) != -1
|
||||
|
||||
seed = random.randint(0, 100000000)
|
||||
@@ -135,9 +135,9 @@ def addBlock(
|
||||
model.add(layers.ReLU())
|
||||
if top:
|
||||
if pooling_same:
|
||||
model.add(pool_func(padding="same", strides=(1, 1)))
|
||||
model.add(pool_func(pool_size=(2,2), padding="same", strides=(1, 1)))
|
||||
else:
|
||||
model.add(pool_func())
|
||||
model.add(pool_func(pool_size=(2,2)))
|
||||
model.add(layers.BatchNormalization())
|
||||
model.add(layers.LeakyReLU())
|
||||
model.add(layers.Dropout(0.4))
|
||||
@@ -172,7 +172,7 @@ model.compile(
|
||||
|
||||
his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
|
||||
NotifyServerCallback(),
|
||||
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
|
||||
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)])
|
||||
|
||||
acc = his.history["accuracy"]
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import numpy as np
|
||||
|
||||
class NotifyServerCallback(tf.keras.callbacks.Callback):
|
||||
def on_epoch_end(self, epoch, log, *args, **kwargs):
|
||||
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}')
|
||||
requests.get(f'{{ .Host }}/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["val_accuracy"]}&head_id={{.HeadId}}')
|
||||
|
||||
|
||||
DATA_DIR = "{{ .DataDir }}"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { goto } from '$app/navigation';
|
||||
import { rdelete } from '$lib/requests.svelte';
|
||||
|
||||
type User = {
|
||||
token: string;
|
||||
@@ -33,6 +34,10 @@ export function createUserStore() {
|
||||
if (value) {
|
||||
localStorage.setItem('user', JSON.stringify(value));
|
||||
} else {
|
||||
if (user) {
|
||||
// Request the deletion of the token
|
||||
rdelete('/user/token/logoff', {});
|
||||
}
|
||||
localStorage.removeItem('user');
|
||||
}
|
||||
user = value;
|
||||
|
||||
@@ -215,7 +215,7 @@
|
||||
</div>
|
||||
{:else if m.status == -3 || m.status == -4}
|
||||
<BaseModelInfo model={m} />
|
||||
<form on:submit|preventDefault={resetModel}>
|
||||
<form on:submit={resetModel}>
|
||||
Failed Prepare for training.<br />
|
||||
<div class="spacer"></div>
|
||||
<MessageSimple bind:this={resetMessages} />
|
||||
|
||||
Reference in New Issue
Block a user