andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

runner-go (#102)

Browse Source 
Reviewed-on: #102
Co-authored-by: Andre Henriques <andr3h3nriqu3s@gmail.com>
Co-committed-by: Andre Henriques <andr3h3nriqu3s@gmail.com>
This commit is contained in:
Andre Henriques 2024-05-10 02:13:02 +01:00 committed by andr3
parent edd1e4c123
commit 0ac6ac8dce
44 changed files with 6609 additions and 511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -9,10 +9,11 @@ require (
github.com/google/uuid v1.6.0
github.com/lib/pq v1.10.9
golang.org/x/crypto v0.19.0
github.com/BurntSushi/toml v1.3.2
github.com/goccy/go-json v0.10.2
)
require (
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/charmbracelet/lipgloss v0.9.1 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
@ -20,7 +21,6 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.19.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/pgx v3.6.2+incompatible // indirect

16
logic/db_types/classes.go
View File

@ -6,3 +6,19 @@ const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)
type ModelClassStatus int
const (
CLASS_STATUS_TO_TRAIN ModelClassStatus = iota + 1
CLASS_STATUS_TRAINING
CLASS_STATUS_TRAINED
)
type ModelClass struct {
Id string `db:"mc.id" json:"id"`
ModelId string `db:"mc.model_id" json:"model_id"`
Name string `db:"mc.name" json:"name"`
ClassOrder int `db:"mc.class_order" json:"class_order"`
Status int `db:"mc.status" json:"status"`
}

95
logic/db_types/definitions.go Normal file
View File

@ -0,0 +1,95 @@
package dbtypes
import (
"time"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
)
type DefinitionStatus int
const (
DEFINITION_STATUS_CANCELD_TRAINING DefinitionStatus = -4
DEFINITION_STATUS_FAILED_TRAINING = -3
DEFINITION_STATUS_PRE_INIT = 1
DEFINITION_STATUS_INIT = 2
DEFINITION_STATUS_TRAINING = 3
DEFINITION_STATUS_PAUSED_TRAINING = 6
DEFINITION_STATUS_TRANIED = 4
DEFINITION_STATUS_READY = 5
)
type Definition struct {
Id string `db:"md.id" json:"id"`
ModelId string `db:"md.model_id" json:"model_id"`
Accuracy float64 `db:"md.accuracy" json:"accuracy"`
TargetAccuracy int `db:"md.target_accuracy" json:"target_accuracy"`
Epoch int `db:"md.epoch" json:"epoch"`
Status int `db:"md.status" json:"status"`
CreatedOn time.Time `db:"md.created_on" json:"created"`
EpochProgress int `db:"md.epoch_progress" json:"epoch_progress"`
}
type SortByAccuracyDefinitions []*Definition
func (nf SortByAccuracyDefinitions) Len() int { return len(nf) }
func (nf SortByAccuracyDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf SortByAccuracyDefinitions) Less(i, j int) bool {
return nf[i].Accuracy < nf[j].Accuracy
}
func GetDefinition(db db.Db, definition_id string) (definition Definition, err error) {
err = GetDBOnce(db, &definition, "model_definition as md where id=$1;", definition_id)
return
}
func MakeDefenition(db db.Db, model_id string, target_accuracy int) (definition Definition, err error) {
var NewDefinition = struct {
ModelId string `db:"model_id"`
TargetAccuracy int `db:"target_accuracy"`
}{ModelId: model_id, TargetAccuracy: target_accuracy}
id, err := InsertReturnId(db, &NewDefinition, "model_definition", "id")
if err != nil {
return
}
return GetDefinition(db, id)
}
func (d Definition) UpdateStatus(db db.Db, status DefinitionStatus) (err error) {
_, err = db.Exec("update model_definition set status=$1 where id=$2", status, d.Id)
return
}
func (d Definition) MakeLayer(db db.Db, layer_order int, layer_type LayerType, shape string) (layer Layer, err error) {
var NewLayer = struct {
DefinitionId string `db:"def_id"`
LayerOrder int `db:"layer_order"`
LayerType LayerType `db:"layer_type"`
Shape string `db:"shape"`
}{
DefinitionId: d.Id,
LayerOrder: layer_order,
LayerType: layer_type,
Shape: shape,
}
id, err := InsertReturnId(db, &NewLayer, "model_definition_layer", "id")
if err != nil {
return
}
return GetLayer(db, id)
}
func (d Definition) GetLayers(db db.Db, filter string, args ...any) (layer []*Layer, err error) {
args = append(args, d.Id)
return GetDbMultitple[Layer](db, "model_definition_layer as mdl where mdl.def_id=$1 "+filter, args...)
}
func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64, epoch int) (err error) {
d.Accuracy = accuracy
d.Epoch += epoch
_, err = db.Exec("update model_definition set epoch=$1, accuracy=$2 where id=$3", d.Epoch, d.Accuracy, d.Id)
return
}

69
logic/db_types/layer.go Normal file
View File

@ -0,0 +1,69 @@
package dbtypes
import (
"encoding/json"
"fmt"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
"github.com/charmbracelet/log"
)
type LayerType int
const (
LAYER_INPUT LayerType = 1
LAYER_DENSE = 2
LAYER_FLATTEN = 3
LAYER_SIMPLE_BLOCK = 4
)
type Layer struct {
Id string `db:"mdl.id" json:"id"`
DefinitionId string `db:"mdl.def_id" json:"definition_id"`
LayerOrder int `db:"mdl.layer_order" json:"layer_order"`
LayerType LayerType `db:"mdl.layer_type" json:"layer_type"`
Shape string `db:"mdl.shape" json:"shape"`
ExpType int `db:"mdl.exp_type" json:"exp_type"`
}
func (x *Layer) ShapeToSize() {
v := x.GetShape()
switch x.LayerType {
case LAYER_INPUT:
x.Shape = fmt.Sprintf("%d,%d", v[1], v[2])
case LAYER_DENSE:
x.Shape = fmt.Sprintf("(%d)", v[0])
default:
x.Shape = "ERROR"
}
}
func ShapeToString(args ...int) string {
text, err := json.Marshal(args)
if err != nil {
log.Error("json err!", "err", err)
panic("Could not generate Shape")
}
return string(text)
}
func StringToShape(str string) (shape []int64) {
err := json.Unmarshal([]byte(str), &shape)
if err != nil {
log.Error("json err!", "err", err)
panic("Could not parse Shape")
}
return
}
func (l Layer) GetShape() []int64 {
if l.Shape == "" {
return []int64{}
}
return StringToShape(l.Shape)
}
func GetLayer(db db.Db, layer_id string) (layer Layer, err error) {
err = GetDBOnce(db, &layer, "model_definition_layer as mdl where mdl.id=$1", layer_id)
return
}

97
logic/db_types/types.go
View File

@ -2,16 +2,18 @@ package dbtypes
import (
"errors"
"fmt"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
)
type ModelStatus int
const (
FAILED_TRAINING = -4
FAILED_TRAINING ModelStatus = -4
FAILED_PREPARING_TRAINING = -3
FAILED_PREPARING_ZIP_FILE = -2
FAILED_PREPARING = -1
PREPARING = 1
CONFIRM_PRE_TRAINING = 2
PREPARING_ZIP_FILE = 3
@ -26,15 +28,6 @@ const (
type ModelDefinitionStatus int
type LayerType int
const (
LAYER_INPUT LayerType = 1
LAYER_DENSE = 2
LAYER_FLATTEN = 3
LAYER_SIMPLE_BLOCK = 4
)
const (
MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
MODEL_DEFINITION_STATUS_FAILED_TRAINING = -3
@ -46,14 +39,6 @@ const (
MODEL_DEFINITION_STATUS_READY = 5
)
type ModelClassStatus int
const (
MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
MODEL_CLASS_STATUS_TRAINING = 2
MODEL_CLASS_STATUS_TRAINED = 3
)
type ModelHeadStatus int
const (
@ -65,17 +50,16 @@ const (
)
type BaseModel struct {
Name string
Status int
Id string
ModelType int `db:"model_type"`
ImageModeRaw string `db:"color_mode"`
ImageMode int `db:"0"`
Width int
Height int
Format string
CanTrain int `db:"can_train"`
Name string `json:"name"`
Status int `json:"status"`
Id string `json:"id"`
ModelType int `db:"model_type" json:"model_type"`
ImageModeRaw string `db:"color_mode" json:"image_more_raw"`
ImageMode int `db:"0" json:"image_mode"`
Width int `json:"width"`
Height int `json:"height"`
Format string `json:"format"`
CanTrain int `db:"can_train" json:"can_train"`
}
var ModelNotFoundError = errors.New("Model not found error")
@ -97,6 +81,59 @@ func (m BaseModel) CanEval() bool {
return true
}
// DO NOT Pass un filtered data on filters
func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
n_args := []any{m.Id}
n_args = append(n_args, args...)
return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
}
func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
n_args := []any{m.Id}
n_args = append(n_args, args...)
return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
}
func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
return
}
type DataPoint struct {
Id string `json:"id"`
Class int `json:"class"`
Path string `json:"path"`
}
func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
rows, err := db.Query(
"select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner "+
"join model_classes as mc on mc.id = mdp.class_id "+
"where mc.model_id = $1 and mdp.model_mode=$2;",
m.Id, mode)
if err != nil {
return
}
defer rows.Close()
data = []DataPoint{}
for rows.Next() {
var id string
var class_order int
var file_path string
if err = rows.Scan(&id, &class_order, &file_path); err != nil {
return
}
data = append(data, DataPoint{
Id: id,
Path: file_path,
Class: class_order,
})
}
return
}
func StringToImageMode(colorMode string) int {
switch colorMode {
case "greyscale":

17
logic/db_types/user.go
View File

@ -14,10 +14,19 @@ const (
)
type User struct {
Id string `db:"u.id"`
Username string `db:"u.username"`
Email string `db:"u.email"`
UserType int `db:"u.user_type"`
Id string `db:"u.id" json:"id"`
Username string `db:"u.username" json:"username"`
Email string `db:"u.email" json:"email"`
UserType int `db:"u.user_type" json:"user_type"`
}
func UserFromId(db db.Db, id string) (*User, error) {
var user User
err := GetDBOnce(db, &user, "users as u where u.id=$1", id)
if err != nil {
return nil, err
}
return &user, nil
}
func UserFromToken(db db.Db, token string) (*User, error) {

6
logic/models/classes/main.go
View File

@ -7,15 +7,15 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
)
type ModelClass struct {
type ModelClassJSON struct {
Id string `json:"id"`
ModelId string `json:"model_id" db:"model_id"`
Name string `json:"name"`
Status int `json:"status"`
}
func ListClasses(c BasePack, model_id string) (cls []*ModelClass, err error) {
return GetDbMultitple[ModelClass](c.GetDb(), "model_classes where model_id=$1", model_id)
func ListClasses(c BasePack, model_id string) (cls []*ModelClassJSON, err error) {
return GetDbMultitple[ModelClassJSON](c.GetDb(), "model_classes where model_id=$1", model_id)
}
func ModelHasDataPoints(db db.Db, model_id string) (result bool, err error) {

4
logic/models/data.go
View File

@ -495,7 +495,7 @@ func handleDataUpload(handle *Handle) {
return c.E500M("Could not create class", err)
}
var modelClass model_classes.ModelClass
var modelClass model_classes.ModelClassJSON
err = GetDBOnce(c, &modelClass, "model_classes where id=$1;", id)
if err != nil {
return c.E500M("Failed to get class information but class was creted", err)
@ -704,7 +704,7 @@ func handleDataUpload(handle *Handle) {
return c.Error500(err)
}
} else {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return c.Error500(err)
}

2
logic/models/delete.go
View File

@ -51,7 +51,7 @@ func handleDelete(handle *Handle) {
return c.E500M("Faield to get model", err)
}
switch model.Status {
switch ModelStatus(model.Status) {
case FAILED_TRAINING:
fallthrough
case FAILED_PREPARING_ZIP_FILE:

2
logic/models/edit.go
View File

@ -35,7 +35,7 @@ func handleEdit(handle *Handle) {
}
type ReturnType struct {
Classes []*model_classes.ModelClass `json:"classes"`
Classes []*model_classes.ModelClassJSON `json:"classes"`
HasData bool `json:"has_data"`
NumberOfInvalidImages int `json:"number_of_invalid_images"`
}

118
logic/models/train/remote_train.go Normal file
View File

@ -0,0 +1,118 @@
package models_train
import (
"errors"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
"github.com/goccy/go-json"
)
func PrepareTraining(handler *Handle, b BasePack, task Task, runner_id string) (err error) {
l := b.GetLogger()
model, err := GetBaseModel(b.GetDb(), *task.ModelId)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
l.Error("Failed to get model information", "err", err)
return err
}
if model.Status != TRAINING {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
return errors.New("Model not in the right status")
}
// TODO do this when the runner says it's OK
//task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
// TODO move this to the runner part as well
var dat struct {
NumberOfModels int
Accuracy int
}
err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
}
if model.ModelType == 2 {
full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
l.Error("Failed to generate defintions", "err", full_error)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
} else {
error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if error != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
}
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
return
}
func CleanUpFailed(b BasePack, task *Task) {
db := b.GetDb()
l := b.GetLogger()
model, err := GetBaseModel(db, *task.ModelId)
if err != nil {
l.Error("Failed to get model", "err", err)
} else {
err = model.UpdateStatus(db, FAILED_TRAINING)
if err != nil {
l.Error("Failed to get status", err)
}
}
// Set the class status to trained
err = SetModelClassStatus(b, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to set class status")
return
}
}
func CleanUpFailedRetrain(b BasePack, task *Task) {
db := b.GetDb()
l := b.GetLogger()
model, err := GetBaseModel(db, *task.ModelId)
if err != nil {
l.Error("Failed to get model", "err", err)
} else {
err = model.UpdateStatus(db, FAILED_TRAINING)
if err != nil {
l.Error("Failed to get status", err)
}
}
ResetClasses(b, model)
ModelUpdateStatus(b, model.Id, READY_RETRAIN_FAILED)
var defData struct {
Id string `db:"md.id"`
TargetAcuuracy float64 `db:"md.target_accuracy"`
}
err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
if err != nil {
log.Error("failed to get def data", err)
return
}
_, err_ := db.Exec("delete from exp_model_head where def_id=$1 and status in (2,3)", defData.Id)
if err_ != nil {
panic(err_)
}
}

2
logic/models/train/reset.go
View File

@ -17,7 +17,7 @@ func handleRest(handle *Handle) {
return c.E500M("Failed to get model", err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
if model.Status != FAILED_PREPARING_TRAINING && model.Status != int(FAILED_TRAINING) {
return c.JsonBadRequest("Model is not in status that be reset")
}

451
logic/models/train/train.go
View File

@ -39,16 +39,6 @@ func getDir() string {
return dir
}
// This function creates a new model_definition
func MakeDefenition(db db.Db, model_id string, target_accuracy int) (id string, err error) {
var NewDefinition = struct {
ModelId string `db:"model_id"`
TargetAccuracy int `db:"target_accuracy"`
}{ModelId: model_id, TargetAccuracy: target_accuracy}
return InsertReturnId(db, &NewDefinition, "model_definition", "id")
}
func ModelDefinitionUpdateStatus(c BasePack, id string, status ModelDefinitionStatus) (err error) {
_, err = c.GetDb().Exec("update model_definition set status = $1 where id = $2", status, id)
return
@ -111,6 +101,10 @@ func setModelClassStatus(c BasePack, status ModelClassStatus, filter string, arg
return
}
func SetModelClassStatus(c BasePack, status ModelClassStatus, filter string, args ...any) (err error) {
return setModelClassStatus(c, status, filter, args...)
}
func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool) (count int, err error) {
db := c.GetDb()
@ -118,14 +112,14 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
count = co.Count
if count == 0 {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
}
@ -137,7 +131,7 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
return generateCvsExp(c, run_path, model_id, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@ -167,40 +161,23 @@ func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool)
return
}
func trainDefinition(c BasePack, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
func trainDefinition(c BasePack, model *BaseModel, def Definition, load_prev bool) (accuracy float64, err error) {
l := c.GetLogger()
db := c.GetDb()
l.Warn("About to start training definition")
accuracy = 0
layers, err := db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
layers, err := def.GetLayers(c.GetDb(), " order by layer_order asc;")
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
LayerNum int
}
got := []layerrow{}
i := 1
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape); err != nil {
return
}
row.Shape = shapeToSize(row.Shape)
row.LayerNum = 1
got = append(got, row)
i = i + 1
for _, layer := range layers {
layer.ShapeToSize()
}
// Generate run folder
run_path := path.Join("/tmp", model.Id, "defs", definition_id)
run_path := path.Join("/tmp", model.Id, "defs", def.Id)
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
@ -225,17 +202,17 @@ func trainDefinition(c BasePack, model *BaseModel, definition_id string, load_pr
}
// Copy result around
result_path := path.Join("savedData", model.Id, "defs", definition_id)
result_path := path.Join("savedData", model.Id, "defs", def.Id)
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"Layers": layers,
"Size": layers[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"RunPath": run_path,
"ColorMode": model.ImageMode,
"Model": model,
"EPOCH_PER_RUN": EPOCH_PER_RUN,
"DefId": definition_id,
"DefId": def.Id,
"LoadPrev": load_prev,
"LastModelRunPath": path.Join(getDir(), result_path, "model.keras"),
"SaveModelPath": path.Join(getDir(), result_path),
@ -287,7 +264,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING)
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@ -296,7 +273,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
count := co.Count
if count == 0 {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN)
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
} else if doPanic {
@ -305,7 +282,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
return generateCvsExpandExp(c, run_path, model_id, offset, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
@ -339,7 +316,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
// This is to load some extra data so that the model has more things to train on
//
data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINED, count*10)
if err != nil {
return
}
@ -362,7 +339,7 @@ func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset i
return
}
func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
func trainDefinitionExpandExp(c BasePack, model *BaseModel, def Definition, load_prev bool) (accuracy float64, err error) {
accuracy = 0
l := c.GetLogger()
@ -377,7 +354,7 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
}
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](c.GetDb(), "exp_model_head where def_id=$1 and (status = 2 or status = 3)", definition_id)
heads, err := GetDbMultitple[ExpHead](c.GetDb(), "exp_model_head where def_id=$1 and (status = 2 or status = 3)", def.Id)
if err != nil {
return
} else if len(heads) == 0 {
@ -396,62 +373,49 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
return
}
layers, err := c.GetDb().Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
layers, err := def.GetLayers(c.GetDb(), " order by layer_order asc;")
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
ExpType int
LayerNum int
}
got := []layerrow{}
var got []*Layer
i := 1
var last *layerrow = nil
var last *Layer = nil
got_2 := false
var first *layerrow = nil
var first *Layer = nil
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape, &row.ExpType); err != nil {
return
}
for _, layer := range layers {
layer.ShapeToSize()
// Keep track of the first layer so we can keep the size of the image
if first == nil {
first = &row
first = layer
}
row.LayerNum = i
row.Shape = shapeToSize(row.Shape)
if row.ExpType == 2 {
if layer.ExpType == 2 {
if !got_2 {
got = append(got, *last)
got = append(got, last)
got_2 = true
}
got = append(got, row)
got = append(got, layer)
}
last = &row
last = layer
i += 1
}
got = append(got, layerrow{
got = append(got, &Layer{
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d", exp.End-exp.Start+1),
ExpType: 2,
LayerNum: i,
LayerOrder: len(got),
})
l.Info("Got layers", "layers", got)
// Generate run folder
run_path := path.Join("/tmp", model.Id+"-defs-"+definition_id+"-retrain")
run_path := path.Join("/tmp", model.Id+"-defs-"+def.Id+"-retrain")
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
@ -482,7 +446,7 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
}
// Copy result around
result_path := path.Join("savedData", model.Id, "defs", definition_id)
result_path := path.Join("savedData", model.Id, "defs", def.Id)
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
@ -538,7 +502,7 @@ func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string
return
}
func trainDefinitionExp(c BasePack, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
func trainDefinitionExp(c BasePack, model *BaseModel, def Definition, load_prev bool) (accuracy float64, err error) {
accuracy = 0
l := c.GetLogger()
db := c.GetDb()
@ -554,7 +518,7 @@ func trainDefinitionExp(c BasePack, model *BaseModel, definition_id string, load
}
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](db, "exp_model_head where def_id=$1 and (status = 2 or status = 3)", definition_id)
heads, err := GetDbMultitple[ExpHead](db, "exp_model_head where def_id=$1 and (status = 2 or status = 3)", def.Id)
if err != nil {
return
} else if len(heads) == 0 {
@ -572,42 +536,24 @@ func trainDefinitionExp(c BasePack, model *BaseModel, definition_id string, load
return
}
layers, err := db.Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
layers, err := def.GetLayers(db, " order by layer_order asc;")
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
ExpType int
LayerNum int
for _, layer := range layers {
layer.ShapeToSize()
}
got := []layerrow{}
i := 1
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape, &row.ExpType); err != nil {
return
}
row.LayerNum = i
row.Shape = shapeToSize(row.Shape)
got = append(got, row)
i += 1
}
got = append(got, layerrow{
layers = append(layers, &Layer{
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d", exp.End-exp.Start+1),
ExpType: 2,
LayerNum: i,
LayerOrder: len(layers),
})
// Generate run folder
run_path := path.Join("/tmp", model.Id+"-defs-"+definition_id)
run_path := path.Join("/tmp", model.Id+"-defs-"+def.Id)
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
@ -634,11 +580,11 @@ func trainDefinitionExp(c BasePack, model *BaseModel, definition_id string, load
}
// Copy result around
result_path := path.Join("savedData", model.Id, "defs", definition_id)
result_path := path.Join("savedData", model.Id, "defs", def.Id)
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"Layers": layers,
"Size": layers[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"HeadId": exp.Id,
"RunPath": run_path,
@ -706,21 +652,6 @@ func remove[T interface{}](lst []T, i int) []T {
return append(lst[:i], lst[i+1:]...)
}
type TrainModelRow struct {
id string
target_accuracy int
epoch int
acuracy float64
}
type TraingModelRowDefinitions []TrainModelRow
func (nf TraingModelRowDefinitions) Len() int { return len(nf) }
func (nf TraingModelRowDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf TraingModelRowDefinitions) Less(i, j int) bool {
return nf[i].acuracy < nf[j].acuracy
}
type ToRemoveList []int
func (nf ToRemoveList) Len() int { return len(nf) }
@ -733,32 +664,18 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
db := c.GetDb()
l := c.GetLogger()
definitionsRows, err := db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
defs_, err := model.GetDefinitions(db, "and md.status=$2", MODEL_DEFINITION_STATUS_INIT)
if err != nil {
l.Error("Failed to train Model! Err:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
l.Error("Failed to train Model!", "err", err)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer definitionsRows.Close()
var definitions TraingModelRowDefinitions = []TrainModelRow{}
var defs SortByAccuracyDefinitions = defs_
for definitionsRows.Next() {
var rowv TrainModelRow
rowv.acuracy = 0
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
l.Error("Failed to train Model Could not read definition from db!Err:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
definitions = append(definitions, rowv)
}
if len(definitions) == 0 {
if len(defs) == 0 {
l.Error("No Definitions defined!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@ -767,35 +684,32 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
for {
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinition(c, model, def.id, !firstRound)
for i, def := range defs {
ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinition(c, model, *def, !firstRound)
if err != nil {
l.Error("Failed to train definition!Err:", "err", err)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
def.epoch += EPOCH_PER_RUN
def.Epoch += EPOCH_PER_RUN
accuracy = accuracy * 100
def.acuracy = float64(accuracy)
def.Accuracy = float64(accuracy)
definitions[i].epoch += EPOCH_PER_RUN
definitions[i].acuracy = accuracy
if accuracy >= float64(def.target_accuracy) {
if accuracy >= float64(def.TargetAccuracy) {
l.Info("Found a definition that reaches target_accuracy!")
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.Epoch, def.Id)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.Id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
@ -803,17 +717,17 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
break
}
if def.epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.target_accuracy)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if def.Epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.TargetAccuracy)
ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.Epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.Id)
if err != nil {
l.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return err
}
}
@ -828,28 +742,26 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
l.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
definitions = remove(definitions, n)
defs = remove(defs, n)
}
len_def := len(definitions)
len_def := len(defs)
if len_def == 0 {
break
}
if len_def == 1 {
} else if len_def == 1 {
continue
}
sort.Sort(sort.Reverse(definitions))
sort.Sort(sort.Reverse(defs))
acc := definitions[0].acuracy - 20.0
acc := defs[0].Accuracy - 20.0
l.Info("Training models, Highest acc", "acc", definitions[0].acuracy, "mod_acc", acc)
l.Info("Training models, Highest acc", "acc", defs[0].Accuracy, "mod_acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.acuracy < acc {
for i, def := range defs {
if def.Accuracy < acc {
toRemove = append(toRemove, i)
}
}
@ -859,8 +771,8 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
sort.Sort(sort.Reverse(toRemove))
for _, n := range toRemove {
l.Warn("Removing definition not fast enough learning", "n", n)
ModelDefinitionUpdateStatus(c, definitions[n].id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
definitions = remove(definitions, n)
ModelDefinitionUpdateStatus(c, defs[n].Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
defs = remove(defs, n)
}
}
@ -868,7 +780,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err != nil {
l.Error("DB: failed to read definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer rows.Close()
@ -876,7 +788,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if !rows.Next() {
// TODO Make the Model status have a message
l.Error("All definitions failed to train!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@ -884,14 +796,14 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err = rows.Scan(&id); err != nil {
l.Error("Failed to read id:")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
if _, err = db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
l.Error("Failed to update model definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@ -899,7 +811,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err != nil {
l.Error("Failed to select model_definition to delete")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
defer to_delete.Close()
@ -909,7 +821,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if err = to_delete.Scan(&id); err != nil {
l.Error("Failed to scan the id of a model_definition to delete")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@ -919,7 +831,7 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
l.Error("Failed to delete model_definition")
l.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
return
}
@ -928,33 +840,18 @@ func trainModel(c BasePack, model *BaseModel) (err error) {
return
}
type TrainModelRowUsable struct {
Id string
TargetAccuracy int `db:"target_accuracy"`
Epoch int
Acuracy float64 `db:"0"`
}
type TrainModelRowUsables []*TrainModelRowUsable
func (nf TrainModelRowUsables) Len() int { return len(nf) }
func (nf TrainModelRowUsables) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf TrainModelRowUsables) Less(i, j int) bool {
return nf[i].Acuracy < nf[j].Acuracy
}
func trainModelExp(c BasePack, model *BaseModel) (err error) {
l := c.GetLogger()
db := c.GetDb()
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](db, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
defs_, err := model.GetDefinitions(db, " and status=$2;", MODEL_DEFINITION_STATUS_INIT)
if err != nil {
l.Error("Failed to get definitions")
return
}
if len(definitions) == 0 {
var defs SortByAccuracyDefinitions = defs_
if len(defs) == 0 {
l.Error("No Definitions defined!")
return errors.New("No Definitions found")
}
@ -964,9 +861,9 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
for {
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
for i, def := range defs {
ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinitionExp(c, model, def.Id, !firstRound)
accuracy, err := trainDefinitionExp(c, model, *def, !firstRound)
if err != nil {
l.Error("Failed to train definition!Err:", "err", err)
ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
@ -975,10 +872,10 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
}
def.Epoch += EPOCH_PER_RUN
accuracy = accuracy * 100
def.Acuracy = float64(accuracy)
def.Accuracy = float64(accuracy)
definitions[i].Epoch += EPOCH_PER_RUN
definitions[i].Acuracy = accuracy
defs[i].Epoch += EPOCH_PER_RUN
defs[i].Accuracy = accuracy
if accuracy >= float64(def.TargetAccuracy) {
l.Info("Found a definition that reaches target_accuracy!")
@ -1028,10 +925,10 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
l.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
definitions = remove(definitions, n)
defs = remove(defs, n)
}
len_def := len(definitions)
len_def := len(defs)
if len_def == 0 {
break
@ -1039,14 +936,14 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
continue
}
sort.Sort(sort.Reverse(definitions))
acc := definitions[0].Acuracy - 20.0
sort.Sort(sort.Reverse(defs))
acc := defs[0].Accuracy - 20.0
l.Info("Training models, Highest acc", "acc", definitions[0].Acuracy, "mod_acc", acc)
l.Info("Training models, Highest acc", "acc", defs[0].Accuracy, "mod_acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.Acuracy < acc {
for i, def := range defs {
if def.Accuracy < acc {
toRemove = append(toRemove, i)
}
}
@ -1056,8 +953,8 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
sort.Sort(sort.Reverse(toRemove))
for _, n := range toRemove {
l.Warn("Removing definition not fast enough learning", "n", n)
ModelDefinitionUpdateStatus(c, definitions[n].Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
definitions = remove(definitions, n)
ModelDefinitionUpdateStatus(c, defs[n].Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
defs = remove(defs, n)
}
}
@ -1066,12 +963,18 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
err = GetDBOnce(db, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("All definitions failed to train! And Failed to set class status")
return err
}
err = model.UpdateStatus(db, FAILED_TRAINING)
if err != nil {
l.Error("All definitions failed to train! And Failed to set model status")
return err
}
l.Error("All definitions failed to train!")
return err
} else if err != nil {
@ -1100,8 +1003,8 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
return err
}
if err = splitModel(c, model); err != nil {
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
if err = SplitModel(c, model); err != nil {
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to split the model! And Failed to set class status")
return err
@ -1112,7 +1015,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
}
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to set class status")
return err
@ -1133,7 +1036,7 @@ func trainModelExp(c BasePack, model *BaseModel) (err error) {
return
}
func splitModel(c BasePack, model *BaseModel) (err error) {
func SplitModel(c BasePack, model *BaseModel) (err error) {
db := c.GetDb()
l := c.GetLogger()
@ -1270,9 +1173,8 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
}
db := c.GetDb()
l := c.GetLogger()
def_id, err := MakeDefenition(db, model.Id, target_accuracy)
def, err := MakeDefenition(db, model.Id, target_accuracy)
if err != nil {
failed()
return
@ -1281,29 +1183,17 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
order := 1
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height))
//_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(model.Width, model.Height, model.ImageMode))
_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height))
if err != nil {
failed()
return
}
order++
} else {
err = MakeLayer(db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
if err != nil {
failed()
return
}
order++
}
loop := max(int((math.Log(float64(model.Width)) / math.Log(float64(10)))), 1)
loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
for i := 0; i < loop; i++ {
err = MakeLayer(db, def_id, order, LAYER_SIMPLE_BLOCK, "")
_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
failed()
@ -1311,16 +1201,19 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
}
}
err = MakeLayer(db, def_id, order, LAYER_FLATTEN, "")
_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
if err != nil {
failed()
return
}
order++
loop = max(int((math.Log(float64(number_of_classes))/math.Log(float64(10)))/2), 1)
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
order++
if err != nil {
failed()
@ -1328,13 +1221,7 @@ func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, numbe
}
}
err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
if err != nil {
failed()
return
}
return nil
return def.UpdateStatus(db, DEFINITION_STATUS_INIT)
}
func generateDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
@ -1393,27 +1280,22 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
return
}
def_id, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
def, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
if err != nil {
failed()
return
}
def_id := def.Id
order := 1
width := model.Width
height := model.Height
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
err = MakeLayerExpandable(c.GetDb(), def_id, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height), 1)
if err != nil {
failed()
return
}
err = MakeLayerExpandable(c.GetDb(), def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1)
order++
// handle the errors inside the pervious if block
@ -1431,7 +1313,7 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
log.Info("Size of the simple block", "loop", loop)
//loop = max(loop, 3)
loop = max(loop, min(2, model.ImageMode))
for i := 0; i < loop; i++ {
err = MakeLayerExpandable(db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1)
@ -1451,7 +1333,7 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
order++
// Flatten the blocks into dense
err = MakeLayerExpandable(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*2), 1)
err = MakeLayerExpandable(db, def_id, order, LAYER_DENSE, ShapeToString(number_of_classes*2), 1)
if err != nil {
failed()
return
@ -1465,7 +1347,7 @@ func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy
loop = max(loop, 3)
for i := 0; i < loop; i++ {
err = MakeLayerExpandable(db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)*2), 2)
err = MakeLayerExpandable(db, def_id, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)*2), 2)
order++
if err != nil {
failed()
@ -1533,7 +1415,7 @@ func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy
}
func ResetClasses(c BasePack, model *BaseModel) {
_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TO_TRAIN, MODEL_CLASS_STATUS_TRAINING, model.Id)
_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
c.GetLogger().Error("Error while reseting the classes", "error", err)
}
@ -1544,35 +1426,35 @@ func trainExpandable(c *Context, model *BaseModel) {
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(c, model.Id, int(FAILED_TRAINING))
ResetClasses(c, model)
}
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
defs_, err := model.GetDefinitions(c, " and status=$2", MODEL_DEFINITION_STATUS_READY)
if err != nil {
failed("Failed to get definitions")
return
}
if len(definitions) != 1 {
var defs SortByAccuracyDefinitions = defs_
if len(defs) != 1 {
failed("There should only be one definition available!")
return
}
firstRound := true
def := definitions[0]
def := defs[0]
epoch := 0
for {
acc, err := trainDefinitionExp(c, model, def.Id, !firstRound)
acc, err := trainDefinitionExp(c, model, *def, !firstRound)
if err != nil {
failed("Failed to train definition!")
return
}
epoch += EPOCH_PER_RUN
if float64(acc*100) >= float64(def.Acuracy) {
if float64(acc*100) >= float64(def.Accuracy) {
c.Logger.Info("Found a definition that reaches target_accuracy!")
_, err = c.Db.Exec("update exp_model_head set status=$1 where def_id=$2 and status=$3;", MODEL_HEAD_STATUS_READY, def.Id, MODEL_HEAD_STATUS_TRAINING)
@ -1588,7 +1470,7 @@ func trainExpandable(c *Context, model *BaseModel) {
}
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
@ -1648,7 +1530,7 @@ func RunTaskTrain(b BasePack, task Task) (err error) {
if err != nil {
l.Error("Failed to train model", "err", err)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
ModelUpdateStatus(b, model.Id, int(FAILED_TRAINING))
return
}
@ -1677,22 +1559,18 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
task.UpdateStatusLog(b, TASK_RUNNING, "Model retraining")
var defData struct {
Id string `db:"md.id"`
TargetAcuuracy float64 `db:"md.target_accuracy"`
}
err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
defs, err := model.GetDefinitions(db, "")
if err != nil {
failed()
return
}
def := *defs[0]
failed = func() {
ResetClasses(b, model)
ModelUpdateStatus(b, model.Id, READY_RETRAIN_FAILED)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model failed retraining")
_, err_ := db.Exec("delete from exp_model_head where def_id=$1 and status in (2,3)", defData.Id)
_, err_ := db.Exec("delete from exp_model_head where def_id=$1 and status in (2,3)", def.Id)
if err_ != nil {
panic(err_)
}
@ -1703,21 +1581,21 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
var epocs = 0
// TODO make max epochs come from db
// TODO re increase the target accuracy
for acc*100 < defData.TargetAcuuracy-5 && epocs < 10 {
for acc*100 < float64(def.TargetAccuracy)-5 && epocs < 10 {
// This is something I have to check
acc, err = trainDefinitionExpandExp(b, model, defData.Id, epocs > 0)
acc, err = trainDefinitionExpandExp(b, model, def, epocs > 0)
if err != nil {
failed()
return
}
l.Info("Retrained model", "accuracy", acc, "target", defData.TargetAcuuracy)
l.Info("Retrained model", "accuracy", acc, "target", def.TargetAccuracy)
epocs += 1
}
if acc*100 < defData.TargetAcuuracy {
l.Error("Model never achived targetd accuracy", "acc", acc*100, "target", defData.TargetAcuuracy)
if acc*100 < float64(def.TargetAccuracy)-5 {
l.Error("Model never achived targetd accuracy", "acc", acc*100, "target", def.TargetAccuracy)
failed()
return
}
@ -1731,7 +1609,14 @@ func RunTaskRetrain(b BasePack, task Task) (err error) {
l.Info("Model updaded")
_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINED, MODEL_CLASS_STATUS_TRAINING, model.Id)
_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINED, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
l.Error("Error while updating the classes", "error", err)
failed()
return
}
_, err = db.Exec("update exp_model_head set status=$1 where status=$2 and def_id=$3", MODEL_HEAD_STATUS_READY, MODEL_HEAD_STATUS_TRAINING, def.Id)
if err != nil {
l.Error("Error while updating the classes", "error", err)
failed()
@ -1861,7 +1746,7 @@ func handleTrain(handle *Handle) {
c,
"model_classes where model_id=$1 and status=$2 order by class_order asc",
model.Id,
MODEL_CLASS_STATUS_TO_TRAIN,
CLASS_STATUS_TO_TRAIN,
)
if err != nil {
_err := c.RollbackTx()
@ -1882,7 +1767,7 @@ func handleTrain(handle *Handle) {
//Update the classes
{
_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", MODEL_CLASS_STATUS_TRAINING, MODEL_CLASS_STATUS_TO_TRAIN, model.Id)
_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINING, CLASS_STATUS_TO_TRAIN, model.Id)
if err != nil {
_err := c.RollbackTx()
if _err != nil {

2
logic/tasks/index.go
View File

@ -8,4 +8,6 @@ func HandleTasks(handle *Handle) {
handleUpload(handle)
handleList(handle)
handleRequests(handle)
handleRemoteRunner(handle)
handleRunnerData(handle)
}

960
logic/tasks/runner.go Normal file
View File

@ -0,0 +1,960 @@
package tasks
import (
"os"
"path"
"sync"
"time"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
)
func verifyRunner(c *Context, dat *JustId) (runner *Runner, e *Error) {
runner, err := GetRunner(c, dat.Id)
if err == NotFoundError {
e = c.JsonBadRequest("Could not find runner, please register runner first")
return
} else if err != nil {
e = c.E500M("Failed to get information about the runner", err)
return
}
if runner.Token != *c.Token {
return nil, c.SendJSONStatus(401, "Only runners can use this funcion")
}
return
}
type VerifyTask struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
}
type RunnerTrainDef struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
}
func verifyTask(x *Handle, c *Context, dat *VerifyTask) (task *Task, error *Error) {
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] == nil {
return nil, c.JsonBadRequest("Runner not active")
}
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
if runner_data["task"] == nil {
return nil, c.SendJSONStatus(404, "No active task")
}
return runner_data["task"].(*Task), nil
}
func clearRunnerTask(x *Handle, runner_id string) {
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[runner_id].(map[string]interface{})
runner_data["task"] = nil
runners[runner_id] = runner_data
x.DataMap["runners"] = runners
}
func handleRemoteRunner(x *Handle) {
type RegisterRunner struct {
Token string `json:"token" validate:"required"`
Type RunnerType `json:"type" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/register", User_Normal, func(c *Context, dat *RegisterRunner) *Error {
if *c.Token != dat.Token {
// TODO do admin
return c.E500M("Please make sure that the token is the same that is being registered", nil)
}
c.Logger.Info("test", "dat", dat)
var runner Runner
err := GetDBOnce(c, &runner, "remote_runner as ru where token=$1", dat.Token)
if err != NotFoundError && err != nil {
return c.E500M("Failed to get information remote runners", err)
}
if err != NotFoundError {
return c.JsonBadRequest("Token is already registered by a runner")
}
// TODO get id from token passed by when doing admin
var userId = c.User.Id
var new_runner = struct {
Type RunnerType
UserId string `db:"user_id"`
Token string
}{
Type: dat.Type,
Token: dat.Token,
UserId: userId,
}
id, err := InsertReturnId(c, &new_runner, "remote_runner", "id")
if err != nil {
return c.E500M("Failed to create remote runner", err)
}
return c.SendJSON(struct {
Id string `json:"id"`
}{
Id: id,
})
})
// TODO remove runner
PostAuthJson(x, "/tasks/runner/init", User_Normal, func(c *Context, dat *JustId) *Error {
runner, error := verifyRunner(c, dat)
if error != nil {
return error
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] != nil {
c.Logger.Info("Logger trying to register but already registerd")
c.ShowMessage = false
return c.SendJSON("Ok")
}
var new_runner = map[string]interface{}{}
new_runner["last_time_check"] = time.Now()
new_runner["runner_info"] = runner
runners[dat.Id] = new_runner
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/active", User_Normal, func(c *Context, dat *JustId) *Error {
_, error := verifyRunner(c, dat)
if error != nil {
return error
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
if runners[dat.Id] == nil {
return c.JsonBadRequest("Runner not active")
}
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
if runner_data["task"] == nil {
c.ShowMessage = false
return c.SendJSONStatus(404, "No active task")
}
c.ShowMessage = false
// This should be a task obj
return c.SendJSON(runner_data["task"])
})
PostAuthJson(x, "/tasks/runner/ready", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
err := task.UpdateStatus(c, TASK_RUNNING, "Task Running on Runner")
if err != nil {
return c.E500M("Failed to set task status", err)
}
return c.SendJSON("Ok")
})
type TaskFail struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
Reason string `json:"reason" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/fail", User_Normal, func(c *Context, dat *TaskFail) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
err := task.UpdateStatus(c, TASK_FAILED_RUNNING, dat.Reason)
if err != nil {
return c.E500M("Failed to set task status", err)
}
// Do extra clean up on tasks
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
CleanUpFailed(c, task)
case int(TASK_TYPE_RETRAINING):
CleanUpFailedRetrain(c, task)
case int(TASK_TYPE_CLASSIFICATION):
// DO nothing
default:
panic("Do not know how to handle this")
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/defs", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
var status DefinitionStatus
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
status = DEFINITION_STATUS_INIT
case int(TASK_TYPE_RETRAINING):
fallthrough
case int(TASK_TYPE_CLASSIFICATION):
status = DEFINITION_STATUS_READY
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
defs, err := model.GetDefinitions(c, "and md.status=$2", status)
if err != nil {
return c.E500M("Failed to get the model definitions", err)
}
return c.SendJSON(defs)
})
PostAuthJson(x, "/tasks/runner/classes", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
classes, err := model.GetClasses(c, "and status in ($2, $3) order by mc.class_order asc", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
case int(TASK_TYPE_RETRAINING):
classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
case int(TASK_TYPE_CLASSIFICATION):
classes, err := model.GetClasses(c, "and status=$2 order by mc.class_order asc", CLASS_STATUS_TRAINED)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(classes)
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
})
type RunnerTrainDefStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Status DefinitionStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/def/status", User_Normal, func(c *Context, dat *RunnerTrainDefStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
err = def.UpdateStatus(c, dat.Status)
if err != nil {
return c.E500M("Failed to update model status", err)
}
return c.SendJSON("Ok")
})
type RunnerTrainDefHeadStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Status ModelHeadStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/def/head/status", User_Normal, func(c *Context, dat *RunnerTrainDefHeadStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
_, err = c.Exec("update exp_model_head set status=$1 where def_id=$2;", dat.Status, def.Id)
if err != nil {
log.Error("Failed to train definition!")
return c.E500M("Failed to train definition", err)
}
return c.SendJSON("Ok")
})
type RunnerRetrainDefHeadStatus struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
HeadId string `json:"defId" validate:"required"`
Status ModelHeadStatus `json:"status" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/retrain/def/head/status", User_Normal, func(c *Context, dat *RunnerRetrainDefHeadStatus) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_RETRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
if err := UpdateStatus(c.GetDb(), "exp_model_head", dat.HeadId, MODEL_DEFINITION_STATUS_TRAINING); err != nil {
return c.E500M("Failed to update head status", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/def/layers", User_Normal, func(c *Context, dat *RunnerTrainDef) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
// Do nothing
case int(TASK_TYPE_RETRAINING):
// Do nothing
default:
c.Logger.Error("Task not is not the right type to get the layers", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the layers")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
layers, err := def.GetLayers(c, " order by layer_order asc")
if err != nil {
return c.E500M("Failed to get layers", err)
}
return c.SendJSON(layers)
})
PostAuthJson(x, "/tasks/runner/train/datapoints", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
// DO nothing
case int(TASK_TYPE_RETRAINING):
// DO nothing
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
training_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
testing_points, err := model.DataPoints(c, DATA_POINT_MODE_TRAINING)
if err != nil {
return c.E500M("Failed to get the model classes", err)
}
return c.SendJSON(struct {
Testing []DataPoint `json:"testing"`
Training []DataPoint `json:"training"`
}{
Testing: testing_points,
Training: training_points,
})
})
type RunnerTrainDefEpoch struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
DefId string `json:"defId" validate:"required"`
Epoch int `json:"epoch" validate:"required"`
Accuracy float64 `json:"accuracy" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/train/epoch", User_Normal, func(c *Context, dat *RunnerTrainDefEpoch) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
def, err := GetDefinition(c, dat.DefId)
if err != nil {
return c.E500M("Failed to get definition information", err)
}
err = def.UpdateAfterEpoch(c, dat.Accuracy, dat.Epoch)
if err != nil {
return c.E500M("Failed to update model", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/mark-failed", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
_, err := c.Exec(
"update model_definition set status=$1 "+
"where model_id=$2 and status in ($3, $4)",
MODEL_DEFINITION_STATUS_CANCELD_TRAINING,
task.ModelId,
MODEL_DEFINITION_STATUS_TRAINING,
MODEL_DEFINITION_STATUS_PAUSED_TRAINING,
)
if err != nil {
return c.E500M("Failed to mark definition as failed", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/model", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
//DO NOTHING
case int(TASK_TYPE_RETRAINING):
//DO NOTHING
case int(TASK_TYPE_CLASSIFICATION):
//DO NOTHING
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model information", err)
}
return c.SendJSON(model)
})
PostAuthJson(x, "/tasks/runner/heads", User_Normal, func(c *Context, dat *RunnerTrainDef) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{Id: dat.Id, TaskId: dat.TaskId})
if error != nil {
return error
}
type ExpHead struct {
Id string `json:"id"`
Start int `db:"range_start" json:"start"`
End int `db:"range_end" json:"end"`
}
switch task.TaskType {
case int(TASK_TYPE_TRAINING):
fallthrough
case int(TASK_TYPE_RETRAINING):
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](c, "exp_model_head where def_id=$1 and status in (2,3)", dat.DefId)
if err != nil {
return c.E500M("Failed getting active heads", err)
}
return c.SendJSON(heads)
case int(TASK_TYPE_CLASSIFICATION):
heads, err := GetDbMultitple[ExpHead](c, "exp_model_head where def_id=$1", dat.DefId)
if err != nil {
return c.E500M("Failed getting active heads", err)
}
return c.SendJSON(heads)
default:
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
})
PostAuthJson(x, "/tasks/runner/train_exp/class/status/train", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
return c.E500M("Failed to get model", err)
}
err = SetModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return c.E500M("Failed update status", err)
}
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
var def Definition
err = GetDBOnce(c, &def, "model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
return c.SendJSON("Ok")
} else if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
return c.E500M("Failed to get model definition", err)
}
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
return c.E500M("Failed to update model definition", err)
}
to_delete, err := c.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if err = to_delete.Scan(&id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
// Set the class status to trained
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1;", model.Id)
if err != nil {
c.Logger.Error("Failed to set class status")
return c.E500M("Failed to set class status", err)
}
if err = model.UpdateStatus(c, READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to update status of model", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
})
type RunnerClassDone struct {
Id string `json:"id" validate:"required"`
TaskId string `json:"taskId" validate:"required"`
Result string `json:"result" validate:"required"`
}
PostAuthJson(x, "/tasks/runner/class/done", User_Normal, func(c *Context, dat *RunnerClassDone) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, &VerifyTask{
Id: dat.Id,
TaskId: dat.TaskId,
})
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_CLASSIFICATION) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
err := task.SetResultText(c, dat.Result)
if err != nil {
return c.E500M("Failed to update the task", err)
}
err = task.UpdateStatus(c, TASK_DONE, "Task completed")
if err != nil {
return c.E500M("Failed to update task", err)
}
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/train_exp/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_TRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
// TODO add check the to the model
var def Definition
err = GetDBOnce(c, &def, "model_definition as md where model_id=$1 and status=$2 order by accuracy desc limit 1;", task.ModelId, DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "All definition failed to train!")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
} else if err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to get model definition")
return c.E500M("Failed to get model definition", err)
}
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to update model definition")
return c.E500M("Failed to update model definition", err)
}
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.GetLogger().Error("Failed to select model_definition to delete")
return c.E500M("Failed to select model definition to delete", err)
}
for _, d := range to_delete {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
// TODO Check if returning also works here
if _, err = c.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to delete unsed definitions", err)
}
if err = SplitModel(c, model); err != nil {
err = SetModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
c.Logger.Error("Failed to split the model! And Failed to set class status")
return c.E500M("Failed to split the model", err)
}
c.Logger.Error("Failed to split the model")
return c.E500M("Failed to split the model", err)
}
// Set the class status to trained
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
c.Logger.Error("Failed to set class status")
return c.E500M("Failed to set class status", err)
}
c.Logger.Warn("Removing base model for", "model", model.Id, "def", def.Id)
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model"))
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model.keras"))
if err = model.UpdateStatus(c, READY); err != nil {
model.UpdateStatus(c, FAILED_TRAINING)
task.UpdateStatusLog(c, TASK_FAILED_RUNNING, "Failed to delete unsed definitions")
return c.E500M("Failed to update status of model", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
mutex := x.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
var runners map[string]interface{} = x.DataMap["runners"].(map[string]interface{})
var runner_data map[string]interface{} = runners[dat.Id].(map[string]interface{})
runner_data["task"] = nil
runners[dat.Id] = runner_data
x.DataMap["runners"] = runners
return c.SendJSON("Ok")
})
PostAuthJson(x, "/tasks/runner/retrain/done", User_Normal, func(c *Context, dat *VerifyTask) *Error {
_, error := verifyRunner(c, &JustId{Id: dat.Id})
if error != nil {
return error
}
task, error := verifyTask(x, c, dat)
if error != nil {
return error
}
if task.TaskType != int(TASK_TYPE_RETRAINING) {
c.Logger.Error("Task not is not the right type to get the definitions", "task type", task.TaskType)
return c.JsonBadRequest("Task is not the right type go get the definitions")
}
model, err := GetBaseModel(c, *task.ModelId)
if err != nil {
c.Logger.Error("Failed to get model", "err", err)
return c.E500M("Failed to get mode", err)
}
err = SetModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
return c.E500M("Failed to set class status", err)
}
defs, err := model.GetDefinitions(c, "")
if err != nil {
return c.E500M("Failed to get definitions", err)
}
_, err = c.Exec("update exp_model_head set status=$1 where status=$2 and def_id=$3", MODEL_HEAD_STATUS_READY, MODEL_HEAD_STATUS_TRAINING, defs[0].Id)
if err != nil {
return c.E500M("Failed to set head status", err)
}
err = model.UpdateStatus(c, READY)
if err != nil {
return c.E500M("Failed to set class status", err)
}
task.UpdateStatusLog(c, TASK_DONE, "Model finished training")
clearRunnerTask(x, dat.Id)
return c.SendJSON("Ok")
})
}

104
logic/tasks/runner/runner.go
View File

@ -5,6 +5,7 @@ import (
"math"
"os"
"runtime/debug"
"sync"
"time"
"github.com/charmbracelet/log"
@ -90,6 +91,57 @@ func runner(config Config, db db.Db, task_channel chan Task, index int, back_cha
}
}
/**
* Handle remote runner
*/
func handleRemoteTask(handler *Handle, base BasePack, runner_id string, task Task) {
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
TimeFormat: time.Kitchen,
Prefix: fmt.Sprintf("Runner pre %s", runner_id),
})
defer func() {
if r := recover(); r != nil {
logger.Error("Runner failed to setup for runner", "due to", r, "stack", string(debug.Stack()))
// TODO maybe create better failed task
task.UpdateStatusLog(base, TASK_FAILED_RUNNING, "Failed to setup task for runner")
}
}()
err := task.UpdateStatus(base, TASK_PICKED_UP, "Failed to setup task for runner")
if err != nil {
logger.Error("Failed to mark task as PICK UP")
return
}
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
defer mutex.Unlock()
switch task.TaskType {
case int(TASK_TYPE_RETRAINING):
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
case int(TASK_TYPE_TRAINING):
if err := PrepareTraining(handler, base, task, runner_id); err != nil {
logger.Error("Failed to prepare for training", "err", err)
}
case int(TASK_TYPE_CLASSIFICATION):
runners := handler.DataMap["runners"].(map[string]interface{})
runner := runners[runner_id].(map[string]interface{})
runner["task"] = &task
runners[runner_id] = runner
handler.DataMap["runners"] = runners
default:
logger.Error("Not sure what to do panicing", "taskType", task.TaskType)
panic("not sure what to do")
}
}
/**
* Tells the orcchestator to look at the task list from time to time
*/
@ -125,7 +177,7 @@ func attentionSeeker(config Config, back_channel chan int) {
/**
* Manages what worker should to Work
*/
func RunnerOrchestrator(db db.Db, config Config) {
func RunnerOrchestrator(db db.Db, config Config, handler *Handle) {
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
@ -133,6 +185,14 @@ func RunnerOrchestrator(db db.Db, config Config) {
Prefix: "Runner Orchestrator Logger",
})
setupHandle(handler)
base := BasePackStruct{
Db: db,
Logger: logger,
Host: config.Hostname,
}
gpu_workers := config.GpuWorker.NumberOfWorkers
logger.Info("Starting runners")
@ -144,12 +204,12 @@ func RunnerOrchestrator(db db.Db, config Config) {
defer func() {
if r := recover(); r != nil {
logger.Error("Recovered in Orchestrator restarting", "due to", r)
logger.Error("Recovered in Orchestrator restarting", "due to", r, "stack", string(debug.Stack()))
for x := range task_runners {
close(task_runners[x])
}
close(back_channel)
go RunnerOrchestrator(db, config)
go RunnerOrchestrator(db, config, handler)
}
}()
@ -159,6 +219,10 @@ func RunnerOrchestrator(db db.Db, config Config) {
for i := 0; i < gpu_workers; i++ {
task_runners[i] = make(chan Task, 10)
task_runners_used[i] = false
AddLocalRunner(handler, LocalRunner{
RunnerNum: i + 1,
Task: nil,
})
go runner(config, db, task_runners[i], i+1, back_channel)
}
@ -166,6 +230,7 @@ func RunnerOrchestrator(db db.Db, config Config) {
for i := range back_channel {
if i != 0 {
if i > 0 {
logger.Info("Runner freed", "runner", i)
task_runners_used[i-1] = false
@ -175,6 +240,8 @@ func RunnerOrchestrator(db db.Db, config Config) {
task_runners_used[i] = false
go runner(config, db, task_runners[i], i+1, back_channel)
}
AddLocalTask(handler, int(math.Abs(float64(i))), nil)
}
if task_to_dispatch == nil {
var task TaskT
@ -197,11 +264,38 @@ func RunnerOrchestrator(db db.Db, config Config) {
}
}
if task_to_dispatch != nil && task_to_dispatch.TaskType != int(TASK_TYPE_DELETE_USER) {
// TODO split tasks into cpu tasks and GPU tasks
mutex := handler.DataMap["runners_mutex"].(*sync.Mutex)
mutex.Lock()
remote_runners := handler.DataMap["runners"].(map[string]interface{})
for k, v := range remote_runners {
runner_data := v.(map[string]interface{})
runner_info := runner_data["runner_info"].(*Runner)
if runner_data["task"] != nil {
continue
}
if runner_info.UserId != task_to_dispatch.UserId {
continue
}
go handleRemoteTask(handler, base, k, *task_to_dispatch)
task_to_dispatch = nil
break
}
mutex.Unlock()
}
if task_to_dispatch != nil {
for i := 0; i < len(task_runners_used); i += 1 {
if !task_runners_used[i] {
task_runners[i] <- *task_to_dispatch
task_runners_used[i] = true
AddLocalTask(handler, i+1, task_to_dispatch)
task_to_dispatch = nil
break
}
@ -211,6 +305,6 @@ func RunnerOrchestrator(db db.Db, config Config) {
}
}
func StartRunners(db db.Db, config Config) {
go RunnerOrchestrator(db, config)
func StartRunners(db db.Db, config Config, handler *Handle) {
go RunnerOrchestrator(db, config, handler)
}

51
logic/tasks/runner/utils.go Normal file
View File

@ -0,0 +1,51 @@
package task_runner
import (
"sync"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
type LocalRunner struct {
RunnerNum int `json:"id"`
Task *Task `json:"task"`
}
type LocalRunners map[int]*LocalRunner
func LockRunners(handler *Handle, t string) *sync.Mutex {
req := t + "_runners_mutex"
if t == "" {
req = "runners_mutex"
}
mutex := handler.DataMap[req].(*sync.Mutex)
mutex.Lock()
return mutex
}
func setupHandle(handler *Handle) {
// Setup Remote Runner data
handler.DataMap["runners"] = map[string]interface{}{}
handler.DataMap["runners_mutex"] = &sync.Mutex{}
// Setup Local Runner data
handler.DataMap["local_runners"] = &LocalRunners{}
handler.DataMap["local_runners_mutex"] = &sync.Mutex{}
}
func AddLocalRunner(handler *Handle, runner LocalRunner) {
mutex := LockRunners(handler, "local")
defer mutex.Unlock()
runners := handler.DataMap["local_runners"].(*LocalRunners)
(*runners)[runner.RunnerNum] = &runner
}
func AddLocalTask(handler *Handle, runner_id int, task *Task) {
mutex := LockRunners(handler, "local")
defer mutex.Unlock()
runners := handler.DataMap["local_runners"].(*LocalRunners)
(*(*runners)[runner_id]).Task = task
}

25
logic/tasks/runner_data.go Normal file
View File

@ -0,0 +1,25 @@
package tasks
import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/runner"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleRunnerData(x *Handle) {
type NonType struct{}
PostAuthJson(x, "/tasks/runner/info", User_Admin, func(c *Context, dat *NonType) *Error {
mutex_remote := LockRunners(x, "")
defer mutex_remote.Unlock()
mutex_local := LockRunners(x, "local")
defer mutex_local.Unlock()
return c.SendJSON(struct {
RemoteRunners map[string]interface{} `json:"remoteRunners"`
LocalRunner *LocalRunners `json:"localRunners"`
}{
RemoteRunners: x.DataMap["runners"].(map[string]interface{}),
LocalRunner: x.DataMap["local_runners"].(*LocalRunners),
})
})
}

29
logic/tasks/utils/runner.go Normal file
View File

@ -0,0 +1,29 @@
package tasks_utils
import (
"time"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
)
type RunnerType int64
const (
RUNNER_TYPE_GPU RunnerType = iota + 1
)
type Runner struct {
Id string `json:"id" db:"ru.id"`
UserId string `json:"user_id" db:"ru.user_id"`
Token string `json:"token" db:"ru.token"`
Type RunnerType `json:"type" db:"ru.type"`
CreateOn time.Time `json:"createOn" db:"ru.created_on"`
}
func GetRunner(db db.Db, id string) (ru *Runner, err error) {
var runner Runner
err = dbtypes.GetDBOnce(db, &runner, "remote_runner as ru where ru.id=$1", id)
ru = &runner
return
}

6
logic/tasks/utils/utils.go
View File

@ -101,7 +101,11 @@ func (t Task) SetResult(base BasePack, result any) (err error) {
if err != nil {
return
}
_, err = base.GetDb().Exec("update tasks set result=$1 where id=$2", text, t.Id)
return t.SetResultText(base, string(text))
}
func (t Task) SetResultText(base BasePack, text string) (err error) {
_, err = base.GetDb().Exec("update tasks set result=$1 where id=$2", []byte(text), t.Id)
return
}

11
logic/users/users.go
View File

@ -241,6 +241,17 @@ func UsersEndpints(db db.Db, handle *Handle) {
return c.SendJSON(userReturn)
})
PostAuthJson(handle, "/user/info/get", User_Admin, func(c *Context, dat *JustId) *Error {
var user *User
user, err := UserFromId(c, dat.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "User not found")
} else if err != nil {
return c.E500M("Could not get user", err)
}
return c.SendJSON(user)
})
// Handles updating users
type UpdateUserData struct {
Id string `json:"id"`

4
logic/utils/config.go
View File

@ -114,12 +114,16 @@ func (c *Config) Cleanup(db db.Db) {
tasks[i].UpdateStatus(base, TASK_FAILED_RUNNING, "Task inturupted by server restart please try again")
_, err = db.Exec("update models set status=$1 where id=$2", READY_RETRAIN_FAILED, tasks[i].ModelId)
failLog(err)
_, err = db.Exec("update model_classes set status=$1 where model_id=$2 and status=$3", CLASS_STATUS_TO_TRAIN, tasks[i].ModelId, CLASS_STATUS_TRAINING)
failLog(err)
continue
}
if tasks[i].TaskType == int(TASK_TYPE_TRAINING) {
tasks[i].UpdateStatus(base, TASK_FAILED_RUNNING, "Task inturupted by server restart please try again")
_, err = db.Exec("update models set status=$1 where id=$2", FAILED_TRAINING, tasks[i].ModelId)
failLog(err)
_, err = db.Exec("update model_classes set status=$1 where model_id=$2 and status=$3", CLASS_STATUS_TO_TRAIN, tasks[i].ModelId, CLASS_STATUS_TRAINING)
failLog(err)
continue
}
}

4
logic/utils/handler.go
View File

@ -374,7 +374,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
c.SetReportCaller(true)
c.Logger.Warn("Request failed with a bad request", "dat", dat)
c.SetReportCaller(false)
return c.ErrorCode(nil, 404, dat)
return c.SendJSONStatus(http.StatusBadRequest, dat)
}
func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
@ -449,7 +449,7 @@ func (x Handle) createContext(handler *Handle, r *http.Request, w http.ResponseW
logger := log.NewWithOptions(os.Stdout, log.Options{
ReportCaller: true,
ReportTimestamp: true,
TimeFormat: time.Kitchen,
TimeFormat: time.DateTime,
Prefix: r.URL.Path,
})

4
main.go
View File

@ -36,11 +36,11 @@ func main() {
log.Info("Config loaded!", "config", config)
config.GenerateToken(db)
StartRunners(db, config)
//TODO check if file structure exists to save data
handle := NewHandler(db, config)
StartRunners(db, config, handle)
config.Cleanup(db)
// TODO Handle this in other way

2
nginx.dev.conf
View File

@ -13,7 +13,7 @@ http {
server {
listen 8000;
client_max_body_size 1G;
client_max_body_size 5G;
location / {
proxy_http_version 1.1;

11
sql/tasks.sql
View File

@ -38,3 +38,14 @@ create table if not exists tasks_dependencies (
main_id uuid references tasks (id) on delete cascade not null,
dependent_id uuid references tasks (id) on delete cascade not null
);
create table if not exists remote_runner (
id uuid primary key default gen_random_uuid(),
user_id uuid references users (id) on delete cascade not null,
token text not null,
-- 1: GPU
type integer,
created_on timestamp default current_timestamp
);

6
views/py/python_model_template.py
View File

@ -135,9 +135,9 @@ def addBlock(
model.add(layers.ReLU())
if top:
if pooling_same:
model.add(pool_func(padding="same", strides=(1, 1)))
model.add(pool_func(pool_size=(2,2), padding="same", strides=(1, 1)))
else:
model.add(pool_func())
model.add(pool_func(pool_size=(2,2)))
model.add(layers.BatchNormalization())
model.add(layers.LeakyReLU())
model.add(layers.Dropout(0.4))
@ -172,7 +172,7 @@ model.compile(
his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
NotifyServerCallback(),
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)])
acc = his.history["accuracy"]

BIN
webpage/bun.lockb
View File

Binary file not shown.

4125
webpage/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

9
webpage/package.json
View File

@ -15,7 +15,8 @@
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "3.0.0",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/d3": "^7.4.3",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
@ -25,7 +26,7 @@
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"sass": "^1.75.0",
"svelte": "5.0.0-next.104",
"svelte": "^5.0.0-next.104",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
@ -33,6 +34,8 @@
},
"type": "module",
"dependencies": {
"chart.js": "^4.4.2"
"chart.js": "^4.4.2",
"d3": "^7.9.0",
"highlight.js": "^11.9.0"
}
}

10
webpage/src/NavBar.svelte
View File

@ -15,8 +15,18 @@
{/if}
<li class="expand"></li>
{#if userStore.user}
{#if userStore.user.user_type == 2}
<li>
<a href="/admin/runners">
<span class="bi bi-cpu-fill"></span>
Runner
</a>
</li>
{/if}
<li>
<a href="/user/info"> <span class="bi bi-person-fill"></span> {userStore.user.username} </a>
</li>
<li>
<a href="/logout"> <span class="bi bi-box-arrow-right"></span> Logout </a>
</li>
{:else}

2
webpage/src/app.html
View File

@ -9,6 +9,8 @@
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github.min.css">
<link href="https://fonts.googleapis.com/css2?family=Andada+Pro:ital,wght@0,400..840;1,400..840&family=Bebas+Neue&family=Fira+Code:wght@300..700&display=swap" rel="stylesheet">
<link
href="https://fonts.googleapis.com/css2?family=Andada+Pro:ital,wght@0,400..840;1,400..840&family=Bebas+Neue&display=swap"
rel="stylesheet"

2
webpage/src/routes/UserStore.svelte.ts
View File

@ -1,7 +1,7 @@
import { goto } from '$app/navigation';
import { rdelete } from '$lib/requests.svelte';
type User = {
export type User = {
token: string;
id: string;
user_type: number;

348
webpage/src/routes/admin/runners/+page.svelte Normal file
View File

@ -0,0 +1,348 @@
<script lang="ts">
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import { post, showMessage } from 'src/lib/requests.svelte';
import { userStore } from 'src/routes/UserStore.svelte';
import { onMount } from 'svelte';
import * as d3 from 'd3';
import type { Base } from './types';
import CardInfo from './CardInfo.svelte';
let width = $state(0);
let height = $state(0);
function drag(simulation: any) {
function dragstarted(event: any, d: any) {
if (!event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
selected = d.data;
}
function dragged(event: any, d: any) {
d.fx = event.x;
d.fy = event.y;
}
function dragended(event: any, d: any) {
if (!event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
return d3.drag().on('start', dragstarted).on('drag', dragged).on('end', dragended);
}
let graph: HTMLDivElement;
let selected: Base | undefined = $state();
async function getData() {
const dataObj: Base = {
name: 'API',
type: 'api',
children: []
};
if (!dataObj.children) throw new Error();
const localRunners: Base[] = [];
const remotePairs: Record<string, Base[]> = {};
try {
let data = await post('tasks/runner/info', {});
if (Object.keys(data.localRunners).length > 0) {
for (const objId of Object.keys(data.localRunners)) {
localRunners.push({ name: objId, type: 'local_runner' });
}
dataObj.children.push({
name: 'local runners',
type: 'runner_group',
children: localRunners
});
}
for (const objId of Object.keys(data.remoteRunners)) {
let obj = data.remoteRunners[objId];
if (remotePairs[obj.runner_info.user_id as string]) {
remotePairs[obj.runner_info.user_id as string].push({
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
});
} else {
remotePairs[data.remoteRunners[objId].runner_info.user_id] = [
{
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
}
];
}
}
dataObj.children.push({
name: 'remote runners',
type: 'runner_group',
task: undefined,
children: Object.keys(remotePairs).map(
(name) =>
({
name,
type: 'user_group',
task: undefined,
children: remotePairs[name]
}) as Base
)
});
} catch (ex) {
showMessage(ex, notificationStore, 'Failed to get Runner information');
return;
}
const root = d3.hierarchy(dataObj);
const links = root.links();
const nodes = root.descendants();
console.log(root, links, nodes);
const simulation = d3
.forceSimulation(nodes)
.force(
'link',
d3
.forceLink(links)
.id((d: any) => d.id)
.distance((d: any) => {
let data = d.source.data as Base;
switch (data.type) {
case 'api':
return 150;
case 'runner_group':
return 90;
case 'user_group':
return 80;
case 'runner':
case 'local_runner':
return 20;
default:
throw new Error();
}
})
.strength(1)
)
.force('charge', d3.forceManyBody().strength(-1000))
.force('x', d3.forceX())
.force('y', d3.forceY());
const svg = d3
.create('svg')
.attr('width', width)
.attr('height', height - 62)
.attr('viewBox', [-width / 2, -height / 2, width, height])
.attr('style', 'max-width: 100%; height: auto;');
// Append links.
const link = svg
.append('g')
.attr('stroke', '#999')
.attr('stroke-opacity', 0.6)
.selectAll('line')
.data(links)
.join('line');
const database_svg = `
<svg xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-database" viewBox="0 0 32 32">
<path transform="scale(2)" d="M4.318 2.687C5.234 2.271 6.536 2 8 2s2.766.27 3.682.687C12.644 3.125 13 3.627 13 4c0 .374-.356.875-1.318 1.313C10.766 5.729 9.464 6 8 6s-2.766-.27-3.682-.687C3.356 4.875 3 4.373 3 4c0-.374.356-.875 1.318-1.313M13 5.698V7c0 .374-.356.875-1.318 1.313C10.766 8.729 9.464 9 8 9s-2.766-.27-3.682-.687C3.356 7.875 3 7.373 3 7V5.698c.271.202.58.378.904.525C4.978 6.711 6.427 7 8 7s3.022-.289 4.096-.777A5 5 0 0 0 13 5.698M14 4c0-1.007-.875-1.755-1.904-2.223C11.022 1.289 9.573 1 8 1s-3.022.289-4.096.777C2.875 2.245 2 2.993 2 4v9c0 1.007.875 1.755 1.904 2.223C4.978 15.71 6.427 16 8 16s3.022-.289 4.096-.777C13.125 14.755 14 14.007 14 13zm-1 4.698V10c0 .374-.356.875-1.318 1.313C10.766 11.729 9.464 12 8 12s-2.766-.27-3.682-.687C3.356 10.875 3 10.373 3 10V8.698c.271.202.58.378.904.525C4.978 9.71 6.427 10 8 10s3.022-.289 4.096-.777A5 5 0 0 0 13 8.698m0 3V13c0 .374-.356.875-1.318 1.313C10.766 14.729 9.464 15 8 15s-2.766-.27-3.682-.687C3.356 13.875 3 13.373 3 13v-1.302c.271.202.58.378.904.525C4.978 12.71 6.427 13 8 13s3.022-.289 4.096-.777c.324-.147.633-.323.904-.525"/>
</svg>
`;
const cpu_svg = `
<svg stroke="white" fill="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-cpu-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M6.5 6a.5.5 0 0 0-.5.5v3a.5.5 0 0 0 .5.5h3a.5.5 0 0 0 .5-.5v-3a.5.5 0 0 0-.5-.5z"/>
<path transform="scale(2)" d="M5.5.5a.5.5 0 0 0-1 0V2A2.5 2.5 0 0 0 2 4.5H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2v1H.5a.5.5 0 0 0 0 1H2A2.5 2.5 0 0 0 4.5 14v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14h1v1.5a.5.5 0 0 0 1 0V14a2.5 2.5 0 0 0 2.5-2.5h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14v-1h1.5a.5.5 0 0 0 0-1H14A2.5 2.5 0 0 0 11.5 2V.5a.5.5 0 0 0-1 0V2h-1V.5a.5.5 0 0 0-1 0V2h-1V.5a.5.5 0 0 0-1 0V2h-1zm1 4.5h3A1.5 1.5 0 0 1 11 6.5v3A1.5 1.5 0 0 1 9.5 11h-3A1.5 1.5 0 0 1 5 9.5v-3A1.5 1.5 0 0 1 6.5 5"/>
</svg>
`;
const user_svg = `
<svg fill="white" stroke="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-person-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M3 14s-1 0-1-1 1-4 6-4 6 3 6 4-1 1-1 1zm5-6a3 3 0 1 0 0-6 3 3 0 0 0 0 6"/>
</svg>
`;
const inbox_fill = `
<svg fill="white" stroke="white" xmlns="http://www.w3.org/2000/svg" stroke-width="0.2" width="32" height="32" fill="currentColor" class="bi bi-inbox-fill" viewBox="0 0 32 32">
<path transform="scale(2)" d="M4.98 4a.5.5 0 0 0-.39.188L1.54 8H6a.5.5 0 0 1 .5.5 1.5 1.5 0 1 0 3 0A.5.5 0 0 1 10 8h4.46l-3.05-3.812A.5.5 0 0 0 11.02 4zm-1.17-.437A1.5 1.5 0 0 1 4.98 3h6.04a1.5 1.5 0 0 1 1.17.563l3.7 4.625a.5.5 0 0 1 .106.374l-.39 3.124A1.5 1.5 0 0 1 14.117 13H1.883a1.5 1.5 0 0 1-1.489-1.314l-.39-3.124a.5.5 0 0 1 .106-.374z"/>
</svg>
`;
const node = svg
.append('g')
.attr('fill', '#fff')
.attr('stroke', '#000')
.attr('stroke-width', 1.5)
.selectAll('g')
.data(nodes)
.join('g')
.attr('style', 'cursor: pointer;')
.call(drag(simulation) as any)
.on('click', (e) => {
console.log('test');
function findData(obj: HTMLElement) {
if ((obj as any).__data__) {
return (obj as any).__data__;
}
if (!obj.parentElement) {
throw new Error();
}
return findData(obj.parentElement);
}
let obj = findData(e.srcElement);
console.log(obj);
selected = obj.data;
});
node
.append('circle')
.attr('fill', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
return '#caf0f8';
case 'runner_group':
return '#00b4d8';
case 'user_group':
return '#0000ff';
case 'runner':
case 'local_runner':
return '#03045e';
default:
throw new Error();
}
})
.attr('stroke', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
case 'user_group':
case 'runner_group':
return '#fff';
case 'runner':
case 'local_runner':
// TODO make this relient on the stauts
return '#000';
default:
throw new Error();
}
})
.attr('r', (d: any) => {
let data = d.data as Base;
switch (data.type) {
case 'api':
return 30;
case 'runner_group':
return 20;
case 'user_group':
return 25;
case 'runner':
case 'local_runner':
return 30;
default:
throw new Error();
}
})
.append('title')
.text((d: any) => d.data.name);
node
.filter((d) => {
return ['api', 'local_runner', 'runner', 'user_group', 'runner_group'].includes(
d.data.type
);
})
.append('g')
.html((d) => {
switch (d.data.type) {
case 'api':
return database_svg;
case 'user_group':
return user_svg;
case 'runner_group':
return inbox_fill;
case 'local_runner':
case 'runner':
return cpu_svg;
default:
throw new Error();
}
});
simulation.on('tick', () => {
link
.attr('x1', (d: any) => d.source.x)
.attr('y1', (d: any) => d.source.y)
.attr('x2', (d: any) => d.target.x)
.attr('y2', (d: any) => d.target.y);
node
.select('circle')
.attr('cx', (d: any) => d.x)
.attr('cy', (d: any) => d.y);
node
.select('svg')
.attr('x', (d: any) => d.x - 16)
.attr('y', (d: any) => d.y - 16);
});
//invalidation.then(() => simulation.stop());
graph.appendChild(svg.node() as any);
}
$effect(() => {
console.log(selected);
});
onMount(() => {
// Check if logged in and admin
if (!userStore.user || userStore.user.user_type != 2) {
goto('/');
return;
}
getData();
});
</script>
<svelte:window bind:innerWidth={width} bind:innerHeight={height} />
<svelte:head>
<title>Runners</title>
</svelte:head>
<div class="graph-container">
<div class="graph" bind:this={graph}></div>
{#if selected}
<div class="selected">
<CardInfo item={selected} />
</div>
{/if}
</div>
<style lang="css">
.graph-container {
position: relative;
.selected {
position: absolute;
right: 40px;
top: 40px;
width: 20%;
height: auto;
padding: 20px;
background: white;
border-radius: 20px;
box-shadow: 1px 1px 8px 2px #22222244;
}
}
</style>

90
webpage/src/routes/admin/runners/CardInfo.svelte Normal file
View File

@ -0,0 +1,90 @@
<script lang="ts">
import { post, showMessage } from 'src/lib/requests.svelte';
import type { Base } from './types';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import type { User } from 'src/routes/UserStore.svelte';
import Spinner from 'src/lib/Spinner.svelte';
import Tooltip from 'src/lib/Tooltip.svelte';
let { item }: { item: Base } = $props();
let user_data: User | undefined = $state();
async function getUserData(id: string) {
try {
user_data = await post('user/info/get', { id });
console.log(user_data);
} catch (ex) {
showMessage(ex, notificationStore, 'Could not get user information');
}
}
$effect(() => {
user_data = undefined;
if (item.type == 'user_group') {
getUserData(item.name);
} else if (item.type == 'runner') {
getUserData(item.parent ?? '');
}
});
</script>
{#if item.type == 'api'}
<h3>API</h3>
{:else if item.type == 'runner_group'}
<h3>Runner Group</h3>
This reprents a the group of {item.name}.
{:else if item.type == 'user_group'}
<h3>User</h3>
{#if user_data}
All Runners connected to this node bellong to <span class="accent">{user_data.username}</span>
{:else}
<div style="text-align: center;">
<Spinner />
</div>
{/if}
{:else if item.type == 'local_runner'}
<h3>Local Runner</h3>
This is a local runner
<div>
{#if item.task}
test
{:else}
Not running any task
{/if}
</div>
{:else if item.type == 'runner'}
<h3>Runner</h3>
{#if user_data}
<p>
This is a remote runner. This runner is owned by<span class="accent"
>{user_data?.username}</span
>
</p>
<div>
{#if item.task}
This runner is runing a <Tooltip title={item.task.id}>task</Tooltip>
{:else}
Not running any task
{/if}
</div>
{:else}
<div style="text-align: center;">
<Spinner />
</div>
{/if}
{:else}
{item.type}
{/if}
<style lang="scss">
h3 {
text-align: center;
margin: 0;
}
.accent {
background: #22222222;
padding: 1px;
border-radius: 5px;
}
</style>

10
webpage/src/routes/admin/runners/types.ts Normal file
View File

@ -0,0 +1,10 @@
import type { Task } from 'src/routes/models/edit/tasks/types';
export type BaseType = 'api' | 'runner_group' | 'user_group' | 'runner' | 'local_runner';
export type Base = {
name: string;
type: BaseType;
children?: Base[];
task?: Task;
parent?: string;
};

25
webpage/src/routes/models/+page.svelte
View File

@ -1,26 +1,22 @@
<script lang="ts">
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { onMount } from 'svelte';
import { get } from '$lib/requests.svelte';
import { get, showMessage } from '$lib/requests.svelte';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import Spinner from 'src/lib/Spinner.svelte';
let list = $state<
{
| {
name: string;
id: string;
}[]
>([]);
let message: MessageSimple;
| undefined
>(undefined);
onMount(async () => {
try {
list = await get('models');
} catch (e) {
if (e instanceof Response) {
message.display(await e.json());
} else {
message.display('Could not request list of models');
}
showMessage(e, notificationStore, 'Could not request list of models');
}
});
</script>
@ -30,7 +26,7 @@
</svelte:head>
<main>
<MessageSimple bind:this={message} />
{#if list}
{#if list.length > 0}
<div class="list-header">
<h2>My Models</h2>
@ -65,6 +61,11 @@
<a class="button padded" href="/models/add"> Create a new model </a>
</div>
{/if}
{:else}
<div style="text-align: center;">
<Spinner />
</div>
{/if}
</main>
<style lang="scss">

14
webpage/src/routes/models/add/+page.svelte
View File

@ -1,15 +1,13 @@
<script lang="ts">
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { postFormData } from 'src/lib/requests.svelte';
import { postFormData, showMessage } from 'src/lib/requests.svelte';
import { goto } from '$app/navigation';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
import 'src/styles/forms.css';
let submitted = $state(false);
let message: MessageSimple;
let buttonClicked: Promise<void> = $state(Promise.resolve());
let data = $state<{
@ -21,7 +19,6 @@
});
async function onSubmit() {
message.display('');
buttonClicked = new Promise<void>(() => {});
if (!data.file || !data.name) return;
@ -34,11 +31,7 @@
let id = await postFormData('models/add', formData);
goto(`/models/edit?id=${id}`);
} catch (e) {
if (e instanceof Response) {
message.display(await e.json());
} else {
message.display('Was not able to create model');
}
showMessage(e, notificationStore, 'Was not able to create model');
}
buttonClicked = Promise.resolve();
@ -75,7 +68,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={message} />
{#await buttonClicked}
<div class="text-center">File Uploading</div>
{:then}

2
webpage/src/routes/models/edit/+page.svelte
View File

@ -215,7 +215,7 @@
</div>
{:else if m.status == -3 || m.status == -4}
<BaseModelInfo model={m} />
<form on:submit={resetModel}>
<form on:submit|preventDefault={resetModel}>
Failed Prepare for training.<br />
<div class="spacer"></div>
<MessageSimple bind:this={resetMessages} />

92
webpage/src/routes/models/edit/RunModel.svelte
View File

@ -1,11 +1,14 @@
<script lang="ts">
import { post, postFormData } from 'src/lib/requests.svelte';
import { post, postFormData, showMessage } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher, onDestroy } from 'svelte';
import Spinner from 'src/lib/Spinner.svelte';
import type { Task } from './TasksTable.svelte';
import type { Task } from './tasks/TasksTable.svelte';
import Tabs from 'src/lib/Tabs.svelte';
import hljs from 'highlight.js';
import { notificationStore } from 'src/lib/NotificationsStore.svelte';
let { model } = $props<{ model: Model }>();
@ -19,8 +22,6 @@
let last_task: string | undefined = $state();
let last_task_timeout: number | null = null;
let messages: MessageSimple;
async function reloadLastTimeout() {
if (!last_task) {
return;
@ -42,7 +43,6 @@
async function submit() {
if (!file) return;
messages.clear();
let form = new FormData();
form.append('json_data', JSON.stringify({ id: model.id }));
@ -56,11 +56,7 @@
file = undefined;
last_task_timeout = setTimeout(() => reloadLastTimeout());
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display('Could not run the model');
}
showMessage(e, notificationStore, 'Could not run the model');
}
dispatch('upload');
@ -73,7 +69,72 @@
});
</script>
<form on:submit|preventDefault={submit}>
<Tabs active="upload" let:isActive>
<div class="buttons" slot="buttons" let:setActive let:isActive>
<button class="tab" class:selected={isActive('upload')} on:click={setActive('upload')}>
Upload
</button>
<button class="tab" class:selected={isActive('api')} on:click={setActive('api')}> Api </button>
</div>
<div class="content" class:selected={isActive('api')}>
<div class="codeinfo">
To perform an image classfication please follow the example bellow:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`let form = new FormData();
form.append('json_data', JSON.stringify({ id: '${model.id}' }));
form.append('file', file, 'file');
const headers = new Headers();
headers.append('response-type', 'application/json');
headers.append('token', token);
const r = await fetch('${window.location.protocol}//${window.location.hostname}/api/tasks/start/image', {
method: 'POST',
headers: headers,
body: form
});`,
{ language: 'javascript' }
).value}</pre>
On Success the request will return a json with this format:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`{ id "00000000-0000-0000-0000-000000000000" }`,
{ language: 'json' }
).value}</pre>
This id can be used to query the API for the result of the task:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`const headers = new Headers();
headers.append('content-type', 'application/json');
headers.append('token', token);
const r = await fetch('${window.location.protocol}//${window.location.hostname}/api/tasks/task', {
method: 'POST',
headers: headers,
body: JSON.stringify({ id: '00000000-0000-0000-0000-000000000000' })
});`,
{ language: 'javascript' }
).value}</pre>
Once the task shows the status as 4 then the data can be obatined in the result field: The successful
return value has this type:
<pre style="font-family: Fira Code;">{@html hljs.highlight(
`{
"id": string,
"user_id": string,
"model_id": string,
"status": number,
"status_message": string,
"user_confirmed": number,
"compacted": number,
"type": number,
"extra_task_info": string,
"result": string,
"created": string
}`,
{ language: 'javascript' }
).value}</pre>
</div>
</div>
<div class="content" class:selected={isActive('upload')}>
<form on:submit|preventDefault={submit} style="box-shadow: none;">
<fieldset class="file-upload">
<label for="file">Image</label>
<div class="form-msg">Run image through them model and get the result</div>
@ -86,7 +147,6 @@
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={messages} />
<button> Run </button>
{#if run}
{#await _result}
@ -109,3 +169,11 @@
{/await}
{/if}
</form>
</div>
</Tabs>
<style lang="scss">
.codeinfo {
padding: 20px;
}
</style>

48
webpage/src/routes/models/edit/tasks/Stats.svelte
View File

@ -18,7 +18,6 @@
PointElement,
LineElement
} from 'chart.js';
import ModelData from '../ModelData.svelte';
Chart.register(
Title,
@ -57,7 +56,9 @@
}
let pie: HTMLCanvasElement;
let pie2: HTMLCanvasElement;
let pieChart: Chart<'pie'> | undefined;
let pie2Chart: Chart<'pie'> | undefined;
function createPie(s: TasksStatsDay) {
if (pieChart) {
pieChart.destroy();
@ -72,23 +73,31 @@
'Classfication Failure',
'Classfication Preparing',
'Classfication Running',
'Classfication Unknown',
'Non Classfication Error',
'Non Classfication Success'
'Classfication Unknown'
],
datasets: [
{
label: 'Total',
data: [
t.c_error,
t.c_success,
t.c_failure,
t.c_pre_running,
t.c_running,
t.c_unknown,
t.nc_error,
t.nc_success
data: [t.c_error, t.c_success, t.c_failure, t.c_pre_running, t.c_running, t.c_unknown]
}
]
},
options: {
animation: false
}
});
if (pie2Chart) {
pieChart.destroy();
}
pie2Chart = new Chart(pie2, {
type: 'pie',
data: {
labels: ['Non Classfication Error', 'Non Classfication Success'],
datasets: [
{
label: 'Total',
data: [t.nc_error, t.nc_success]
}
]
},
@ -147,9 +156,14 @@
<h1>Statistics (Day)</h1>
<h2>Total</h2>
<div class="pies">
<div>
<canvas bind:this={pie}></canvas>
</div>
<div>
<canvas bind:this={pie2}></canvas>
</div>
</div>
<h2>Hourly</h2>
<div>
@ -160,4 +174,12 @@
canvas {
width: 100%;
}
.pies {
display: flex;
align-content: stretch;
div {
width: 50%;
}
}
</style>

14
webpage/src/routes/models/edit/tasks/TasksTable.svelte
View File

@ -1,16 +1,4 @@
<script lang="ts" context="module">
export type Task = {
id: string;
user_id: string;
model_id: string;
status: number;
status_message: string;
user_confirmed: number;
compacted: number;
type: number;
created: string;
result: string;
};
export const TaskType = {
TASK_FAILED_RUNNING: -2,
TASK_FAILED_CREATION: -1,
@ -55,6 +43,8 @@
import MessageSimple from 'src/lib/MessageSimple.svelte';
import Tooltip from 'src/lib/Tooltip.svelte';
import type { Task } from './types';
let { model }: { model: Model } = $props();
let page = $state(0);

12
webpage/src/routes/models/edit/tasks/types.ts Normal file
View File

@ -0,0 +1,12 @@
export type Task = {
id: string;
user_id: string;
model_id: string;
status: number;
status_message: string;
user_confirmed: number;
compacted: number;
type: number;
created: string;
result: string;
};

7
webpage/vite.config.ts
View File

@ -2,5 +2,10 @@ import { sveltekit } from '@sveltejs/kit/vite';
import { defineConfig } from 'vite';
export default defineConfig({
plugins: [sveltekit()]
plugins: [sveltekit()],
build: {
commonjsOptions: {
esmExternals: true
}
}
});