andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: moved model page to svelte

Browse Source
This commit is contained in:
Andre Henriques 2024-03-02 12:45:49 +00:00
parent e990b832d3
commit 30c5b57378
15 changed files with 1150 additions and 768 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
logic/models/classes/list.go
View File

@ -38,10 +38,10 @@ func models_data_list_json(w http.ResponseWriter, r *http.Request, c *Context) *
} }
type baserow struct { type baserow struct {
Id string Id string `json:"id"`
File_Path string File_Path string `json:"file_path"`
Model_Mode int Model_Mode int `json:"model_mode"`
Status int Status int `json:"status"`
} }
rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10) rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10)

277
logic/models/edit.go
View File

@ -43,141 +43,9 @@ func handleJson(w http.ResponseWriter, r *http.Request, c *Context) *Error {
/* /*
// Handle errors
// All errors will be negative
if model.Status < 0 {
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
return nil
}
switch model.Status { switch model.Status {
case READY:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
case TRAINING: case TRAINING:
type defrow struct {
Id string
Status int
EpochProgress int
Epoch int
Accuracy float64
}
defs := []defrow{}
if model.Type == 2 {
def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
} else {
def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
}
type layerdef struct {
id string
LayerType int
Shape string
}
layers := []layerdef{}
for _, def := range defs {
if def.Status == MODEL_DEFINITION_STATUS_TRAINING {
rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
for rows.Next() {
var layerdef layerdef
err = rows.Scan(&layerdef.id, &layerdef.LayerType, &layerdef.Shape)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef)
}
if model.Type == 2 {
type lastLayerType struct {
Id string
Range_start int
Range_end int
}
var lastLayer lastLayerType
err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef{
id: lastLayer.Id,
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1),
})
}
break
}
}
sep_mod := 100
if len(layers) > 8 {
sep_mod = 100 - (len(layers)-8)*10
}
if sep_mod < 10 {
sep_mod = 10
}
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
"Defs": defs,
"Layers": layers,
"SepMod": sep_mod,
}))
case PREPARING_ZIP_FILE:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
default:
fmt.Printf("Unkown Status: %d\n", model.Status)
return Error500(nil)
} }
return nil
*/ */
} }
@ -230,6 +98,151 @@ func handleEdit(handle *Handle) {
}) })
}) })
handle.Get("/models/edit/definitions", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
if c.Mode != JSON {
return c.ErrorCode(nil, 400, AnyMap{})
}
id, err := GetIdFromUrl(r, "id")
if err != nil {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
}
model, err := GetBaseModel(c.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
}
type defrow struct {
Id string
Status int
EpochProgress int
Epoch int
Accuracy float64
}
defs := []defrow{}
if model.ModelType == 2 {
def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
} else {
def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
}
type layerdef struct {
id string
LayerType int `json:"layer_type"`
Shape string `json:"shape"`
}
layers := []layerdef{}
for _, def := range defs {
if def.Status == MODEL_DEFINITION_STATUS_TRAINING {
rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
for rows.Next() {
var layerdef layerdef
err = rows.Scan(&layerdef.id, &layerdef.LayerType, &layerdef.Shape)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef)
}
if model.ModelType == 2 {
type lastLayerType struct {
Id string
Range_start int
Range_end int
}
var lastLayer lastLayerType
err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef{
id: lastLayer.Id,
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1),
})
}
break
}
}
type Definitions struct {
Id string `json:"id"`
Status int `json:"status"`
EpochProgress int `json:"epoch_progress"`
Epoch int `json:"epoch"`
Accuracy float64 `json:"accuracy"`
Layers *[]layerdef `json:"layers"`
}
defsToReturn := make([]Definitions, len(defs), len(defs))
setLayers := false
for i, def := range defs {
var lay *[]layerdef = nil
if def.Status == MODEL_DEFINITION_STATUS_TRAINING && !setLayers {
lay = &layers
setLayers = true
}
defsToReturn[i] = Definitions{
Id: def.Id,
Status: def.Status,
EpochProgress: def.EpochProgress,
Epoch: def.Epoch,
Accuracy: def.Accuracy,
Layers: lay,
}
}
return c.SendJSON(defsToReturn)
})
handle.Get("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Get("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil

215
logic/models/run.go
View File

@ -39,7 +39,7 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image {
func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) { func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) {
order = 0 order = 0
err = nil err = nil
tf_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil) tf_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "model"), []string{"serve"}, nil)
@ -63,55 +63,55 @@ func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.
} }
func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) { func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) {
err = nil
order = 0
base_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "base", "model"), []string{"serve"}, nil) err = nil
order = 0
base_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "base", "model"), []string{"serve"}, nil)
//results := base_model.Exec([]tf.Output{ //results := base_model.Exec([]tf.Output{
base_results := base_model.Exec([]tf.Output{ base_results := base_model.Exec([]tf.Output{
base_model.Op("StatefulPartitionedCall", 0), base_model.Op("StatefulPartitionedCall", 0),
}, map[tf.Output]*tf.Tensor{ }, map[tf.Output]*tf.Tensor{
//base_model.Op("serving_default_rescaling_input", 0): inputImage, //base_model.Op("serving_default_rescaling_input", 0): inputImage,
base_model.Op("serving_default_input_1", 0): inputImage, base_model.Op("serving_default_input_1", 0): inputImage,
}) })
type head struct { type head struct {
Id string Id string
Range_start int Range_start int
} }
heads, err := GetDbMultitple[head](c, "exp_model_head where def_id=$1;", def_id) heads, err := GetDbMultitple[head](c, "exp_model_head where def_id=$1;", def_id)
if err != nil { if err != nil {
return return
} }
var vmax float32 = 0.0 var vmax float32 = 0.0
for _, element := range heads { for _, element := range heads {
head_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "head", element.Id, "model"), []string{"serve"}, nil) head_model := tg.LoadModel(path.Join("savedData", model.Id, "defs", def_id, "head", element.Id, "model"), []string{"serve"}, nil)
results := head_model.Exec([]tf.Output{ results := head_model.Exec([]tf.Output{
head_model.Op("StatefulPartitionedCall", 0), head_model.Op("StatefulPartitionedCall", 0),
}, map[tf.Output]*tf.Tensor{ }, map[tf.Output]*tf.Tensor{
head_model.Op("serving_default_input_2", 0): base_results[0], head_model.Op("serving_default_input_2", 0): base_results[0],
}) })
var predictions = results[0].Value().([][]float32)[0] var predictions = results[0].Value().([][]float32)[0]
for i, v := range predictions { for i, v := range predictions {
if v > vmax { if v > vmax {
order = element.Range_start + i order = element.Range_start + i
vmax = v vmax = v
} }
} }
} }
// TODO runthe head model // TODO runthe head model
c.Logger.Info("Got", "heads", len(heads)) c.Logger.Info("Got", "heads", len(heads))
return return
} }
func handleRun(handle *Handle) { func handleRun(handle *Handle) {
@ -120,8 +120,123 @@ func handleRun(handle *Handle) {
return nil return nil
} }
if c.Mode == JSON { if c.Mode == JSON {
// TODO improve message
return ErrorCode(nil, 400, nil) read_form, err := r.MultipartReader()
if err != nil {
// TODO improve message
return ErrorCode(nil, 400, nil)
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Invalid multipart data")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Models not found");
} else if err != nil {
return c.Error500(err)
}
if model.Status != READY {
return c.JsonBadRequest("Model not ready to run images")
}
def := JustId{}
err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id)
if err == NotFoundError {
return c.JsonBadRequest("Could not find definition")
} else if err != nil {
return c.Error500(err)
}
def_id := def.Id
// TODO create a database table with tasks
run_path := path.Join("/tmp", model.Id, "runs")
os.MkdirAll(run_path, os.ModePerm)
img_path := path.Join(run_path, "img."+model.Format)
img_file, err := os.Create(img_path)
if err != nil {
return c.Error500(err)
}
defer img_file.Close()
img_file.Write(file)
if !testImgForModel(c, model, img_path) {
return c.JsonBadRequest("Provided image does not match the model")
}
root := tg.NewRoot()
var tf_img *image.Image = nil
switch model.Format {
case "png":
tf_img = ReadPNG(root, img_path, int64(model.ImageMode))
case "jpeg":
tf_img = ReadJPG(root, img_path, int64(model.ImageMode))
default:
panic("Not sure what to do with '" + model.Format + "'")
}
exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{})
inputImage, err := tf.NewTensor(exec_results[0].Value())
if err != nil {
return c.Error500(err)
}
vi := -1
if model.ModelType == 2 {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, err = runModelExp(c, model, def_id, inputImage)
if err != nil {
return c.Error500(err)
}
} else {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, err = runModelNormal(c, model, def_id, inputImage)
if err != nil {
return c.Error500(err)
}
}
os.RemoveAll(run_path)
rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi)
if err != nil {
return c.Error500(err)
}
if !rows.Next() {
return c.SendJSON(nil)
}
var name string
if err = rows.Scan(&name); err != nil {
return c.Error500(err)
}
return c.SendJSON(name)
} }
read_form, err := r.MultipartReader() read_form, err := r.MultipartReader()
@ -220,21 +335,21 @@ func handleRun(handle *Handle) {
return Error500(err) return Error500(err)
} }
vi := -1 vi := -1
if model.ModelType == 2 { if model.ModelType == 2 {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, err = runModelExp(c, model, def_id, inputImage) vi, err = runModelExp(c, model, def_id, inputImage)
if err != nil { if err != nil {
return c.Error500(err); return c.Error500(err)
} }
} else { } else {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, err = runModelNormal(c, model, def_id, inputImage) vi, err = runModelNormal(c, model, def_id, inputImage)
if err != nil { if err != nil {
return c.Error500(err); return c.Error500(err)
} }
} }
os.RemoveAll(run_path) os.RemoveAll(run_path)

103
logic/models/train/reset.go
View File

@ -10,53 +10,82 @@ import (
) )
func handleRest(handle *Handle) { func handleRest(handle *Handle) {
handle.Delete("/models/train/reset", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Delete("/models/train/reset", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil
} }
if c.Mode == JSON { if c.Mode == JSON {
panic("handle JSON /models/train/reset") var dat struct {
} Id string `json:"id"`
}
if err := c.ToJSON(r, &dat); err != nil {
return err;
}
f, err := MyParseForm(r) model, err := GetBaseModel(c.Db, dat.Id)
if err != nil { if err == ModelNotFoundError {
// TODO improve response return c.JsonBadRequest("Model not found");
return c.ErrorCode(nil, 400, c.AddMap(nil)) } else if err != nil {
} // TODO improve response
return c.Error500(err)
}
if !CheckId(f, "id") { if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
// TODO improve response return c.JsonBadRequest("Model is not in status that be reset")
return c.ErrorCode(nil, 400, c.AddMap(nil)) }
}
id := f.Get("id") os.RemoveAll(path.Join("savedData", model.Id, "defs"))
model, err := GetBaseModel(handle.Db, id) _, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err == ModelNotFoundError { if err != nil {
// TODO improve response
return c.Error500(err)
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
return c.SendJSON(model.Id)
}
f, err := MyParseForm(r)
if err != nil {
// TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil))
}
if !CheckId(f, "id") {
// TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil))
}
id := f.Get("id")
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found", "NotFoundMessage": "Model not found",
"GoBackLink": "/models", "GoBackLink": "/models",
}) })
} else if err != nil { } else if err != nil {
// TODO improve response // TODO improve response
return c.Error500(err) return c.Error500(err)
} }
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING { if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
// TODO improve response // TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil)) return c.ErrorCode(nil, 400, c.AddMap(nil))
} }
os.RemoveAll(path.Join("savedData", model.Id, "defs")) os.RemoveAll(path.Join("savedData", model.Id, "defs"))
_, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id) _, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil { if err != nil {
// TODO improve response // TODO improve response
return c.Error500(err) return c.Error500(err)
} }
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id=" + model.Id, c.Mode, w, r) Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil return nil
}) })
} }

199
logic/models/train/train.go
View File

@ -42,9 +42,9 @@ func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionSt
return return
} }
func UpdateStatus (c *Context, table string, id string, status int) (err error) { func UpdateStatus(c *Context, table string, id string, status int) (err error) {
_, err = c.Db.Exec(fmt.Sprintf("update %s set status = $1 where id = $2", table), status, id) _, err = c.Db.Exec(fmt.Sprintf("update %s set status = $1 where id = $2", table), status, id)
return return
} }
func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType, shape string) (err error) { func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType, shape string) (err error) {
@ -246,7 +246,7 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
return return
} }
UpdateStatus(c, "exp_model_head", exp.id, MODEL_DEFINITION_STATUS_TRAINING) UpdateStatus(c, "exp_model_head", exp.id, MODEL_DEFINITION_STATUS_TRAINING)
layers, err := c.Db.Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id) layers, err := c.Db.Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil { if err != nil {
@ -283,7 +283,7 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load
got = append(got, layerrow{ got = append(got, layerrow{
LayerType: LAYER_DENSE, LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d", exp.end-exp.start + 1), Shape: fmt.Sprintf("%d", exp.end-exp.start+1),
ExpType: 2, ExpType: 2,
LayerNum: i, LayerNum: i,
}) })
@ -625,14 +625,14 @@ func trainModelExp(c *Context, model *BaseModel) {
var rowv TrainModelRow var rowv TrainModelRow
rowv.acuracy = 0 rowv.acuracy = 0
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil { if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
failed("Failed to train Model Could not read definition from db!") failed("Failed to train Model Could not read definition from db!")
return return
} }
definitions = append(definitions, rowv) definitions = append(definitions, rowv)
} }
if len(definitions) == 0 { if len(definitions) == 0 {
failed("No Definitions defined!") failed("No Definitions defined!")
return return
} }
@ -661,13 +661,13 @@ func trainModelExp(c *Context, model *BaseModel) {
c.Logger.Info("Found a definition that reaches target_accuracy!") c.Logger.Info("Found a definition that reaches target_accuracy!")
_, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id) _, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
if err != nil { if err != nil {
failed("Failed to train definition!") failed("Failed to train definition!")
return return
} }
_, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) _, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if err != nil { if err != nil {
failed("Failed to train definition!") failed("Failed to train definition!")
return return
} }
@ -684,7 +684,7 @@ func trainModelExp(c *Context, model *BaseModel) {
_, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id) _, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
if err != nil { if err != nil {
failed("Failed to train definition!") failed("Failed to train definition!")
return return
} }
} }
@ -737,30 +737,30 @@ func trainModelExp(c *Context, model *BaseModel) {
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err != nil { if err != nil {
failed("DB: failed to read definition") failed("DB: failed to read definition")
return return
} }
defer rows.Close() defer rows.Close()
if !rows.Next() { if !rows.Next() {
failed("All definitions failed to train!") failed("All definitions failed to train!")
return return
} }
var id string var id string
if err = rows.Scan(&id); err != nil { if err = rows.Scan(&id); err != nil {
failed("Failed to read id") failed("Failed to read id")
return return
} }
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil { if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
failed("Failed to update model definition") failed("Failed to update model definition")
return return
} }
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil { if err != nil {
failed("Failed to select model_definition to delete") failed("Failed to select model_definition to delete")
return return
} }
defer to_delete.Close() defer to_delete.Close()
@ -768,7 +768,7 @@ func trainModelExp(c *Context, model *BaseModel) {
for to_delete.Next() { for to_delete.Next() {
var id string var id string
if to_delete.Scan(&id); err != nil { if to_delete.Scan(&id); err != nil {
failed("Failed to scan the id of a model_definition to delete") failed("Failed to scan the id of a model_definition to delete")
return return
} }
os.RemoveAll(path.Join("savedData", model.Id, "defs", id)) os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
@ -776,24 +776,24 @@ func trainModelExp(c *Context, model *BaseModel) {
// TODO Check if returning also works here // TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
failed("Failed to delete model_definition") failed("Failed to delete model_definition")
return return
} }
if err = splitModel(c, model); err != nil { if err = splitModel(c, model); err != nil {
failed("Failed to split the model") failed("Failed to split the model")
return return
} }
// There should only be one def availabale
def := JustId{}
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil { // There should only be one def availabale
return def := JustId{}
}
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
// Remove the base model return
c.Logger.Warn("Removing base model for", "model", model.Id, "def", def.Id) }
// Remove the base model
c.Logger.Warn("Removing base model for", "model", model.Id, "def", def.Id)
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model")) os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model"))
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model.keras")) os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model.keras"))
@ -802,17 +802,17 @@ func trainModelExp(c *Context, model *BaseModel) {
func splitModel(c *Context, model *BaseModel) (err error) { func splitModel(c *Context, model *BaseModel) (err error) {
def := JustId{} def := JustId{}
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil { if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
return return
} }
head := JustId{} head := JustId{}
if err = GetDBOnce(c, &head, "exp_model_head where def_id=$1", def.Id); err != nil { if err = GetDBOnce(c, &head, "exp_model_head where def_id=$1", def.Id); err != nil {
return return
} }
// Generate run folder // Generate run folder
run_path := path.Join("/tmp", model.Id, "defs", def.Id) run_path := path.Join("/tmp", model.Id, "defs", def.Id)
@ -821,7 +821,7 @@ func splitModel(c *Context, model *BaseModel) (err error) {
if err != nil { if err != nil {
return return
} }
// TODO reneable it // TODO reneable it
// defer os.RemoveAll(run_path) // defer os.RemoveAll(run_path)
// Create python script // Create python script
@ -838,9 +838,9 @@ func splitModel(c *Context, model *BaseModel) (err error) {
// Copy result around // Copy result around
result_path := path.Join(getDir(), "savedData", model.Id, "defs", def.Id) result_path := path.Join(getDir(), "savedData", model.Id, "defs", def.Id)
// TODO maybe move this to a select count(*) // TODO maybe move this to a select count(*)
// Get only fixed lawers // Get only fixed lawers
layers, err := c.Db.Query("select exp_type from model_definition_layer where def_id=$1 and exp_type=$2 order by layer_order asc;", def.Id, 1) layers, err := c.Db.Query("select exp_type from model_definition_layer where def_id=$1 and exp_type=$2 order by layer_order asc;", def.Id, 1)
if err != nil { if err != nil {
return return
@ -848,24 +848,24 @@ func splitModel(c *Context, model *BaseModel) (err error) {
defer layers.Close() defer layers.Close()
type layerrow struct { type layerrow struct {
ExpType int ExpType int
} }
count := -1 count := -1
for layers.Next() { for layers.Next() {
count += 1 count += 1
} }
if count == -1 { if count == -1 {
err = errors.New("Can not get layers") err = errors.New("Can not get layers")
return return
} }
log.Warn("Spliting model", "def", def.Id, "head", head.Id, "count", count) log.Warn("Spliting model", "def", def.Id, "head", head.Id, "count", count)
basePath := path.Join(result_path, "base") basePath := path.Join(result_path, "base")
headPath := path.Join(result_path, "head", head.Id) headPath := path.Join(result_path, "head", head.Id)
if err = os.MkdirAll(basePath, os.ModePerm); err != nil { if err = os.MkdirAll(basePath, os.ModePerm); err != nil {
return return
@ -876,10 +876,10 @@ func splitModel(c *Context, model *BaseModel) (err error) {
} }
if err = tmpl.Execute(f, AnyMap{ if err = tmpl.Execute(f, AnyMap{
"SplitLen": count, "SplitLen": count,
"ModelPath": path.Join(result_path, "model.keras"), "ModelPath": path.Join(result_path, "model.keras"),
"BaseModelPath": basePath, "BaseModelPath": basePath,
"HeadModelPath": headPath, "HeadModelPath": headPath,
}); err != nil { }); err != nil {
return return
} }
@ -892,7 +892,7 @@ func splitModel(c *Context, model *BaseModel) (err error) {
c.Logger.Info("Python finished running") c.Logger.Info("Python finished running")
return return
} }
func removeFailedDataPoints(c *Context, model *BaseModel) (err error) { func removeFailedDataPoints(c *Context, model *BaseModel) (err error) {
@ -1089,7 +1089,7 @@ func ExpModelHeadUpdateStatus(db *sql.DB, id string, status ModelDefinitionStatu
// This generates a definition // This generates a definition
func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy int, number_of_classes int, complexity int) *Error { func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy int, number_of_classes int, complexity int) *Error {
c.Logger.Info("Generating expandable new definition for model", "id", model.Id, "complexity", complexity) c.Logger.Info("Generating expandable new definition for model", "id", model.Id, "complexity", complexity)
var err error = nil var err error = nil
failed := func() *Error { failed := func() *Error {
@ -1133,13 +1133,13 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
// Create the blocks // Create the blocks
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10)))) loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if model.Width < 50 && model.Height < 50 { if model.Width < 50 && model.Height < 50 {
loop = 0 loop = 0
} }
log.Info("Size of the simple block", "loop", loop) log.Info("Size of the simple block", "loop", loop)
//loop = max(loop, 3) //loop = max(loop, 3)
for i := 0; i < loop; i++ { for i := 0; i < loop; i++ {
err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1) err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1)
@ -1165,9 +1165,9 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2) loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
log.Info("Size of the dense layers", "loop", loop) log.Info("Size of the dense layers", "loop", loop)
// loop = max(loop, 3) // loop = max(loop, 3)
for i := 0; i < loop; i++ { for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i))) err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
@ -1176,7 +1176,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
return failed() return failed()
} }
} }
_, err = CreateExpModelHead(c, def_id, 0, number_of_classes-1, MODEL_DEFINITION_STATUS_INIT) _, err = CreateExpModelHead(c, def_id, 0, number_of_classes-1, MODEL_DEFINITION_STATUS_INIT)
if err != nil { if err != nil {
return failed() return failed()
@ -1190,6 +1190,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
return nil return nil
} }
// TODO make this json friendy
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error { func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id) cls, err := model_classes.ListClasses(c.Db, model.Id)
if err != nil { if err != nil {
@ -1225,13 +1226,77 @@ func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy
return nil return nil
} }
func handle_models_train_json(w http.ResponseWriter, r *http.Request, c *Context) *Error {
var dat struct {
Id string `json:"id"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
if err_ := c.ToJSON(r, &dat); err_ != nil {
return err_
}
if dat.Id == "" {
return c.JsonBadRequest("Please provide a id")
}
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.Error500(err)
}
if model.Status != CONFIRM_PRE_TRAINING {
return c.JsonBadRequest("Model in invalid status for training")
}
if modelTypeId == 2 {
full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
}
if modelTypeId == 2 {
go trainModelExp(c, model)
} else {
go trainModel(c, model)
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
}
return c.SendJSON(model.Id)
}
func handleTrain(handle *Handle) { func handleTrain(handle *Handle) {
handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil
} }
if c.Mode == JSON { if c.Mode == JSON {
panic("TODO /models/train JSON") return handle_models_train_json(w, r, c)
} }
r.ParseForm() r.ParseForm()

38
logic/utils/handler.go
View File

@ -387,6 +387,7 @@ func (c Context) SendJSONStatus(status int, dat any) *Error {
} }
func (c Context) JsonBadRequest(dat any) *Error { func (c Context) JsonBadRequest(dat any) *Error {
c.Logger.Warn("Request failed with a bad request", "dat", dat)
return c.SendJSONStatus(http.StatusBadRequest, dat) return c.SendJSONStatus(http.StatusBadRequest, dat)
} }
@ -620,6 +621,7 @@ func (x Handle) ReadFiles(pathTest string, baseFilePath string, fileType string,
}) })
} }
// TODO remove this
func (x Handle) ReadTypesFiles(pathTest string, baseFilePath string, fileTypes []string, contentTypes []string) { func (x Handle) ReadTypesFiles(pathTest string, baseFilePath string, fileTypes []string, contentTypes []string) {
http.HandleFunc(pathTest, func(w http.ResponseWriter, r *http.Request) { http.HandleFunc(pathTest, func(w http.ResponseWriter, r *http.Request) {
user_path := r.URL.Path[len(pathTest):] user_path := r.URL.Path[len(pathTest):]
@ -656,6 +658,42 @@ func (x Handle) ReadTypesFiles(pathTest string, baseFilePath string, fileTypes [
}) })
} }
func (x Handle) ReadTypesFilesApi(pathTest string, baseFilePath string, fileTypes []string, contentTypes []string) {
http.HandleFunc("/api" + pathTest, func(w http.ResponseWriter, r *http.Request) {
r.URL.Path = strings.Replace(r.URL.Path, "/api", "", 1)
user_path := r.URL.Path[len(pathTest):]
found := false
index := -1
for i, fileType := range fileTypes {
if strings.HasSuffix(user_path, fileType) {
found = true
index = i
break
}
}
if !found {
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("File not found"))
return
}
bytes, err := os.ReadFile(path.Join(baseFilePath, pathTest, user_path))
if err != nil {
fmt.Println(err)
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to load file"))
return
}
w.Header().Set("Content-Type", contentTypes[index])
w.Write(bytes)
})
}
func NewHandler(db *sql.DB) *Handle { func NewHandler(db *sql.DB) *Handle {
var gets []HandleFunc var gets []HandleFunc

2
main.go
View File

@ -48,7 +48,7 @@ func main() {
handle.StaticFiles("/js/", ".js", "text/javascript") handle.StaticFiles("/js/", ".js", "text/javascript")
handle.ReadFiles("/imgs/", "views", ".png", "image/png;") handle.ReadFiles("/imgs/", "views", ".png", "image/png;")
handle.ReadTypesFiles("/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"}) handle.ReadTypesFiles("/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"})
handle.ReadTypesFiles("/api/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"}) handle.ReadTypesFilesApi("/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"})
handle.GetHTML("/", AnswerTemplate("index.html", nil, 0)) handle.GetHTML("/", AnswerTemplate("index.html", nil, 0))

2
webpage/src/app.html
View File

@ -4,6 +4,8 @@
<meta charset="utf-8" /> <meta charset="utf-8" />
<link rel="icon" href="%sveltekit.assets%/favicon.png" /> <link rel="icon" href="%sveltekit.assets%/favicon.png" />
<meta name="viewport" content="width=device-width, initial-scale=1" /> <meta name="viewport" content="width=device-width, initial-scale=1" />
<link href="https://fonts.googleapis.com/css2?family=Roboto:ital,wght@0,100;0,300;0,400;0,500;0,700;0,900;1,100;1,300;1,400;1,500;1,700;1,900&display=swap" rel="stylesheet">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.1/font/bootstrap-icons.css">
%sveltekit.head% %sveltekit.head%
</head> </head>
<body data-sveltekit-preload-data="hover"> <body data-sveltekit-preload-data="hover">

5
webpage/src/lib/MessageSimple.svelte
View File

@ -14,6 +14,11 @@
let timeout: number | undefined = undefined; let timeout: number | undefined = undefined;
export function clear() {
if (timeout) clearTimeout(timeout);
message = undefined;
}
export function display( export function display(
msg: string, msg: string,
options?: { options?: {

17
webpage/src/lib/requests.svelte.ts
View File

@ -16,7 +16,10 @@ export async function get(url: string) {
headers: headers, headers: headers,
}); });
if (r.status !== 200) { if (r.status === 401) {
userStore.user = undefined;
goto("/login")
} else if (r.status !== 200) {
throw r; throw r;
} }
@ -35,8 +38,11 @@ export async function post(url: string, body: any) {
headers: headers, headers: headers,
body: JSON.stringify(body), body: JSON.stringify(body),
}); });
if (r.status !== 200) { if (r.status === 401) {
userStore.user = undefined;
goto("/login")
} else if (r.status !== 200) {
throw r; throw r;
} }
@ -56,7 +62,10 @@ export async function rdelete(url: string, body: any) {
body: JSON.stringify(body), body: JSON.stringify(body),
}); });
if (r.status !== 200) { if (r.status === 401) {
userStore.user = undefined;
goto("/login")
} else if (r.status !== 200) {
throw r; throw r;
} }

706
webpage/src/routes/models/edit/+page.svelte
View File

@ -1,25 +1,25 @@
<script lang="ts" context="module"> <script lang="ts" context="module">
export type Model = { export type Model = {
id: string; id: string;
name: string; name: string;
color_mode: string; color_mode: string;
width: number; width: number;
height: number; height: number;
status: number; status: number;
} };
export type Layer = { export type Layer = {
layer_type: number; layer_type: number;
shape: string; shape: string;
} };
export type Definitions = { export type Definitions = {
epoch: number; epoch: number;
epoch_progress: number; epoch_progress: number;
status: number; status: number;
accuracy: number; accuracy: number;
layers?: Layer[]; layers?: Layer[];
} };
</script> </script>
<script lang="ts"> <script lang="ts">
@ -28,12 +28,14 @@
import BaseModelInfo from "./BaseModelInfo.svelte"; import BaseModelInfo from "./BaseModelInfo.svelte";
import DeleteModel from "./DeleteModel.svelte"; import DeleteModel from "./DeleteModel.svelte";
import { goto } from "$app/navigation"; import { goto } from "$app/navigation";
import { get } from "src/lib/requests.svelte"; import { get, rdelete } from "src/lib/requests.svelte";
import MessageSimple from "$lib/MessageSimple.svelte"
import 'src/styles/forms.css'
import ModelData from "./ModelData.svelte"; import ModelData from "./ModelData.svelte";
import DeleteZip from "./DeleteZip.svelte"; import DeleteZip from "./DeleteZip.svelte";
import 'src/styles/forms.css'
import RunModel from "./RunModel.svelte";
let model: Promise<Model> = $state(new Promise(() => {})); let model: Promise<Model> = $state(new Promise(() => {}));
let definitions: Promise<Definitions[]> = $state(new Promise(() => {})); let definitions: Promise<Definitions[]> = $state(new Promise(() => {}));
@ -43,10 +45,18 @@
try { try {
let temp_model: Model = await get(`models/edit?id=${id}`); let temp_model: Model = await get(`models/edit?id=${id}`);
console.log(temp_model)
if (temp_model.status == 3) { if (temp_model.status == 3) {
setTimeout(getModel, 2000); setTimeout(getModel, 2000);
} }
if (temp_model.status == 4) {
setTimeout(getModel, 5000);
definitions = await get(`models/edit/definitions?id=${id}`);
}
model = Promise.resolve(temp_model); model = Promise.resolve(temp_model);
} catch (e) { } catch (e) {
if (e instanceof Response) { if (e instanceof Response) {
@ -69,457 +79,253 @@
getModel(); getModel();
}); });
let resetMessages: MessageSimple;
async function resetModel() { async function resetModel() {
throw new Error("TODO"); resetMessages.display("");
let _model = await model;
try {
await rdelete('models/train/reset', {
"id": _model.id
});
getModel();
} catch (e) {
if (e instanceof Response) {
resetMessages.display(await e.json())
} else {
resetMessages.display("Could not reset model!")
}
}
} }
// Auto reload after 2s when model.status 3,4 // Auto reload after 2s when model.status 3,4
</script> </script>
<svelte:head> <svelte:head>
{#await model} {#await model}
<title> <title>Model</title>
Model {:then m}
</title> {#if m}
{:then m} <title>
{#if m} Model: {m.name}
<title> </title>
Model: {m.name} {:else}
</title> <title>Model</title>
{:else} {/if}
<title> {/await}
Model
</title>
{/if}
{/await}
</svelte:head> </svelte:head>
<!-- {{/* Is called from a diffrent endpoint so that it does not matter where this is from :) which means that . can mean what ever I want */}}
{{ define "data-model-create-class-table-table" }}
<div>
<table>
<thead>
<tr>
<th>
File Path
</th>
<th>
Mode
</th>
<th>
<!-- Img -- >
</th>
<th>
<!-- Status -- >
</th>
</tr>
</thead>
<tbody>
{{range .List}}
<tr>
<td>
{{ if eq .FilePath "id://" }}
Managed
{{ else }}
{{.FilePath}}
{{ end }}
</td>
<td>
{{ if (eq .Mode 2) }}
Testing
{{ else }}
Training
{{ end }}
</td>
<td class="text-center">
{{ if startsWith .FilePath "id://" }}
<img src="/savedData/{{ $.Model.Id }}/data/{{ .Id }}.{{ $.Model.Format }}" height="30px" width="30px" style="object-fit: contain;" />
{{ else }}
TODO
img {{ .FilePath }}
{{ end }}
</td>
<td class="text-center">
{{ if eq .Status 1 }}
<span class="bi bi-check-circle-fill" style="color: green"></span>
{{ else }}
<span class="bi bi-exclamation-triangle-fill" style="color: red"></span>
{{ end }}
</td>
</tr>
{{end}}
</tbody>
</table>
<div class="flex justify-center align-center">
<div class="grow-1 flex justify-end align-center ">
{{ if gt .Page 0 }}
<button
hx-get="/models/data/list?id={{ .Id }}&page={{ add .Page -1 }}"
hx-target=".content[data-tab='{{ .Name }}']"
hx-swap="innerHTML"
hx-headers='{"REQUEST-TYPE": "html"}'
data-tab="{{ .Name }}">
Prev
</button>
{{ end }}
</div>
<div style="padding: 10px;">
{{ .Page }}
</div>
<div class="grow-1 flex justify-start align-center">
{{ if .ShowNext }}
<button
hx-get="/models/data/list?id={{ .Id }}&page={{ add .Page 1 }}"
hx-target=".content[data-tab='{{ .Name }}']"
hx-swap="innerHTML"
hx-headers='{"REQUEST-TYPE": "html"}'
data-tab="{{ .Name }}">
Next
</button>
{{ end }}
</div>
</div>
</div>
{{ end }}
-->
<!--
{{ define "data-model-create-class-table" }}
{{ if eq (len .Classes) 0 }}
TODO CREATE TABLE
{{else}}
<div class="tabs-header">
{{/* Handle the case where there are to many buttons */}}
<div class="header">
{{ range .Classes }}
{{/* TODO Auto Load 1st */}}
<button
hx-get="/models/data/list?id={{ .Id }}"
hx-target=".content[data-tab='{{ .Name }}']"
hx-swap="innerHTML"
hx-headers='{"REQUEST-TYPE": "html"}'
hx-trigger="click"
class="tab"
data-tab="{{ .Name }}">
{{ .Name }}
</button>
{{ end }}
</div>
{{ range $i, $a := .Classes }}
{{ if eq $i 0}}
<div
hx-get="/models/data/list?id={{ .Id }}"
hx-target=".content[data-tab='{{ $a.Name }}']"
hx-swap="innerHTML"
hx-headers='{"REQUEST-TYPE": "html"}'
hx-trigger="load"
class="content"
data-tab="{{ $a.Name }}">
</div>
{{ else }}
<div
class="content"
data-tab="{{ $a.Name }}" >
</div>
{{ end }}
{{ end }}
</div>
{{end}}
{{ end }}
-->
<!--
{{ define "train-model-card" }}
<form hx-post="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
{{ if .HasData }}
{{ if .NumberOfInvalidImages }}
{{ if gt .NumberOfInvalidImages 0 }}
<p class="danger">
There are images {{ .NumberOfInvalidImages }} that were loaded that do not have the correct format. These images will be delete when the model trains.
</p>
<input type="hidden" value="{{ .NumberOfInvalidImages }}" name="id" />
{{ end }}
{{ end }}
{{ if .ErrorMessage }}
<p class="danger">
{{ .ErrorMessage }}
</p>
{{ end }}
{{/* TODO expading mode */}}
<input type="hidden" value="{{ .Model.Id }}" name="id" />
<fieldset>
<legend>
Model Type
</legend>
<div class="input-radial">
<input id="model_type_simple" value="simple" name="model_type" type="radio" checked />
<label for="model_type_simple">Simple</label><br/>
<input id="model_type_expandable" value="expandable" name="model_type" type="radio" />
<label for="model_type_expandable">Expandable</label>
</div>
</fieldset>
{{/* TODO allow more models to be created */}}
<fieldset>
<label for="number_of_models">Number of Models</label>
<input id="number_of_models" type="number" name="number_of_models" value="1" />
</fieldset>
{{/* TODO to Change the acc */}}
<fieldset>
<label for="accuracy">Target accuracy</label>
<input id="accuracy" type="number" name="accuracy" value="95" />
</fieldset>
{{/* TODO allow to chose the base of the model */}}
{{/* TODO allow to change the shape of the model */}}
<button>
Train
</button>
{{ else }}
<h2>
To train the model please provide data to the model first
</h2>
{{ end }}
</form>
{{ end }}
-->
<!-- <!--
{{ define "run-model-card" }} {{ define "run-model-card" }}
<form hx-headers='{"REQUEST-TYPE": "html"}' enctype="multipart/form-data" hx-post="/models/run" hx-swap="outerHTML">
<input type="hidden" name="id" value={{.Model.Id}} />
<fieldset class="file-upload" >
<label for="file">Image</label>
<div class="form-msg">
Run image through them model and get the result
</div>
<div class="icon-holder">
<button class="icon">
<img replace="icon" src="/imgs/upload-icon.png" />
<span replace="File Selected">
Image File
</span>
</button>
{{ if .ImageError }}
<span class="form-msg error">
The provided image was not valid for this model
</span>
{{ end }}
<input id="file" name="file" type="file" required accept="image/png" />
</div>
</fieldset>
<button>
Run
</button>
{{ if .NotFound }}
<div class="result">
<h1>
The class was not found
</h1>
</div>
{{ else if .Result }}
<div>
<h1>
Result
</h1>
The image was classified as {{.Result}}
</div>
{{ end }}
</form>
{{ end }} {{ end }}
--> -->
<main> <main>
{#await model} {#await model}
Loading Loading
{:then m} {:then m}
{#if m.status == 1} {#if m.status == 1}
<div> <div>
<h1 class="text-center"> <h1 class="text-center">
{ m.name } {m.name}
</h1> </h1>
<!-- TODO add cool animation --> <!-- TODO add cool animation -->
<h2 class="text-center"> <h2 class="text-center">Preparing the model</h2>
Preparing the model </div>
</h2> {:else if m.status == -1}
</div> <div>
{:else if m.status == -1} <h1 class="text-center">
<div> {m.name}
<h1 class="text-center"> </h1>
{m.name} <!-- TODO improve message -->
</h1> <h2 class="text-center">Failed to prepare model</h2>
<!-- TODO improve message -->
<h2 class="text-center">
Failed to prepare model
</h2>
<div> <div>TODO button delete</div>
TODO button delete
</div> <!--form hx-delete="/models/delete">
<!--form hx-delete="/models/delete">
<input type="hidden" name="id" value="{{ .Model.Id }}" /> <input type="hidden" name="id" value="{{ .Model.Id }}" />
<button class="danger"> <button class="danger">
Delete Delete
</button> </button>
</form--> </form-->
</div> </div>
<!-- PRE TRAINING STATUS --> <!-- PRE TRAINING STATUS -->
{:else if m.status == 2 } {:else if m.status == 2}
<BaseModelInfo model={m} /> <BaseModelInfo model={m} />
<ModelData model={m} on:reload={getModel} /> <ModelData model={m} on:reload={getModel} />
<!-- {{ template "train-model-card" . }} --> <!-- {{ template "train-model-card" . }} -->
<DeleteModel model={m} /> <DeleteModel model={m} />
{:else if m.status == -2 } {:else if m.status == -2}
<BaseModelInfo model={m} /> <BaseModelInfo model={m} />
<DeleteZip model={m} on:reload={getModel} /> <DeleteZip model={m} on:reload={getModel} />
<DeleteModel model={m} /> <DeleteModel model={m} />
{:else if m.status == 3 } {:else if m.status == 3}
<BaseModelInfo model={m} /> <BaseModelInfo model={m} />
<div class="card"> <div class="card">
<!-- TODO improve this --> <!-- TODO improve this -->
Processing zip file... Processing zip file...
</div> </div>
{:else if m.status == -3 || m.status == -4} {:else if m.status == -3 || m.status == -4}
<BaseModelInfo model={m} /> <BaseModelInfo model={m} />
<form on:submit={resetModel}> <form on:submit={resetModel}>
Failed Prepare for training.<br/> Failed Prepare for training.<br />
<div class="spacer" ></div> <div class="spacer"></div>
<button class="danger"> <MessageSimple bind:this={resetMessages} />
Try Again <button class="danger"> Try Again </button>
<button> </form>
</form> <DeleteModel model={m} />
<DeleteModel model={m} /> {:else if m.status == 4}
{:else if m.status == 4} <BaseModelInfo model={m} />
<BaseModelInfo model={m} /> <!-- TODO request /models/edit?id={m.id} -->
<!-- TODO request /models/edit?id={m.id} --> <div class="card">
<div class="card"> <!-- TODO improve this -->
<!-- TODO improve this --> Training the model...<br />
Training the model...<br/> <!-- TODO Add progress status on definitions -->
<!-- TODO Add progress status on definitions --> {#await definitions}
{#await definitions} Loading
Loading {:then defs}
{:then defs} <table>
<table> <thead>
<thead> <tr>
<tr> <th> Done Progress </th>
<th> <th> Training Round Progress </th>
Done Progress <th> Accuracy </th>
</th> <th> Status </th>
<th> </tr>
Training Round Progress </thead>
</th> <tbody>
<th> {#each defs as def}
Accuracy <tr>
</th> <td>
<th> {def.epoch}
Status </td>
</th> <td>
</tr> {def.epoch_progress}/20
</thead> </td>
<tbody> <td>
{#each defs as def} {def.accuracy}%
<tr> </td>
<td> <td style="text-align: center;">
{def.epoch} {#if def.status == 2}
</td> <span class="bi bi-book" style="color: green;"></span>
<td> {:else if [3, 6, -3].includes(def.status)}
{def.epoch_progress}/20 <span
</td> class="bi bi-book-half"
<td> style="color: {{
{def.accuracy}% '3': 'green',
</td> '-3': 'red',
<td style="text-align: center;"> '6': 'orange'
{#if def.status == 2} }[String(def.status)]};"
<span class="bi bi-book" style="color: green;"></span> ></span>
{:else if [3,6,-3].includes(def.status) } {:else}
<span class="bi bi-book-half" style="color: {{ {def.status}
'3': 'green', {/if}
'-3': 'red', </td>
'6': 'orange', </tr>
}[String(def.status)]};"></span> {#if def.status == 3 && def.layers}
{:else} <tr>
{def.status} <td colspan="4">
{/if} <svg viewBox="0 200 1000 600">
</td> {#each def.layers as layer, i}
</tr> {@const sep_mod =
{#if def.status == 3 && def.layers} def.layers.length > 8
<tr> ? Math.max(10, 100 - (def.layers.length - 8) * 10)
<td colspan="4"> : 100}
<svg viewBox="0 200 1000 600"> {#if layer.layer_type == 1}
{#each def.layers as layer, i} <polygon
{@const sep_mod = def.layers.length > 8 ? Math.max(10, 100 - (def.layers.length - 8) * 10) : 100} points="50,450 200,250 200,550 50,750"
{#if layer.layer_type == 1} stroke="black"
<polygon stroke-width="2"
points="50,450 200,250 200,550 50,750" fill="green"
stroke="black" ></polygon>
stroke-width="2" {:else if layer.layer_type == 4}
fill="green"></polygon> <polygon
{:else if layer.layer_type == 4} points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
<polygon i * sep_mod},550 {50 + i * sep_mod},750"
points="{50 + (i * sep_mod)},450 {200 + (i * sep_mod)},250 {200 + (i * sep_mod)},550 {50 + (i * sep_mod)},750" stroke="black"
stroke="black" stroke-width="2"
stroke-width="2" fill="orange"
fill="orange"> >
</polygon> </polygon>
{:else if layer.layer_type == 3} {:else if layer.layer_type == 3}
<polygon <polygon
points="{50 + (i * sep_mod)},450 {200 + (i * sep_mod)},250 {200 + (i * sep_mod)}},550 {50 + (i * sep_mod)},750" points="{50 + i * sep_mod},450 {200 + i * sep_mod},250 {200 +
stroke="black" i * sep_mod},550 {50 + i * sep_mod},750"
stroke-width="2" stroke="black"
fill="red"> stroke-width="2"
</polygon> fill="red"
{:else if layer.layer_type == 3} >
<polygon </polygon>
points="{50 + (i * sep_mod)},550 {200 + (i * sep_mod)},350 {200 + (i * sep_mod)},450 {50 + (i * sep_mod)},650" {:else if layer.layer_type == 2}
stroke="black" <polygon
stroke-width="2" points="{50 + i * sep_mod},550 {200 + i * sep_mod},350 {200 +
fill="blue"> i * sep_mod},450 {50 + i * sep_mod},650"
</polygon> stroke="black"
{:else} stroke-width="2"
<div> fill="blue"
{layer.layer_type} >
{layer.shape} </polygon>
</div> {:else}
{/if} <div>
{/each} {layer.layer_type}
</svg> {layer.shape}
</td> </div>
</tr> {/if}
{/if} {/each}
{/each} </svg>
</tbody> </td>
</table> </tr>
{/await} {/if}
{{/* TODO Add ability to stop training */}} {/each}
</div> </tbody>
{:else if m.status == 5} </table>
<BaseModelInfo model={m} /> {/await}
TODO run model <!-- TODO Add ability to stop training -->
<!-- </div>
<form hx-delete="/models/train/reset" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML"> {:else if m.status == 5}
Failed Prepare for training.<br/> <BaseModelInfo model={m} />
<div class="spacer" ></div> <RunModel model={m} />
<input type="hidden" name="id" value="{{ .Model.Id }}" /> <DeleteModel model={m} />
<button class="danger"> {:else}
Try Again <h1>Unknown Status of the model.</h1>
</button> {/if}
</form> {/await}
-->
<DeleteModel model={m} />
{:else}
<h1>
Unknown Status of the model.
</h1>
{/if}
{/await}
</main> </main>
<style lang="scss"> <style lang="scss">
main { main {
padding: 20px 15vw; padding: 20px 15vw;
} }
table {
width: 100%;
box-shadow: 0 2px 8px 1px #66666622;
border-radius: 10px;
border-collapse: collapse;
overflow: hidden;
}
table thead {
background: #60606022;
}
table tr td,
table tr th {
border-left: 1px solid #22222244;
padding: 15px;
}
table tr td:first-child,
table tr th:first-child {
border-left: none;
}
table tr td button,
table tr td .button {
padding: 5px 10px;
box-shadow: 0 2px 5px 1px #66666655;
}
</style> </style>

9
webpage/src/routes/models/edit/ModelData.svelte
View File

@ -12,10 +12,12 @@
import MessageSimple from "src/lib/MessageSimple.svelte"; import MessageSimple from "src/lib/MessageSimple.svelte";
import { createEventDispatcher } from "svelte"; import { createEventDispatcher } from "svelte";
import ModelTable from "./ModelTable.svelte"; import ModelTable from "./ModelTable.svelte";
import TrainModel from "./TrainModel.svelte";
let { model } = $props<{model: Model}>(); let { model } = $props<{model: Model}>();
let classes: Class[] = $state([]); let classes: Class[] = $state([]);
let has_data: boolean = $state(false);
let file: File | undefined = $state(); let file: File | undefined = $state();
@ -61,6 +63,7 @@
let data = await get(`models/edit/classes?id=${model.id}`); let data = await get(`models/edit/classes?id=${model.id}`);
classes = data.classes classes = data.classes
numberOfInvalidImages = data.number_of_invalid_images; numberOfInvalidImages = data.number_of_invalid_images;
has_data = data.has_data;
} catch { } catch {
return; return;
} }
@ -150,7 +153,7 @@
</form> </form>
</div> </div>
<div class="content" class:selected={isActive("create-class")}> <div class="content" class:selected={isActive("create-class")}>
<ModelTable classes={classes} /> <ModelTable {classes} {model} />
</div> </div>
<div class="content" class:selected={isActive("api")}> <div class="content" class:selected={isActive("api")}>
TODO TODO
@ -177,7 +180,7 @@
</button> </button>
</div> </div>
<div class="content" class:selected={isActive("create-class")}> <div class="content" class:selected={isActive("create-class")}>
<ModelTable classes={classes} /> <ModelTable {classes} {model} />
</div> </div>
<div class="content" class:selected={isActive("api")}> <div class="content" class:selected={isActive("api")}>
TODO TODO
@ -185,3 +188,5 @@
</Tabs> </Tabs>
{/if} {/if}
</div> </div>
<TrainModel number_of_invalid_images={numberOfInvalidImages} {model} {has_data} on:reload={() => dispatch('reload')} />

164
webpage/src/routes/models/edit/ModelTable.svelte
View File

@ -1,15 +1,28 @@
<script lang="ts" context="module">
export type Image = {
file_path: string;
mode: number;
status: number;
};
</script>
<script lang="ts"> <script lang="ts">
import Tabs from 'src/lib/Tabs.svelte'; import Tabs from 'src/lib/Tabs.svelte';
import type { Class } from './ModelData.svelte'; import type { Class } from './ModelData.svelte';
import { get } from 'src/lib/requests.svelte'; import { get } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
let selected_class: Class | undefined = $state(); let selected_class: Class | undefined = $state();
let { classes } = $props<{ classes: Class[] }>(); let { classes, model } = $props<{ classes: Class[]; model: Model }>();
function setActiveClass(c: Class, tb_fn: (name: string) => (() => void)) { let page = $state(0);
let showNext = $state(false);
let image_list = $state<Image[]>([]);
function setActiveClass(c: Class, tb_fn: (name: string) => () => void) {
selected_class = c; selected_class = c;
console.log("test", c, classes, c.name) console.log('test', c, classes, c.name);
tb_fn(c.name)(); tb_fn(c.name)();
} }
@ -17,27 +30,35 @@
selected_class = classes[0]; selected_class = classes[0];
}); });
async function getList() { async function getList() {
console.log(selected_class); console.log(selected_class);
try {
let url = new URLSearchParams();
url.append('id', selected_class?.id ?? '');
let res = await get('models/data/list?' + url.toString()); try {
console.log(res); let url = new URLSearchParams();
} catch (e) { url.append('id', selected_class?.id ?? '');
console.error("TODO notify user", e); url.append('page', `${page}`);
}
}
$effect(() => { let res = await get('models/data/list?' + url.toString());
if (selected_class) { showNext = res.showNext;
getList(); image_list = res.image_list;
}
})
console.log(image_list);
} catch (e) {
console.error('TODO notify user', e);
}
}
$effect(() => {
if (selected_class) {
page = 0;
}
});
$effect(() => {
if (selected_class) {
getList();
}
});
</script> </script>
{#if classes.length == 0} {#if classes.length == 0}
@ -56,8 +77,109 @@
</button> </button>
{/each} {/each}
</div> </div>
<div class="content selected">
<table>
<thead>
<tr>
<th> File Path </th>
<th> Mode </th>
<th>
<!-- Img -->
</th>
<th>
<!-- Status -->
</th>
</tr>
</thead>
<tbody>
{#each image_list as image}
<tr>
<td>
{#if image.file_path == 'id://'}
Managed
{:else}
{image.file_path}
{/if}
</td>
<td>
{#if image.mode == 2}
Testing
{:else}
Training
{/if}
</td>
<td class="text-center">
{#if image.file_path == 'id://'}
<img
alt=""
src="/api/savedData/{model.id}/data/{image.id}.{model.format}"
height="30px"
width="30px"
style="object-fit: contain;"
/>
{:else}
TODO img {image.file_path}
{/if}
</td>
<td class="text-center">
{#if image.status == 1}
<span class="bi bi-check-circle-fill" style="color: green"></span>
{:else}
<span class="bi bi-exclamation-triangle-fill" style="color: red"></span>
{/if}
</td>
</tr>
{/each}
</tbody>
</table>
<div class="flex justify-center align-center">
<div class="grow-1 flex justify-end align-center">
{#if page > 0}
<button on:click={() => (page -= 1)}> Prev </button>
{/if}
</div>
<div style="padding: 10px;">
{page}
</div>
<div class="grow-1 flex justify-start align-center">
{#if showNext}
<button on:click={() => (page += 1)}> Next </button>
{/if}
</div>
</div>
</div>
</Tabs> </Tabs>
{/if} {/if}
<style lang="scss"> <style lang="scss">
table {
width: 100%;
box-shadow: 0 2px 8px 1px #66666622;
border-radius: 10px;
border-collapse: collapse;
overflow: hidden;
}
table thead {
background: #60606022;
}
table tr td,
table tr th {
border-left: 1px solid #22222244;
padding: 15px;
}
table tr td:first-child,
table tr th:first-child {
border-left: none;
}
table tr td button,
table tr td .button {
padding: 5px 10px;
box-shadow: 0 2px 5px 1px #66666655;
}
</style> </style>

78
webpage/src/routes/models/edit/RunModel.svelte Normal file
View File

@ -0,0 +1,78 @@
<script lang="ts">
import { postFormData } from "src/lib/requests.svelte";
import type { Model } from "./+page.svelte";
import FileUpload from "src/lib/FileUpload.svelte";
import MessageSimple from "src/lib/MessageSimple.svelte";
let {model} = $props<{model: Model}>();
let file: File | undefined = $state();
let result: string | undefined = $state();
let run = $state(false);
let messages: MessageSimple;
async function submit() {
console.log("here", file);
if (!file) return;
messages.clear();
let form = new FormData();
form.append("id", model.id);
form.append("file", file, "file");
run = true;
try {
result = await postFormData('models/run', form);
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display("Could not run the model");
}
}
}
</script>
<form on:submit|preventDefault={submit}>
<fieldset class="file-upload" >
<label for="file">Image</label>
<div class="form-msg">
Run image through them model and get the result
</div>
<FileUpload replace_slot bind:file accept="image/*" >
<img src="/imgs/upload-icon.png" alt="" />
<span>
Upload image
</span>
<div slot="replaced-name">
<span>
Image selected
</span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={messages} />
<button>
Run
</button>
{#if run}
{#if !result}
<div class="result">
<h1>
The class was not found
</h1>
</div>
{:else}
<div>
<h1>
Result
</h1>
The image was classified as {result}
</div>
{/if}
{/if}
</form>

95
webpage/src/routes/models/edit/TrainModel.svelte Normal file
View File

@ -0,0 +1,95 @@
<script lang="ts">
import MessageSimple from 'src/lib/MessageSimple.svelte';
import type { Model } from './+page.svelte';
import { post } from 'src/lib/requests.svelte';
import { createEventDispatcher } from 'svelte';
let { number_of_invalid_images, has_data, model } = $props<{
number_of_invalid_images: number;
has_data: boolean;
model: Model;
}>();
let data = $state({
model_type: 'simple',
number_of_models: 1,
accuracy: 95
});
let submitted = $state(false);
let dispatch = createEventDispatcher<{ reload: void }>();
let messages: MessageSimple;
async function submit() {
submitted = true;
try {
await post('models/train', {
id: model.id,
...data
});
dispatch('reload');
} catch (e) {
if (e instanceof Response) {
messages.display(await e.json());
} else {
messages.display("Could not start the training of the model");
}
}
}
</script>
<form class:submitted on:submit|preventDefault={submit}>
{#if has_data}
{#if number_of_invalid_images > 0}
<p class="danger">
There are images {number_of_invalid_images} that were loaded that do not have the correct format.DeleteZip
These images will be delete when the model trains.
</p>
{/if}
<MessageSimple bind:this={messages} />
<!-- TODO expading mode -->
<fieldset>
<legend> Model Type </legend>
<div class="input-radial">
<input
id="model_type_simple"
value="simple"
name="model_type"
type="radio"
bind:group={data.model_type}
/>
<label for="model_type_simple">Simple</label><br />
<input
id="model_type_expandable"
value="expandable"
name="model_type"
bind:group={data.model_type}
type="radio"
/>
<label for="model_type_expandable">Expandable</label>
</div>
</fieldset>
<!-- TODO allow more models to be created -->
<fieldset>
<label for="number_of_models">Number of Models</label>
<input
id="number_of_models"
type="number"
name="number_of_models"
bind:value={data.number_of_models}
/>
</fieldset>
<!-- TODO to Change the acc -->
<fieldset>
<label for="accuracy">Target accuracy</label>
<input id="accuracy" type="number" name="accuracy" bind:value={data.accuracy} />
</fieldset>
<!-- TODO allow to chose the base of the model -->
<!-- TODO allow to change the shape of the model -->
<button> Train </button>
{:else}
<h2>To train the model please provide data to the model first</h2>
{/if}
</form>