sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: update to go packages versions

Browse Source
This commit is contained in:
Andre Henriques
2024-01-31 21:48:35 +00:00
parent c00bd87cf7
commit d08a0a2a4c
8 changed files with 503 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
logic/models/classes/list.go
View File

@@ -1,7 +1,6 @@
package model_classes
import (
"fmt"
"net/http"
"strconv"
@@ -54,7 +53,7 @@ func HandleList(handle *Handle) {
return Error500(err)
}
rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 10 offset $2;", id, page * 10)
rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 11 offset $2;", id, page * 10)
if err != nil {
return Error500(err)
}
@@ -77,31 +76,16 @@ func HandleList(handle *Handle) {
}
got = append(got, nrow)
}
max_len := min(11, len(got))
rows_count, err := handle.Db.Query("select count(*) from model_data_point where class_id=$1;", id)
if err != nil {
return Error500(err)
}
defer rows_count.Close()
if !rows_count.Next() {
fmt.Printf("select count(*) from model_data_point where class_id='%s';\n", id)
return Error500(nil)
}
count := 0
err = rows_count.Scan(&count)
if err != nil {
return Error500(err)
}
LoadDefineTemplate(w, "/models/edit.html", "data-model-create-class-table-table", c.AddMap(AnyMap{
"List": got,
"Count": count,
"List": got[0:max_len],
"Page": page,
"Id": id,
"Name": name,
"Model": model,
"ShowNext": len(got) == 11,
}))
return nil
})

456
logic/models/train/train.go
View File

@@ -46,6 +46,11 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
return
}
func MakeLayerExpandable(db *sql.DB, def_id string, layer_order int, layer_type LayerType, shape string, exp_type int) (err error) {
_, err = db.Exec("insert into model_definition_layer (def_id, layer_order, layer_type, shape, exp_type) values ($1, $2, $3, $4, $5)", def_id, layer_order, layer_type, shape, exp_type)
return
}
func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
@@ -427,6 +432,205 @@ func trainModel(c *Context, model *BaseModel) {
ModelUpdateStatus(c, model.Id, READY)
}
func trainModelExp(c *Context, model *BaseModel) {
var err error = nil
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
}
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
failed("Failed to trainModel!")
return
}
defer definitionsRows.Close()
var definitions TraingModelRowDefinitions = []TrainModelRow{}
for definitionsRows.Next() {
var rowv TrainModelRow
rowv.acuracy = 0
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
definitions = append(definitions, rowv)
}
if len(definitions) == 0 {
c.Logger.Error("No Definitions defined!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
firstRound := true
finished := false
for {
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinition(c, model, def.id, !firstRound)
if err != nil {
c.Logger.Error("Failed to train definition!Err:", "err", err)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
def.epoch += EPOCH_PER_RUN
accuracy = accuracy * 100
def.acuracy = float64(accuracy)
definitions[i].epoch += EPOCH_PER_RUN
definitions[i].acuracy = accuracy
if accuracy >= float64(def.target_accuracy) {
c.Logger.Info("Found a definition that reaches target_accuracy!")
_, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
_, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
finished = true
break
}
if def.epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.target_accuracy)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
_, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
firstRound = false
if finished {
break
}
sort.Sort(sort.Reverse(toRemove))
c.Logger.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
definitions = remove(definitions, n)
}
len_def := len(definitions)
if len_def == 0 {
break
}
if len_def == 1 {
continue
}
sort.Sort(sort.Reverse(definitions))
acc := definitions[0].acuracy - 20.0
c.Logger.Info("Training models, Highest acc", "acc", definitions[0].acuracy, "mod_acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.acuracy < acc {
toRemove = append(toRemove, i)
}
}
c.Logger.Info("Removing due to accuracy", "toRemove", toRemove)
sort.Sort(sort.Reverse(toRemove))
for _, n := range toRemove {
c.Logger.Warn("Removing definition not fast enough learning", "n", n)
ModelDefinitionUpdateStatus(c, definitions[n].id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
definitions = remove(definitions, n)
}
}
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err != nil {
c.Logger.Error("DB: failed to read definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
defer rows.Close()
if !rows.Next() {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
var id string
if err = rows.Scan(&id); err != nil {
c.Logger.Error("Failed to read id:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
c.Logger.Error("Failed to update model definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.Logger.Error("Failed to select model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if to_delete.Scan(&id); err != nil {
c.Logger.Error("Failed to scan the id of a model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
c.Logger.Error("Failed to delete model_definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
ModelUpdateStatus(c, model.Id, READY)
}
func removeFailedDataPoints(c *Context, model *BaseModel) (err error) {
rows, err := c.Db.Query("select mdp.id from model_data_point as mdp join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model.Id)
if err != nil {
@@ -511,7 +715,7 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
}
}
} else if complexity == 1 {
} else if complexity == 1 || complexity == 2 {
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if loop == 0 {
@@ -542,39 +746,6 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
return failed()
}
}
} else if complexity == 2 {
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
return failed()
}
}
err = MakeLayer(c.Db, def_id, order, LAYER_FLATTEN, "")
if err != nil {
return failed()
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++
if err != nil {
return failed()
}
}
} else {
c.Logger.Error("Unkown complexity", "complexity", complexity)
return failed()
@@ -625,6 +796,163 @@ func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, numb
return nil
}
func CreateExpModelHead(c *Context, def_id string, range_start int, range_end int, status ModelDefinitionStatus) (id string, err error) {
rows, err := c.Db.Query("insert into exp_model_head (def_id, range_start, range_end) values ($1, $2, $3, $4) returning id", def_id, range_start, range_end, status)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
c.Logger.Error("Could not get status of model definition")
err = errors.New("Could not get status of model definition")
return
}
err = rows.Scan(&id)
if err != nil {
return
}
return
}
func ExpModelHeadUpdateStatus(db *sql.DB, id string, status ModelDefinitionStatus) (err error) {
_, err = db.Exec("update model_definition set status = $1 where id = $2", status, id)
return
}
// This generates a definition
func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy int, number_of_classes int, complexity int) *Error {
var err error = nil
failed := func() *Error {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
return c.Error500(err)
}
if complexity == 0 {
return failed()
}
def_id, err := MakeDefenition(c.Db, model.Id, target_accuracy)
if err != nil {
return failed()
}
order := 1
width := model.Width
height := model.Height
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
c.Logger.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
}
err = MakeLayerExpandable(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1)
order++
// handle the errors inside the pervious if block
if err != nil {
return failed()
}
// Create the blocks
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayerExpandable(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "", 1)
order++
if err != nil {
return failed()
}
}
// Flatten the blocks into dense
err = MakeLayerExpandable(c.Db, def_id, order, LAYER_FLATTEN, "", 1)
if err != nil {
return failed()
}
order++
// Flatten the blocks into dense
err = MakeLayerExpandable(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes * 2), 1)
if err != nil {
return failed()
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++
if err != nil {
return failed()
}
}
_, err = CreateExpModelHead(c, def_id, 0, number_of_classes - 1, MODEL_DEFINITION_STATUS_INIT)
if err != nil {
return failed()
}
err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
if err != nil {
return failed()
}
return nil
}
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id)
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
return c.Error500(err)
}
err = removeFailedDataPoints(c, model)
if err != nil {
return c.Error500(err)
}
cls_len := len(cls)
if number_of_models == 1 {
if model.Width > 100 && model.Height > 100 {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
} else {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
}
} else if number_of_models == 3 {
for i := 0; i < number_of_models; i++ {
generateExpandableDefinition(c, model, target_accuracy, cls_len, i)
}
} else {
// TODO handle incrisea the complexity
for i := 0; i < number_of_models; i++ {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 1)
}
}
return nil
}
func handleTrain(handle *Handle) {
handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
@@ -641,35 +969,27 @@ func handleTrain(handle *Handle) {
accuracy := 0
if !CheckId(f, "id") || CheckEmpty(f, "model_type") || !CheckNumber(f, "number_of_models", &number_of_models) || !CheckNumber(f, "accuracy", &accuracy) {
fmt.Println(
!CheckId(f, "id"), CheckEmpty(f, "model_type"), !CheckNumber(f, "number_of_models", &number_of_models), !CheckNumber(f, "accuracy", &accuracy),
)
// TODO improve this response
return ErrorCode(nil, 400, c.AddMap(nil))
}
id := f.Get("id")
model_type := f.Get("model_type")
// Its not used rn
_ = model_type
model_type_id := 1
model_type_form := f.Get("model_type")
// TODO check if the model has data
/*rows, err := handle.Db.Query("select mc.name, mdp.file_path from model_classes as mc join model_data_point as mdp on mdp.class_id = mc.id where mdp.model_mode = 1 and mc.model_id = $1 limit 1;", id)
if err != nil {
return Error500(err)
if model_type_form == "expandable" {
model_type_id = 2
c.Logger.Warn("TODO: handle expandable")
return c.Error400(nil, "TODO: handle expandable!", w, "/models/edit.html", "train-model-card", AnyMap{
"HasData": true,
"ErrorMessage": "TODO: handle expandable!",
})
} else if model_type_form != "simple" {
return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{
"HasData": true,
"ErrorMessage": "Invalid model type!",
})
}
defer rows.Close()
if !rows.Next() {
return Error500(err)
}
var name string
var file_path string
err = rows.Scan(&name, &file_path)
if err != nil {
return Error500(err)
}*/
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
@@ -687,14 +1007,28 @@ func handleTrain(handle *Handle) {
return ErrorCode(nil, 400, c.AddMap(nil))
}
full_error := generateDefinitions(c, model, accuracy, number_of_models)
if full_error != nil {
return full_error
}
if model_type_id == 2 {
full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, accuracy, number_of_models)
if full_error != nil {
return full_error
}
}
go trainModel(c, model)
ModelUpdateStatus(c, model.Id, TRAINING)
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
}
Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil
})