fyp/logic/models/train/train_normal.go

2001 lines
51 KiB
Go

package models_train
import (
"errors"
"fmt"
"io"
"math"
"os"
"os/exec"
"path"
"runtime/debug"
"sort"
"strconv"
"strings"
"text/template"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
my_torch "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch"
modelloader "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/train/torch/modelloader"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
"github.com/goccy/go-json"
"github.com/sugarme/gotch"
"github.com/sugarme/gotch/nn"
torch "github.com/sugarme/gotch/ts"
)
const EPOCH_PER_RUN = 20
const MAX_EPOCH = 100
func shapeToSize(shape string) string {
split := strings.Split(shape, ",")
return strings.Join(split[:len(split)-1], ",")
}
func getDir() string {
dir, err := os.Getwd()
if err != nil {
panic(err)
}
return dir
}
func MakeLayer(db db.Db, def_id string, layer_order int, layer_type LayerType, shape string) (err error) {
_, err = db.Exec("insert into model_definition_layer (def_id, layer_order, layer_type, shape) values ($1, $2, $3, $4)", def_id, layer_order, layer_type, shape)
return
}
func MakeLayerExpandable(db db.Db, def_id string, layer_order int, layer_type LayerType, shape string, exp_type int) (err error) {
_, err = db.Exec("insert into model_definition_layer (def_id, layer_order, layer_type, shape, exp_type) values ($1, $2, $3, $4, $5)", def_id, layer_order, layer_type, shape, exp_type)
return
}
func setModelClassStatus(c BasePack, status ModelClassStatus, filter string, args ...any) (err error) {
_, err = c.GetDb().Exec(fmt.Sprintf("update model_classes set status=%d where %s", status, filter), args...)
return
}
func generateCvsExp(c BasePack, run_path string, model_id string, doPanic bool) (count int, err error) {
db := c.GetDb()
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
count = co.Count
if count == 0 {
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
}
if doPanic {
return 0, errors.New("No model classes available")
}
return generateCvsExp(c, run_path, model_id, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
defer data.Close()
f, err := os.Create(path.Join(run_path, "train.csv"))
if err != nil {
return
}
defer f.Close()
f.Write([]byte("Id,Index\n"))
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil {
return
}
if file_path == "id://" {
f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
return
}
func trainDefinition(c BasePack, m *BaseModel, def *Definition, in_model *my_torch.ContainerModel, classes []*ModelClass) (accuracy float64, model *my_torch.ContainerModel, err error) {
log := c.GetLogger()
db := c.GetDb()
log.Warn("About to start training definition")
model = in_model
accuracy = 0
if model == nil {
var layers []*Layer
layers, err = def.GetLayers(db, " order by layer_order asc")
if err != nil {
return
}
model = my_torch.BuildModel(layers, 0, true)
}
// TODO Make the runner provide this
// device := gotch.CudaIfAvailable()
device := gotch.CPU
result_path := path.Join(getDir(), "savedData", m.Id, "defs", def.Id)
err = os.MkdirAll(result_path, os.ModePerm)
if err != nil {
return
}
model.To(device)
defer model.To(gotch.CPU)
var ds *modelloader.Dataset
ds, err = modelloader.NewDataset(db, m, classes[0].ClassOrder, classes[len(classes)-1].ClassOrder)
if err != nil {
return
}
err = ds.To(device)
if err != nil {
return
}
opt, err := nn.DefaultAdamConfig().Build(model.Vs, 0.001)
if err != nil {
return
}
for epoch := 0; epoch < EPOCH_PER_RUN; epoch++ {
var trainIter *torch.Iter2
trainIter, err = ds.TrainIter(64)
if err != nil {
return
}
// trainIter.ToDevice(device)
log.Info("epoch", "epoch", epoch)
var trainLoss float64 = 0
var trainCorrect float64 = 0
ok := true
for ok {
var item torch.Iter2Item
var loss *torch.Tensor
item, ok = trainIter.Next()
if !ok {
continue
}
pred := model.ForwardT(item.Data, true)
// Calculate loss
loss, err = pred.BinaryCrossEntropyWithLogits(item.Label, &torch.Tensor{}, &torch.Tensor{}, 1, false)
if err != nil {
return
}
loss, err = loss.SetRequiresGrad(true, false)
if err != nil {
return
}
err = opt.ZeroGrad()
if err != nil {
return
}
err = loss.Backward()
if err != nil {
return
}
err = opt.Step()
if err != nil {
return
}
trainLoss = loss.Float64Values()[0]
// Calculate accuracy
var p_pred, p_labels *torch.Tensor
p_pred, err = pred.Argmax([]int64{1}, true, false)
if err != nil {
return
}
p_labels, err = item.Label.Argmax([]int64{1}, true, false)
if err != nil {
return
}
floats := p_pred.Float64Values()
floats_labels := p_labels.Float64Values()
for i := range floats {
if floats[i] == floats_labels[i] {
trainCorrect += 1
}
}
}
log.Info("model training epoch done loss", "loss", trainLoss, "correct", trainCorrect, "out", ds.TrainImagesSize, "accuracy", trainCorrect/float64(ds.TrainImagesSize))
/*correct := int64(0)
//torch.NoGrad(func() {
ok = true
testIter := ds.TestIter(64)
for ok {
var item torch.Iter2Item
item, ok = testIter.Next()
if !ok {
continue
}
output := model.Forward(item.Data)
var pred, labels *torch.Tensor
pred, err = output.Argmax([]int64{1}, true, false)
if err != nil {
return
}
labels, err = item.Label.Argmax([]int64{1}, true, false)
if err != nil {
return
}
floats := pred.Float64Values()
floats_labels := labels.Float64Values()
for i := range floats {
if floats[i] == floats_labels[i] {
correct += 1
}
}
}
accuracy = float64(correct) / float64(ds.TestImagesSize)
log.Info("Eval accuracy", "accuracy", accuracy)
err = def.UpdateAfterEpoch(db, accuracy*100)
if err != nil {
return
}*/
//})
}
err = my_torch.SaveModel(model, path.Join(result_path, "model.dat"))
if err != nil {
return
}
log.Info("Model finished training!", "accuracy", accuracy)
return
}
func generateCvsExpandExp(c BasePack, run_path string, model_id string, offset int, doPanic bool) (count_re int, err error) {
l, db := c.GetLogger(), c.GetDb()
var co struct {
Count int `db:"count(*)"`
}
err = GetDBOnce(db, &co, "model_classes where model_id=$1 and status=$2;", model_id, CLASS_STATUS_TRAINING)
if err != nil {
return
}
l.Info("test here", "count", co)
count_re = co.Count
count := co.Count
if count == 0 {
err = setModelClassStatus(c, CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, CLASS_STATUS_TO_TRAIN)
if err != nil {
return
} else if doPanic {
return 0, errors.New("No model classes available")
}
return generateCvsExpandExp(c, run_path, model_id, offset, true)
}
data, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINING)
if err != nil {
return
}
defer data.Close()
f, err := os.Create(path.Join(run_path, "train.csv"))
if err != nil {
return
}
defer f.Close()
f.Write([]byte("Id,Index\n"))
count = 0
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil {
return
}
if file_path == "id://" {
f.Write([]byte(id + "," + strconv.Itoa(class_order-offset) + "\n"))
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
count += 1
}
//
// This is to load some extra data so that the model has more things to train on
//
data_other, err := db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, CLASS_STATUS_TRAINED, count*10)
if err != nil {
return
}
defer data_other.Close()
for data_other.Next() {
var id string
var class_order int
var file_path string
if err = data_other.Scan(&id, &class_order, &file_path); err != nil {
return
}
if file_path == "id://" {
f.Write([]byte(id + "," + strconv.Itoa(-2) + "\n"))
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
return
}
func trainDefinitionExpandExp(c BasePack, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
accuracy = 0
l := c.GetLogger()
l.Warn("About to retrain model")
// Get untrained models heads
type ExpHead struct {
Id string
Start int `db:"range_start"`
End int `db:"range_end"`
}
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](c.GetDb(), "exp_model_head where def_id=$1 and (status = 2 or status = 3)", definition_id)
if err != nil {
return
} else if len(heads) == 0 {
log.Error("Failed to get the exp head of the model")
return
} else if len(heads) != 1 {
err = errors.New("This training function can only train one model at the time")
return
}
exp := heads[0]
l.Info("Got exp head", "head", exp)
if err = UpdateStatus(c.GetDb(), "exp_model_head", exp.Id, DEFINITION_STATUS_TRAINING); err != nil {
return
}
layers, err := c.GetDb().Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
ExpType int
LayerNum int
}
got := []layerrow{}
i := 1
var last *layerrow = nil
got_2 := false
var first *layerrow = nil
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape, &row.ExpType); err != nil {
return
}
// Keep track of the first layer so we can keep the size of the image
if first == nil {
first = &row
}
row.LayerNum = i
row.Shape = shapeToSize(row.Shape)
if row.ExpType == 2 {
if !got_2 {
got = append(got, *last)
got_2 = true
}
got = append(got, row)
}
last = &row
i += 1
}
got = append(got, layerrow{
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d", exp.End-exp.Start+1),
ExpType: 2,
LayerNum: i,
})
l.Info("Got layers", "layers", got)
// Generate run folder
run_path := path.Join("/tmp", model.Id+"-defs-"+definition_id+"-retrain")
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
return
}
classCount, err := generateCvsExpandExp(c, run_path, model.Id, exp.Start, false)
if err != nil {
return
}
l.Info("Generated cvs", "classCount", classCount)
// TODO update the run script
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
return
}
defer f.Close()
l.Info("About to run python!")
tmpl, err := template.New("python_model_template_expand.py").ParseFiles("views/py/python_model_template_expand.py")
if err != nil {
return
}
// Copy result around
result_path := path.Join("savedData", model.Id, "defs", definition_id)
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": first.Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"HeadId": exp.Id,
"RunPath": run_path,
"ColorMode": model.ImageMode,
"Model": model,
"EPOCH_PER_RUN": EPOCH_PER_RUN,
"LoadPrev": load_prev,
"BaseModel": path.Join(getDir(), result_path, "base", "model.keras"),
"LastModelRunPath": path.Join(getDir(), result_path, "head", exp.Id, "model.keras"),
"SaveModelPath": path.Join(getDir(), result_path, "head", exp.Id),
"Depth": classCount,
"StartPoint": 0,
"Host": c.GetHost(),
}); err != nil {
return
}
// Run the command
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput()
if err != nil {
l.Warn("Python failed to run", "err", err, "out", string(out))
return
}
l.Info("Python finished running")
if err = os.MkdirAll(result_path, os.ModePerm); err != nil {
return
}
accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val"))
if err != nil {
return
}
defer accuracy_file.Close()
accuracy_file_bytes, err := io.ReadAll(accuracy_file)
if err != nil {
return
}
accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64)
if err != nil {
return
}
os.RemoveAll(run_path)
l.Info("Model finished training!", "accuracy", accuracy)
return
}
func trainDefinitionExp(c BasePack, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) {
accuracy = 0
l := c.GetLogger()
db := c.GetDb()
l.Warn("About to start training definition")
// Get untrained models heads
type ExpHead struct {
Id string
Start int `db:"range_start"`
End int `db:"range_end"`
}
// status = 2 (INIT) 3 (TRAINING)
heads, err := GetDbMultitple[ExpHead](db, "exp_model_head where def_id=$1 and (status = 2 or status = 3)", definition_id)
if err != nil {
return
} else if len(heads) == 0 {
log.Error("Failed to get the exp head of the model")
return
} else if len(heads) != 1 {
log.Error("This training function can only train one model at the time")
err = errors.New("This training function can only train one model at the time")
return
}
exp := heads[0]
if err = UpdateStatus(db, "exp_model_head", exp.Id, DEFINITION_STATUS_TRAINING); err != nil {
return
}
layers, err := db.Query("select layer_type, shape, exp_type from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
ExpType int
LayerNum int
}
got := []layerrow{}
i := 1
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape, &row.ExpType); err != nil {
return
}
row.LayerNum = i
row.Shape = shapeToSize(row.Shape)
got = append(got, row)
i += 1
}
got = append(got, layerrow{
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d", exp.End-exp.Start+1),
ExpType: 2,
LayerNum: i,
})
// Generate run folder
run_path := path.Join("/tmp", model.Id+"-defs-"+definition_id)
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
return
}
classCount, err := generateCvsExp(c, run_path, model.Id, false)
if err != nil {
return
}
// TODO update the run script
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
return
}
defer f.Close()
tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
if err != nil {
return
}
// Copy result around
result_path := path.Join("savedData", model.Id, "defs", definition_id)
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"HeadId": exp.Id,
"RunPath": run_path,
"ColorMode": model.ImageMode,
"Model": model,
"EPOCH_PER_RUN": EPOCH_PER_RUN,
"LoadPrev": load_prev,
"LastModelRunPath": path.Join(getDir(), result_path, "model.keras"),
"SaveModelPath": path.Join(getDir(), result_path),
"Depth": classCount,
"StartPoint": 0,
"Host": c.GetHost(),
}); err != nil {
return
}
// Run the command
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput()
if err != nil {
l.Debug(string(out))
return
}
l.Info("Python finished running")
if err = os.MkdirAll(result_path, os.ModePerm); err != nil {
return
}
accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val"))
if err != nil {
return
}
defer accuracy_file.Close()
accuracy_file_bytes, err := io.ReadAll(accuracy_file)
if err != nil {
return
}
accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64)
if err != nil {
return
}
os.RemoveAll(run_path)
l.Info("Model finished training!", "accuracy", accuracy)
return
}
func remove[T interface{}](lst []T, i int) []T {
lng := len(lst)
if i >= lng {
return []T{}
}
if i+1 >= lng {
return lst[:lng-1]
}
if i == 0 {
return lst[1:]
}
return append(lst[:i], lst[i+1:]...)
}
type TrainModelRow struct {
id string
target_accuracy int
epoch int
acuracy float64
}
type TraingModelRowDefinitions []TrainModelRow
func (nf TraingModelRowDefinitions) Len() int { return len(nf) }
func (nf TraingModelRowDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf TraingModelRowDefinitions) Less(i, j int) bool {
return nf[i].acuracy < nf[j].acuracy
}
type ToRemoveList []int
func (nf ToRemoveList) Len() int { return len(nf) }
func (nf ToRemoveList) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf ToRemoveList) Less(i, j int) bool {
return nf[i] < nf[j]
}
func trainModel(c BasePack, model *BaseModel) (err error) {
db := c.GetDb()
log := c.GetLogger()
fail := func(err error) {
log.Error("Failed to train Model!", "err", err, "stack", string(debug.Stack()))
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
}
defs, err := model.GetDefinitions(db, "and md.status=$2", DEFINITION_STATUS_INIT)
if err != nil {
fail(err)
return
}
var definitions SortByAccuracyDefinitions = defs
if len(definitions) == 0 {
fail(errors.New("No definitons defined!"))
return
}
finished := false
models := map[string]*my_torch.ContainerModel{}
classes, err := model.GetClasses(db, " and status=$2 order by mc.class_order asc", CLASS_STATUS_TO_TRAIN)
for {
// Keep track of definitions that did not train fast enough
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
err := def.UpdateStatus(c, DEFINITION_STATUS_TRAINING)
if err != nil {
log.Error("Could not make model into training", "err", err)
def.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
accuracy, ml_model, err := trainDefinition(c, model, def, models[def.Id], classes)
if err != nil {
log.Error("Failed to train definition!Err:", "err", err)
def.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
models[def.Id] = ml_model
if accuracy >= float64(def.TargetAccuracy) {
log.Info("Found a definition that reaches target_accuracy!")
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, DEFINITION_STATUS_TRANIED, def.Epoch, def.Id)
if err != nil {
log.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return err
}
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", DEFINITION_STATUS_CANCELD_TRAINING, def.Id, model.Id, DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
log.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return err
}
finished = true
break
}
if def.Epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.TargetAccuracy)
def.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.Epoch, DEFINITION_STATUS_PAUSED_TRAINING, def.Id)
if err != nil {
log.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return err
}
}
if finished {
break
}
sort.Sort(sort.Reverse(toRemove))
log.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
// Clean up unsed models
models[definitions[n].Id] = nil
definitions = remove(definitions, n)
}
len_def := len(definitions)
if len_def == 0 {
break
}
if len_def == 1 {
continue
}
sort.Sort(sort.Reverse(definitions))
acc := definitions[0].Accuracy - 20.0
log.Info("Training models, Highest acc", "acc", definitions[0].Accuracy, "mod_acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.Accuracy < acc {
toRemove = append(toRemove, i)
}
}
log.Info("Removing due to accuracy", "toRemove", toRemove)
sort.Sort(sort.Reverse(toRemove))
for _, n := range toRemove {
log.Warn("Removing definition not fast enough learning", "n", n)
definitions[n].UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
models[definitions[n].Id] = nil
definitions = remove(definitions, n)
}
}
var def Definition
err = GetDBOnce(c, &def, "model_definition as md where md.model_id=$1 and md.status=$2 order by md.accuracy desc limit 1;", model.Id, DEFINITION_STATUS_TRANIED)
if err != nil {
if err == NotFoundError {
log.Error("All definitions failed to train!")
} else {
log.Error("DB: failed to read definition", "err", err)
}
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
if err = def.UpdateStatus(c, DEFINITION_STATUS_READY); err != nil {
log.Error("Failed to update model definition", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
to_delete, err := db.Query("select id from model_definition where status != $1 and model_id=$2", DEFINITION_STATUS_READY, model.Id)
if err != nil {
log.Error("Failed to select model_definition to delete")
log.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if err = to_delete.Scan(&id); err != nil {
log.Error("Failed to scan the id of a model_definition to delete", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", DEFINITION_STATUS_READY, model.Id); err != nil {
log.Error("Failed to delete model_definition")
log.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
ModelUpdateStatus(c, model.Id, READY)
return
}
type TrainModelRowUsable struct {
Id string
TargetAccuracy int `db:"target_accuracy"`
Epoch int
Acuracy float64 `db:"0"`
}
type TrainModelRowUsables []*TrainModelRowUsable
func (nf TrainModelRowUsables) Len() int { return len(nf) }
func (nf TrainModelRowUsables) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf TrainModelRowUsables) Less(i, j int) bool {
return nf[i].Acuracy < nf[j].Acuracy
}
func trainModelExp(c BasePack, model *BaseModel) (err error) {
l := c.GetLogger()
db := c.GetDb()
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](db, "model_definition where status=$1 and model_id=$2", DEFINITION_STATUS_INIT, model.Id)
if err != nil {
l.Error("Failed to get definitions")
return
}
if len(definitions) == 0 {
l.Error("No Definitions defined!")
return errors.New("No Definitions found")
}
firstRound := true
finished := false
for {
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
Definition{Id: def.Id}.UpdateStatus(c, DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinitionExp(c, model, def.Id, !firstRound)
if err != nil {
l.Error("Failed to train definition!Err:", "err", err)
Definition{Id: def.Id}.UpdateStatus(c, DEFINITION_STATUS_TRAINING)
toRemove = append(toRemove, i)
continue
}
def.Epoch += EPOCH_PER_RUN
accuracy = accuracy * 100
def.Acuracy = float64(accuracy)
definitions[i].Epoch += EPOCH_PER_RUN
definitions[i].Acuracy = accuracy
if accuracy >= float64(def.TargetAccuracy) {
l.Info("Found a definition that reaches target_accuracy!")
_, err = db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, DEFINITION_STATUS_TRANIED, def.Epoch, def.Id)
if err != nil {
l.Error("Failed to train definition!")
return err
}
_, err = db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", DEFINITION_STATUS_CANCELD_TRAINING, def.Id, model.Id, DEFINITION_STATUS_FAILED_TRAINING)
if err != nil {
l.Error("Failed to train definition!")
return err
}
_, err = db.Exec("update exp_model_head set status=$1 where def_id=$2;", MODEL_HEAD_STATUS_READY, def.Id)
if err != nil {
l.Error("Failed to train definition!")
return err
}
finished = true
break
}
if def.Epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.TargetAccuracy)
Definition{Id: def.Id}.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
toRemove = append(toRemove, i)
continue
}
_, err = db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.Epoch, DEFINITION_STATUS_PAUSED_TRAINING, def.Id)
if err != nil {
l.Error("Failed to train definition!")
return err
}
}
firstRound = false
if finished {
break
}
sort.Sort(sort.Reverse(toRemove))
l.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
definitions = remove(definitions, n)
}
len_def := len(definitions)
if len_def == 0 {
break
} else if len_def == 1 {
continue
}
sort.Sort(sort.Reverse(definitions))
acc := definitions[0].Acuracy - 20.0
l.Info("Training models, Highest acc", "acc", definitions[0].Acuracy, "mod_acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.Acuracy < acc {
toRemove = append(toRemove, i)
}
}
l.Info("Removing due to accuracy", "toRemove", toRemove)
sort.Sort(sort.Reverse(toRemove))
for _, n := range toRemove {
l.Warn("Removing definition not fast enough learning", "n", n)
Definition{Id: definitions[n].Id}.UpdateStatus(c, DEFINITION_STATUS_FAILED_TRAINING)
definitions = remove(definitions, n)
}
}
var dat JustId
err = GetDBOnce(db, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
// Set the class status to trained
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("All definitions failed to train! And Failed to set class status")
return err
}
l.Error("All definitions failed to train!")
return err
} else if err != nil {
l.Error("All definitions failed to train!")
return err
}
if _, err = db.Exec("update model_definition set status=$1 where id=$2;", DEFINITION_STATUS_READY, dat.Id); err != nil {
l.Error("Failed to update model definition")
return err
}
to_delete, err := GetDbMultitple[JustId](db, "model_definition where status!=$1 and model_id=$2", DEFINITION_STATUS_READY, model.Id)
if err != nil {
l.Error("Failed to select model_definition to delete")
return err
}
for _, d := range to_delete {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
// TODO Check if returning also works here
if _, err = db.Exec("delete from model_definition where status!=$1 and model_id=$2;", DEFINITION_STATUS_READY, model.Id); err != nil {
l.Error("Failed to delete model_definition")
return err
}
if err = splitModel(c, model); err != nil {
err = setModelClassStatus(c, CLASS_STATUS_TO_TRAIN, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to split the model! And Failed to set class status")
return err
}
l.Error("Failed to split the model")
return err
}
// Set the class status to trained
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
l.Error("Failed to set class status")
return err
}
// There should only be one def availabale
def := JustId{}
if err = GetDBOnce(db, &def, "model_definition where model_id=$1", model.Id); err != nil {
return
}
// Remove the base model
l.Warn("Removing base model for", "model", model.Id, "def", def.Id)
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model"))
os.RemoveAll(path.Join("savedData", model.Id, "defs", def.Id, "model.keras"))
ModelUpdateStatus(c, model.Id, READY)
return
}
func splitModel(c BasePack, model *BaseModel) (err error) {
db := c.GetDb()
l := c.GetLogger()
def := JustId{}
if err = GetDBOnce(db, &def, "model_definition where model_id=$1", model.Id); err != nil {
return
}
head := JustId{}
if err = GetDBOnce(db, &head, "exp_model_head where def_id=$1", def.Id); err != nil {
return
}
// Generate run folder
run_path := path.Join("/tmp", model.Id+"-defs-"+def.Id+"-split")
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
return
}
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
return
}
defer f.Close()
tmpl, err := template.New("python_split_model_template.py").ParseFiles("views/py/python_split_model_template.py")
if err != nil {
return
}
// Copy result around
result_path := path.Join(getDir(), "savedData", model.Id, "defs", def.Id)
// TODO maybe move this to a select count(*)
// Get only fixed lawers
layers, err := db.Query("select exp_type from model_definition_layer where def_id=$1 and exp_type=$2 order by layer_order asc;", def.Id, 1)
if err != nil {
return
}
defer layers.Close()
type layerrow struct {
ExpType int
}
count := -1
for layers.Next() {
count += 1
}
if count == -1 {
err = errors.New("Can not get layers")
return
}
log.Warn("Spliting model", "def", def.Id, "head", head.Id, "count", count)
basePath := path.Join(result_path, "base")
headPath := path.Join(result_path, "head", head.Id)
if err = os.MkdirAll(basePath, os.ModePerm); err != nil {
return
}
if err = os.MkdirAll(headPath, os.ModePerm); err != nil {
return
}
if err = tmpl.Execute(f, AnyMap{
"SplitLen": count,
"ModelPath": path.Join(result_path, "model.keras"),
"BaseModelPath": basePath,
"HeadModelPath": headPath,
}); err != nil {
return
}
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput()
if err != nil {
l.Debug(string(out))
return
}
os.RemoveAll(run_path)
l.Info("Python finished running")
return
}
// This generates a definition
func generateDefinition(c BasePack, model *BaseModel, target_accuracy int, number_of_classes int, complexity int) (err error) {
failed := func() {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
}
db := c.GetDb()
l := c.GetLogger()
def, err := MakeDefenition(db, model.Id, target_accuracy)
if err != nil {
failed()
return
}
order := 1
// Note the shape of the first layer defines the import size
//_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(model.Width, model.Height, model.ImageMode))
_, err = def.MakeLayer(db, order, LAYER_INPUT, ShapeToString(3, model.Width, model.Height))
if err != nil {
failed()
return
}
order++
if complexity == 0 {
_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
if err != nil {
failed()
return
}
order++
loop := int(math.Log2(float64(number_of_classes)))
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
order++
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
return
}
}
} else if complexity == 1 || complexity == 2 {
loop := max(1, int((math.Log(float64(model.Width)) / math.Log(float64(10)))))
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
failed()
return
}
}
_, err = def.MakeLayer(db, order, LAYER_FLATTEN, "")
if err != nil {
failed()
return
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
_, err = def.MakeLayer(db, order, LAYER_DENSE, ShapeToString(number_of_classes*(loop-i)))
order++
if err != nil {
failed()
return
}
}
} else {
l.Error("Unkown complexity", "complexity", complexity)
failed()
return
}
return def.UpdateStatus(db, DEFINITION_STATUS_INIT)
}
func generateDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
cls, err := model.GetClasses(c, "")
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
return
}
cls_len := len(cls)
if number_of_models == 1 {
if model.Width < 100 && model.Height < 100 && cls_len < 30 {
err = generateDefinition(c, model, target_accuracy, cls_len, 0)
} else if model.Width > 100 && model.Height > 100 {
err = generateDefinition(c, model, target_accuracy, cls_len, 2)
} else {
err = generateDefinition(c, model, target_accuracy, cls_len, 1)
}
if err != nil {
return
}
} else {
for i := 0; i < number_of_models; i++ {
err = generateDefinition(c, model, target_accuracy, cls_len, min(i, 2))
if err != nil {
return
}
}
}
return nil
}
func ExpModelHeadUpdateStatus(db db.Db, id string, status DefinitionStatus) (err error) {
_, err = db.Exec("update model_definition set status = $1 where id = $2", status, id)
return
}
// This generates a definition
func generateExpandableDefinition(c BasePack, model *BaseModel, target_accuracy int, number_of_classes int, complexity int) (err error) {
l := c.GetLogger()
db := c.GetDb()
l.Info("Generating expandable new definition for model", "id", model.Id, "complexity", complexity)
failed := func() {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
}
if complexity == 0 {
failed()
return
}
def, err := MakeDefenition(c.GetDb(), model.Id, target_accuracy)
if err != nil {
failed()
return
}
order := 1
width := model.Width
height := model.Height
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
l.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
}
err = MakeLayerExpandable(c.GetDb(), def.Id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height), 1)
order++
// handle the errors inside the pervious if block
if err != nil {
failed()
return
}
// Create the blocks
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
/*if model.Width < 50 && model.Height < 50 {
loop = 0
}*/
log.Info("Size of the simple block", "loop", loop)
//loop = max(loop, 3)
for i := 0; i < loop; i++ {
err = MakeLayerExpandable(db, def.Id, order, LAYER_SIMPLE_BLOCK, "", 1)
order++
if err != nil {
failed()
return
}
}
// Flatten the blocks into dense
err = MakeLayerExpandable(db, def.Id, order, LAYER_FLATTEN, "", 1)
if err != nil {
failed()
return
}
order++
// Flatten the blocks into dense
err = MakeLayerExpandable(db, def.Id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*2), 1)
if err != nil {
failed()
return
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
log.Info("Size of the dense layers", "loop", loop)
// loop = max(loop, 3)
for i := 0; i < loop; i++ {
err = MakeLayer(db, def.Id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++
if err != nil {
failed()
return
}
}
var newHead = struct {
DefId string `db:"def_id"`
RangeStart int `db:"range_start"`
RangeEnd int `db:"range_end"`
Status DefinitionStatus `db:"status"`
}{
def.Id, 0, number_of_classes - 1, DEFINITION_STATUS_INIT,
}
_, err = InsertReturnId(c.GetDb(), &newHead, "exp_model_head", "id")
if err != nil {
failed()
return
}
err = def.UpdateStatus(c, DEFINITION_STATUS_INIT)
if err != nil {
failed()
return
}
return
}
// TODO make this json friendy
func generateExpandableDefinitions(c BasePack, model *BaseModel, target_accuracy int, number_of_models int) (err error) {
cls, err := model.GetClasses(c, "")
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
return
}
cls_len := len(cls)
if number_of_models == 1 {
if model.Width > 100 && model.Height > 100 {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
} else {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
}
} else if number_of_models == 3 {
for i := 0; i < number_of_models; i++ {
generateExpandableDefinition(c, model, target_accuracy, cls_len, i)
}
} else {
// TODO handle incrisea the complexity
for i := 0; i < number_of_models; i++ {
generateExpandableDefinition(c, model, target_accuracy, cls_len, 2)
}
}
return nil
}
func ResetClasses(c BasePack, model *BaseModel) {
_, err := c.GetDb().Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TO_TRAIN, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
c.GetLogger().Error("Error while reseting the classes", "error", err)
}
}
func trainExpandable(c *Context, model *BaseModel) {
var err error = nil
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ResetClasses(c, model)
}
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", DEFINITION_STATUS_READY, model.Id)
if err != nil {
failed("Failed to get definitions")
return
}
if len(definitions) != 1 {
failed("There should only be one definition available!")
return
}
firstRound := true
def := definitions[0]
epoch := 0
for {
acc, err := trainDefinitionExp(c, model, def.Id, !firstRound)
if err != nil {
failed("Failed to train definition!")
return
}
epoch += EPOCH_PER_RUN
if float64(acc*100) >= float64(def.Acuracy) {
c.Logger.Info("Found a definition that reaches target_accuracy!")
_, err = c.Db.Exec("update exp_model_head set status=$1 where def_id=$2 and status=$3;", MODEL_HEAD_STATUS_READY, def.Id, MODEL_HEAD_STATUS_TRAINING)
if err != nil {
failed("Failed to train definition!")
return
}
break
} else if def.Epoch > MAX_EPOCH {
failed(fmt.Sprintf("Failed to train definition! Accuracy less %f < %d\n", acc*100, def.TargetAccuracy))
return
}
}
// Set the class status to trained
err = setModelClassStatus(c, CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
}
ModelUpdateStatus(c, model.Id, READY)
}
func RunTaskTrain(b BasePack, task Task) (err error) {
l := b.GetLogger()
model, err := GetBaseModel(b.GetDb(), *task.ModelId)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model information")
l.Error("Failed to get model information", "err", err)
return err
} else if model.Status != TRAINING {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model not in the correct status for training")
return errors.New("Model not in the right status")
}
task.UpdateStatusLog(b, TASK_RUNNING, "Training model")
var dat struct {
NumberOfModels int
Accuracy int
}
err = json.Unmarshal([]byte(task.ExtraTaskInfo), &dat)
if err != nil {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed to get model extra information")
}
if model.ModelType == 2 {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "TODO expandable models")
ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
panic("todo")
full_error := generateExpandableDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
l.Error("Failed to generate defintions", "err", full_error)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
} else {
full_error := generateDefinitions(b, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
l.Error("Failed to generate defintions", "err", full_error)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
return errors.New("Failed to generate definitions")
}
}
if model.ModelType == 2 {
err = trainModelExp(b, model)
} else {
err = trainModel(b, model)
}
if err != nil {
l.Error("Failed to train model", "err", err)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Failed generate model")
ModelUpdateStatus(b, model.Id, FAILED_TRAINING)
return
}
task.UpdateStatusLog(b, TASK_DONE, "Model finished training")
return
}
func RunTaskRetrain(b BasePack, task Task) (err error) {
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "TODO retrain with torch")
panic("TODO")
model, err := GetBaseModel(b.GetDb(), *task.ModelId)
if err != nil {
return err
} else if model.Status != READY_RETRAIN {
return errors.New("Model in invalid status for re-training")
}
l := b.GetLogger()
db := b.GetDb()
failed := func() {
ResetClasses(b, model)
ModelUpdateStatus(b, model.Id, READY_RETRAIN_FAILED)
task.UpdateStatusLog(b, TASK_FAILED_RUNNING, "Model failed retraining")
l.Error("Failed to retrain", "err", err)
}
task.UpdateStatusLog(b, TASK_RUNNING, "Model retraining")
var defData struct {
Id string `db:"md.id"`
TargetAcuuracy float64 `db:"md.target_accuracy"`
}
err = GetDBOnce(db, &defData, "models as m inner join model_definition as md on m.id = md.model_id where m.id=$1;", task.ModelId)
if err != nil {
failed()
return
}
var acc float64 = 0
var epocs = 0
// TODO make max epochs come from db
for acc*100 < defData.TargetAcuuracy && epocs < 20 {
// This is something I have to check
acc, err = trainDefinitionExpandExp(b, model, defData.Id, epocs > 0)
if err != nil {
failed()
return
}
l.Info("Retrained model", "accuracy", acc, "target", defData.TargetAcuuracy)
epocs += 1
}
if acc*100 < defData.TargetAcuuracy {
l.Error("Model never achived targetd accuracy", "acc", acc*100, "target", defData.TargetAcuuracy)
failed()
return
}
// TODO check accuracy
err = UpdateStatus(db, "models", model.Id, READY)
if err != nil {
failed()
return
}
l.Info("Model updaded")
_, err = db.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINED, CLASS_STATUS_TRAINING, model.Id)
if err != nil {
l.Error("Error while updating the classes", "error", err)
failed()
return
}
task.UpdateStatusLog(b, TASK_DONE, "Model finished retraining")
return
}
func handleTrain(handle *Handle) {
type TrainReq struct {
Id string `json:"id" validate:"required"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
PostAuthJson(handle, "/models/train", User_Normal, func(c *Context, dat *TrainReq) *Error {
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == NotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.E500M("Failed to get model information", err)
} else if model.CanTrain == 0 {
return c.JsonBadRequest("Model can not be trained!")
}
if model.Status != CONFIRM_PRE_TRAINING {
return c.JsonBadRequest("Model in invalid status for training")
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
return c.E500M("Failed to update model_status", err)
}
text, err := json.Marshal(struct {
NumberOfModels int
Accuracy int
}{
NumberOfModels: dat.NumberOfModels,
Accuracy: dat.Accuracy,
})
if err != nil {
return c.E500M("Failed create data", err)
}
type CreateNewTask struct {
UserId string `db:"user_id"`
ModelId string `db:"model_id"`
TaskType TaskType `db:"task_type"`
Status int `db:"status"`
ExtraTaskInfo string `db:"extra_task_info"`
}
newTask := CreateNewTask{
UserId: c.User.Id,
ModelId: model.Id,
TaskType: TASK_TYPE_TRAINING,
Status: 1,
ExtraTaskInfo: string(text),
}
id, err := InsertReturnId(c, &newTask, "tasks", "id")
if err != nil {
return c.E500M("Failed to create task", err)
}
return c.SendJSON(id)
})
PostAuthJson(handle, "/model/train/retrain", User_Normal, func(c *Context, dat *JustId) *Error {
model, err := GetBaseModel(c.Db, dat.Id)
if err == NotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.E500M("Faield to get model", err)
} else if model.Status != READY && model.Status != READY_RETRAIN_FAILED && model.Status != READY_ALTERATION_FAILED {
return c.JsonBadRequest("Model in invalid status for re-training")
} else if model.CanTrain == 0 {
return c.JsonBadRequest("Model can not be trained!")
}
c.Logger.Info("Expanding definitions for models", "id", model.Id)
classesUpdated := false
failed := func() *Error {
if classesUpdated {
ResetClasses(c, model)
}
ModelUpdateStatus(c, model.Id, READY_RETRAIN_FAILED)
return c.E500M("Failed to retrain model", err)
}
var def struct {
Id string
TargetAccuracy int `db:"target_accuracy"`
}
err = GetDBOnce(c, &def, "model_definition where model_id=$1;", model.Id)
if err != nil {
return failed()
}
type C struct {
Id string
ClassOrder int `db:"class_order"`
}
err = c.StartTx()
if err != nil {
return failed()
}
classes, err := GetDbMultitple[C](
c,
"model_classes where model_id=$1 and status=$2 order by class_order asc",
model.Id,
CLASS_STATUS_TO_TRAIN,
)
if err != nil {
_err := c.RollbackTx()
if _err != nil {
c.Logger.Error("Two errors happended rollback failed", "err", _err)
}
return failed()
}
if len(classes) == 0 {
c.Logger.Error("No classes are available!")
_err := c.RollbackTx()
if _err != nil {
c.Logger.Error("Two errors happended rollback failed", "err", _err)
}
return failed()
}
//Update the classes
{
_, err = c.Exec("update model_classes set status=$1 where status=$2 and model_id=$3", CLASS_STATUS_TRAINING, CLASS_STATUS_TO_TRAIN, model.Id)
if err != nil {
_err := c.RollbackTx()
if _err != nil {
c.Logger.Error("Two errors happended rollback failed", "err", _err)
}
return failed()
}
err = c.CommitTx()
if err != nil {
_err := c.RollbackTx()
if _err != nil {
c.Logger.Error("Two errors happended rollback failed", "err", _err)
}
return failed()
}
classesUpdated = true
}
var newHead = struct {
DefId string `db:"def_id"`
RangeStart int `db:"range_start"`
RangeEnd int `db:"range_end"`
Status DefinitionStatus `db:"status"`
}{
def.Id, classes[0].ClassOrder, classes[len(classes)-1].ClassOrder, DEFINITION_STATUS_INIT,
}
_, err = InsertReturnId(c.GetDb(), &newHead, "exp_model_head", "id")
if err != nil {
return failed()
}
_, err = c.Db.Exec("update models set status=$1 where id=$2;", READY_RETRAIN, model.Id)
if err != nil {
return c.E500M("Failed to update model status", err)
}
newTask := struct {
UserId string `db:"user_id"`
ModelId string `db:"model_id"`
TaskType TaskType `db:"task_type"`
Status int `db:"status"`
}{
UserId: c.User.Id,
ModelId: model.Id,
TaskType: TASK_TYPE_RETRAINING,
Status: 1,
}
id, err := InsertReturnId(c, &newTask, "tasks", "id")
if err != nil {
return c.E500M("Failed to create task", err)
}
return c.SendJSON(JustId{Id: id})
})
handle.Get("/model/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
model_id := f.Get("model_id")
def_id := f.Get("definition")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select md.status from model_definition as md where md.model_id=$1 and md.id=$2", model_id, def_id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
if !rows.Next() {
c.Logger.Error("Could not get status of model definition")
return c.Error500(nil)
}
var status int
err = rows.Scan(&status)
if err != nil {
return c.Error500(err)
}
if status != 3 {
c.Logger.Warn("Definition not on status 3(training)", "status", status)
return c.JsonBadRequest("Definition not on status 3(training)")
}
c.Logger.Debug("Updated model_definition!", "model", model_id, "progress", epoch, "accuracy", accuracy)
_, err = c.Db.Exec("update model_definition set epoch_progress=$1, accuracy=$2 where id=$3", epoch, accuracy, def_id)
if err != nil {
return c.Error500(err)
}
c.ShowMessage = false
return nil
})
handle.Get("/model/head/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "head_id") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
head_id := f.Get("head_id")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select hd.status from exp_model_head as hd where hd.id=$1;", head_id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
if !rows.Next() {
c.Logger.Error("Could not get status of model head")
return c.Error500(nil)
}
var status int
err = rows.Scan(&status)
if err != nil {
return c.Error500(err)
}
if status != 3 {
c.Logger.Warn("Head not on status 3(training)", "status", status)
return c.JsonBadRequest("Head not on status 3(training)")
}
c.Logger.Debug("Updated model_head!", "head", head_id, "progress", epoch, "accuracy", accuracy)
_, err = c.Db.Exec("update exp_model_head set epoch_progress=$1, accuracy=$2 where id=$3", epoch, accuracy, head_id)
if err != nil {
return c.Error500(err)
}
c.ShowMessage = false
return nil
})
}