sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: removed non svelte front page

Browse Source
This commit is contained in:
Andre Henriques
2024-03-09 10:52:08 +00:00
parent 0d37ba8d59
commit 274d7d22aa
32 changed files with 614 additions and 3695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
logic/models/train/train.go
View File

@@ -6,12 +6,12 @@ import (
"fmt"
"io"
"math"
"net/http"
"os"
"os/exec"
"path"
"sort"
"strconv"
"strings"
"text/template"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
@@ -23,6 +23,19 @@ import (
const EPOCH_PER_RUN = 20
const MAX_EPOCH = 100
func shapeToSize(shape string) string {
split := strings.Split(shape, ",")
return strings.Join(split[:len(split)-1], ",")
}
func getDir() string {
dir, err := os.Getwd()
if err != nil {
panic(err)
}
return dir
}
func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) {
id = ""
rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
@@ -702,13 +715,13 @@ func trainModelExp(c *Context, model *BaseModel) {
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
}
var definitions TrainModelRowUsables
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
failed("Failed to get definitions");
return
}
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
failed("Failed to get definitions")
return
}
if len(definitions) == 0 {
failed("No Definitions defined!")
return
@@ -810,38 +823,38 @@ func trainModelExp(c *Context, model *BaseModel) {
}
}
// Set the class status to trained
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
}
var dat JustId
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
var dat JustId
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
failed("All definitions failed to train!")
return
} else if err != nil {
} else if err != nil {
failed("DB: failed to read definition")
return
}
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil {
failed("Failed to update model definition")
return
}
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
failed("Failed to select model_definition to delete")
return
}
}
for _, d := range(to_delete) {
for _, d := range to_delete {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
@@ -1293,163 +1306,81 @@ func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy
return nil
}
func handle_models_train_json(w http.ResponseWriter, r *http.Request, c *Context) *Error {
var dat struct {
Id string `json:"id"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
if err_ := c.ToJSON(r, &dat); err_ != nil {
return err_
}
if dat.Id == "" {
return c.JsonBadRequest("Please provide a id")
}
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.Error500(err)
}
if model.Status != CONFIRM_PRE_TRAINING {
return c.JsonBadRequest("Model in invalid status for training")
}
if modelTypeId == 2 {
full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
}
if modelTypeId == 2 {
go trainModelExp(c, model)
} else {
go trainModel(c, model)
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
}
return c.SendJSON(model.Id)
}
func handleTrain(handle *Handle) {
handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Post("/models/train", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
return handle_models_train_json(w, r, c)
var dat struct {
Id string `json:"id"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
r.ParseForm()
f := r.Form
number_of_models := 0
accuracy := 0
if !CheckId(f, "id") || CheckEmpty(f, "model_type") || !CheckNumber(f, "number_of_models", &number_of_models) || !CheckNumber(f, "accuracy", &accuracy) {
// TODO improve this response
return ErrorCode(nil, 400, c.AddMap(nil))
if err_ := c.ToJSON(&dat); err_ != nil {
return err_
}
id := f.Get("id")
model_type_id := 1
model_type_form := f.Get("model_type")
if model_type_form == "expandable" {
model_type_id = 2
} else if model_type_form != "simple" {
return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{
"HasData": true,
"ErrorMessage": "Invalid model type!",
})
if dat.Id == "" {
return c.JsonBadRequest("Please provide a id")
}
model, err := GetBaseModel(handle.Db, id)
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, c.AddMap(AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
}))
return c.JsonBadRequest("Model not found")
} else if err != nil {
// TODO improve this response
return Error500(err)
return c.Error500(err)
}
if model.Status != CONFIRM_PRE_TRAINING {
// TODO improve this response
return ErrorCode(nil, 400, c.AddMap(nil))
return c.JsonBadRequest("Model in invalid status for training")
}
if model_type_id == 2 {
full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models)
if modelTypeId == 2 {
full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, accuracy, number_of_models)
full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
}
if model_type_id == 2 {
if modelTypeId == 2 {
go trainModelExp(c, model)
} else {
go trainModel(c, model)
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id)
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
return c.Error500(err)
}
Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil
return c.SendJSON(model.Id)
})
handle.Get("/model/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
// TODO check auth level
if c.Mode != NORMAL {
// This should only handle normal requests
c.Logger.Warn("This function only works with normal")
return c.UnsafeErrorCode(nil, 400, nil)
}
f := r.URL.Query()
handle.Get("/model/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or definition or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
@@ -1458,9 +1389,7 @@ func handleTrain(handle *Handle) {
def_id := f.Get("definition")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
c.Logger.Warn("Epoch is not a number")
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select md.status from model_definition as md where md.model_id=$1 and md.id=$2", model_id, def_id)
@@ -1482,8 +1411,7 @@ func handleTrain(handle *Handle) {
if status != 3 {
c.Logger.Warn("Definition not on status 3(training)", "status", status)
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Definition not on status 3(training)")
}
c.Logger.Info("Updated model_definition!", "model", model_id, "progress", epoch, "accuracy", accuracy)
@@ -1495,21 +1423,13 @@ func handleTrain(handle *Handle) {
return nil
})
handle.Get("/model/head/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
// TODO check auth level
if c.Mode != NORMAL {
// This should only handle normal requests
c.Logger.Warn("This function only works with normal")
return c.UnsafeErrorCode(nil, 400, nil)
}
f := r.URL.Query()
handle.Get("/model/head/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "head_id") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or head_id or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
@@ -1517,9 +1437,7 @@ func handleTrain(handle *Handle) {
head_id := f.Get("head_id")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
c.Logger.Warn("Epoch is not a number")
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select hd.status from exp_model_head as hd where hd.id=$1;", head_id)
@@ -1541,8 +1459,7 @@ func handleTrain(handle *Handle) {
if status != 3 {
c.Logger.Warn("Head not on status 3(training)", "status", status)
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Head not on status 3(training)")
}
c.Logger.Info("Updated model_head!", "head", head_id, "progress", epoch, "accuracy", accuracy)