sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: did more clean up

Browse Source
This commit is contained in:
Andre Henriques
2024-04-14 15:19:32 +01:00
parent 8b13afba48
commit e7eeccd09e
20 changed files with 283 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
logic/db_types/classes.go Normal file
View File

@@ -0,0 +1,8 @@
package dbtypes
type DATA_POINT_MODE int
const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)

10
logic/models/utils/types.go → logic/db_types/types.go
View File

@@ -1,4 +1,4 @@
package models_utils
package dbtypes
import (
"database/sql"
@@ -100,10 +100,10 @@ func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
}
func (m BaseModel) CanEval() bool {
if m.Status != READY && m.Status != READY_RETRAIN && m.Status != READY_RETRAIN_FAILED && m.Status != READY_ALTERATION && m.Status != READY_ALTERATION_FAILED {
return false
}
return true
if m.Status != READY && m.Status != READY_RETRAIN && m.Status != READY_RETRAIN_FAILED && m.Status != READY_ALTERATION && m.Status != READY_ALTERATION_FAILED {
return false
}
return true
}
func StringToImageMode(colorMode string) int {

2
logic/models/add.go
View File

@@ -11,7 +11,7 @@ import (
"os"
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

6
logic/models/classes/data_point.go
View File

@@ -3,6 +3,8 @@ package model_classes
import (
"database/sql"
"errors"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
)
var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")
@@ -23,6 +25,6 @@ func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT
}
func UpdateDataPointStatus(db *sql.DB, data_point_id string, status int, message *string) (err error) {
_, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id)
return
_, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id)
return
}

49
logic/models/classes/list.go
View File

@@ -1,43 +1,26 @@
package model_classes
import (
"strconv"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func HandleList(handle *Handle) {
handle.Get("/models/data/list", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
id, err := GetIdFromUrl(c, "id")
if err != nil {
return c.JsonBadRequest("Model Class not found!")
}
page := 0
if c.R.URL.Query().Has("page") {
page_url := c.R.URL.Query().Get("page")
page_url_number, err := strconv.Atoi(page_url)
if err != nil {
return c.JsonBadRequest("Page is not a number")
}
page = page_url_number
}
type DataList struct {
Id string `json:"id" validate:"required"`
Page int `json:"page"`
}
PostAuthJson(handle, "/models/data/list", User_Normal, func(c *Context, dat *DataList) *Error {
var class_row struct {
Name string
Model_id string
}
err = GetDBOnce(c, &class_row, "model_classes where id=$1", id)
err := GetDBOnce(c, &class_row, "model_classes where id=$1", dat.Id)
if err == NotFoundError {
return c.JsonBadRequest("Model Class not found!")
} else if err != nil {
return c.Error500(err)
return c.E500M("Failed to get classes", err)
}
type baserow struct {
@@ -47,23 +30,21 @@ func HandleList(handle *Handle) {
Status int `json:"status"`
}
rows, err := GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10)
rows, err := GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", dat.Id, dat.Page*10)
if err != nil {
return c.Error500(err)
}
type ReturnType struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
return c.E500M("Failed to get classes", err)
}
max_len := min(11, len(rows))
c.ShowMessage = false
return c.SendJSON(ReturnType{
return c.SendJSON(struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
}{
ImageList: rows[0:max_len],
Page: page,
Page: dat.Page,
ShowNext: len(rows) == 11,
})
})

8
logic/models/classes/types.go
View File

@@ -1,8 +0,0 @@
package model_classes
type DATA_POINT_MODE int
const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)

5
logic/models/data.go
View File

@@ -14,7 +14,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@@ -68,9 +67,9 @@ func fileProcessor(
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
mode := DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
mode = DATA_POINT_MODE_TESTING
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)

26
logic/models/delete.go
View File

@@ -6,7 +6,6 @@ import (
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@@ -14,44 +13,35 @@ func deleteModelJSON(c *Context, id string) *Error {
c.Logger.Warnf("Removing model with id: %s", id)
_, err := c.Db.Exec("delete from models where id=$1;", id)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to delete models", err)
}
model_path := path.Join("./savedData", id)
c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path)
err = os.RemoveAll(model_path)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to remove data", err)
}
return c.SendJSON(id)
}
func handleDelete(handle *Handle) {
handle.Delete("/models/delete", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
var dat struct {
Id string `json:"id" validate:"required"`
Name *string `json:"name,omitempty"`
}
if err_ := c.ToJSON(&dat); err_ != nil {
return err_
}
type DeleteModel struct {
Id string `json:"id" validate:"required"`
Name *string `json:"name,omitempty"`
}
DeleteAuthJson(handle, "/models/delete", User_Normal, func(c *Context, dat *DeleteModel) *Error {
var model struct {
Id string
Name string
Status int
}
err := GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found!")
} else if err != nil {
return c.Error500(err)
return c.E500M("Faield to get model", err)
}
switch model.Status {

1
logic/models/edit.go
View File

@@ -5,7 +5,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

14
logic/models/list.go
View File

@@ -6,15 +6,7 @@ import (
)
// Auth level set when path is definied as 1
func handleStats(c *Context) *Error {
var b struct {
Id string `json:"id" validate:"required"`
}
if _err := c.ToJSON(&b); _err != nil {
return _err
}
func handleStats(c *Context, b *JustId) *Error {
type Row struct {
Name string `db:"mc.name" json:"name"`
Training string `db:"count(mdp.id) filter (where mdp.model_mode=1)" json:"training"`
@@ -23,7 +15,7 @@ func handleStats(c *Context) *Error {
rows, err := GetDbMultitple[Row](c, "model_data_point as mdp inner join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 group by mc.name order by mc.name asc;", b.Id)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to get stats", err)
}
c.ShowMessage = false
@@ -50,5 +42,5 @@ func handleList(handle *Handle) {
return c.SendJSON(got)
})
handle.PostAuth("/models/class/stats", 1, handleStats)
PostAuthJson(handle, "/models/class/stats", User_Normal, handleStats)
}

1
logic/models/run.go
View File

@@ -6,7 +6,6 @@ import (
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
tf "github.com/galeone/tensorflow/tensorflow/go"

2
logic/models/test.go
View File

@@ -6,7 +6,7 @@ import (
_ "image/png"
"os"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

21
logic/models/train/reset.go
View File

@@ -4,29 +4,17 @@ import (
"os"
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleRest(handle *Handle) {
handle.Delete("/models/train/reset", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
var dat struct {
Id string `json:"id"`
}
if err := c.ToJSON(&dat); err != nil {
return err
}
DeleteAuthJson(handle, "/models/train/reset", User_Normal, func(c *Context, dat *JustId) *Error {
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
// TODO improve response
return c.Error500(err)
return c.E500M("Failed to get model", err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
@@ -37,8 +25,7 @@ func handleRest(handle *Handle) {
_, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil {
// TODO improve response
return c.Error500(err)
return c.E500M("Failed to delete model", err)
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)

9
logic/models/train/train.go
View File

@@ -16,7 +16,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
)
@@ -80,7 +79,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e
}
count = co.Count
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2;", model_id, model_classes.DATA_POINT_MODE_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2;", model_id, DATA_POINT_MODE_TRAINING)
if err != nil {
return
}
@@ -139,7 +138,7 @@ func generateCvsExp(c *Context, run_path string, model_id string, doPanic bool)
return generateCvsExp(c, run_path, model_id, true)
}
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
return
}
@@ -303,7 +302,7 @@ func generateCvsExpandExp(c *Context, run_path string, model_id string, offset i
return generateCvsExpandExp(c, run_path, model_id, offset, true)
}
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
return
}
@@ -337,7 +336,7 @@ func generateCvsExpandExp(c *Context, run_path string, model_id string, offset i
// This is to load some extra data so that the model has more things to train on
//
data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
if err != nil {
return
}

31
logic/tasks/handleUpload.go
View File

@@ -9,7 +9,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@@ -48,7 +47,7 @@ func handleUpload(handler *Handle) {
ModelId string `json:"id" validate:"required"`
}
_err := c.ParseJson(&requestData, json_data)
_err := c.ParseJson(&requestData, json_data)
if _err != nil {
return _err
}
@@ -88,9 +87,9 @@ func handleUpload(handler *Handle) {
}
id, err := InsertReturnId(c, &newTask, "tasks", "id")
if err != nil {
if err != nil {
return c.E500M("Error 500", err)
}
}
save_path := path.Join("savedData", model.Id, "tasks")
os.MkdirAll(save_path, os.ModePerm)
@@ -99,26 +98,28 @@ func handleUpload(handler *Handle) {
img_file, err := os.Create(img_path)
if err != nil {
if _err := UpdateTaskStatus(c,id, -1, "Failed to create the file"); _err != nil {
c.Logger.Error("Failed to update tasks")
}
if _err := UpdateTaskStatus(c, id, -1, "Failed to create the file"); _err != nil {
c.Logger.Error("Failed to update tasks")
}
return c.E500M("Failed to create the file", err)
}
defer img_file.Close()
img_file.Write(file)
if !TestImgForModel(c, model, img_path) {
if _err := UpdateTaskStatus(c, id, -1, "The provided image is not a valid image for this model"); _err != nil {
c.Logger.Error("Failed to update tasks")
}
if _err := UpdateTaskStatus(c, id, -1, "The provided image is not a valid image for this model"); _err != nil {
c.Logger.Error("Failed to update tasks")
}
return c.JsonBadRequest(struct {
Message string `json:"message"`
Id string `json:"task_id"`
} { "Provided image does not match the model", id})
Message string `json:"message"`
Id string `json:"task_id"`
}{"Provided image does not match the model", id})
}
UpdateStatus(c, "tasks", id, 1)
UpdateStatus(c, "tasks", id, 1)
return c.SendJSON(struct {Id string `json:"id"`}{id})
return c.SendJSON(struct {
Id string `json:"id"`
}{id})
})
}

1
logic/tasks/list.go
View File

@@ -3,7 +3,6 @@ package tasks
import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

21
logic/utils/handler.go
View File

@@ -11,7 +11,6 @@ import (
"time"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
"github.com/charmbracelet/log"
"github.com/go-playground/validator/v10"
"github.com/goccy/go-json"
@@ -67,7 +66,7 @@ func (x *Handle) Get(path string, fn func(c *Context) *Error) {
x.gets = append(x.gets, HandleFunc{path, fn})
}
func (x *Handle) GetAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) GetAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@@ -80,7 +79,7 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
x.posts = append(x.posts, HandleFunc{path, fn})
}
func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) PostAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@@ -92,7 +91,7 @@ func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error
func PostAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(int(authLevel)) {
if !c.CheckAuthLevel(authLevel) {
return nil
}
@@ -112,7 +111,7 @@ func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
x.deletes = append(x.deletes, HandleFunc{path, fn})
}
func (x *Handle) DeleteAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) DeleteAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@@ -122,7 +121,7 @@ func (x *Handle) DeleteAuth(path string, authLevel int, fn func(c *Context) *Err
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel int, fn func(c *Context, obj *T) *Error) {
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@@ -160,13 +159,13 @@ func handleLoop(array []HandleFunc, context *Context) {
handleError(&Error{404, "Endpoint not found"}, context)
}
func (c *Context) CheckAuthLevel(authLevel int) bool {
func (c *Context) CheckAuthLevel(authLevel dbtypes.UserType) bool {
if authLevel > 0 {
if c.User == nil {
contextlessLogoff(c.Writer)
return false
}
if c.User.UserType < authLevel {
if c.User.UserType < int(authLevel) {
c.Writer.WriteHeader(http.StatusUnauthorized)
e := c.SendJSON("Not Authorized")
if e != nil {
@@ -321,15 +320,15 @@ func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
return c.SendJSONStatus(http.StatusBadRequest, dat)
}
func (c *Context) GetModelFromId(id_path string) (*BaseModel, *Error) {
func (c *Context) GetModelFromId(id_path string) (*dbtypes.BaseModel, *Error) {
id, err := GetIdFromUrl(c, id_path)
if err != nil {
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
}
model, err := GetBaseModel(c.Db, id)
if err == ModelNotFoundError {
model, err := dbtypes.GetBaseModel(c.Db, id)
if err == dbtypes.ModelNotFoundError {
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return nil, c.Error500(err)