sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: added way to add new images

Browse Source
This commit is contained in:
Andre Henriques
2024-03-09 09:41:16 +00:00
parent 4a95f0211d
commit 0d37ba8d59
12 changed files with 510 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
logic/models/classes/main.go
View File

@@ -3,36 +3,19 @@ package model_classes
import (
"database/sql"
"errors"
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
type ModelClass struct {
Id string `json:"id"`
ModelId string `json:"model_id"`
ModelId string `json:"model_id" db:"model_id"`
Name string `json:"name"`
Status int `json:"status"`
}
func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
if err != nil {
return
}
defer rows.Close()
cls = []ModelClass{}
for rows.Next() {
var model ModelClass
err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
if err != nil {
return
}
cls = append(cls, model)
}
return
func ListClasses(c *Context, model_id string) (cls []*ModelClass, err error) {
return GetDbMultitple[ModelClass](c, "model_classes where model_id=$1", model_id)
}
func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {

233
logic/models/data.go
View File

@@ -151,6 +151,122 @@ func processZipFile(c *Context, model *BaseModel) {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
}
func processZipFileExpand(c *Context, model *BaseModel) {
var err error
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, READY_FAILED)
}
reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip"))
if err != nil {
failed("Faield to proccess zip file failed to open reader\n")
return
}
defer reader.Close()
training := []string{}
testing := []string{}
for _, file := range reader.Reader.File {
paths := strings.Split(file.Name, "/")
if paths[1] == "" {
continue
}
if paths[0] != "training" && paths[0] != "testing" {
failed(fmt.Sprintf("Invalid file '%s' TODO add msg to response!!!", file.Name))
return
}
if paths[0] != "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
failed("testing and training are diferent")
return
}
base_path := path.Join("savedData", model.Id, "data")
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
failed("Failed to create base_path dir")
return
}
ids := map[string]string{}
for i, name := range training {
id, err := model_classes.CreateClass(c.Db, model.Id, i, name)
if err != nil {
failed(fmt.Sprintf("Failed to create class '%s' on db\n", name))
return
}
ids[name] = id
}
for _, file := range reader.Reader.File {
if file.Name[len(file.Name)-1] == '/' {
continue
}
data, err := reader.Open(file.Name)
if err != nil {
failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name))
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name))
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
if err != nil {
failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id))
return
}
file_path := path.Join(base_path, data_point_id+"."+model.Format)
f, err := os.Create(file_path)
if err != nil {
failed(fmt.Sprintf("Could not create file %s\n", file_path))
return
}
defer f.Close()
f.Write(file_data)
if !testImgForModel(c, model, file_path) {
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
failed(fmt.Sprintf("Failed to update data point status"))
return
}
}
}
c.Logger.Info("Added data to model", "id", model.Id)
ModelUpdateStatus(c, model.Id, READY)
}
func handleDataUpload(handle *Handle) {
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
@@ -267,6 +383,73 @@ func handleDataUpload(handle *Handle) {
return nil
})
// ------
// ------ CLASS DATA UPLOAD
// ------
handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
read_form, err := r.MultipartReader()
if err != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
c.Logger.Info("Trying to expand model", "id", id)
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "expand_data.zip"))
if err != nil {
return Error500(err)
}
defer f.Close()
f.Write(file)
ModelUpdateStatus(c, id, READY_ALTERATION)
go processZipFileExpand(c, model)
return c.SendJSON(model.Id)
})
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
@@ -274,7 +457,7 @@ func handleDataUpload(handle *Handle) {
if c.Mode == JSON {
type ModelData struct {
Id string `json:"id"`
Id string `json:"id"`
}
var dat ModelData
@@ -285,32 +468,52 @@ func handleDataUpload(handle *Handle) {
model, err := GetBaseModel(handle.Db, dat.Id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found");
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.SendJSONStatus(http.StatusNotFound, "Model not in the correct status")
delete_path := "base_data.zip"
if model.Status == READY_FAILED {
delete_path = "expand_data.zip"
} else if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.JsonBadRequest("Model not in the correct status")
}
err = os.Remove(path.Join("savedData", model.Id, "base_data.zip"))
err = os.Remove(path.Join("savedData", model.Id, delete_path))
if err != nil {
return Error500(err)
}
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
if err != nil {
return Error500(err)
}
if model.Status != READY_FAILED {
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
if err != nil {
return Error500(err)
}
} else {
c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding")
}
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
if err != nil {
return Error500(err)
}
if model.Status != READY_FAILED {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
if err != nil {
return Error500(err)
}
} else {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
if err != nil {
return Error500(err)
}
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
return c.SendJSON(model.Id)
if model.Status != READY_FAILED {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
} else {
ModelUpdateStatus(c, model.Id, READY)
}
return c.SendJSON(model.Id)
}
f, err := MyParseForm(r)

10
logic/models/edit.go
View File

@@ -75,7 +75,7 @@ func handleEdit(handle *Handle) {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(handle.Db, id)
cls, err := model_classes.ListClasses(c, id)
if err != nil {
return c.Error500(err)
}
@@ -86,9 +86,9 @@ func handleEdit(handle *Handle) {
}
type ReturnType struct {
Classes []model_classes.ModelClass `json:"classes"`
HasData bool `json:"has_data"`
NumberOfInvalidImages int `json:"number_of_invalid_images"`
Classes []*model_classes.ModelClass `json:"classes"`
HasData bool `json:"has_data"`
NumberOfInvalidImages int `json:"number_of_invalid_images"`
}
return c.SendJSON(ReturnType{
@@ -314,7 +314,7 @@ func handleEdit(handle *Handle) {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(handle.Db, id)
cls, err := model_classes.ListClasses(c, id)
if err != nil {
return c.Error500(err)
}

1
logic/models/run.go
View File

@@ -103,7 +103,6 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
var predictions = results[0].Value().([][]float32)[0]
for i, v := range predictions {
c.Logger.Info("This is test", "v", v)
if v > vmax {
order = element.Range_start + i
vmax = v

52
logic/models/train/train.go
View File

@@ -102,7 +102,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e
}
func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) {
_, err = c.Db.Exec("update model_classes set stauts = $1 where "+filter, args...)
_, err = c.Db.Exec(fmt.Sprintf("update model_classes set status=%d where %s", status, filter), args...)
return
}
@@ -485,6 +485,7 @@ func (nf ToRemoveList) Less(i, j int) bool {
}
func trainModel(c *Context, model *BaseModel) {
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
c.Logger.Error("Failed to train Model! Err:")
@@ -783,9 +784,7 @@ func trainModelExp(c *Context, model *BaseModel) {
if len_def == 0 {
break
}
if len_def == 1 {
} else if len_def == 1 {
continue
}
@@ -811,44 +810,38 @@ func trainModelExp(c *Context, model *BaseModel) {
}
}
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
failed("DB: failed to read definition")
failed("Failed to set class status")
return
}
defer rows.Close()
var dat JustId
if !rows.Next() {
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
failed("All definitions failed to train!")
return
}
var id string
if err = rows.Scan(&id); err != nil {
failed("Failed to read id")
} else if err != nil {
failed("DB: failed to read definition")
return
}
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil {
failed("Failed to update model definition")
return
}
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
failed("Failed to select model_definition to delete")
return
}
defer to_delete.Close()
}
for to_delete.Next() {
var id string
if to_delete.Scan(&id); err != nil {
failed("Failed to scan the id of a model_definition to delete")
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
for _, d := range(to_delete) {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
@@ -863,7 +856,6 @@ func trainModelExp(c *Context, model *BaseModel) {
// There should only be one def availabale
def := JustId{}
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
return
}
@@ -1099,7 +1091,7 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
}
func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
@@ -1267,7 +1259,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
// TODO make this json friendy
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response

6
logic/models/utils/types.go
View File

@@ -28,6 +28,8 @@ const (
PREPARING_ZIP_FILE = 3
TRAINING = 4
READY = 5
READY_ALTERATION = 6
READY_FAILED = -6
)
type ModelDefinitionStatus int
@@ -56,8 +58,8 @@ type ModelClassStatus int
const (
MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
MODEL_CLASS_STATUS_TRAINING = 2
MODEL_CLASS_STATUS_TRAINED = 3
MODEL_CLASS_STATUS_TRAINING = 2
MODEL_CLASS_STATUS_TRAINED = 3
)
var ModelNotFoundError = errors.New("Model not found error")

9
logic/models/utils/utils.go
View File

@@ -1,17 +1,14 @@
package models_utils
import (
"fmt"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
// TODO make this return and caller handle error
func ModelUpdateStatus(c *Context, id string, status int) {
_, err := c.Db.Exec("update models set status = $1 where id = $2", status, id)
_, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
panic("TODO handle better")
c.Logger.Error("Failed to update model status", "err", err)
c.Logger.Warn("TODO Maybe handle better")
}
}