chore: added way to add new images
This commit is contained in:
parent
4a95f0211d
commit
0d37ba8d59
@ -3,36 +3,19 @@ package model_classes
|
|||||||
import (
|
import (
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
"errors"
|
||||||
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
|
||||||
|
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ModelClass struct {
|
type ModelClass struct {
|
||||||
Id string `json:"id"`
|
Id string `json:"id"`
|
||||||
ModelId string `json:"model_id"`
|
ModelId string `json:"model_id" db:"model_id"`
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
|
Status int `json:"status"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
|
func ListClasses(c *Context, model_id string) (cls []*ModelClass, err error) {
|
||||||
|
return GetDbMultitple[ModelClass](c, "model_classes where model_id=$1", model_id)
|
||||||
rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
cls = []ModelClass{}
|
|
||||||
|
|
||||||
for rows.Next() {
|
|
||||||
var model ModelClass
|
|
||||||
err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
|
|
||||||
if err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
cls = append(cls, model)
|
|
||||||
}
|
|
||||||
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {
|
func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {
|
||||||
|
@ -151,6 +151,122 @@ func processZipFile(c *Context, model *BaseModel) {
|
|||||||
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
|
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func processZipFileExpand(c *Context, model *BaseModel) {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
failed := func(msg string) {
|
||||||
|
c.Logger.Error(msg, "err", err)
|
||||||
|
ModelUpdateStatus(c, model.Id, READY_FAILED)
|
||||||
|
}
|
||||||
|
|
||||||
|
reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip"))
|
||||||
|
if err != nil {
|
||||||
|
failed("Faield to proccess zip file failed to open reader\n")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer reader.Close()
|
||||||
|
|
||||||
|
training := []string{}
|
||||||
|
testing := []string{}
|
||||||
|
|
||||||
|
for _, file := range reader.Reader.File {
|
||||||
|
|
||||||
|
paths := strings.Split(file.Name, "/")
|
||||||
|
|
||||||
|
if paths[1] == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if paths[0] != "training" && paths[0] != "testing" {
|
||||||
|
failed(fmt.Sprintf("Invalid file '%s' TODO add msg to response!!!", file.Name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if paths[0] != "training" {
|
||||||
|
training = InsertIfNotPresent(training, paths[1])
|
||||||
|
} else if paths[0] != "testing" {
|
||||||
|
testing = InsertIfNotPresent(testing, paths[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !reflect.DeepEqual(testing, training) {
|
||||||
|
failed("testing and training are diferent")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
base_path := path.Join("savedData", model.Id, "data")
|
||||||
|
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
|
||||||
|
failed("Failed to create base_path dir")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ids := map[string]string{}
|
||||||
|
|
||||||
|
for i, name := range training {
|
||||||
|
id, err := model_classes.CreateClass(c.Db, model.Id, i, name)
|
||||||
|
if err != nil {
|
||||||
|
failed(fmt.Sprintf("Failed to create class '%s' on db\n", name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ids[name] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range reader.Reader.File {
|
||||||
|
if file.Name[len(file.Name)-1] == '/' {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := reader.Open(file.Name)
|
||||||
|
if err != nil {
|
||||||
|
failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer data.Close()
|
||||||
|
file_data, err := io.ReadAll(data)
|
||||||
|
if err != nil {
|
||||||
|
failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO check if the file is a valid photo that matched the defined photo on the database
|
||||||
|
|
||||||
|
parts := strings.Split(file.Name, "/")
|
||||||
|
|
||||||
|
mode := model_classes.DATA_POINT_MODE_TRAINING
|
||||||
|
if parts[0] == "testing" {
|
||||||
|
mode = model_classes.DATA_POINT_MODE_TESTING
|
||||||
|
}
|
||||||
|
|
||||||
|
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
|
||||||
|
if err != nil {
|
||||||
|
failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
file_path := path.Join(base_path, data_point_id+"."+model.Format)
|
||||||
|
f, err := os.Create(file_path)
|
||||||
|
if err != nil {
|
||||||
|
failed(fmt.Sprintf("Could not create file %s\n", file_path))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
f.Write(file_data)
|
||||||
|
|
||||||
|
if !testImgForModel(c, model, file_path) {
|
||||||
|
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
|
||||||
|
c.Logger.Warn("Not failling updating data point to status -1")
|
||||||
|
message := "Image did not have valid format for the model"
|
||||||
|
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
|
||||||
|
failed(fmt.Sprintf("Failed to update data point status"))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Logger.Info("Added data to model", "id", model.Id)
|
||||||
|
ModelUpdateStatus(c, model.Id, READY)
|
||||||
|
}
|
||||||
|
|
||||||
func handleDataUpload(handle *Handle) {
|
func handleDataUpload(handle *Handle) {
|
||||||
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
||||||
if !CheckAuthLevel(1, w, r, c) {
|
if !CheckAuthLevel(1, w, r, c) {
|
||||||
@ -267,6 +383,73 @@ func handleDataUpload(handle *Handle) {
|
|||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// ------
|
||||||
|
// ------ CLASS DATA UPLOAD
|
||||||
|
// ------
|
||||||
|
handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
||||||
|
if !CheckAuthLevel(1, w, r, c) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
read_form, err := r.MultipartReader()
|
||||||
|
if err != nil {
|
||||||
|
return c.JsonBadRequest("Please provide a valid form data request!")
|
||||||
|
}
|
||||||
|
|
||||||
|
var id string
|
||||||
|
var file []byte
|
||||||
|
|
||||||
|
for {
|
||||||
|
part, err_part := read_form.NextPart()
|
||||||
|
if err_part == io.EOF {
|
||||||
|
break
|
||||||
|
} else if err_part != nil {
|
||||||
|
return c.JsonBadRequest("Please provide a valid form data request!")
|
||||||
|
}
|
||||||
|
if part.FormName() == "id" {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
buf.ReadFrom(part)
|
||||||
|
id = buf.String()
|
||||||
|
}
|
||||||
|
if part.FormName() == "file" {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
buf.ReadFrom(part)
|
||||||
|
file = buf.Bytes()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Logger.Info("Trying to expand model", "id", id)
|
||||||
|
|
||||||
|
model, err := GetBaseModel(handle.Db, id)
|
||||||
|
if err == ModelNotFoundError {
|
||||||
|
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||||
|
} else if err != nil {
|
||||||
|
return Error500(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO work in allowing the model to add new in the pre ready moment
|
||||||
|
if model.Status != READY {
|
||||||
|
return c.JsonBadRequest("Model not in the correct state to add a more classes")
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO mk this path configurable
|
||||||
|
dir_path := path.Join("savedData", id)
|
||||||
|
|
||||||
|
f, err := os.Create(path.Join(dir_path, "expand_data.zip"))
|
||||||
|
if err != nil {
|
||||||
|
return Error500(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
f.Write(file)
|
||||||
|
|
||||||
|
ModelUpdateStatus(c, id, READY_ALTERATION)
|
||||||
|
|
||||||
|
go processZipFileExpand(c, model)
|
||||||
|
|
||||||
|
return c.SendJSON(model.Id)
|
||||||
|
})
|
||||||
|
|
||||||
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
||||||
if !CheckAuthLevel(1, w, r, c) {
|
if !CheckAuthLevel(1, w, r, c) {
|
||||||
return nil
|
return nil
|
||||||
@ -274,7 +457,7 @@ func handleDataUpload(handle *Handle) {
|
|||||||
if c.Mode == JSON {
|
if c.Mode == JSON {
|
||||||
|
|
||||||
type ModelData struct {
|
type ModelData struct {
|
||||||
Id string `json:"id"`
|
Id string `json:"id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var dat ModelData
|
var dat ModelData
|
||||||
@ -285,32 +468,52 @@ func handleDataUpload(handle *Handle) {
|
|||||||
|
|
||||||
model, err := GetBaseModel(handle.Db, dat.Id)
|
model, err := GetBaseModel(handle.Db, dat.Id)
|
||||||
if err == ModelNotFoundError {
|
if err == ModelNotFoundError {
|
||||||
return c.SendJSONStatus(http.StatusNotFound, "Model not found");
|
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
return Error500(err)
|
return Error500(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if model.Status != FAILED_PREPARING_ZIP_FILE {
|
delete_path := "base_data.zip"
|
||||||
return c.SendJSONStatus(http.StatusNotFound, "Model not in the correct status")
|
|
||||||
|
if model.Status == READY_FAILED {
|
||||||
|
delete_path = "expand_data.zip"
|
||||||
|
} else if model.Status != FAILED_PREPARING_ZIP_FILE {
|
||||||
|
return c.JsonBadRequest("Model not in the correct status")
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.Remove(path.Join("savedData", model.Id, "base_data.zip"))
|
err = os.Remove(path.Join("savedData", model.Id, delete_path))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Error500(err)
|
return Error500(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
|
if model.Status != READY_FAILED {
|
||||||
if err != nil {
|
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
|
||||||
return Error500(err)
|
if err != nil {
|
||||||
}
|
return Error500(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding")
|
||||||
|
}
|
||||||
|
|
||||||
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
|
if model.Status != READY_FAILED {
|
||||||
if err != nil {
|
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
|
||||||
return Error500(err)
|
if err != nil {
|
||||||
}
|
return Error500(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
|
||||||
|
if err != nil {
|
||||||
|
return Error500(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
|
if model.Status != READY_FAILED {
|
||||||
return c.SendJSON(model.Id)
|
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
|
||||||
|
} else {
|
||||||
|
ModelUpdateStatus(c, model.Id, READY)
|
||||||
|
}
|
||||||
|
|
||||||
|
return c.SendJSON(model.Id)
|
||||||
}
|
}
|
||||||
|
|
||||||
f, err := MyParseForm(r)
|
f, err := MyParseForm(r)
|
||||||
|
@ -75,7 +75,7 @@ func handleEdit(handle *Handle) {
|
|||||||
return c.Error500(err)
|
return c.Error500(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cls, err := model_classes.ListClasses(handle.Db, id)
|
cls, err := model_classes.ListClasses(c, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return c.Error500(err)
|
return c.Error500(err)
|
||||||
}
|
}
|
||||||
@ -86,9 +86,9 @@ func handleEdit(handle *Handle) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type ReturnType struct {
|
type ReturnType struct {
|
||||||
Classes []model_classes.ModelClass `json:"classes"`
|
Classes []*model_classes.ModelClass `json:"classes"`
|
||||||
HasData bool `json:"has_data"`
|
HasData bool `json:"has_data"`
|
||||||
NumberOfInvalidImages int `json:"number_of_invalid_images"`
|
NumberOfInvalidImages int `json:"number_of_invalid_images"`
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.SendJSON(ReturnType{
|
return c.SendJSON(ReturnType{
|
||||||
@ -314,7 +314,7 @@ func handleEdit(handle *Handle) {
|
|||||||
return c.Error500(err)
|
return c.Error500(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
cls, err := model_classes.ListClasses(handle.Db, id)
|
cls, err := model_classes.ListClasses(c, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return c.Error500(err)
|
return c.Error500(err)
|
||||||
}
|
}
|
||||||
|
@ -103,7 +103,6 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
|
|||||||
var predictions = results[0].Value().([][]float32)[0]
|
var predictions = results[0].Value().([][]float32)[0]
|
||||||
|
|
||||||
for i, v := range predictions {
|
for i, v := range predictions {
|
||||||
c.Logger.Info("This is test", "v", v)
|
|
||||||
if v > vmax {
|
if v > vmax {
|
||||||
order = element.Range_start + i
|
order = element.Range_start + i
|
||||||
vmax = v
|
vmax = v
|
||||||
|
@ -102,7 +102,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) {
|
func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) {
|
||||||
_, err = c.Db.Exec("update model_classes set stauts = $1 where "+filter, args...)
|
_, err = c.Db.Exec(fmt.Sprintf("update model_classes set status=%d where %s", status, filter), args...)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -485,6 +485,7 @@ func (nf ToRemoveList) Less(i, j int) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func trainModel(c *Context, model *BaseModel) {
|
func trainModel(c *Context, model *BaseModel) {
|
||||||
|
|
||||||
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
|
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
c.Logger.Error("Failed to train Model! Err:")
|
c.Logger.Error("Failed to train Model! Err:")
|
||||||
@ -783,9 +784,7 @@ func trainModelExp(c *Context, model *BaseModel) {
|
|||||||
|
|
||||||
if len_def == 0 {
|
if len_def == 0 {
|
||||||
break
|
break
|
||||||
}
|
} else if len_def == 1 {
|
||||||
|
|
||||||
if len_def == 1 {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -811,44 +810,38 @@ func trainModelExp(c *Context, model *BaseModel) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
|
// Set the class status to trained
|
||||||
|
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failed("DB: failed to read definition")
|
failed("Failed to set class status")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer rows.Close()
|
|
||||||
|
|
||||||
if !rows.Next() {
|
var dat JustId
|
||||||
|
|
||||||
|
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
|
||||||
|
if err == NotFoundError {
|
||||||
failed("All definitions failed to train!")
|
failed("All definitions failed to train!")
|
||||||
return
|
return
|
||||||
}
|
} else if err != nil {
|
||||||
|
failed("DB: failed to read definition")
|
||||||
var id string
|
|
||||||
if err = rows.Scan(&id); err != nil {
|
|
||||||
failed("Failed to read id")
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
|
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil {
|
||||||
failed("Failed to update model definition")
|
failed("Failed to update model definition")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
|
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failed("Failed to select model_definition to delete")
|
failed("Failed to select model_definition to delete")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
defer to_delete.Close()
|
|
||||||
|
|
||||||
for to_delete.Next() {
|
for _, d := range(to_delete) {
|
||||||
var id string
|
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
|
||||||
if to_delete.Scan(&id); err != nil {
|
}
|
||||||
failed("Failed to scan the id of a model_definition to delete")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
|
|
||||||
}
|
|
||||||
|
|
||||||
// TODO Check if returning also works here
|
// TODO Check if returning also works here
|
||||||
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
|
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
|
||||||
@ -863,7 +856,6 @@ func trainModelExp(c *Context, model *BaseModel) {
|
|||||||
|
|
||||||
// There should only be one def availabale
|
// There should only be one def availabale
|
||||||
def := JustId{}
|
def := JustId{}
|
||||||
|
|
||||||
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
|
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@ -1099,7 +1091,7 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
|
|||||||
}
|
}
|
||||||
|
|
||||||
func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
|
func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
|
||||||
cls, err := model_classes.ListClasses(c.Db, model.Id)
|
cls, err := model_classes.ListClasses(c, model.Id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
|
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
|
||||||
// TODO improve this response
|
// TODO improve this response
|
||||||
@ -1267,7 +1259,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
|
|||||||
|
|
||||||
// TODO make this json friendy
|
// TODO make this json friendy
|
||||||
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
|
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
|
||||||
cls, err := model_classes.ListClasses(c.Db, model.Id)
|
cls, err := model_classes.ListClasses(c, model.Id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
|
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
|
||||||
// TODO improve this response
|
// TODO improve this response
|
||||||
|
@ -28,6 +28,8 @@ const (
|
|||||||
PREPARING_ZIP_FILE = 3
|
PREPARING_ZIP_FILE = 3
|
||||||
TRAINING = 4
|
TRAINING = 4
|
||||||
READY = 5
|
READY = 5
|
||||||
|
READY_ALTERATION = 6
|
||||||
|
READY_FAILED = -6
|
||||||
)
|
)
|
||||||
|
|
||||||
type ModelDefinitionStatus int
|
type ModelDefinitionStatus int
|
||||||
@ -56,8 +58,8 @@ type ModelClassStatus int
|
|||||||
|
|
||||||
const (
|
const (
|
||||||
MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
|
MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
|
||||||
MODEL_CLASS_STATUS_TRAINING = 2
|
MODEL_CLASS_STATUS_TRAINING = 2
|
||||||
MODEL_CLASS_STATUS_TRAINED = 3
|
MODEL_CLASS_STATUS_TRAINED = 3
|
||||||
)
|
)
|
||||||
|
|
||||||
var ModelNotFoundError = errors.New("Model not found error")
|
var ModelNotFoundError = errors.New("Model not found error")
|
||||||
|
@ -1,17 +1,14 @@
|
|||||||
package models_utils
|
package models_utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
|
|
||||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||||
)
|
)
|
||||||
|
|
||||||
// TODO make this return and caller handle error
|
// TODO make this return and caller handle error
|
||||||
func ModelUpdateStatus(c *Context, id string, status int) {
|
func ModelUpdateStatus(c *Context, id string, status int) {
|
||||||
_, err := c.Db.Exec("update models set status = $1 where id = $2", status, id)
|
_, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Println("Failed to update model status")
|
c.Logger.Error("Failed to update model status", "err", err)
|
||||||
fmt.Println(err)
|
c.Logger.Warn("TODO Maybe handle better")
|
||||||
panic("TODO handle better")
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
<div class="tab-buttons">
|
<div class="tab-buttons">
|
||||||
<slot name="buttons" {setActive} {isActive} />
|
<slot name="buttons" {setActive} {isActive} />
|
||||||
</div>
|
</div>
|
||||||
<slot {isActive} />
|
<slot {isActive} {active} />
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<style lang="scss">
|
<style lang="scss">
|
||||||
@ -29,6 +29,11 @@
|
|||||||
.tab-buttons {
|
.tab-buttons {
|
||||||
display: flex;
|
display: flex;
|
||||||
overflow-x: scroll;
|
overflow-x: scroll;
|
||||||
|
width: 100%;
|
||||||
|
|
||||||
|
:global(.buttons) {
|
||||||
|
width: 100%;
|
||||||
|
}
|
||||||
|
|
||||||
:global(.tab) {
|
:global(.tab) {
|
||||||
padding: 5px;
|
padding: 5px;
|
||||||
|
@ -6,6 +6,8 @@
|
|||||||
width: number;
|
width: number;
|
||||||
height: number;
|
height: number;
|
||||||
status: number;
|
status: number;
|
||||||
|
model_type: number;
|
||||||
|
format: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type Layer = {
|
export type Layer = {
|
||||||
@ -286,9 +288,20 @@
|
|||||||
{/await}
|
{/await}
|
||||||
<!-- TODO Add ability to stop training -->
|
<!-- TODO Add ability to stop training -->
|
||||||
</div>
|
</div>
|
||||||
{:else if m.status == 5}
|
{:else if [5, 6, -6].includes(m.status)}
|
||||||
<BaseModelInfo model={m} />
|
<BaseModelInfo model={m} />
|
||||||
<RunModel model={m} />
|
<RunModel model={m} />
|
||||||
|
{#if m.status == 6}
|
||||||
|
<div class="card">
|
||||||
|
Model expading... Processing ZIP file
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{#if m.status == -6}
|
||||||
|
<DeleteZip model={m} on:reload={getModel} expand />
|
||||||
|
{/if}
|
||||||
|
{#if m.model_type == 2}
|
||||||
|
<ModelData model={m} on:reload={getModel} />
|
||||||
|
{/if}
|
||||||
<DeleteModel model={m} />
|
<DeleteModel model={m} />
|
||||||
{:else}
|
{:else}
|
||||||
<h1>Unknown Status of the model.</h1>
|
<h1>Unknown Status of the model.</h1>
|
||||||
|
@ -6,12 +6,12 @@
|
|||||||
|
|
||||||
let message: MessageSimple;
|
let message: MessageSimple;
|
||||||
|
|
||||||
let { model } = $props<{model: Model}>();
|
let { model, expand } = $props<{model: Model, expand?: boolean}>();
|
||||||
|
|
||||||
const dispatch = createEventDispatcher<{reload: void}>();
|
const dispatch = createEventDispatcher<{reload: void}>();
|
||||||
|
|
||||||
async function deleteZip() {
|
async function deleteZip() {
|
||||||
message.display("");
|
message.clear();
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await rdelete("models/data/delete-zip-file", { id: model.id });
|
await rdelete("models/data/delete-zip-file", { id: model.id });
|
||||||
@ -28,9 +28,15 @@
|
|||||||
|
|
||||||
|
|
||||||
<form on:submit|preventDefault={deleteZip}>
|
<form on:submit|preventDefault={deleteZip}>
|
||||||
Failed to proccess the zip file.<br/>
|
{#if expand}
|
||||||
Delete file and proccess again.<br/>
|
Failed to proccess the zip file.<br/>
|
||||||
<br/>
|
Delete file and upload a correct version do add more classes.<br/>
|
||||||
|
<br/>
|
||||||
|
{:else}
|
||||||
|
Failed to proccess the zip file.<br/>
|
||||||
|
Delete file and proccess again.<br/>
|
||||||
|
<br/>
|
||||||
|
{/if}
|
||||||
<div class="spacer" ></div>
|
<div class="spacer" ></div>
|
||||||
<MessageSimple bind:this={message} />
|
<MessageSimple bind:this={message} />
|
||||||
<button class="danger">
|
<button class="danger">
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
export type Class = {
|
export type Class = {
|
||||||
name: string;
|
name: string;
|
||||||
id: string;
|
id: string;
|
||||||
|
status: number;
|
||||||
}
|
}
|
||||||
</script>
|
</script>
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
@ -153,7 +154,7 @@
|
|||||||
</form>
|
</form>
|
||||||
</div>
|
</div>
|
||||||
<div class="content" class:selected={isActive("create-class")}>
|
<div class="content" class:selected={isActive("create-class")}>
|
||||||
<ModelTable {classes} {model} />
|
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
|
||||||
</div>
|
</div>
|
||||||
<div class="content" class:selected={isActive("api")}>
|
<div class="content" class:selected={isActive("api")}>
|
||||||
TODO
|
TODO
|
||||||
@ -180,7 +181,7 @@
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<div class="content" class:selected={isActive("create-class")}>
|
<div class="content" class:selected={isActive("create-class")}>
|
||||||
<ModelTable {classes} {model} />
|
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
|
||||||
</div>
|
</div>
|
||||||
<div class="content" class:selected={isActive("api")}>
|
<div class="content" class:selected={isActive("api")}>
|
||||||
TODO
|
TODO
|
||||||
|
@ -3,14 +3,20 @@
|
|||||||
file_path: string;
|
file_path: string;
|
||||||
mode: number;
|
mode: number;
|
||||||
status: number;
|
status: number;
|
||||||
|
id: string;
|
||||||
};
|
};
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<script lang="ts">
|
<script lang="ts">
|
||||||
import Tabs from 'src/lib/Tabs.svelte';
|
import Tabs from 'src/lib/Tabs.svelte';
|
||||||
import type { Class } from './ModelData.svelte';
|
import type { Class } from './ModelData.svelte';
|
||||||
import { get } from 'src/lib/requests.svelte';
|
import { get, postFormData } from 'src/lib/requests.svelte';
|
||||||
import type { Model } from './+page.svelte';
|
import type { Model } from './+page.svelte';
|
||||||
|
import FileUpload from 'src/lib/FileUpload.svelte';
|
||||||
|
import MessageSimple from 'src/lib/MessageSimple.svelte';
|
||||||
|
import { createEventDispatcher } from 'svelte';
|
||||||
|
|
||||||
|
const dispatch = createEventDispatcher<{reload: void}>();
|
||||||
|
|
||||||
let selected_class: Class | undefined = $state();
|
let selected_class: Class | undefined = $state();
|
||||||
|
|
||||||
@ -22,7 +28,6 @@
|
|||||||
|
|
||||||
function setActiveClass(c: Class, tb_fn: (name: string) => () => void) {
|
function setActiveClass(c: Class, tb_fn: (name: string) => () => void) {
|
||||||
selected_class = c;
|
selected_class = c;
|
||||||
console.log('test', c, classes, c.name);
|
|
||||||
tb_fn(c.name)();
|
tb_fn(c.name)();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -31,8 +36,6 @@
|
|||||||
});
|
});
|
||||||
|
|
||||||
async function getList() {
|
async function getList() {
|
||||||
console.log(selected_class);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let url = new URLSearchParams();
|
let url = new URLSearchParams();
|
||||||
url.append('id', selected_class?.id ?? '');
|
url.append('id', selected_class?.id ?? '');
|
||||||
@ -59,101 +62,231 @@
|
|||||||
getList();
|
getList();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let file: File | undefined = $state();
|
||||||
|
let uploadImage: MessageSimple;
|
||||||
|
let uploading = $state(Promise.resolve());
|
||||||
|
|
||||||
|
async function uploadZip() {
|
||||||
|
uploadImage.clear();
|
||||||
|
if (!file) return;
|
||||||
|
|
||||||
|
uploading = new Promise(() => {});
|
||||||
|
|
||||||
|
let form = new FormData();
|
||||||
|
form.append('id', model.id);
|
||||||
|
form.append('file', file, 'upload.zip');
|
||||||
|
|
||||||
|
try {
|
||||||
|
await postFormData('models/data/class/upload', form);
|
||||||
|
dispatch('reload');
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof Response) {
|
||||||
|
uploadImage.display(await e.json());
|
||||||
|
} else {
|
||||||
|
uploadImage.display('');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
uploading = Promise.resolve();
|
||||||
|
}
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
{#if classes.length == 0}
|
{#if classes.length == 0}
|
||||||
TODO CREATE TABLE
|
TODO CREATE TABLE
|
||||||
{:else}
|
{:else}
|
||||||
<Tabs active={classes[0]?.name} let:isActive>
|
<Tabs active={classes[0]?.name} let:isActive>
|
||||||
<div slot="buttons" let:setActive let:isActive>
|
<div class="buttons" slot="buttons" let:setActive let:isActive>
|
||||||
<!-- TODO Auto Load 1st -->
|
<!-- TODO Auto Load 1st -->
|
||||||
{#each classes as item}
|
<div>
|
||||||
<button
|
{#each classes as item}
|
||||||
on:click={() => setActiveClass(item, setActive)}
|
<button
|
||||||
class="tab"
|
on:click={() => setActiveClass(item, setActive)}
|
||||||
class:selected={isActive(item.name)}
|
class="tab"
|
||||||
>
|
class:selected={isActive(item.name)}
|
||||||
{item.name}
|
>
|
||||||
</button>
|
{item.name}
|
||||||
{/each}
|
</button>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
<button on:click={() => {
|
||||||
|
setActive("-----New Class-----")();
|
||||||
|
selected_class = undefined;
|
||||||
|
}}>
|
||||||
|
<span class="bi bi-plus" />
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<div class="content selected">
|
{#if selected_class == undefined && isActive('-----New Class-----')}
|
||||||
<table>
|
<div class="content selected">
|
||||||
<thead>
|
<h2>
|
||||||
<tr>
|
Add New Class
|
||||||
<th> File Path </th>
|
</h2>
|
||||||
<th> Mode </th>
|
<form on:submit|preventDefault={uploadZip}>
|
||||||
<th>
|
<fieldset class="file-upload" >
|
||||||
<!-- Img -->
|
<label for="file">Data file</label>
|
||||||
</th>
|
<div class="form-msg">
|
||||||
<th>
|
Please provide a file that has the training and testing data<br/>
|
||||||
<!-- Status -->
|
The file must have 2 folders one with testing images and one with training images. <br/>
|
||||||
</th>
|
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
|
||||||
</tr>
|
The class folders must have the images for the classes.
|
||||||
</thead>
|
<pre>
|
||||||
<tbody>
|
training\
|
||||||
{#each image_list as image}
|
class1\
|
||||||
<tr>
|
img1.png
|
||||||
<td>
|
img2.png
|
||||||
{#if image.file_path == 'id://'}
|
img2.png
|
||||||
Managed
|
...
|
||||||
{:else}
|
class2\
|
||||||
{image.file_path}
|
img1.png
|
||||||
{/if}
|
img2.png
|
||||||
</td>
|
img2.png
|
||||||
<td>
|
...
|
||||||
{#if image.mode == 2}
|
...
|
||||||
Testing
|
testing\
|
||||||
{:else}
|
class1\
|
||||||
Training
|
img1.png
|
||||||
{/if}
|
img2.png
|
||||||
</td>
|
img2.png
|
||||||
<td class="text-center">
|
...
|
||||||
{#if image.file_path == 'id://'}
|
class2\
|
||||||
<img
|
img1.png
|
||||||
alt=""
|
img2.png
|
||||||
src="/api/savedData/{model.id}/data/{image.id}.{model.format}"
|
img2.png
|
||||||
height="30px"
|
...
|
||||||
width="30px"
|
...
|
||||||
style="object-fit: contain;"
|
</pre>
|
||||||
/>
|
</div>
|
||||||
{:else}
|
<FileUpload replace_slot bind:file={file} accept="application/zip" notExpand >
|
||||||
TODO img {image.file_path}
|
<img src="/imgs/upload-icon.png" alt="" />
|
||||||
{/if}
|
<span>
|
||||||
</td>
|
Upload Zip File
|
||||||
<td class="text-center">
|
</span>
|
||||||
{#if image.status == 1}
|
<div slot="replaced" style="display: inline;">
|
||||||
<span class="bi bi-check-circle-fill" style="color: green"></span>
|
<img src="/imgs/upload-icon.png" alt="" />
|
||||||
{:else}
|
<span>
|
||||||
<span class="bi bi-exclamation-triangle-fill" style="color: red"></span>
|
File selected
|
||||||
{/if}
|
</span>
|
||||||
</td>
|
</div>
|
||||||
</tr>
|
</FileUpload>
|
||||||
{/each}
|
</fieldset>
|
||||||
</tbody>
|
<MessageSimple bind:this={uploadImage} />
|
||||||
</table>
|
{#await uploading}
|
||||||
<div class="flex justify-center align-center">
|
<button disabled>
|
||||||
<div class="grow-1 flex justify-end align-center">
|
Uploading
|
||||||
{#if page > 0}
|
</button>
|
||||||
<button on:click={() => (page -= 1)}> Prev </button>
|
{:then}
|
||||||
{/if}
|
<button>
|
||||||
</div>
|
Add
|
||||||
|
</button>
|
||||||
|
{/await}
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{#if selected_class}
|
||||||
|
<div class="content selected">
|
||||||
|
{#if model.model_type == 2}
|
||||||
|
{#if selected_class?.status == 1}
|
||||||
|
<h2>
|
||||||
|
Class to train
|
||||||
|
</h2>
|
||||||
|
{:else if selected_class?.status == 2}
|
||||||
|
<h2>
|
||||||
|
Class training
|
||||||
|
</h2>
|
||||||
|
{:else if selected_class?.status == 3}
|
||||||
|
<h2>
|
||||||
|
Class trained
|
||||||
|
</h2>
|
||||||
|
{/if}
|
||||||
|
{/if}
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th> File Path </th>
|
||||||
|
<th> Mode </th>
|
||||||
|
<th>
|
||||||
|
<!-- Img -->
|
||||||
|
</th>
|
||||||
|
<th>
|
||||||
|
<!-- Status -->
|
||||||
|
</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{#each image_list as image}
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
{#if image.file_path == 'id://'}
|
||||||
|
Managed
|
||||||
|
{:else}
|
||||||
|
{image.file_path}
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{#if image.mode == 2}
|
||||||
|
Testing
|
||||||
|
{:else}
|
||||||
|
Training
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
<td class="text-center">
|
||||||
|
{#if image.file_path == 'id://'}
|
||||||
|
<img
|
||||||
|
alt=""
|
||||||
|
src="/api/savedData/{model.id}/data/{image.id}.{model.format}"
|
||||||
|
height="30px"
|
||||||
|
width="30px"
|
||||||
|
style="object-fit: contain;"
|
||||||
|
/>
|
||||||
|
{:else}
|
||||||
|
TODO img {image.file_path}
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
<td class="text-center">
|
||||||
|
{#if image.status == 1}
|
||||||
|
<span class="bi bi-check-circle-fill" style="color: green"></span>
|
||||||
|
{:else}
|
||||||
|
<span class="bi bi-exclamation-triangle-fill" style="color: red"></span>
|
||||||
|
{/if}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<div class="flex justify-center align-center">
|
||||||
|
<div class="grow-1 flex justify-end align-center">
|
||||||
|
{#if page > 0}
|
||||||
|
<button on:click={() => (page -= 1)}> Prev </button>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
<div style="padding: 10px;">
|
<div style="padding: 10px;">
|
||||||
{page}
|
{page}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="grow-1 flex justify-start align-center">
|
<div class="grow-1 flex justify-start align-center">
|
||||||
{#if showNext}
|
{#if showNext}
|
||||||
<button on:click={() => (page += 1)}> Next </button>
|
<button on:click={() => (page += 1)}> Next </button>
|
||||||
{/if}
|
{/if}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
{/if}
|
||||||
</Tabs>
|
</Tabs>
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
<style lang="scss">
|
<style lang="scss">
|
||||||
|
|
||||||
|
.buttons {
|
||||||
|
width: 100%;
|
||||||
|
display: flex;
|
||||||
|
justify-content: space-between;
|
||||||
|
|
||||||
|
&>button {
|
||||||
|
margin: 3px 5px;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
table {
|
table {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
box-shadow: 0 2px 8px 1px #66666622;
|
box-shadow: 0 2px 8px 1px #66666622;
|
||||||
|
Loading…
Reference in New Issue
Block a user