andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: started working #36 and closes #12

Browse Source
This commit is contained in:
Andre Henriques 2023-10-06 12:13:19 +01:00
parent 16a6ae844b
commit 8d5f2a829a
10 changed files with 299 additions and 290 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
logic/models/add.go
View File

@ -4,38 +4,41 @@ import (
"bytes" "bytes"
"fmt" "fmt"
"image" "image"
_ "image/png"
"image/color" "image/color"
_ "image/jpeg"
_ "image/png"
"io" "io"
"net/http" "net/http"
"os" "os"
"path" "path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
) )
func loadBaseImage(handle *Handle, id string) { func loadBaseImage(c *Context, id string) {
// TODO handle more types than png // TODO handle more types than png
infile, err := os.Open(path.Join("savedData", id, "baseimage.png")) infile, err := os.Open(path.Join("savedData", id, "baseimage.png"))
if err != nil { if err != nil {
// TODO better logging c.Logger.Errorf("Failed to read image for model with id %s\n", id)
fmt.Println(err) c.Logger.Error(err)
fmt.Printf("Failed to read image for model with id %s\n", id) ModelUpdateStatus(c, id, FAILED_PREPARING)
ModelUpdateStatus(handle, id, -1)
return return
} }
defer infile.Close() defer infile.Close()
src, format, err := image.Decode(infile) src, format, err := image.Decode(infile)
if err != nil { if err != nil {
// TODO better logging c.Logger.Errorf("Failed to decode image for model with id %s\n", id)
fmt.Println(err) c.Logger.Error(err)
fmt.Printf("Failed to load image for model with id %s\n", id) ModelUpdateStatus(c, id, FAILED_PREPARING)
ModelUpdateStatus(handle, id, -1)
return return
} }
if format != "png" { switch format {
case "png":
case "jpeg":
break
default:
// TODO better logging // TODO better logging
fmt.Printf("Found unkown format '%s'\n", format) fmt.Printf("Found unkown format '%s'\n", format)
panic("Handle diferent files than .png") panic("Handle diferent files than .png")
@ -51,6 +54,8 @@ func loadBaseImage(handle *Handle, id string) {
fallthrough fallthrough
case color.GrayModel: case color.GrayModel:
model_color = "greyscale" model_color = "greyscale"
case color.YCbCrModel:
model_color = "rgb"
default: default:
fmt.Println("Do not know how to handle this color model") fmt.Println("Do not know how to handle this color model")
@ -58,8 +63,6 @@ func loadBaseImage(handle *Handle, id string) {
fmt.Println("Color is rgb") fmt.Println("Color is rgb")
} else if src.ColorModel() == color.NRGBAModel { } else if src.ColorModel() == color.NRGBAModel {
fmt.Println("Color is nrgb") fmt.Println("Color is nrgb")
} else if src.ColorModel() == color.YCbCrModel {
fmt.Println("Color is ycbcr")
} else if src.ColorModel() == color.AlphaModel { } else if src.ColorModel() == color.AlphaModel {
fmt.Println("Color is alpha") fmt.Println("Color is alpha")
} else if src.ColorModel() == color.CMYKModel { } else if src.ColorModel() == color.CMYKModel {
@ -68,31 +71,30 @@ func loadBaseImage(handle *Handle, id string) {
fmt.Println("Other so assuming color") fmt.Println("Other so assuming color")
} }
ModelUpdateStatus(handle, id, -1) ModelUpdateStatus(c, id, -1)
return return
} }
// Note: this also updates the status to 2 // Note: this also updates the status to 2
_, err = handle.Db.Exec("update models set width=$1, height=$2, color_mode=$3, status=$4 where id=$5", width, height, model_color, CONFIRM_PRE_TRAINING, id) _, err = c.Db.Exec("update models set width=$1, height=$2, color_mode=$3, format=$4, status=$5 where id=$6", width, height, model_color, format, CONFIRM_PRE_TRAINING, id)
if err != nil { if err != nil {
// TODO better logging c.Logger.Error("Could not update model")
fmt.Println(err) c.Logger.Error(err)
fmt.Printf("Could not update model\n") ModelUpdateStatus(c, id, -1)
ModelUpdateStatus(handle, id, -1)
return return
} }
} }
func handleAdd(handle *Handle) { func handleAdd(handle *Handle) {
handle.GetHTML("/models/add", AnswerTemplate("models/add.html", nil, 1)) handle.GetHTML("/models/add", AnswerTemplate("models/add.html", nil, 1))
// TODO json
handle.Post("/models/add", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Post("/models/add", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if c.Mode == JSON {
panic("TODO JSON")
}
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil
} }
if c.Mode == JSON {
// TODO json
panic("TODO JSON")
}
read_form, err := r.MultipartReader() read_form, err := r.MultipartReader()
if err != nil { if err != nil {
@ -176,7 +178,7 @@ handle.GetHTML("/models/add", AnswerTemplate("models/add.html", nil, 1))
f.Write(file) f.Write(file)
fmt.Printf("Created model with id %s! Started to proccess image!\n", id) fmt.Printf("Created model with id %s! Started to proccess image!\n", id)
go loadBaseImage(handle, id) go loadBaseImage(c, id)
Redirect("/models/edit?id="+id, c.Mode, w, r) Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil return nil

318
logic/models/data.go
View File

@ -13,139 +13,145 @@ import (
"strings" "strings"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
) )
func InsertIfNotPresent(ss []string, s string) []string { func InsertIfNotPresent(ss []string, s string) []string {
i := sort.SearchStrings(ss, s) i := sort.SearchStrings(ss, s)
if len(ss) > i && ss[i] == s { if len(ss) > i && ss[i] == s {
return ss return ss
} }
ss = append(ss, "") ss = append(ss, "")
copy(ss[i+1:], ss[i:]) copy(ss[i+1:], ss[i:])
ss[i] = s ss[i] = s
return ss return ss
} }
func processZipFile(handle *Handle, model_id string) { func processZipFile(c *Context, model *BaseModel) {
reader, err := zip.OpenReader(path.Join("savedData", model_id, "base_data.zip")) reader, err := zip.OpenReader(path.Join("savedData", model.Id, "base_data.zip"))
if err != nil { if err != nil {
// TODO add msg to error // TODO add msg to error
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Faield to proccess zip file failed to open reader\n") fmt.Printf("Faield to proccess zip file failed to open reader\n")
fmt.Println(err) fmt.Println(err)
return return
} }
defer reader.Close() defer reader.Close()
training := []string{} training := []string{}
testing := []string{} testing := []string{}
for _, file := range reader.Reader.File { for _, file := range reader.Reader.File {
paths := strings.Split(file.Name, "/") paths := strings.Split(file.Name, "/")
if paths[1] == "" { if paths[1] == "" {
continue continue
}
if paths[0] != "training" && paths[0] != "testing" {
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
if paths[0] != "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
fmt.Printf("testing and training are diferent\n")
fmt.Println(testing)
fmt.Println(training)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
base_path := path.Join("savedData", model_id, "data")
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
fmt.Printf("Failed to create base_path dir\n")
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
ids := map[string]string{}
for i, name := range training {
id, err := model_classes.CreateClass(handle.Db, model_id, i, name)
if err != nil {
fmt.Printf("Failed to create class '%s' on db\n", name)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
ids[name] = id
}
for _, file := range reader.Reader.File {
if file.Name[len(file.Name) - 1] == '/' {
continue
}
data, err := reader.Open(file.Name)
if err != nil {
fmt.Printf("Could not open file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
} }
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
fmt.Printf("Could not read file file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database if paths[0] != "training" && paths[0] != "testing" {
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
parts := strings.Split(file.Name, "/") if paths[0] != "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
mode := model_classes.DATA_POINT_MODE_TRAINING if !reflect.DeepEqual(testing, training) {
if parts[0] == "testing" { fmt.Printf("testing and training are diferent\n")
mode = model_classes.DATA_POINT_MODE_TESTING fmt.Println(testing)
} fmt.Println(training)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
data_point_id, err := model_classes.AddDataPoint(handle.Db, ids[parts[1]], "id://", mode) base_path := path.Join("savedData", model.Id, "data")
if err != nil { if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
fmt.Printf("Failed to add data point for %s\n", model_id) fmt.Printf("Failed to create base_path dir\n")
fmt.Println(err) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE) return
return }
}
file_path := path.Join(base_path, data_point_id + ".png") ids := map[string]string{}
for i, name := range training {
id, err := model_classes.CreateClass(c.Db, model.Id, i, name)
if err != nil {
fmt.Printf("Failed to create class '%s' on db\n", name)
ModelUpdateStatus(c, id, FAILED_PREPARING_ZIP_FILE)
return
}
ids[name] = id
}
for _, file := range reader.Reader.File {
if file.Name[len(file.Name)-1] == '/' {
continue
}
data, err := reader.Open(file.Name)
if err != nil {
fmt.Printf("Could not open file in zip %s\n", file.Name)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
fmt.Printf("Could not read file file in zip %s\n", file.Name)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
if err != nil {
fmt.Printf("Failed to add data point for %s\n", model.Id)
fmt.Println(err)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
file_path := path.Join(base_path, data_point_id+".png")
f, err := os.Create(file_path) f, err := os.Create(file_path)
if err != nil { if err != nil {
fmt.Printf("Could not create file %s\n", file_path) fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return return
} }
defer f.Close() defer f.Close()
f.Write(file_data) f.Write(file_data)
}
fmt.Printf("Added data to model '%s'!\n", model_id) if !testImgForModel(c, model, file_path) {
ModelUpdateStatus(handle, model_id, CONFIRM_PRE_TRAINING) c.Logger.Errorf("Image did not have valid format for model %s\n", file_path)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
}
}
fmt.Printf("Added data to model '%s'!\n", model.Id)
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
} }
func handleDataUpload(handle *Handle) { func handleDataUpload(handle *Handle) {
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil
} }
if c.Mode == JSON { if c.Mode == JSON {
// TODO improve message // TODO improve message
return ErrorCode(nil, 400, nil) return ErrorCode(nil, 400, nil)
@ -179,15 +185,15 @@ func handleDataUpload(handle *Handle) {
} }
} }
_, err = GetBaseModel(handle.Db, id) model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError { if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{ return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found", "NotFoundMessage": "Model not found",
"GoBackLink": "/models", "GoBackLink": "/models",
}) })
} else if err != nil { } else if err != nil {
return Error500(err) return Error500(err)
} }
// TODO mk this path configurable // TODO mk this path configurable
dir_path := path.Join("savedData", id) dir_path := path.Join("savedData", id)
@ -200,65 +206,65 @@ func handleDataUpload(handle *Handle) {
f.Write(file) f.Write(file)
ModelUpdateStatus(handle, id, PREPARING_ZIP_FILE) ModelUpdateStatus(c, id, PREPARING_ZIP_FILE)
go processZipFile(handle, id) go processZipFile(c, model)
Redirect("/models/edit?id="+id, c.Mode, w, r) Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil return nil
}) })
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) { if !CheckAuthLevel(1, w, r, c) {
return nil return nil
} }
if c.Mode == JSON { if c.Mode == JSON {
panic("Handle delete zip file json") panic("Handle delete zip file json")
} }
f, err := MyParseForm(r) f, err := MyParseForm(r)
if err != nil { if err != nil {
return ErrorCode(err, 400, c.AddMap(nil)) return ErrorCode(err, 400, c.AddMap(nil))
} }
if !CheckId(f, "id") { if !CheckId(f, "id") {
return ErrorCode(err, 400, c.AddMap(nil)) return ErrorCode(err, 400, c.AddMap(nil))
} }
id := f.Get("id") id := f.Get("id")
model, err := GetBaseModel(handle.Db, id) model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError { if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{ return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found", "NotFoundMessage": "Model not found",
"GoBackLink": "/models", "GoBackLink": "/models",
}) })
} else if err != nil { } else if err != nil {
return Error500(err) return Error500(err)
} }
if model.Status != FAILED_PREPARING_ZIP_FILE { if model.Status != FAILED_PREPARING_ZIP_FILE {
// TODO add message // TODO add message
return ErrorCode(nil, 400, c.AddMap(nil)) return ErrorCode(nil, 400, c.AddMap(nil))
} }
err = os.Remove(path.Join("savedData", id, "base_data.zip")); err = os.Remove(path.Join("savedData", id, "base_data.zip"))
if err != nil { if err != nil {
return Error500(err) return Error500(err)
} }
err = os.RemoveAll(path.Join("savedData", id, "data")); err = os.RemoveAll(path.Join("savedData", id, "data"))
if err != nil { if err != nil {
return Error500(err) return Error500(err)
} }
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
if err != nil { if err != nil {
return Error500(err) return Error500(err)
} }
ModelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING) ModelUpdateStatus(c, id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id="+id, c.Mode, w, r) Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil return nil
}) })
} }

11
logic/models/delete.go
View File

@ -81,15 +81,14 @@ func handleDelete(handle *Handle) {
} }
switch model.Status { switch model.Status {
case FAILED_TRAINING: case FAILED_TRAINING: fallthrough
fallthrough case FAILED_PREPARING_ZIP_FILE: fallthrough
case FAILED_PREPARING_TRAINING: case FAILED_PREPARING_TRAINING: fallthrough
fallthrough
case FAILED_PREPARING: case FAILED_PREPARING:
deleteModel(handle, id, w, c, model) deleteModel(handle, id, w, c, model)
return nil return nil
case READY:
fallthrough case READY: fallthrough
case CONFIRM_PRE_TRAINING: case CONFIRM_PRE_TRAINING:
if CheckEmpty(f, "name") { if CheckEmpty(f, "name") {
return c.Error400(nil, "Name is empty", w, "/models/edit.html", "delete-model-card", AnyMap{ return c.Error400(nil, "Name is empty", w, "/models/edit.html", "delete-model-card", AnyMap{

16
logic/models/test.go
View File

@ -25,10 +25,6 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
c.Logger.Errorf("Failed to decode image for model with id %s\nErr:%s", model.Id, err) c.Logger.Errorf("Failed to decode image for model with id %s\nErr:%s", model.Id, err)
return return
} }
if format != "png" {
c.Logger.Errorf("Found unkown format '%s' while testing an image\n", format)
return
}
var model_color string var model_color string
@ -36,10 +32,11 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
width, height := bounds.Max.X, bounds.Max.Y width, height := bounds.Max.X, bounds.Max.Y
switch src.ColorModel() { switch src.ColorModel() {
case color.Gray16Model: case color.Gray16Model: fallthrough
fallthrough
case color.GrayModel: case color.GrayModel:
model_color = "greyscale" model_color = "greyscale"
case color.YCbCrModel:
model_color = "rgb"
default: default:
c.Logger.Error("Do not know how to handle this color model") c.Logger.Error("Do not know how to handle this color model")
@ -47,8 +44,6 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
c.Logger.Info("Color is rgb") c.Logger.Info("Color is rgb")
} else if src.ColorModel() == color.NRGBAModel { } else if src.ColorModel() == color.NRGBAModel {
c.Logger.Info("Color is nrgb") c.Logger.Info("Color is nrgb")
} else if src.ColorModel() == color.YCbCrModel {
c.Logger.Info("Color is ycbcr")
} else if src.ColorModel() == color.AlphaModel { } else if src.ColorModel() == color.AlphaModel {
c.Logger.Info("Color is alpha") c.Logger.Info("Color is alpha")
} else if src.ColorModel() == color.CMYKModel { } else if src.ColorModel() == color.CMYKModel {
@ -69,5 +64,10 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
return return
} }
if format != model.Format {
c.Logger.Warn("Image format does not match model", format, model.Format)
return
}
return true return true
} }

14
logic/models/train/reset.go
View File

@ -21,30 +21,30 @@ func handleRest(handle *Handle) {
f, err := MyParseForm(r) f, err := MyParseForm(r)
if err != nil { if err != nil {
// TODO improve response // TODO improve response
return ErrorCode(nil, 400, c.AddMap(nil)) return c.ErrorCode(nil, 400, c.AddMap(nil))
} }
if !CheckId(f, "id") { if !CheckId(f, "id") {
// TODO improve response // TODO improve response
return ErrorCode(nil, 400, c.AddMap(nil)) return c.ErrorCode(nil, 400, c.AddMap(nil))
} }
id := f.Get("id") id := f.Get("id")
model, err := GetBaseModel(handle.Db, id) model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError { if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{ return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found", "NotFoundMessage": "Model not found",
"GoBackLink": "/models", "GoBackLink": "/models",
}) })
} else if err != nil { } else if err != nil {
// TODO improve response // TODO improve response
return Error500(err) return c.Error500(err)
} }
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING { if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
// TODO improve response // TODO improve response
return ErrorCode(nil, 400, c.AddMap(nil)) return c.ErrorCode(nil, 400, c.AddMap(nil))
} }
os.RemoveAll(path.Join("savedData", model.Id, "defs")) os.RemoveAll(path.Join("savedData", model.Id, "defs"))
@ -52,10 +52,10 @@ func handleRest(handle *Handle) {
_, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id) _, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil { if err != nil {
// TODO improve response // TODO improve response
return Error500(err) return c.Error500(err)
} }
ModelUpdateStatus(handle, model.Id, CONFIRM_PRE_TRAINING) ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id=" + model.Id, c.Mode, w, r) Redirect("/models/edit?id=" + model.Id, c.Mode, w, r)
return nil return nil
}) })

138
logic/models/train/train.go
View File

@ -46,8 +46,8 @@ const (
LAYER_FLATTEN = 3 LAYER_FLATTEN = 3
) )
func ModelDefinitionUpdateStatus(handle *Handle, id string, status ModelDefinitionStatus) (err error) { func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionStatus) (err error) {
_, err = handle.Db.Exec("update model_definition set status = $1 where id = $2", status, id) _, err = c.Db.Exec("update model_definition set status = $1 where id = $2", status, id)
return return
} }
@ -56,15 +56,15 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
return return
} }
func generateCvs(handle *Handle, run_path string, model_id string) (count int, err error) { func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
classes, err := handle.Db.Query("select count(*) from model_classes where model_id=$1;", model_id) classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
if err != nil { return } if err != nil { return }
defer classes.Close() defer classes.Close()
if !classes.Next() { return } if !classes.Next() { return }
if err = classes.Scan(&count); err != nil { return } if err = classes.Scan(&count); err != nil { return }
data, err := handle.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id) data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
if err != nil { return } if err != nil { return }
defer data.Close() defer data.Close()
@ -88,9 +88,9 @@ func generateCvs(handle *Handle, run_path string, model_id string) (count int,
return return
} }
func trainDefinition(handle *Handle, model *BaseModel, definition_id string) (accuracy float64, err error) { func trainDefinition(c *Context, model *BaseModel, definition_id string) (accuracy float64, err error) {
accuracy = 0 accuracy = 0
layers, err := handle.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id) layers, err := c.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil { if err != nil {
return return
} }
@ -120,7 +120,7 @@ func trainDefinition(handle *Handle, model *BaseModel, definition_id string) (ac
return return
} }
_, err = generateCvs(handle, run_path, model.Id) _, err = generateCvs(c, run_path, model.Id)
if err != nil { return } if err != nil { return }
// Create python script // Create python script
@ -185,12 +185,12 @@ func trainDefinition(handle *Handle, model *BaseModel, definition_id string) (ac
return return
} }
func trainModel(handle *Handle, model *BaseModel) { func trainModel(c *Context, model *BaseModel) {
definitionsRows, err := handle.Db.Query("select id, target_accuracy from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) definitionsRows, err := c.Db.Query("select id, target_accuracy from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil { if err != nil {
fmt.Printf("Failed to trainModel!Err:\n") c.Logger.Error("Failed to trainModel!Err:")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
defer definitionsRows.Close() defer definitionsRows.Close()
@ -205,27 +205,26 @@ func trainModel(handle *Handle, model *BaseModel) {
for definitionsRows.Next() { for definitionsRows.Next() {
var rowv row var rowv row
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy); err != nil { if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy); err != nil {
fmt.Printf("Failed to trainModel!Err:\n") c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
definitions = append(definitions, rowv) definitions = append(definitions, rowv)
} }
if len(definitions) == 0 { if len(definitions) == 0 {
fmt.Printf("Failed to trainModel!Err:\n") c.Logger.Error("No Definitions defined!")
fmt.Println(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING)
return return
} }
for _, def := range definitions { for _, def := range definitions {
accuracy, err := trainDefinition(handle, model, def.id) accuracy, err := trainDefinition(c, model, def.id)
if err != nil { if err != nil {
fmt.Printf("Failed to train definition!Err:\n") c.Logger.Error("Failed to train definition!Err:")
fmt.Println(err) c.Logger.Error(err)
ModelDefinitionUpdateStatus(handle, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
continue continue
} }
@ -233,55 +232,55 @@ func trainModel(handle *Handle, model *BaseModel) {
if int_accuracy < def.target_accuracy { if int_accuracy < def.target_accuracy {
fmt.Printf("Failed to train definition! Accuracy less %d < %d\n", int_accuracy, def.target_accuracy) fmt.Printf("Failed to train definition! Accuracy less %d < %d\n", int_accuracy, def.target_accuracy)
ModelDefinitionUpdateStatus(handle, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
continue continue
} }
_, err = handle.Db.Exec("update model_definition set accuracy=$1, status=$2 where id=$3", int_accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.id) _, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2 where id=$3", int_accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.id)
if err != nil { if err != nil {
fmt.Printf("Failed to train definition!Err:\n") fmt.Printf("Failed to train definition!Err:\n")
fmt.Println(err) fmt.Println(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
} }
rows, err := handle.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED) rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err != nil { if err != nil {
fmt.Printf("Db err select!Err:\n") c.Logger.Error("DB: failed to read definition")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
defer rows.Close() defer rows.Close()
if !rows.Next() { if !rows.Next() {
// TODO improve message // TODO Make the Model status have a message
fmt.Printf("All definitions failed to train!") c.Logger.Error("All definitions failed to train!")
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
var id string var id string
if err = rows.Scan(&id); err != nil { if err = rows.Scan(&id); err != nil {
fmt.Printf("Db err!Err:\n") c.Logger.Error("Failed to read id:")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
if _, err = handle.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil { if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
fmt.Printf("Db err!Err:\n") c.Logger.Error("Failed to update model definition")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
to_delete, err := handle.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id) to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil { if err != nil {
fmt.Printf("Db err!Err:\n") c.Logger.Error("Failed to select model_definition to delete")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
defer to_delete.Close() defer to_delete.Close()
@ -289,22 +288,23 @@ func trainModel(handle *Handle, model *BaseModel) {
for to_delete.Next() { for to_delete.Next() {
var id string var id string
if to_delete.Scan(&id);err != nil { if to_delete.Scan(&id);err != nil {
fmt.Printf("Db err!Err:\n") c.Logger.Error("Failed to scan the id of a model_definition to delete")
fmt.Println(err) c.Logger.Error(err)
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
os.RemoveAll(path.Join("savedData", model.Id, "defs", id)) os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
} }
if _, err = handle.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil { // TODO Check if returning also works here
fmt.Printf("Db err!Err:\n") if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
fmt.Println(err) c.Logger.Error("Failed to delete model_definition")
ModelUpdateStatus(handle, model.Id, FAILED_TRAINING) c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return return
} }
ModelUpdateStatus(handle, model.Id, READY) ModelUpdateStatus(c, model.Id, READY)
} }
func handleTrain(handle *Handle) { func handleTrain(handle *Handle) {
@ -371,18 +371,18 @@ func handleTrain(handle *Handle) {
cls, err := model_classes.ListClasses(handle.Db, model.Id) cls, err := model_classes.ListClasses(handle.Db, model.Id)
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
var fid string var fid string
for i := 0; i < number_of_models; i++ { for i := 0; i < number_of_models; i++ {
def_id, err := MakeDefenition(handle.Db, model.Id, accuracy) def_id, err := MakeDefenition(handle.Db, model.Id, accuracy)
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
if fid == "" { if fid == "" {
@ -392,42 +392,42 @@ func handleTrain(handle *Handle) {
// TODO change shape of it depends on the type of the image // TODO change shape of it depends on the type of the image
err = MakeLayer(handle.Db, def_id, 1, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height)) err = MakeLayer(handle.Db, def_id, 1, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
err = MakeLayer(handle.Db, def_id, 4, LAYER_FLATTEN, fmt.Sprintf("%d,1", len(cls))) err = MakeLayer(handle.Db, def_id, 4, LAYER_FLATTEN, fmt.Sprintf("%d,1", len(cls)))
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3)) err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3))
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
// Using sparce // Using sparce
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d, 1", len(cls))) err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d, 1", len(cls)))
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
err = ModelDefinitionUpdateStatus(handle, def_id, MODEL_DEFINITION_STATUS_INIT) err = ModelDefinitionUpdateStatus(c, def_id, MODEL_DEFINITION_STATUS_INIT)
if err != nil { if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response // TODO improve this response
return Error500(err) return c.Error500(err)
} }
} }
// TODO start training with id fid // TODO start training with id fid
go trainModel(handle, model) go trainModel(c, model)
ModelUpdateStatus(handle, model.Id, TRAINING) ModelUpdateStatus(c, model.Id, TRAINING)
Redirect("/models/edit?id="+model.Id, c.Mode, w, r) Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil return nil
}) })

19
logic/models/utils/types.go
View File

@ -13,6 +13,7 @@ type BaseModel struct {
ImageMode int ImageMode int
Width int Width int
Height int Height int
Format string
} }
const ( const (
@ -31,31 +32,29 @@ const (
var ModelNotFoundError = errors.New("Model not found error") var ModelNotFoundError = errors.New("Model not found error")
func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) { func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
rows, err := db.Query("select name, status, id, width, height, color_mode from models where id=$1;", id) rows, err := db.Query("select name, status, id, width, height, color_mode, format from models where id=$1;", id)
if err != nil { if err != nil { return }
return
}
defer rows.Close() defer rows.Close()
if !rows.Next() { if !rows.Next() { return nil, ModelNotFoundError }
return nil, ModelNotFoundError
}
base = &BaseModel{} base = &BaseModel{}
var colorMode string var colorMode string
err = rows.Scan(&base.Name, &base.Status, &base.Id, &base.Width, &base.Height, &colorMode) err = rows.Scan(&base.Name, &base.Status, &base.Id, &base.Width, &base.Height, &colorMode, &base.Format)
if err != nil { if err != nil {
return nil, err return nil, err
} }
base.ImageMode = StringToImageMode(colorMode) base.ImageMode = StringToImageMode(colorMode)
return return
} }
func StringToImageMode(colorMode string) (int){ func StringToImageMode(colorMode string) int {
switch colorMode { switch colorMode {
case "greyscale": case "greyscale":
return 1 return 1
case "rgb":
return 3
default: default:
panic("unkown color mode") panic("unkown color mode")
} }

4
logic/models/utils/utils.go
View File

@ -7,8 +7,8 @@ import (
) )
// TODO make this return and caller handle error // TODO make this return and caller handle error
func ModelUpdateStatus(handle *Handle, id string, status int) { func ModelUpdateStatus(c *Context, id string, status int) {
_, err := handle.Db.Exec("update models set status = $1 where id = $2", status, id) _, err := c.Db.Exec("update models set status = $1 where id = $2", status, id)
if err != nil { if err != nil {
fmt.Println("Failed to update model status") fmt.Println("Failed to update model status")
fmt.Println(err) fmt.Println(err)

8
logic/utils/handler.go
View File

@ -344,6 +344,7 @@ type Context struct {
User *dbtypes.User User *dbtypes.User
Mode AnswerType Mode AnswerType
Logger *log.Logger Logger *log.Logger
Db *sql.DB
} }
func (c Context) Error400(err error, message string, w http.ResponseWriter, path string, base string, data AnyMap) *Error { func (c Context) Error400(err error, message string, w http.ResponseWriter, path string, base string, data AnyMap) *Error {
@ -409,7 +410,7 @@ func (c *Context) requireAuth(w http.ResponseWriter, r *http.Request) bool {
var LogoffError = errors.New("Invalid token!") var LogoffError = errors.New("Invalid token!")
func (x Handle) createContext(mode AnswerType, r *http.Request) (*Context, error) { func (x Handle) createContext(handler *Handle, mode AnswerType, r *http.Request) (*Context, error) {
var token *string var token *string
@ -438,7 +439,7 @@ func (x Handle) createContext(mode AnswerType, r *http.Request) (*Context, error
Prefix: r.URL.Path, Prefix: r.URL.Path,
}) })
return &Context{token, user, mode, logger}, nil return &Context{token, user, mode, logger, handler.Db}, nil
} }
// TODO check if I can use http.Redirect // TODO check if I can use http.Redirect
@ -516,6 +517,7 @@ func ErrorCode(err error, code int, data AnyMap) *Error {
} }
func Error500(err error) *Error { func Error500(err error) *Error {
log.Warn("This function is deprecated please use the one provided by context")
return ErrorCode(err, http.StatusInternalServerError, nil) return ErrorCode(err, http.StatusInternalServerError, nil)
} }
@ -563,7 +565,7 @@ func NewHandler(db *sql.DB) *Handle {
//TODO JSON //TODO JSON
//Login state //Login state
context, err := x.createContext(ans, r) context, err := x.createContext(x, ans, r)
if err != nil { if err != nil {
Logoff(ans, w, r) Logoff(ans, w, r)
return return

3
sql/models.sql
View File

@ -13,7 +13,8 @@ create table if not exists models (
width integer, width integer,
height integer, height integer,
color_mode varchar (20) color_mode varchar (20),
format varchar (20)
); );
-- drop table if exists model_data_point; -- drop table if exists model_data_point;