sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: removed non svelte front page

Browse Source
This commit is contained in:
Andre Henriques
2024-03-09 10:52:08 +00:00
parent 0d37ba8d59
commit 274d7d22aa
32 changed files with 614 additions and 3695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
logic/models/add.go
View File

@@ -86,90 +86,14 @@ func loadBaseImage(c *Context, id string) {
}
func handleAdd(handle *Handle) {
handle.GetHTML("/models/add", AnswerTemplate("models/add.html", nil, 1))
handle.Post("/models/add", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Post("/models/add", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
read_form, err := r.MultipartReader()
if err != nil {
return c.JsonErrorBadRequest(err, "Please provide a valid multipart Reader request")
}
var name string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return &Error{Code: http.StatusBadRequest}
}
if part.FormName() == "name" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
name = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
if name == "" || len(file) == 0 {
return c.JsonBadRequest("Name is empty or file is empty")
}
row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id)
if err != nil {
return c.Error500(err)
}
if row.Next() {
return c.JsonBadRequest("Model with that name already exists!")
}
row, err = handle.Db.Query("insert into models (user_id, name) values ($1, $2) returning id", c.User.Id, name)
if err != nil || !row.Next() {
return c.Error500(err)
}
var id string
err = row.Scan(&id)
if err != nil {
return c.Error500(err)
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
err = os.Mkdir(dir_path, os.ModePerm)
if err != nil {
return c.Error500(err)
}
f, err := os.Create(path.Join(dir_path, "baseimage.png"))
if err != nil {
return c.Error500(err)
}
defer f.Close()
f.Write(file)
c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id)
go loadBaseImage(c, id)
return c.SendJSON(id)
}
read_form, err := r.MultipartReader()
read_form, err := c.R.MultipartReader()
if err != nil {
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
return nil
return c.JsonErrorBadRequest(err, "Please provide a valid multipart Reader request")
}
var name string
@@ -195,8 +119,7 @@ func handleAdd(handle *Handle) {
}
if name == "" || len(file) == 0 {
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
return nil
return c.JsonBadRequest("Name is empty or file is empty")
}
row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id)
@@ -205,27 +128,14 @@ func handleAdd(handle *Handle) {
}
if row.Next() {
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(AnyMap{
"NameFoundError": true,
"Name": name,
}))
return nil
return c.JsonBadRequest("Model with that name already exists!")
}
_, err = handle.Db.Exec("insert into models (user_id, name) values ($1, $2)", c.User.Id, name)
if err != nil {
row, err = handle.Db.Query("insert into models (user_id, name) values ($1, $2) returning id", c.User.Id, name)
if err != nil || !row.Next() {
return c.Error500(err)
}
row, err = handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id)
if err != nil {
return c.Error500(err)
}
if !row.Next() {
return &Error{Code: http.StatusInternalServerError}
}
var id string
err = row.Scan(&id)
if err != nil {
@@ -239,6 +149,7 @@ func handleAdd(handle *Handle) {
if err != nil {
return c.Error500(err)
}
f, err := os.Create(path.Join(dir_path, "baseimage.png"))
if err != nil {
return c.Error500(err)
@@ -250,7 +161,6 @@ func handleAdd(handle *Handle) {
c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id)
go loadBaseImage(c, id)
Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil
return c.SendJSON(id)
})
}

147
logic/models/classes/list.go
View File

@@ -1,148 +1,69 @@
package model_classes
import (
"net/http"
"strconv"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func models_data_list_json(w http.ResponseWriter, r *http.Request, c *Context) *Error {
id, err := GetIdFromUrl(r, "id")
if err != nil {
return c.JsonBadRequest("Model Class not found!")
}
page := 0
if r.URL.Query().Has("page") {
page_url := r.URL.Query().Get("page")
page_url_number, err := strconv.Atoi(page_url)
if err != nil {
return c.JsonBadRequest("Page is not a number")
}
page = page_url_number
}
var class_row struct {
Name string
Model_id string
}
err = utils.GetDBOnce(c, &class_row, "model_classes where id=$1", id)
if err == NotFoundError {
return c.JsonBadRequest("Model Class not found!")
} else if err != nil {
return c.Error500(err)
}
type baserow struct {
Id string `json:"id"`
File_Path string `json:"file_path"`
Model_Mode int `json:"model_mode"`
Status int `json:"status"`
}
rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10)
if err != nil {
return c.Error500(err)
}
type ReturnType struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
}
max_len := min(11, len(rows))
return c.SendJSON(ReturnType{
ImageList: rows[0:max_len],
Page: page,
ShowNext: len(rows) == 11,
})
}
func HandleList(handle *Handle) {
handle.Get("/models/data/list", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Get("/models/data/list", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
return models_data_list_json(w, r, c)
}
id, err := GetIdFromUrl(r, "id")
id, err := GetIdFromUrl(c, "id")
if err != nil {
return ErrorCode(err, 400, c.AddMap(nil))
return c.JsonBadRequest("Model Class not found!")
}
page := 0
if r.URL.Query().Has("page") {
page_url := r.URL.Query().Get("page")
if c.R.URL.Query().Has("page") {
page_url := c.R.URL.Query().Get("page")
page_url_number, err := strconv.Atoi(page_url)
if err != nil {
return ErrorCode(err, http.StatusBadRequest, c.AddMap(nil))
return c.JsonBadRequest("Page is not a number")
}
page = page_url_number
}
class_rows, err := handle.Db.Query("select name, model_id from model_classes where id=$1;", id)
if err != nil {
return Error500(err)
}
defer class_rows.Close()
if !class_rows.Next() {
return ErrorCode(nil, 404, c.AddMap(nil))
var class_row struct {
Name string
Model_id string
}
name := ""
model_id := ""
if err = class_rows.Scan(&name, &model_id); err != nil {
return Error500(nil)
err = utils.GetDBOnce(c, &class_row, "model_classes where id=$1", id)
if err == NotFoundError {
return c.JsonBadRequest("Model Class not found!")
} else if err != nil {
return c.Error500(err)
}
model, err := GetBaseModel(c.Db, model_id)
if err != nil {
return Error500(err)
}
rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 11 offset $2;", id, page*10)
if err != nil {
return Error500(err)
}
defer rows.Close()
type baserow struct {
Id string
FilePath string
Mode int
Status int
Id string `json:"id"`
File_Path string `json:"file_path"`
Model_Mode int `json:"model_mode"`
Status int `json:"status"`
}
got := []baserow{}
for rows.Next() {
nrow := baserow{}
err = rows.Scan(&nrow.Id, &nrow.FilePath, &nrow.Mode, &nrow.Status)
if err != nil {
return Error500(err)
}
got = append(got, nrow)
rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10)
if err != nil {
return c.Error500(err)
}
max_len := min(11, len(got))
type ReturnType struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
}
LoadDefineTemplate(w, "/models/edit.html", "data-model-create-class-table-table", c.AddMap(AnyMap{
"List": got[0:max_len],
"Page": page,
"Id": id,
"Name": name,
"Model": model,
"ShowNext": len(got) == 11,
}))
return nil
max_len := min(11, len(rows))
return c.SendJSON(ReturnType{
ImageList: rows[0:max_len],
Page: page,
ShowNext: len(rows) == 11,
})
})
}

343
logic/models/data.go
View File

@@ -152,8 +152,8 @@ func processZipFile(c *Context, model *BaseModel) {
}
func processZipFileExpand(c *Context, model *BaseModel) {
var err error
var err error
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, READY_FAILED)
@@ -161,7 +161,7 @@ func processZipFileExpand(c *Context, model *BaseModel) {
reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip"))
if err != nil {
failed("Faield to proccess zip file failed to open reader\n")
failed("Faield to proccess zip file failed to open reader\n")
return
}
defer reader.Close()
@@ -257,8 +257,8 @@ func processZipFileExpand(c *Context, model *BaseModel) {
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
failed(fmt.Sprintf("Failed to update data point status"))
return
failed(fmt.Sprintf("Failed to update data point status"))
return
}
}
}
@@ -268,130 +268,12 @@ func processZipFileExpand(c *Context, model *BaseModel) {
}
func handleDataUpload(handle *Handle) {
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Post("/models/data/upload", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
read_form, err := r.MultipartReader()
if err != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "base_data.zip"))
if err != nil {
return Error500(err)
}
defer f.Close()
f.Write(file)
ModelUpdateStatus(c, id, PREPARING_ZIP_FILE)
go processZipFile(c, model)
return c.SendJSON(model.Id)
}
read_form, err := r.MultipartReader()
if err != nil {
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
return nil
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return &Error{Code: http.StatusBadRequest}
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
} else if err != nil {
return Error500(err)
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "base_data.zip"))
if err != nil {
return Error500(err)
}
defer f.Close()
f.Write(file)
ModelUpdateStatus(c, id, PREPARING_ZIP_FILE)
go processZipFile(c, model)
Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil
})
// ------
// ------ CLASS DATA UPLOAD
// ------
handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
read_form, err := r.MultipartReader()
read_form, err := c.R.MultipartReader()
if err != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
@@ -418,26 +300,86 @@ func handleDataUpload(handle *Handle) {
}
}
c.Logger.Info("Trying to expand model", "id", id)
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "base_data.zip"))
if err != nil {
return c.Error500(err)
}
defer f.Close()
f.Write(file)
ModelUpdateStatus(c, id, PREPARING_ZIP_FILE)
go processZipFile(c, model)
return c.SendJSON(model.Id)
})
// ------
// ------ CLASS DATA UPLOAD
// ------
handle.Post("/models/data/class/upload", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
read_form, err := c.R.MultipartReader()
if err != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
c.Logger.Info("Trying to expand model", "id", id)
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
return c.Error500(err)
}
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
}
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "expand_data.zip"))
if err != nil {
return Error500(err)
return c.Error500(err)
}
defer f.Close()
@@ -450,115 +392,68 @@ func handleDataUpload(handle *Handle) {
return c.SendJSON(model.Id)
})
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Delete("/models/data/delete-zip-file", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
type ModelData struct {
Id string `json:"id"`
}
var dat ModelData
if err := c.ToJSON(r, &dat); err != nil {
return err
}
model, err := GetBaseModel(handle.Db, dat.Id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
delete_path := "base_data.zip"
if model.Status == READY_FAILED {
delete_path = "expand_data.zip"
} else if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.JsonBadRequest("Model not in the correct status")
}
err = os.Remove(path.Join("savedData", model.Id, delete_path))
if err != nil {
return Error500(err)
}
if model.Status != READY_FAILED {
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
if err != nil {
return Error500(err)
}
} else {
c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding")
}
if model.Status != READY_FAILED {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
if err != nil {
return Error500(err)
}
} else {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
if err != nil {
return Error500(err)
}
}
if model.Status != READY_FAILED {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
} else {
ModelUpdateStatus(c, model.Id, READY)
}
return c.SendJSON(model.Id)
type ModelData struct {
Id string `json:"id"`
}
f, err := MyParseForm(r)
if err != nil {
return ErrorCode(err, 400, c.AddMap(nil))
var dat ModelData
if err := c.ToJSON(&dat); err != nil {
return err
}
if !CheckId(f, "id") {
return ErrorCode(err, 400, c.AddMap(nil))
}
id := f.Get("id")
model, err := GetBaseModel(handle.Db, id)
model, err := GetBaseModel(handle.Db, dat.Id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
return c.Error500(err)
}
if model.Status != FAILED_PREPARING_ZIP_FILE {
// TODO add message
return ErrorCode(nil, 400, c.AddMap(nil))
delete_path := "base_data.zip"
if model.Status == READY_FAILED {
delete_path = "expand_data.zip"
} else if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.JsonBadRequest("Model not in the correct status")
}
err = os.Remove(path.Join("savedData", id, "base_data.zip"))
err = os.Remove(path.Join("savedData", model.Id, delete_path))
if err != nil {
return Error500(err)
return c.Error500(err)
}
err = os.RemoveAll(path.Join("savedData", id, "data"))
if err != nil {
return Error500(err)
if model.Status != READY_FAILED {
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
if err != nil {
return c.Error500(err)
}
} else {
c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding")
}
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
if err != nil {
return Error500(err)
if model.Status != READY_FAILED {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
if err != nil {
return c.Error500(err)
}
} else {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
if err != nil {
return c.Error500(err)
}
}
ModelUpdateStatus(c, id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil
if model.Status != READY_FAILED {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
} else {
ModelUpdateStatus(c, model.Id, READY)
}
return c.SendJSON(model.Id)
})
}

152
logic/models/delete.go
View File

@@ -4,40 +4,12 @@ import (
"net/http"
"os"
"path"
"strconv"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
utils "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func deleteModel(handle *Handle, id string, w http.ResponseWriter, c *Context, model BaseModel) {
c.Logger.Warnf("Removing model with id: %s", id)
_, err := handle.Db.Exec("delete from models where id=$1;", id)
if err != nil {
c.Logger.Error(err)
panic("TODO handle better deleteModel failed delete database query")
}
model_path := path.Join("./savedData", id)
c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path)
err = os.RemoveAll(model_path)
if err != nil {
c.Logger.Error(err)
panic("TODO handle better deleteModel failed to delete folder")
}
if c.Mode == HTML {
// TODO move this to a constant so i don't forget
w.WriteHeader(309)
c.Mode = HTMLFULL
}
LoadBasedOnAnswer(c.Mode, w, "/models/delete.html", c.AddMap(AnyMap{
"Model": model,
}))
}
func deleteModelJSON(c *Context, id string) *Error {
c.Logger.Warnf("Removing model with id: %s", id)
_, err := c.Db.Exec("delete from models where id=$1;", id)
@@ -56,94 +28,29 @@ func deleteModelJSON(c *Context, id string) *Error {
}
func handleDelete(handle *Handle) {
handle.Delete("/models/delete", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if c.Mode == JSON {
var dat struct {
Id string `json:"id" validate:"required"`
Name *string `json:"name,omitempty"`
}
if err_ := c.ToJSON(r, &dat); err_ != nil {
return err_
}
var model struct {
Id string
Name string
Status int
}
err := utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found!")
} else if err != nil {
return c.Error500(err)
}
switch model.Status {
case FAILED_TRAINING:
fallthrough
case FAILED_PREPARING_ZIP_FILE:
fallthrough
case FAILED_PREPARING_TRAINING:
fallthrough
case FAILED_PREPARING:
return deleteModelJSON(c, dat.Id)
case READY:
fallthrough
case CONFIRM_PRE_TRAINING:
if dat.Name == nil {
return c.JsonBadRequest("Provided name does not match the model name")
}
if *dat.Name != model.Name {
return c.JsonBadRequest("Provided name does not match the model name")
}
return deleteModelJSON(c, dat.Id)
default:
c.Logger.Warn("Do not know how to handle model in status", "status", model.Status)
return c.JsonBadRequest("Model in invalid status")
}
handle.Delete("/models/delete", func(c *Context) *Error {
if c.CheckAuthLevel(1) {
return nil
}
var dat struct {
Id string `json:"id" validate:"required"`
Name *string `json:"name,omitempty"`
}
// This is required to parse delete forms with bodies
f, err := MyParseForm(r)
if err != nil {
return c.ErrorCode(err, 400, nil)
if err_ := c.ToJSON(&dat); err_ != nil {
return err_
}
if !CheckId(f, "id") {
return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
var model struct {
Id string
Name string
Status int
}
id := f.Get("id")
// TODO handle admin users
rows, err := handle.Db.Query("select name, status from models where id=$1 and user_id=$2;", id, c.User.Id)
if err != nil {
return c.ErrorCode(err, http.StatusInternalServerError, nil)
}
defer rows.Close()
if !rows.Next() {
c.Logger.Warn("Could not find model for", id, c.User.Id)
return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
}
model := BaseModel{}
model.Id = id
if err = rows.Scan(&model.Name, &model.Status); err != nil {
err := utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found!")
} else if err != nil {
return c.Error500(err)
}
@@ -155,32 +62,23 @@ func handleDelete(handle *Handle) {
case FAILED_PREPARING_TRAINING:
fallthrough
case FAILED_PREPARING:
deleteModel(handle, id, w, c, model)
return nil
return deleteModelJSON(c, dat.Id)
case READY:
fallthrough
case CONFIRM_PRE_TRAINING:
if CheckEmpty(f, "name") {
return c.Error400(nil, "Name is empty", w, "/models/edit.html", "delete-model-card", AnyMap{
"NameDoesNotMatch": true,
"Model": model,
})
if dat.Name == nil {
return c.JsonBadRequest("Provided name does not match the model name")
}
name := f.Get("name")
if name != model.Name {
LoadDefineTemplate(w, "/models/edit.html", "delete-model-card", c.AddMap(AnyMap{
"NameDoesNotMatch": true,
"Model": model,
}))
return nil
if *dat.Name != model.Name {
return c.JsonBadRequest("Provided name does not match the model name")
}
deleteModel(handle, id, w, c, model)
return nil
return deleteModelJSON(c, dat.Id)
default:
panic("Do not know how to handle model in status:" + strconv.Itoa(model.Status))
c.Logger.Warn("Do not know how to handle model in status", "status", model.Status)
return c.JsonBadRequest("Model in invalid status")
}
})
}

308
logic/models/edit.go
View File

@@ -2,7 +2,6 @@ package models
import (
"fmt"
"net/http"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
@@ -10,79 +9,30 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleJson(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
id, err := GetIdFromUrl(r, "id")
if err != nil {
return c.JsonBadRequest("Model not found")
}
type rowmodel struct {
Name string `json:"name"`
Status int `json:"status"`
Id string `json:"id"`
Width *int `json:"width"`
Height *int `json:"height"`
Color_mode *string `json:"color_mode"`
Format string `json:"format"`
Model_type int `json:"model_type"`
}
var model rowmodel = rowmodel{}
err = utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2", id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "Model not found")
} else if err != nil {
return c.Error500(err)
}
return c.SendJSON(model)
/*
switch model.Status {
case TRAINING:
}
*/
}
func handleEdit(handle *Handle) {
handle.Get("/models/edit/classes", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Get("/models/edit/classes", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode != JSON {
return c.ErrorCode(nil, 400, AnyMap{})
}
id, err := GetIdFromUrl(r, "id")
if err != nil {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
}
model, err := GetBaseModel(c.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
}
model, err_ := c.GetModelFromId("id")
if err_ != nil {
return err_
}
wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id)
if err != nil {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(c, id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
return c.Error500(err)
}
has_data, err := model_classes.ModelHasDataPoints(handle.Db, id)
has_data, err := model_classes.ModelHasDataPoints(handle.Db, model.Id)
if err != nil {
return Error500(err)
return c.Error500(err)
}
type ReturnType struct {
@@ -98,25 +48,15 @@ func handleEdit(handle *Handle) {
})
})
handle.Get("/models/edit/definitions", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Get("/models/edit/definitions", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode != JSON {
return c.ErrorCode(nil, 400, AnyMap{})
}
id, err := GetIdFromUrl(r, "id")
if err != nil {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
}
model, err := GetBaseModel(c.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return c.Error500(err)
}
model, err_ := c.GetModelFromId("id")
if err_ != nil {
return err_
}
type defrow struct {
Id string
@@ -243,217 +183,39 @@ func handleEdit(handle *Handle) {
return c.SendJSON(defsToReturn)
})
handle.Get("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Get("/models/edit", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
return handleJson(w, r, c)
if !c.CheckAuthLevel(1) {
return nil
}
id, err := GetIdFromUrl(r, "id")
id, err := GetIdFromUrl(c, "id")
if err != nil {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
}
// TODO handle admin users
rows, err := handle.Db.Query("select name, status, width, height, color_mode, format, model_type from models where id=$1 and user_id=$2;", id, c.User.Id)
if err != nil {
return Error500(err)
}
defer rows.Close()
if !rows.Next() {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
return c.JsonBadRequest("Model not found")
}
type rowmodel struct {
Name string
Status int
Id string
Width *int
Height *int
Color_mode *string
Format string
Type int
Name string `json:"name"`
Status int `json:"status"`
Id string `json:"id"`
Width *int `json:"width"`
Height *int `json:"height"`
Color_mode *string `json:"color_mode"`
Format string `json:"format"`
Model_type int `json:"model_type"`
}
var model rowmodel = rowmodel{}
model.Id = id
err = rows.Scan(&model.Name, &model.Status, &model.Width, &model.Height, &model.Color_mode, &model.Format, &model.Type)
if err != nil {
return Error500(err)
err = utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2", id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "Model not found")
} else if err != nil {
return c.Error500(err)
}
// Handle errors
// All errors will be negative
if model.Status < 0 {
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
return nil
}
switch model.Status {
case PREPARING:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
case CONFIRM_PRE_TRAINING:
wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id)
if err != nil {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(c, id)
if err != nil {
return c.Error500(err)
}
has_data, err := model_classes.ModelHasDataPoints(handle.Db, id)
if err != nil {
return Error500(err)
}
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
"Classes": cls,
"HasData": has_data,
"NumberOfInvalidImages": wrong_number,
}))
case READY:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
case TRAINING:
type defrow struct {
Id string
Status int
EpochProgress int
Epoch int
Accuracy float64
}
defs := []defrow{}
if model.Type == 2 {
def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
} else {
def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id)
if err != nil {
return c.Error500(err)
}
defer def_rows.Close()
for def_rows.Next() {
var def defrow
err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy)
if err != nil {
return c.Error500(err)
}
defs = append(defs, def)
}
}
type layerdef struct {
id string
LayerType int
Shape string
}
layers := []layerdef{}
for _, def := range defs {
if def.Status == MODEL_DEFINITION_STATUS_TRAINING {
rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id)
if err != nil {
return c.Error500(err)
}
defer rows.Close()
for rows.Next() {
var layerdef layerdef
err = rows.Scan(&layerdef.id, &layerdef.LayerType, &layerdef.Shape)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef)
}
if model.Type == 2 {
type lastLayerType struct {
Id string
Range_start int
Range_end int
}
var lastLayer lastLayerType
err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id)
if err != nil {
return c.Error500(err)
}
layers = append(layers, layerdef{
id: lastLayer.Id,
LayerType: LAYER_DENSE,
Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1),
})
}
break
}
}
sep_mod := 100
if len(layers) > 8 {
sep_mod = 100 - (len(layers)-8)*10
}
if sep_mod < 10 {
sep_mod = 10
}
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
"Defs": defs,
"Layers": layers,
"SepMod": sep_mod,
}))
case PREPARING_ZIP_FILE:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
default:
fmt.Printf("Unkown Status: %d\n", model.Status)
return Error500(nil)
}
return nil
return c.SendJSON(model)
})
}

56
logic/models/list.go
View File

@@ -1,66 +1,26 @@
package models
import (
"net/http"
"git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleList(handle *Handle) {
handle.Get("/models", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Get("/models", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
// TODO handle admin
if c.Mode == JSON {
rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id)
if err != nil {
return Error500(err)
}
defer rows.Close()
type Row struct {
Name string `json:"name"`
Id string `json:"id"`
}
got, err := utils.GetDbMultitple[Row](c, "models where user_id=$1", c.User.Id);
if err != nil {
c.Logger.Warn("HERE 6")
return c.Error500(nil)
}
return c.SendJSON(got)
type Row struct {
Name string `json:"name"`
Id string `json:"id"`
}
rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id)
got, err := utils.GetDbMultitple[Row](c, "models where user_id=$1", c.User.Id)
if err != nil {
return Error500(err)
}
defer rows.Close()
type row struct {
Name string
Id string
return c.Error500(nil)
}
got := []row{}
for rows.Next() {
var r row
err = rows.Scan(&r.Id, &r.Name)
if err != nil {
return Error500(err)
}
got = append(got, r)
}
LoadBasedOnAnswer(c.Mode, w, "/models/list.html", c.AddMap(AnyMap{
"List": got,
}))
return nil
return c.SendJSON(got)
})
}

195
logic/models/run.go
View File

@@ -2,9 +2,7 @@ package models
import (
"bytes"
"fmt"
"io"
"net/http"
"os"
"path"
@@ -118,143 +116,14 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
}
func handleRun(handle *Handle) {
handle.Post("/models/run", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Post("/models/run", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
read_form, err := r.MultipartReader()
if err != nil {
// TODO improve message
return ErrorCode(nil, 400, nil)
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Invalid multipart data")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Models not found")
} else if err != nil {
return c.Error500(err)
}
if model.Status != READY {
return c.JsonBadRequest("Model not ready to run images")
}
def := JustId{}
err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id)
if err == NotFoundError {
return c.JsonBadRequest("Could not find definition")
} else if err != nil {
return c.Error500(err)
}
def_id := def.Id
// TODO create a database table with tasks
run_path := path.Join("/tmp", model.Id, "runs")
os.MkdirAll(run_path, os.ModePerm)
img_path := path.Join(run_path, "img."+model.Format)
img_file, err := os.Create(img_path)
if err != nil {
return c.Error500(err)
}
defer img_file.Close()
img_file.Write(file)
if !testImgForModel(c, model, img_path) {
return c.JsonBadRequest("Provided image does not match the model")
}
root := tg.NewRoot()
var tf_img *image.Image = nil
switch model.Format {
case "png":
tf_img = ReadPNG(root, img_path, int64(model.ImageMode))
case "jpeg":
tf_img = ReadJPG(root, img_path, int64(model.ImageMode))
default:
panic("Not sure what to do with '" + model.Format + "'")
}
exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{})
inputImage, err := tf.NewTensor(exec_results[0].Value())
if err != nil {
return c.Error500(err)
}
vi := -1
var confidence float32 = 0
if model.ModelType == 2 {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, confidence, err = runModelExp(c, model, def_id, inputImage)
if err != nil {
return c.Error500(err)
}
} else {
c.Logger.Info("Running model normal", "model", model.Id, "def", def_id)
vi, confidence, err = runModelNormal(c, model, def_id, inputImage)
if err != nil {
return c.Error500(err)
}
}
os.RemoveAll(run_path)
rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi)
if err != nil {
return c.Error500(err)
}
if !rows.Next() {
return c.SendJSON(nil)
}
var name string
if err = rows.Scan(&name); err != nil {
return c.Error500(err)
}
returnValue := struct {
Class string `json:"class"`
Confidence float32 `json:"confidence"`
}{
Class: name,
Confidence: confidence,
}
return c.SendJSON(returnValue)
}
read_form, err := r.MultipartReader()
read_form, err := c.R.MultipartReader()
if err != nil {
// TODO improve message
return ErrorCode(nil, 400, nil)
return c.JsonBadRequest("Invalid muilpart body")
}
var id string
@@ -265,7 +134,7 @@ func handleRun(handle *Handle) {
if err_part == io.EOF {
break
} else if err_part != nil {
return &Error{Code: http.StatusBadRequest}
return c.JsonBadRequest("Invalid multipart data")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
@@ -281,27 +150,21 @@ func handleRun(handle *Handle) {
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
return c.JsonBadRequest("Models not found")
} else if err != nil {
return Error500(err)
return c.Error500(err)
}
if model.Status != READY {
// TODO improve this
return ErrorCode(nil, 400, c.AddMap(nil))
return c.JsonBadRequest("Model not ready to run images")
}
def := JustId{}
err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id)
if err == NotFoundError {
// TODO improve this
fmt.Printf("Could not find definition\n")
return ErrorCode(nil, 400, c.AddMap(nil))
return c.JsonBadRequest("Could not find definition")
} else if err != nil {
return Error500(err)
return c.Error500(err)
}
def_id := def.Id
@@ -313,19 +176,13 @@ func handleRun(handle *Handle) {
img_file, err := os.Create(img_path)
if err != nil {
return Error500(err)
return c.Error500(err)
}
defer img_file.Close()
img_file.Write(file)
if !testImgForModel(c, model, img_path) {
LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{
"Model": model,
"NotFound": false,
"Result": nil,
"ImageError": true,
}))
return nil
return c.JsonBadRequest("Provided image does not match the model")
}
root := tg.NewRoot()
@@ -344,7 +201,7 @@ func handleRun(handle *Handle) {
exec_results := tg.Exec(root, []tf.Output{tf_img.Value()}, nil, &tf.SessionOptions{})
inputImage, err := tf.NewTensor(exec_results[0].Value())
if err != nil {
return Error500(err)
return c.Error500(err)
}
vi := -1
@@ -368,27 +225,25 @@ func handleRun(handle *Handle) {
rows, err := handle.Db.Query("select name from model_classes where model_id=$1 and class_order=$2;", model.Id, vi)
if err != nil {
return Error500(err)
return c.Error500(err)
}
if !rows.Next() {
LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{
"Model": model,
"NotFound": true,
"Result": nil,
"Confidence": confidence,
}))
return nil
return c.SendJSON(nil)
}
var name string
if err = rows.Scan(&name); err != nil {
return nil
return c.Error500(err)
}
LoadDefineTemplate(w, "/models/edit.html", "run-model-card", c.AddMap(AnyMap{
"Model": model,
"Result": name,
}))
return nil
returnValue := struct {
Class string `json:"class"`
Confidence float32 `json:"confidence"`
}{
Class: name,
Confidence: confidence,
}
return c.SendJSON(returnValue)
})
}

3
logic/models/train/main.go
View File

@@ -7,7 +7,4 @@ import (
func HandleTrainEndpoints(handle *Handle) {
handleTrain(handle)
handleRest(handle)
//TODO remove
handleTest(handle)
}

66
logic/models/train/reset.go
View File

@@ -1,7 +1,6 @@
package models_train
import (
"net/http"
"os"
"path"
@@ -10,82 +9,39 @@ import (
)
func handleRest(handle *Handle) {
handle.Delete("/models/train/reset", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Delete("/models/train/reset", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
var dat struct {
Id string `json:"id"`
}
if err := c.ToJSON(r, &dat); err != nil {
return err;
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found");
} else if err != nil {
// TODO improve response
return c.Error500(err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
return c.JsonBadRequest("Model is not in status that be reset")
}
os.RemoveAll(path.Join("savedData", model.Id, "defs"))
_, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil {
// TODO improve response
return c.Error500(err)
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
return c.SendJSON(model.Id)
var dat struct {
Id string `json:"id"`
}
f, err := MyParseForm(r)
if err != nil {
// TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil))
if err := c.ToJSON(&dat); err != nil {
return err
}
if !CheckId(f, "id") {
// TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil))
}
id := f.Get("id")
model, err := GetBaseModel(handle.Db, id)
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
return c.JsonBadRequest("Model not found")
} else if err != nil {
// TODO improve response
return c.Error500(err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
// TODO improve response
return c.ErrorCode(nil, 400, c.AddMap(nil))
return c.JsonBadRequest("Model is not in status that be reset")
}
os.RemoveAll(path.Join("savedData", model.Id, "defs"))
_, err = handle.Db.Exec("delete from model_definition where model_id=$1", model.Id)
_, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil {
// TODO improve response
return c.Error500(err)
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil
return c.SendJSON(model.Id)
})
}

175
logic/models/train/tensorflow-test.go
View File

@@ -1,175 +0,0 @@
package models_train
import (
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
"strconv"
"strings"
"text/template"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
/*import (
tf "github.com/galeone/tensorflow/tensorflow/go"
tg "github.com/galeone/tfgo"
"github.com/galeone/tfgo/image"
"github.com/galeone/tfgo/image/filter"
"github.com/galeone/tfgo/image/padding"
)*/
func getDir() string {
dir, err := os.Getwd()
if err != nil {
panic(err)
}
return dir
}
func shapeToSize(shape string) string {
split := strings.Split(shape, ",")
return strings.Join(split[:len(split) - 1], ",")
}
func handleTest(handle *Handle) {
handle.Post("/models/train/test", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
id, err := GetIdFromUrl(r, "id")
if err != nil {
return ErrorCode(err, 400, c.AddMap(nil))
}
rows, err := handle.Db.Query("select mc.name, mdp.file_path from model_classes as mc join model_data_point as mdp on mdp.class_id = mc.id where mdp.model_mode = 1 and mc.model_id = $1 limit 1;", id)
if err != nil {
return Error500(err)
}
defer rows.Close()
if !rows.Next() {
return Error500(err)
}
var name string
var file_path string
err = rows.Scan(&name, &file_path)
if err != nil {
return Error500(err)
}
file_path = strings.Replace(file_path, "file://", "", 1)
img_path := path.Join("savedData", id, "data", "training", name, file_path)
fmt.Printf("%s\n", img_path)
definitions, err := handle.Db.Query("select id from model_definition where model_id=$1 and status=2 limit 1;", id)
if err != nil {
return Error500(err)
}
defer definitions.Close()
if !definitions.Next() {
fmt.Println("Did not find definition")
return Error500(nil)
}
var definition_id string
if err = definitions.Scan(&definition_id); err != nil {
return Error500(err)
}
layers, err := handle.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
if err != nil {
return Error500(err)
}
defer layers.Close()
type layerrow struct {
LayerType int
Shape string
}
got := []layerrow{}
for layers.Next() {
var row = layerrow{}
if err = layers.Scan(&row.LayerType, &row.Shape); err != nil {
return Error500(err)
}
row.Shape = shapeToSize(row.Shape)
got = append(got, row)
}
// Generate folder
run_path := path.Join("/tmp", id, "defs", definition_id)
err = os.MkdirAll(run_path, os.ModePerm)
if err != nil {
return Error500(err)
}
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
return Error500(err)
}
defer f.Close()
fmt.Printf("Using path: %s\n", run_path)
tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
if err != nil {
return Error500(err)
}
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", id, "data", "training"),
}); err != nil {
return Error500(err)
}
cmd := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path))
_, err = cmd.Output()
if err != nil {
return Error500(err)
}
result_path := path.Join("savedData", id, "defs", definition_id)
os.MkdirAll(result_path, os.ModePerm)
err = exec.Command("cp", path.Join(run_path, "model.keras"), path.Join(result_path, "model.keras")).Run()
if err != nil {
return Error500(err)
}
accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val"))
if err != nil {
return Error500(err)
}
defer accuracy_file.Close()
accuracy_file_bytes, err := io.ReadAll(accuracy_file)
if err != nil {
return Error500(err)
}
accuracy, err := strconv.ParseFloat(string(accuracy_file_bytes), 64)
if err != nil {
return Error500(err)
}
w.Write([]byte(strconv.FormatFloat(accuracy, 'f', -1, 64)))
return nil
})
}

227
logic/models/train/train.go
View File

@@ -6,12 +6,12 @@ import (
"fmt"
"io"
"math"
"net/http"
"os"
"os/exec"
"path"
"sort"
"strconv"
"strings"
"text/template"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
@@ -23,6 +23,19 @@ import (
const EPOCH_PER_RUN = 20
const MAX_EPOCH = 100
func shapeToSize(shape string) string {
split := strings.Split(shape, ",")
return strings.Join(split[:len(split)-1], ",")
}
func getDir() string {
dir, err := os.Getwd()
if err != nil {
panic(err)
}
return dir
}
func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) {
id = ""
rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
@@ -702,13 +715,13 @@ func trainModelExp(c *Context, model *BaseModel) {
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
}
var definitions TrainModelRowUsables
var definitions TrainModelRowUsables
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
failed("Failed to get definitions");
return
}
definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
failed("Failed to get definitions")
return
}
if len(definitions) == 0 {
failed("No Definitions defined!")
return
@@ -810,38 +823,38 @@ func trainModelExp(c *Context, model *BaseModel) {
}
}
// Set the class status to trained
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
}
var dat JustId
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
var dat JustId
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
failed("All definitions failed to train!")
return
} else if err != nil {
} else if err != nil {
failed("DB: failed to read definition")
return
}
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil {
failed("Failed to update model definition")
return
}
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
failed("Failed to select model_definition to delete")
return
}
}
for _, d := range(to_delete) {
for _, d := range to_delete {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
@@ -1293,163 +1306,81 @@ func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy
return nil
}
func handle_models_train_json(w http.ResponseWriter, r *http.Request, c *Context) *Error {
var dat struct {
Id string `json:"id"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
if err_ := c.ToJSON(r, &dat); err_ != nil {
return err_
}
if dat.Id == "" {
return c.JsonBadRequest("Please provide a id")
}
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
return c.Error500(err)
}
if model.Status != CONFIRM_PRE_TRAINING {
return c.JsonBadRequest("Model in invalid status for training")
}
if modelTypeId == 2 {
full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
}
if modelTypeId == 2 {
go trainModelExp(c, model)
} else {
go trainModel(c, model)
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
}
return c.SendJSON(model.Id)
}
func handleTrain(handle *Handle) {
handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
handle.Post("/models/train", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
if c.Mode == JSON {
return handle_models_train_json(w, r, c)
var dat struct {
Id string `json:"id"`
ModelType string `json:"model_type"`
NumberOfModels int `json:"number_of_models"`
Accuracy int `json:"accuracy"`
}
r.ParseForm()
f := r.Form
number_of_models := 0
accuracy := 0
if !CheckId(f, "id") || CheckEmpty(f, "model_type") || !CheckNumber(f, "number_of_models", &number_of_models) || !CheckNumber(f, "accuracy", &accuracy) {
// TODO improve this response
return ErrorCode(nil, 400, c.AddMap(nil))
if err_ := c.ToJSON(&dat); err_ != nil {
return err_
}
id := f.Get("id")
model_type_id := 1
model_type_form := f.Get("model_type")
if model_type_form == "expandable" {
model_type_id = 2
} else if model_type_form != "simple" {
return c.Error400(nil, "Invalid model type!", w, "/models/edit.html", "train-model-card", AnyMap{
"HasData": true,
"ErrorMessage": "Invalid model type!",
})
if dat.Id == "" {
return c.JsonBadRequest("Please provide a id")
}
model, err := GetBaseModel(handle.Db, id)
modelTypeId := 1
if dat.ModelType == "expandable" {
modelTypeId = 2
} else if dat.ModelType != "simple" {
return c.JsonBadRequest("Invalid model type!")
}
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, c.AddMap(AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
}))
return c.JsonBadRequest("Model not found")
} else if err != nil {
// TODO improve this response
return Error500(err)
return c.Error500(err)
}
if model.Status != CONFIRM_PRE_TRAINING {
// TODO improve this response
return ErrorCode(nil, 400, c.AddMap(nil))
return c.JsonBadRequest("Model in invalid status for training")
}
if model_type_id == 2 {
full_error := generateExpandableDefinitions(c, model, accuracy, number_of_models)
if modelTypeId == 2 {
full_error := generateExpandableDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
} else {
full_error := generateDefinitions(c, model, accuracy, number_of_models)
full_error := generateDefinitions(c, model, dat.Accuracy, dat.NumberOfModels)
if full_error != nil {
return full_error
}
}
if model_type_id == 2 {
if modelTypeId == 2 {
go trainModelExp(c, model)
} else {
go trainModel(c, model)
}
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, model_type_id, model.Id)
_, err = c.Db.Exec("update models set status = $1, model_type = $2 where id = $3", TRAINING, modelTypeId, model.Id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
// TODO improve this response
return Error500(err)
return c.Error500(err)
}
Redirect("/models/edit?id="+model.Id, c.Mode, w, r)
return nil
return c.SendJSON(model.Id)
})
handle.Get("/model/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
// TODO check auth level
if c.Mode != NORMAL {
// This should only handle normal requests
c.Logger.Warn("This function only works with normal")
return c.UnsafeErrorCode(nil, 400, nil)
}
f := r.URL.Query()
handle.Get("/model/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or definition or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
@@ -1458,9 +1389,7 @@ func handleTrain(handle *Handle) {
def_id := f.Get("definition")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
c.Logger.Warn("Epoch is not a number")
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select md.status from model_definition as md where md.model_id=$1 and md.id=$2", model_id, def_id)
@@ -1482,8 +1411,7 @@ func handleTrain(handle *Handle) {
if status != 3 {
c.Logger.Warn("Definition not on status 3(training)", "status", status)
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Definition not on status 3(training)")
}
c.Logger.Info("Updated model_definition!", "model", model_id, "progress", epoch, "accuracy", accuracy)
@@ -1495,21 +1423,13 @@ func handleTrain(handle *Handle) {
return nil
})
handle.Get("/model/head/epoch/update", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
// TODO check auth level
if c.Mode != NORMAL {
// This should only handle normal requests
c.Logger.Warn("This function only works with normal")
return c.UnsafeErrorCode(nil, 400, nil)
}
f := r.URL.Query()
handle.Get("/model/head/epoch/update", func(c *Context) *Error {
f := c.R.URL.Query()
accuracy := 0.0
if !CheckId(f, "head_id") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or head_id or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Invalid: model_id or definition or epoch or accuracy")
}
accuracy = accuracy * 100
@@ -1517,9 +1437,7 @@ func handleTrain(handle *Handle) {
head_id := f.Get("head_id")
epoch, err := strconv.Atoi(f.Get("epoch"))
if err != nil {
c.Logger.Warn("Epoch is not a number")
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Epoch is not a number")
}
rows, err := c.Db.Query("select hd.status from exp_model_head as hd where hd.id=$1;", head_id)
@@ -1541,8 +1459,7 @@ func handleTrain(handle *Handle) {
if status != 3 {
c.Logger.Warn("Head not on status 3(training)", "status", status)
// No need to improve message because this function is only called internaly
return c.UnsafeErrorCode(nil, 400, nil)
return c.JsonBadRequest("Head not on status 3(training)")
}
c.Logger.Info("Updated model_head!", "head", head_id, "progress", epoch, "accuracy", accuracy)

14
logic/models/utils/utils.go
View File

@@ -1,14 +0,0 @@
package models_utils
import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
// TODO make this return and caller handle error
func ModelUpdateStatus(c *Context, id string, status int) {
_, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id)
if err != nil {
c.Logger.Error("Failed to update model status", "err", err)
c.Logger.Warn("TODO Maybe handle better")
}
}