diff --git a/DockerfileDev b/DockerfileDev new file mode 100644 index 0000000..6784fc5 --- /dev/null +++ b/DockerfileDev @@ -0,0 +1,6 @@ +# vi: ft=dockerfile +FROM docker.io/nginx + +ADD nginx.dev.conf /nginx.conf + +CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"] diff --git a/logic/models/add.go b/logic/models/add.go index 0149164..53c680b 100644 --- a/logic/models/add.go +++ b/logic/models/add.go @@ -91,9 +91,79 @@ func handleAdd(handle *Handle) { if !CheckAuthLevel(1, w, r, c) { return nil } + if c.Mode == JSON { - // TODO json - panic("TODO JSON") + read_form, err := r.MultipartReader() + if err != nil { + return c.JsonErrorBadRequest(err, "Please provide a valid multipart Reader request") + } + + var name string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return &Error{Code: http.StatusBadRequest} + } + if part.FormName() == "name" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + name = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + if name == "" || len(file) == 0 { + return c.JsonBadRequest("Name is empty or file is empty") + } + + row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id) + if err != nil { + return c.Error500(err) + } + + if row.Next() { + return c.JsonBadRequest("Model with that name already exists!") + } + + row, err = handle.Db.Query("insert into models (user_id, name) values ($1, $2) returning id", c.User.Id, name) + if err != nil || !row.Next() { + return c.Error500(err) + } + + var id string + err = row.Scan(&id) + if err != nil { + return c.Error500(err) + } + + // TODO mk this path configurable + dir_path := path.Join("savedData", id) + + err = os.Mkdir(dir_path, os.ModePerm) + if err != nil { + return c.Error500(err) + } + + f, err := os.Create(path.Join(dir_path, "baseimage.png")) + if err != nil { + return c.Error500(err) + } + defer f.Close() + + f.Write(file) + + c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id) + go loadBaseImage(c, id) + + return c.SendJSON(id) } read_form, err := r.MultipartReader() diff --git a/logic/models/classes/list.go b/logic/models/classes/list.go index d74d38e..a873953 100644 --- a/logic/models/classes/list.go +++ b/logic/models/classes/list.go @@ -4,89 +4,145 @@ import ( "net/http" "strconv" - . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" + "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" + . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) -func HandleList(handle *Handle) { - handle.Get("/models/data/list", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if !CheckAuthLevel(1, w, r, c) { - return nil - } - if c.Mode == JSON { - panic("TODO JSON on /models/data/list") - } +func models_data_list_json(w http.ResponseWriter, r *http.Request, c *Context) *Error { + id, err := GetIdFromUrl(r, "id") + if err != nil { + return c.JsonBadRequest("Model Class not found!") + } - id, err := GetIdFromUrl(r, "id") - if err != nil { - return ErrorCode(err, 400, c.AddMap(nil)) - } - - page := 0 - if r.URL.Query().Has("page") { - page_url := r.URL.Query().Get("page") - page_url_number, err := strconv.Atoi(page_url) - if err != nil { - return ErrorCode(err, http.StatusBadRequest, c.AddMap(nil)) - } - page = page_url_number - } + page := 0 + if r.URL.Query().Has("page") { + page_url := r.URL.Query().Get("page") + page_url_number, err := strconv.Atoi(page_url) + if err != nil { + return c.JsonBadRequest("Page is not a number") + } + page = page_url_number + } - class_rows, err := handle.Db.Query("select name, model_id from model_classes where id=$1;", id) - if err != nil { - return Error500(err) - } - defer class_rows.Close() + var class_row struct { + Name string + Model_id string + } - if !class_rows.Next() { - return ErrorCode(nil, 404, c.AddMap(nil)) - } + err = utils.GetDBOnce(c, &class_row, "model_classes where id=$1", id) + if err == NotFoundError { + return c.JsonBadRequest("Model Class not found!") + } else if err != nil { + return c.Error500(err) + } - name := "" - model_id := "" - if err = class_rows.Scan(&name, &model_id); err != nil { - return Error500(nil) - } + type baserow struct { + Id string + File_Path string + Model_Mode int + Status int + } - model, err := GetBaseModel(c.Db, model_id) - if err != nil { - return Error500(err) - } + rows, err := utils.GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10) + if err != nil { + return c.Error500(err) + } - rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 11 offset $2;", id, page * 10) - if err != nil { - return Error500(err) - } - defer rows.Close() + type ReturnType struct { + ImageList []*baserow `json:"image_list"` + Page int `json:"page"` + ShowNext bool `json:"showNext"` + } - type baserow struct { - Id string - FilePath string - Mode int - Status int - } + max_len := min(11, len(rows)) - got := []baserow{} - - for rows.Next() { - nrow := baserow{} - err = rows.Scan(&nrow.Id, &nrow.FilePath, &nrow.Mode, &nrow.Status) - if err != nil { - return Error500(err) - } - got = append(got, nrow) - } - - max_len := min(11, len(got)) - - LoadDefineTemplate(w, "/models/edit.html", "data-model-create-class-table-table", c.AddMap(AnyMap{ - "List": got[0:max_len], - "Page": page, - "Id": id, - "Name": name, - "Model": model, - "ShowNext": len(got) == 11, - })) - return nil - }) + return c.SendJSON(ReturnType{ + ImageList: rows[0:max_len], + Page: page, + ShowNext: len(rows) == 11, + }) +} + +func HandleList(handle *Handle) { + handle.Get("/models/data/list", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + if c.Mode == JSON { + return models_data_list_json(w, r, c) + } + + id, err := GetIdFromUrl(r, "id") + if err != nil { + return ErrorCode(err, 400, c.AddMap(nil)) + } + + page := 0 + if r.URL.Query().Has("page") { + page_url := r.URL.Query().Get("page") + page_url_number, err := strconv.Atoi(page_url) + if err != nil { + return ErrorCode(err, http.StatusBadRequest, c.AddMap(nil)) + } + page = page_url_number + } + + class_rows, err := handle.Db.Query("select name, model_id from model_classes where id=$1;", id) + if err != nil { + return Error500(err) + } + defer class_rows.Close() + + if !class_rows.Next() { + return ErrorCode(nil, 404, c.AddMap(nil)) + } + + name := "" + model_id := "" + if err = class_rows.Scan(&name, &model_id); err != nil { + return Error500(nil) + } + + model, err := GetBaseModel(c.Db, model_id) + if err != nil { + return Error500(err) + } + + rows, err := handle.Db.Query("select id, file_path, model_mode, status from model_data_point where class_id=$1 limit 11 offset $2;", id, page*10) + if err != nil { + return Error500(err) + } + defer rows.Close() + + type baserow struct { + Id string + FilePath string + Mode int + Status int + } + + got := []baserow{} + + for rows.Next() { + nrow := baserow{} + err = rows.Scan(&nrow.Id, &nrow.FilePath, &nrow.Mode, &nrow.Status) + if err != nil { + return Error500(err) + } + got = append(got, nrow) + } + + max_len := min(11, len(got)) + + LoadDefineTemplate(w, "/models/edit.html", "data-model-create-class-table-table", c.AddMap(AnyMap{ + "List": got[0:max_len], + "Page": page, + "Id": id, + "Name": name, + "Model": model, + "ShowNext": len(got) == 11, + })) + return nil + }) } diff --git a/logic/models/classes/main.go b/logic/models/classes/main.go index 1381609..762a20d 100644 --- a/logic/models/classes/main.go +++ b/logic/models/classes/main.go @@ -7,78 +7,82 @@ import ( ) type ModelClass struct { - Id string - ModelId string - Name string + Id string `json:"id"` + ModelId string `json:"model_id"` + Name string `json:"name"` } func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) { - - rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id) - if err != nil { - return - } - defer rows.Close() - for rows.Next() { - var model ModelClass - err = rows.Scan(&model.Id, &model.ModelId, &model.Name) - if err != nil { - return - } + rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id) + if err != nil { + return + } + defer rows.Close() - cls = append(cls, model) - } + for rows.Next() { + var model ModelClass + err = rows.Scan(&model.Id, &model.ModelId, &model.Name) + if err != nil { + return + } - return + cls = append(cls, model) + } + + return } func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) { - result = false - rows, err := db.Query("select mdp.id from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 limit 1;", model_id) - if err != nil { - return - } - defer rows.Close() - - return rows.Next(), nil + result = false + rows, err := db.Query("select mdp.id from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 limit 1;", model_id) + if err != nil { + return + } + defer rows.Close() + + return rows.Next(), nil } var ClassAlreadyExists = errors.New("Class aready exists") func CreateClass(db *sql.DB, model_id string, order int, name string) (id string, err error) { - id = "" - rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name) - if err != nil { - return - } - defer rows.Close() + id = "" + rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name) + if err != nil { + return + } + defer rows.Close() - if rows.Next() { - return id, ClassAlreadyExists - } + if rows.Next() { + return id, ClassAlreadyExists + } - rows, err = db.Query("insert into model_classes (model_id, name, class_order) values ($1, $2, $3) returning id;", model_id, name, order) - if err != nil { - return - } - defer rows.Close() + rows, err = db.Query("insert into model_classes (model_id, name, class_order) values ($1, $2, $3) returning id;", model_id, name, order) + if err != nil { + return + } + defer rows.Close() - if !rows.Next() { - return id, errors.New("Insert did not return anything") - } + if !rows.Next() { + return id, errors.New("Insert did not return anything") + } - err = rows.Scan(&id) - return + err = rows.Scan(&id) + return } func GetNumberOfWrongDataPoints(db *sql.DB, model_id string) (number int, err error) { - number = 0 - rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id) - if err != nil { return } - defer rows.Close() - // TODO not an error because if there is no result means that there is no need to count - if !rows.Next() { return } - err = rows.Scan(&number) - return + number = 0 + rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id) + if err != nil { + return + } + defer rows.Close() + // TODO not an error because if there is no result means that there is no need to count + if !rows.Next() { + return + } + err = rows.Scan(&number) + return } diff --git a/logic/models/data.go b/logic/models/data.go index 8aab454..9ef7783 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -126,7 +126,7 @@ func processZipFile(c *Context, model *BaseModel) { return } - file_path := path.Join(base_path, data_point_id + "." + model.Format) + file_path := path.Join(base_path, data_point_id+"."+model.Format) f, err := os.Create(file_path) if err != nil { fmt.Printf("Could not create file %s\n", file_path) @@ -137,13 +137,13 @@ func processZipFile(c *Context, model *BaseModel) { f.Write(file_data) if !testImgForModel(c, model, file_path) { - c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) - c.Logger.Warn("Not failling updating data point to status -1") - message := "Image did not have valid format for the model" - if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { - c.Logger.Error("Failed to update data point status") - ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) - } + c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) + c.Logger.Warn("Not failling updating data point to status -1") + message := "Image did not have valid format for the model" + if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { + c.Logger.Error("Failed to update data point status") + ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) + } } } @@ -156,9 +156,58 @@ func handleDataUpload(handle *Handle) { if !CheckAuthLevel(1, w, r, c) { return nil } + if c.Mode == JSON { - // TODO improve message - return ErrorCode(nil, 400, nil) + read_form, err := r.MultipartReader() + if err != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + + var id string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return c.JsonBadRequest("Please provide a valid form data request!") + } + if part.FormName() == "id" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + id = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + model, err := GetBaseModel(handle.Db, id) + if err == ModelNotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found") + } else if err != nil { + return Error500(err) + } + + // TODO mk this path configurable + dir_path := path.Join("savedData", id) + + f, err := os.Create(path.Join(dir_path, "base_data.zip")) + if err != nil { + return Error500(err) + } + defer f.Close() + + f.Write(file) + + ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) + + go processZipFile(c, model) + + return c.SendJSON(model.Id) } read_form, err := r.MultipartReader() @@ -223,7 +272,45 @@ func handleDataUpload(handle *Handle) { return nil } if c.Mode == JSON { - panic("Handle delete zip file json") + + type ModelData struct { + Id string `json:"id"` + } + + var dat ModelData + + if err := c.ToJSON(r, &dat); err != nil { + return err + } + + model, err := GetBaseModel(handle.Db, dat.Id) + if err == ModelNotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found"); + } else if err != nil { + return Error500(err) + } + + if model.Status != FAILED_PREPARING_ZIP_FILE { + return c.SendJSONStatus(http.StatusNotFound, "Model not in the correct status") + } + + err = os.Remove(path.Join("savedData", model.Id, "base_data.zip")) + if err != nil { + return Error500(err) + } + + err = os.RemoveAll(path.Join("savedData", model.Id, "data")) + if err != nil { + return Error500(err) + } + + _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) + if err != nil { + return Error500(err) + } + + ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) + return c.SendJSON(model.Id) } f, err := MyParseForm(r) diff --git a/logic/models/edit.go b/logic/models/edit.go index 527e710..b4d8c03 100644 --- a/logic/models/edit.go +++ b/logic/models/edit.go @@ -6,14 +6,238 @@ import ( model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" + "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) +func handleJson(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + + id, err := GetIdFromUrl(r, "id") + if err != nil { + return c.JsonBadRequest("Model not found") + } + + type rowmodel struct { + Name string `json:"name"` + Status int `json:"status"` + Id string `json:"id"` + Width *int `json:"width"` + Height *int `json:"height"` + Color_mode *string `json:"color_mode"` + Format string `json:"format"` + Model_type int `json:"model_type"` + } + + var model rowmodel = rowmodel{} + err = utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2", id, c.User.Id) + if err == NotFoundError { + return c.SendJSONStatus(404, "Model not found") + } else if err != nil { + return c.Error500(err) + } + + return c.SendJSON(model) + + /* + + // Handle errors + // All errors will be negative + if model.Status < 0 { + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ + "Model": model, + })) + return nil + } + + switch model.Status { + case READY: + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ + "Model": model, + })) + case TRAINING: + + type defrow struct { + Id string + Status int + EpochProgress int + Epoch int + Accuracy float64 + } + + defs := []defrow{} + + if model.Type == 2 { + def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id) + if err != nil { + return c.Error500(err) + } + defer def_rows.Close() + + for def_rows.Next() { + var def defrow + err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) + if err != nil { + return c.Error500(err) + } + defs = append(defs, def) + } + } else { + def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id) + if err != nil { + return c.Error500(err) + } + defer def_rows.Close() + + for def_rows.Next() { + var def defrow + err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) + if err != nil { + return c.Error500(err) + } + defs = append(defs, def) + } + } + + type layerdef struct { + id string + LayerType int + Shape string + } + + layers := []layerdef{} + + for _, def := range defs { + if def.Status == MODEL_DEFINITION_STATUS_TRAINING { + rows, err := c.Db.Query("select id, layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", def.Id) + if err != nil { + return c.Error500(err) + } + defer rows.Close() + + for rows.Next() { + var layerdef layerdef + err = rows.Scan(&layerdef.id, &layerdef.LayerType, &layerdef.Shape) + if err != nil { + return c.Error500(err) + } + layers = append(layers, layerdef) + } + + if model.Type == 2 { + + type lastLayerType struct { + Id string + Range_start int + Range_end int + } + + var lastLayer lastLayerType + + err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id) + if err != nil { + return c.Error500(err) + } + + layers = append(layers, layerdef{ + id: lastLayer.Id, + LayerType: LAYER_DENSE, + Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1), + }) + } + + break + } + } + + sep_mod := 100 + if len(layers) > 8 { + sep_mod = 100 - (len(layers)-8)*10 + } + + if sep_mod < 10 { + sep_mod = 10 + } + + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ + "Model": model, + "Defs": defs, + "Layers": layers, + "SepMod": sep_mod, + })) + case PREPARING_ZIP_FILE: + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ + "Model": model, + })) + default: + fmt.Printf("Unkown Status: %d\n", model.Status) + return Error500(nil) + } + + + return nil + */ +} + func handleEdit(handle *Handle) { - handle.GetHTML("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + handle.Get("/models/edit/classes", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } + if c.Mode != JSON { + return c.ErrorCode(nil, 400, AnyMap{}) + } + + id, err := GetIdFromUrl(r, "id") + if err != nil { + return c.SendJSONStatus(http.StatusNotFound, "Model not found") + } + + model, err := GetBaseModel(c.Db, id) + if err == ModelNotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found") + } else if err != nil { + return c.Error500(err) + } + + wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id) + if err != nil { + return c.Error500(err) + } + + cls, err := model_classes.ListClasses(handle.Db, id) + if err != nil { + return c.Error500(err) + } + + has_data, err := model_classes.ModelHasDataPoints(handle.Db, id) + if err != nil { + return Error500(err) + } + + type ReturnType struct { + Classes []model_classes.ModelClass `json:"classes"` + HasData bool `json:"has_data"` + NumberOfInvalidImages int `json:"number_of_invalid_images"` + } + + return c.SendJSON(ReturnType{ + Classes: cls, + HasData: has_data, + NumberOfInvalidImages: wrong_number, + }) + }) + + handle.Get("/models/edit", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + + if c.Mode == JSON { + return handleJson(w, r, c) + } id, err := GetIdFromUrl(r, "id") if err != nil { @@ -109,38 +333,37 @@ func handleEdit(handle *Handle) { defs := []defrow{} - if model.Type == 2 { - def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id) - if err != nil { - return c.Error500(err) - } - defer def_rows.Close() + if model.Type == 2 { + def_rows, err := c.Db.Query("select md.id, md.status, md.epoch, h.epoch_progress, h.accuracy from model_definition as md inner join exp_model_head as h on h.def_id = md.id where md.model_id=$1 order by md.created_on asc", model.Id) + if err != nil { + return c.Error500(err) + } + defer def_rows.Close() - for def_rows.Next() { - var def defrow - err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) - if err != nil { - return c.Error500(err) - } - defs = append(defs, def) - } - } else { - def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id) - if err != nil { - return c.Error500(err) - } - defer def_rows.Close() - - for def_rows.Next() { - var def defrow - err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) - if err != nil { - return c.Error500(err) - } - defs = append(defs, def) - } - } + for def_rows.Next() { + var def defrow + err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) + if err != nil { + return c.Error500(err) + } + defs = append(defs, def) + } + } else { + def_rows, err := c.Db.Query("select id, status, epoch, epoch_progress, accuracy from model_definition where model_id=$1 order by created_on asc", model.Id) + if err != nil { + return c.Error500(err) + } + defer def_rows.Close() + for def_rows.Next() { + var def defrow + err = def_rows.Scan(&def.Id, &def.Status, &def.Epoch, &def.EpochProgress, &def.Accuracy) + if err != nil { + return c.Error500(err) + } + defs = append(defs, def) + } + } type layerdef struct { id string @@ -175,7 +398,7 @@ func handleEdit(handle *Handle) { Range_end int } - var lastLayer lastLayerType + var lastLayer lastLayerType err := GetDBOnce(c, &lastLayer, "exp_model_head where def_id=$1 and status=3;", def.Id) if err != nil { @@ -183,9 +406,9 @@ func handleEdit(handle *Handle) { } layers = append(layers, layerdef{ - id: lastLayer.Id, + id: lastLayer.Id, LayerType: LAYER_DENSE, - Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start + 1), + Shape: fmt.Sprintf("%d, 1", lastLayer.Range_end-lastLayer.Range_start+1), }) } @@ -219,4 +442,5 @@ func handleEdit(handle *Handle) { return nil }) + } diff --git a/logic/models/list.go b/logic/models/list.go index 10b10dd..016eda0 100644 --- a/logic/models/list.go +++ b/logic/models/list.go @@ -3,25 +3,43 @@ package models import ( "net/http" + "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) - func handleList(handle *Handle) { - // TODO json - handle.GetHTML("/models", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { - if c.Mode == JSON { - panic("TODO JSON") - } + handle.Get("/models", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } + // TODO handle admin + if c.Mode == JSON { + rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id) + if err != nil { + return Error500(err) + } + defer rows.Close() + + type Row struct { + Name string `json:"name"` + Id string `json:"id"` + } + + got, err := utils.GetDbMultitple[Row](c, "models where user_id=$1", c.User.Id); + if err != nil { + c.Logger.Warn("HERE 6") + return c.Error500(nil) + } + + return c.SendJSON(got) + } + rows, err := handle.Db.Query("select id, name from models where user_id=$1;", c.User.Id) if err != nil { return Error500(err) } - defer rows.Close() + defer rows.Close() type row struct { Name string diff --git a/logic/utils/handler.go b/logic/utils/handler.go index 586e05b..c2b7b29 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -346,6 +346,7 @@ type Context struct { Mode AnswerType Logger *log.Logger Db *sql.DB + Writer http.ResponseWriter } func (c Context) ToJSON(r *http.Request, dat any) *Error { @@ -360,33 +361,40 @@ func (c Context) ToJSON(r *http.Request, dat any) *Error { return nil } -func (c Context) SendJSON(w http.ResponseWriter, dat any) *Error { - w.Header().Add("content-type", "application/json") +func (c Context) SendJSON(dat any) *Error { + c.Writer.Header().Add("content-type", "application/json") text, err := json.Marshal(dat) if err != nil { return c.Error500(err) } - if _, err = w.Write(text); err != nil { + if _, err = c.Writer.Write(text); err != nil { return c.Error500(err) } return nil } -func (c Context) SendJSONStatus(w http.ResponseWriter, status int, dat any) *Error { - w.Header().Add("content-type", "application/json") - w.WriteHeader(status) +func (c Context) SendJSONStatus(status int, dat any) *Error { + c.Writer.Header().Add("content-type", "application/json") + c.Writer.WriteHeader(status) text, err := json.Marshal(dat) if err != nil { return c.Error500(err) } - if _, err = w.Write(text); err != nil { + if _, err = c.Writer.Write(text); err != nil { return c.Error500(err) } return nil } -func (c Context) JsonBadRequest(w http.ResponseWriter, dat any) *Error { - return c.SendJSONStatus(w, http.StatusBadRequest, dat) +func (c Context) JsonBadRequest(dat any) *Error { + return c.SendJSONStatus(http.StatusBadRequest, dat) +} + +func (c Context) JsonErrorBadRequest(err error, dat any) *Error { + c.SetReportCaller(true) + c.Logger.Error("Error while processing request", "err", err, "dat", dat) + c.SetReportCaller(false) + return c.SendJSONStatus(http.StatusBadRequest, dat) } func (c Context) Error400(err error, message string, w http.ResponseWriter, path string, base string, data AnyMap) *Error { @@ -460,7 +468,7 @@ func (c *Context) requireAuth(w http.ResponseWriter, r *http.Request) bool { var LogoffError = errors.New("Invalid token!") -func (x Handle) createContext(handler *Handle, mode AnswerType, r *http.Request) (*Context, error) { +func (x Handle) createContext(handler *Handle, mode AnswerType, r *http.Request, w http.ResponseWriter) (*Context, error) { var token *string @@ -486,12 +494,12 @@ func (x Handle) createContext(handler *Handle, mode AnswerType, r *http.Request) // TODO check that the token is still valid - if token == nil { return &Context{ Mode: mode, Logger: logger, Db: handler.Db, + Writer: w, }, nil } @@ -500,7 +508,7 @@ func (x Handle) createContext(handler *Handle, mode AnswerType, r *http.Request) return nil, errors.Join(err, LogoffError) } - return &Context{token, user, mode, logger, handler.Db}, nil + return &Context{token, user, mode, logger, handler.Db, w}, nil } // TODO check if I can use http.Redirect @@ -664,21 +672,28 @@ func NewHandler(db *sql.DB) *Handle { if r.Header.Get("Request-Type") == "htmlfull" { ans = HTMLFULL } - if r.Header.Get("content-type") == "application/json" { + if r.Header.Get("content-type") == "application/json" || r.Header.Get("response-type") == "application/json" { ans = JSON } - //TODO JSON + if !strings.HasPrefix(r.URL.Path, "/api") { + return + } + + r.URL.Path = strings.Replace(r.URL.Path, "/api", "", 1) + + w.Header().Add("Access-Control-Allow-Origin", "*") + w.Header().Add("Access-Control-Allow-Headers", "*") + w.Header().Add("Access-Control-Allow-Methods", "*") //Login state - context, err := x.createContext(x, ans, r) + context, err := x.createContext(x, ans, r, w) if err != nil { Logoff(ans, w, r) return } - w.Header().Add("Access-Control-Allow-Origin", "*") - w.Header().Add("Access-Control-Allow-Headers", "*") + // context.Logger.Info("Parsing", "path", r.URL.Path) if r.Method == "GET" { x.handleGets(w, r, context) diff --git a/main.go b/main.go index 27348e7..6bfa1ac 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ func main() { panic(err) } defer db.Close() - log.Info("Starting server on :8000!") + log.Info("Starting server on :5002!") //TODO check if file structure exists to save data handle := NewHandler(db) @@ -48,6 +48,7 @@ func main() { handle.StaticFiles("/js/", ".js", "text/javascript") handle.ReadFiles("/imgs/", "views", ".png", "image/png;") handle.ReadTypesFiles("/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"}) + handle.ReadTypesFiles("/api/savedData/", ".", []string{".png", ".jpeg"}, []string{"image/png", "image/jpeg"}) handle.GetHTML("/", AnswerTemplate("index.html", nil, 0)) diff --git a/nginx.dev.conf b/nginx.dev.conf new file mode 100644 index 0000000..eb8d676 --- /dev/null +++ b/nginx.dev.conf @@ -0,0 +1,29 @@ + +events { + worker_connections 1024; +} + +http { + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 8000; + + location / { + proxy_http_version 1.1; + proxy_pass http://127.0.0.1:5001; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + } + + location /api { + + proxy_pass http://127.0.0.1:5002; + } + } +} diff --git a/users.go b/users.go index 5c31f71..81b0c98 100644 --- a/users.go +++ b/users.go @@ -94,7 +94,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { // TODO Give this to the generateToken function token, login := generateToken(db, dat.Email, dat.Password) if !login { - return c.SendJSONStatus(w, http.StatusUnauthorized, "Email or password are incorrect") + return c.SendJSONStatus(http.StatusUnauthorized, "Email or password are incorrect") } user, err := dbtypes.UserFromToken(c.Db, token) @@ -118,7 +118,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { Email: user.Email, } - return c.SendJSON(w, userReturn) + return c.SendJSON(userReturn) } r.ParseForm() @@ -170,7 +170,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { } if len(dat.Username) == 0 || len(dat.Password) == 0 || len(dat.Email) == 0 { - return c.SendJSONStatus(w, http.StatusBadRequest, "Please provide a valid json") + return c.SendJSONStatus(http.StatusBadRequest, "Please provide a valid json") } rows, err := db.Query("select username, email from users where username=$1 or email=$2;", dat.Username, dat.Email) @@ -187,16 +187,16 @@ func usersEndpints(db *sql.DB, handle *Handle) { return c.Error500(err) } if db_email == dat.Email { - return c.SendJSONStatus(w, http.StatusBadRequest, "Email already in use!") + return c.SendJSONStatus(http.StatusBadRequest, "Email already in use!") } if db_username == dat.Username { - return c.SendJSONStatus(w, http.StatusBadRequest, "Username already in use!") + return c.SendJSONStatus(http.StatusBadRequest, "Username already in use!") } panic("Unrechable") } if len([]byte(dat.Password)) > 68 { - return c.JsonBadRequest(w, "Password is to long!") + return c.JsonBadRequest("Password is to long!") } salt := generateSalt() @@ -213,7 +213,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { // TODO Give this to the generateToken function token, login := generateToken(db, dat.Email, dat.Password) if !login { - return c.SendJSONStatus(w, 500, "Could not login after creatting account please try again later") + return c.SendJSONStatus(500, "Could not login after creatting account please try again later") } user, err := dbtypes.UserFromToken(c.Db, token) @@ -237,7 +237,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { Email: user.Email, } - return c.SendJSON(w, userReturn) + return c.SendJSON(userReturn) } r.ParseForm() @@ -357,7 +357,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { } if dat.Id != c.User.Id && c.User.UserType != int(dbtypes.User_Admin) { - return c.SendJSONStatus(w, 401, "You need to be an admin to update another users account") + return c.SendJSONStatus(403, "You need to be an admin to update another users account") } if dat.Id != c.User.Id { @@ -367,7 +367,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { err := utils.GetDBOnce(c, &data, "users where id=$1", dat.Id) if err == NotFoundError { - return c.JsonBadRequest(w, "User does not exist") + return c.JsonBadRequest("User does not exist") } else if err != nil { return c.Error500(err) } @@ -384,9 +384,9 @@ func usersEndpints(db *sql.DB, handle *Handle) { if err != NotFoundError { if data.Id == dat.Id { - return c.JsonBadRequest(w, "Email is the name as the previous one!") + return c.JsonBadRequest("Email is the name as the previous one!") } else { - return c.JsonBadRequest(w, "Email already in use") + return c.JsonBadRequest("Email already in use") } } @@ -414,7 +414,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { UserType: user.User_Type, } - return c.SendJSON(w, toReturnUser) + return c.SendJSON(toReturnUser) }) handle.Post("/user/info/email", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { @@ -461,18 +461,18 @@ func usersEndpints(db *sql.DB, handle *Handle) { } if dat.Password == "" { - return c.JsonBadRequest(w, "Password can not be empty") + return c.JsonBadRequest("Password can not be empty") } if dat.Password != dat.Password2 { - return c.JsonBadRequest(w, "New passwords did not match") + return c.JsonBadRequest("New passwords did not match") } c.Logger.Warn("test", "dat", dat) _, login := generateToken(db, c.User.Email, dat.Old_Password) if !login { - return c.JsonBadRequest(w, "Password is incorrect"); + return c.JsonBadRequest("Password is incorrect"); } salt := generateSalt() @@ -486,7 +486,7 @@ func usersEndpints(db *sql.DB, handle *Handle) { return c.Error500(err) } - return c.SendJSON(w, c.User.Id) + return c.SendJSON(c.User.Id) } r.ParseForm() diff --git a/webpage/src/lib/Tabs.svelte b/webpage/src/lib/Tabs.svelte new file mode 100644 index 0000000..790a526 --- /dev/null +++ b/webpage/src/lib/Tabs.svelte @@ -0,0 +1,57 @@ + + +
+
+ +
+ +
+ + diff --git a/webpage/src/lib/requests.svelte.ts b/webpage/src/lib/requests.svelte.ts index 13738d4..37f52a5 100644 --- a/webpage/src/lib/requests.svelte.ts +++ b/webpage/src/lib/requests.svelte.ts @@ -1,10 +1,12 @@ +import { goto } from '$app/navigation'; import { userStore } from 'routes/UserStore.svelte'; -const API = "http://localhost:8000"; +const API = "/api"; export async function get(url: string) { const headers = new Headers(); //headers.append('content-type', 'application/json'); + headers.append('response-type', 'application/json'); if (userStore.user) { headers.append('token', userStore.user.token); } @@ -40,3 +42,50 @@ export async function post(url: string, body: any) { return r.json(); } + +export async function rdelete(url: string, body: any) { + const headers = new Headers(); + headers.append('content-type', 'application/json'); + if (userStore.user) { + headers.append('token', userStore.user.token); + } + + let r = await fetch(`${API}/${url}`, { + method: 'DELETE', + headers: headers, + body: JSON.stringify(body), + }); + + if (r.status !== 200) { + throw r; + } + + return r.json(); +} + +export async function postFormData(url: string, body: FormData) { + const headers = new Headers(); + //headers.append('content-type', 'multipart/form-data'); + headers.append('response-type', 'application/json'); + if (userStore.user) { + headers.append('token', userStore.user.token); + } + + let r = await fetch(`${API}/${url}`, { + method: 'POST', + headers: headers, + body: body, + }); + + if (r.status == 401) { + userStore.user = undefined; + goto('/login'); + throw new Error("Redirect"); + } + + if (r.status !== 200) { + throw r; + } + + return r.json(); +} diff --git a/webpage/src/routes/models/+page.svelte b/webpage/src/routes/models/+page.svelte index f19c30d..50a3e69 100644 --- a/webpage/src/routes/models/+page.svelte +++ b/webpage/src/routes/models/+page.svelte @@ -1,8 +1,26 @@ @@ -12,6 +30,7 @@
+ {#if list.length > 0}

My Models

@@ -57,3 +76,29 @@
{/if}
+ + diff --git a/webpage/src/routes/models/add/+page.svelte b/webpage/src/routes/models/add/+page.svelte index c8cf3d8..914e87e 100644 --- a/webpage/src/routes/models/add/+page.svelte +++ b/webpage/src/routes/models/add/+page.svelte @@ -1,6 +1,8 @@ diff --git a/webpage/src/routes/models/edit/+page.svelte b/webpage/src/routes/models/edit/+page.svelte new file mode 100644 index 0000000..bb734fb --- /dev/null +++ b/webpage/src/routes/models/edit/+page.svelte @@ -0,0 +1,525 @@ + + + + + + {#await model} + + Model + + {:then m} + {#if m} + + Model: {m.name} + + {:else} + + Model + + {/if} + {/await} + + + + + + + + + + +
+ {#await model} + Loading + {:then m} + {#if m.status == 1} +
+

+ { m.name } +

+ +

+ Preparing the model +

+
+ {:else if m.status == -1} +
+

+ {m.name} +

+ +

+ Failed to prepare model +

+ +
+ TODO button delete +
+ + +
+ + {:else if m.status == 2 } + + + + + {:else if m.status == -2 } + + + + {:else if m.status == 3 } + +
+ + Processing zip file... +
+ {:else if m.status == -3 || m.status == -4} + +
+ Failed Prepare for training.
+
+
+ + diff --git a/webpage/src/routes/models/edit/BaseModelInfo.svelte b/webpage/src/routes/models/edit/BaseModelInfo.svelte new file mode 100644 index 0000000..6961cb2 --- /dev/null +++ b/webpage/src/routes/models/edit/BaseModelInfo.svelte @@ -0,0 +1,43 @@ + + +
+

+ {model.name} +

+
+ +
+
+ Image Type: + {model.color_mode} +
+
+ Image Size: + {model.width}x{model.height} +
+
+
+
+ + diff --git a/webpage/src/routes/models/edit/DeleteModel.svelte b/webpage/src/routes/models/edit/DeleteModel.svelte new file mode 100644 index 0000000..f2a559b --- /dev/null +++ b/webpage/src/routes/models/edit/DeleteModel.svelte @@ -0,0 +1,31 @@ + + +
+
+ + + {#if nameDoesNotMatch } + + Name does not match "{model.name}" + + {/if} +
+ +
diff --git a/webpage/src/routes/models/edit/DeleteZip.svelte b/webpage/src/routes/models/edit/DeleteZip.svelte new file mode 100644 index 0000000..c83e18b --- /dev/null +++ b/webpage/src/routes/models/edit/DeleteZip.svelte @@ -0,0 +1,39 @@ + + + +
+ Failed to proccess the zip file.
+ Delete file and proccess again.
+
+
+ + + diff --git a/webpage/src/routes/models/edit/ModelData.svelte b/webpage/src/routes/models/edit/ModelData.svelte new file mode 100644 index 0000000..5d06dc9 --- /dev/null +++ b/webpage/src/routes/models/edit/ModelData.svelte @@ -0,0 +1,187 @@ + + + +
+

+ Training data +

+ {#if classes.length == 0} +

+ You need to upload data so the model can train. +

+ +
+ + + +
+
+
+
+ +
+ Please provide a file that has the training and testing data
+ The file must have 2 folders one with testing images and one with training images.
+ Each of the folders will contain the classes of the model. The folders must be the same in testing and training. + The class folders must have the images for the classes. +
+    training\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+    testing\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+
+
+ + + + Upload Zip File + +
+ + + File selected + +
+
+
+ + {#await uploading} + + {:then} + + {/await} + +
+
+ +
+
+ TODO +
+
+
+
+ {:else} +

+ You need to upload data so the model can train. +

+ {#if numberOfInvalidImages > 0} +

+ There are images {numberOfInvalidImages} that were loaded that do not have the correct format. These images will be delete when the model trains. +

+ {/if} + +
+ + +
+
+ +
+
+ TODO +
+
+ {/if} +
diff --git a/webpage/src/routes/models/edit/ModelTable.svelte b/webpage/src/routes/models/edit/ModelTable.svelte new file mode 100644 index 0000000..a2ab808 --- /dev/null +++ b/webpage/src/routes/models/edit/ModelTable.svelte @@ -0,0 +1,63 @@ + + +{#if classes.length == 0} + TODO CREATE TABLE +{:else} + +
+ + {#each classes as item} + + {/each} +
+
+{/if} + + diff --git a/webpage/src/styles/app.css b/webpage/src/styles/app.css index 4fd3e93..26c5426 100644 --- a/webpage/src/styles/app.css +++ b/webpage/src/styles/app.css @@ -90,3 +90,14 @@ a.button { .danger { color: red; } + +.card { + box-shadow: 0 2px 5px 1px #66666655; + padding: 20px; + border-radius: 10px; + margin: 20px 0; +} + +.card h3 { + margin-top: 0; +}