package models import ( "archive/zip" "bytes" "fmt" "io" "net/http" "os" "path" "reflect" "sort" "strings" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) func InsertIfNotPresent(ss []string, s string) []string { i := sort.SearchStrings(ss, s) if len(ss) > i && ss[i] == s { return ss } ss = append(ss, "") copy(ss[i+1:], ss[i:]) ss[i] = s return ss } func processZipFile(c *Context, model *BaseModel) { reader, err := zip.OpenReader(path.Join("savedData", model.Id, "base_data.zip")) if err != nil { // TODO add msg to error ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) fmt.Printf("Faield to proccess zip file failed to open reader\n") fmt.Println(err) return } defer reader.Close() training := []string{} testing := []string{} for _, file := range reader.Reader.File { paths := strings.Split(file.Name, "/") if paths[1] == "" { continue } if paths[0] != "training" && paths[0] != "testing" { fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } if paths[0] != "training" { training = InsertIfNotPresent(training, paths[1]) } else if paths[0] != "testing" { testing = InsertIfNotPresent(testing, paths[1]) } } if !reflect.DeepEqual(testing, training) { fmt.Printf("testing and training are diferent\n") fmt.Println(testing) fmt.Println(training) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } base_path := path.Join("savedData", model.Id, "data") if err = os.MkdirAll(base_path, os.ModePerm); err != nil { fmt.Printf("Failed to create base_path dir\n") ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } ids := map[string]string{} for i, name := range training { id, err := model_classes.CreateClass(c.Db, model.Id, i, name) if err != nil { fmt.Printf("Failed to create class '%s' on db\n", name) ModelUpdateStatus(c, id, FAILED_PREPARING_ZIP_FILE) return } ids[name] = id } for _, file := range reader.Reader.File { if file.Name[len(file.Name)-1] == '/' { continue } data, err := reader.Open(file.Name) if err != nil { fmt.Printf("Could not open file in zip %s\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } defer data.Close() file_data, err := io.ReadAll(data) if err != nil { fmt.Printf("Could not read file file in zip %s\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } // TODO check if the file is a valid photo that matched the defined photo on the database parts := strings.Split(file.Name, "/") mode := model_classes.DATA_POINT_MODE_TRAINING if parts[0] == "testing" { mode = model_classes.DATA_POINT_MODE_TESTING } data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode) if err != nil { fmt.Printf("Failed to add data point for %s\n", model.Id) fmt.Println(err) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } file_path := path.Join(base_path, data_point_id+"."+model.Format) f, err := os.Create(file_path) if err != nil { fmt.Printf("Could not create file %s\n", file_path) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } defer f.Close() f.Write(file_data) if !testImgForModel(c, model, file_path) { c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) c.Logger.Warn("Not failling updating data point to status -1") message := "Image did not have valid format for the model" if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { c.Logger.Error("Failed to update data point status") ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) } } } fmt.Printf("Added data to model '%s'!\n", model.Id) ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) } func processZipFileExpand(c *Context, model *BaseModel) { var err error failed := func(msg string) { c.Logger.Error(msg, "err", err) ModelUpdateStatus(c, model.Id, READY_FAILED) } reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip")) if err != nil { failed("Faield to proccess zip file failed to open reader\n") return } defer reader.Close() training := []string{} testing := []string{} for _, file := range reader.Reader.File { paths := strings.Split(file.Name, "/") if paths[1] == "" { continue } if paths[0] != "training" && paths[0] != "testing" { failed(fmt.Sprintf("Invalid file '%s' TODO add msg to response!!!", file.Name)) return } if paths[0] != "training" { training = InsertIfNotPresent(training, paths[1]) } else if paths[0] != "testing" { testing = InsertIfNotPresent(testing, paths[1]) } } if !reflect.DeepEqual(testing, training) { failed("testing and training are diferent") return } base_path := path.Join("savedData", model.Id, "data") if err = os.MkdirAll(base_path, os.ModePerm); err != nil { failed("Failed to create base_path dir") return } ids := map[string]string{} for i, name := range training { id, err := model_classes.CreateClass(c.Db, model.Id, i, name) if err != nil { failed(fmt.Sprintf("Failed to create class '%s' on db\n", name)) return } ids[name] = id } for _, file := range reader.Reader.File { if file.Name[len(file.Name)-1] == '/' { continue } data, err := reader.Open(file.Name) if err != nil { failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name)) return } defer data.Close() file_data, err := io.ReadAll(data) if err != nil { failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name)) return } // TODO check if the file is a valid photo that matched the defined photo on the database parts := strings.Split(file.Name, "/") mode := model_classes.DATA_POINT_MODE_TRAINING if parts[0] == "testing" { mode = model_classes.DATA_POINT_MODE_TESTING } data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode) if err != nil { failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id)) return } file_path := path.Join(base_path, data_point_id+"."+model.Format) f, err := os.Create(file_path) if err != nil { failed(fmt.Sprintf("Could not create file %s\n", file_path)) return } defer f.Close() f.Write(file_data) if !testImgForModel(c, model, file_path) { c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) c.Logger.Warn("Not failling updating data point to status -1") message := "Image did not have valid format for the model" if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { failed(fmt.Sprintf("Failed to update data point status")) return } } } c.Logger.Info("Added data to model", "id", model.Id) ModelUpdateStatus(c, model.Id, READY) } func handleDataUpload(handle *Handle) { handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { read_form, err := r.MultipartReader() if err != nil { return c.JsonBadRequest("Please provide a valid form data request!") } var id string var file []byte for { part, err_part := read_form.NextPart() if err_part == io.EOF { break } else if err_part != nil { return c.JsonBadRequest("Please provide a valid form data request!") } if part.FormName() == "id" { buf := new(bytes.Buffer) buf.ReadFrom(part) id = buf.String() } if part.FormName() == "file" { buf := new(bytes.Buffer) buf.ReadFrom(part) file = buf.Bytes() } } model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { return Error500(err) } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "base_data.zip")) if err != nil { return Error500(err) } defer f.Close() f.Write(file) ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) go processZipFile(c, model) return c.SendJSON(model.Id) } read_form, err := r.MultipartReader() if err != nil { LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) return nil } var id string var file []byte for { part, err_part := read_form.NextPart() if err_part == io.EOF { break } else if err_part != nil { return &Error{Code: http.StatusBadRequest} } if part.FormName() == "id" { buf := new(bytes.Buffer) buf.ReadFrom(part) id = buf.String() } if part.FormName() == "file" { buf := new(bytes.Buffer) buf.ReadFrom(part) file = buf.Bytes() } } model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "base_data.zip")) if err != nil { return Error500(err) } defer f.Close() f.Write(file) ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) go processZipFile(c, model) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) // ------ // ------ CLASS DATA UPLOAD // ------ handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } read_form, err := r.MultipartReader() if err != nil { return c.JsonBadRequest("Please provide a valid form data request!") } var id string var file []byte for { part, err_part := read_form.NextPart() if err_part == io.EOF { break } else if err_part != nil { return c.JsonBadRequest("Please provide a valid form data request!") } if part.FormName() == "id" { buf := new(bytes.Buffer) buf.ReadFrom(part) id = buf.String() } if part.FormName() == "file" { buf := new(bytes.Buffer) buf.ReadFrom(part) file = buf.Bytes() } } c.Logger.Info("Trying to expand model", "id", id) model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { return Error500(err) } // TODO work in allowing the model to add new in the pre ready moment if model.Status != READY { return c.JsonBadRequest("Model not in the correct state to add a more classes") } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "expand_data.zip")) if err != nil { return Error500(err) } defer f.Close() f.Write(file) ModelUpdateStatus(c, id, READY_ALTERATION) go processZipFileExpand(c, model) return c.SendJSON(model.Id) }) handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { type ModelData struct { Id string `json:"id"` } var dat ModelData if err := c.ToJSON(r, &dat); err != nil { return err } model, err := GetBaseModel(handle.Db, dat.Id) if err == ModelNotFoundError { return c.SendJSONStatus(http.StatusNotFound, "Model not found") } else if err != nil { return Error500(err) } delete_path := "base_data.zip" if model.Status == READY_FAILED { delete_path = "expand_data.zip" } else if model.Status != FAILED_PREPARING_ZIP_FILE { return c.JsonBadRequest("Model not in the correct status") } err = os.Remove(path.Join("savedData", model.Id, delete_path)) if err != nil { return Error500(err) } if model.Status != READY_FAILED { err = os.RemoveAll(path.Join("savedData", model.Id, "data")) if err != nil { return Error500(err) } } else { c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding") } if model.Status != READY_FAILED { _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id) if err != nil { return Error500(err) } } else { _, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN) if err != nil { return Error500(err) } } if model.Status != READY_FAILED { ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) } else { ModelUpdateStatus(c, model.Id, READY) } return c.SendJSON(model.Id) } f, err := MyParseForm(r) if err != nil { return ErrorCode(err, 400, c.AddMap(nil)) } if !CheckId(f, "id") { return ErrorCode(err, 400, c.AddMap(nil)) } id := f.Get("id") model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } if model.Status != FAILED_PREPARING_ZIP_FILE { // TODO add message return ErrorCode(nil, 400, c.AddMap(nil)) } err = os.Remove(path.Join("savedData", id, "base_data.zip")) if err != nil { return Error500(err) } err = os.RemoveAll(path.Join("savedData", id, "data")) if err != nil { return Error500(err) } _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) if err != nil { return Error500(err) } ModelUpdateStatus(c, id, CONFIRM_PRE_TRAINING) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) }