package models import ( "archive/zip" "bytes" "fmt" "io" "net/http" "os" "path" "reflect" "sort" "strings" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) func InsertIfNotPresent(ss []string, s string) []string { i := sort.SearchStrings(ss, s) if len(ss) > i && ss[i] == s { return ss } ss = append(ss, "") copy(ss[i+1:], ss[i:]) ss[i] = s return ss } func processZipFile(c *Context, model *BaseModel) { reader, err := zip.OpenReader(path.Join("savedData", model.Id, "base_data.zip")) if err != nil { // TODO add msg to error ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) fmt.Printf("Faield to proccess zip file failed to open reader\n") fmt.Println(err) return } defer reader.Close() training := []string{} testing := []string{} for _, file := range reader.Reader.File { paths := strings.Split(file.Name, "/") if paths[1] == "" { continue } if paths[0] != "training" && paths[0] != "testing" { fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } if paths[0] != "training" { training = InsertIfNotPresent(training, paths[1]) } else if paths[0] != "testing" { testing = InsertIfNotPresent(testing, paths[1]) } } if !reflect.DeepEqual(testing, training) { fmt.Printf("testing and training are diferent\n") fmt.Println(testing) fmt.Println(training) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } base_path := path.Join("savedData", model.Id, "data") if err = os.MkdirAll(base_path, os.ModePerm); err != nil { fmt.Printf("Failed to create base_path dir\n") ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } ids := map[string]string{} for i, name := range training { id, err := model_classes.CreateClass(c.Db, model.Id, i, name) if err != nil { fmt.Printf("Failed to create class '%s' on db\n", name) ModelUpdateStatus(c, id, FAILED_PREPARING_ZIP_FILE) return } ids[name] = id } for _, file := range reader.Reader.File { if file.Name[len(file.Name)-1] == '/' { continue } data, err := reader.Open(file.Name) if err != nil { fmt.Printf("Could not open file in zip %s\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } defer data.Close() file_data, err := io.ReadAll(data) if err != nil { fmt.Printf("Could not read file file in zip %s\n", file.Name) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } // TODO check if the file is a valid photo that matched the defined photo on the database parts := strings.Split(file.Name, "/") mode := model_classes.DATA_POINT_MODE_TRAINING if parts[0] == "testing" { mode = model_classes.DATA_POINT_MODE_TESTING } data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode) if err != nil { fmt.Printf("Failed to add data point for %s\n", model.Id) fmt.Println(err) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } file_path := path.Join(base_path, data_point_id + "." + model.Format) f, err := os.Create(file_path) if err != nil { fmt.Printf("Could not create file %s\n", file_path) ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) return } defer f.Close() f.Write(file_data) if !testImgForModel(c, model, file_path) { c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name) c.Logger.Warn("Not failling updating data point to status -1") message := "Image did not have valid format for the model" if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil { c.Logger.Error("Failed to update data point status") ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE) } } } fmt.Printf("Added data to model '%s'!\n", model.Id) ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING) } func handleDataUpload(handle *Handle) { handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { // TODO improve message return ErrorCode(nil, 400, nil) } read_form, err := r.MultipartReader() if err != nil { LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) return nil } var id string var file []byte for { part, err_part := read_form.NextPart() if err_part == io.EOF { break } else if err_part != nil { return &Error{Code: http.StatusBadRequest} } if part.FormName() == "id" { buf := new(bytes.Buffer) buf.ReadFrom(part) id = buf.String() } if part.FormName() == "file" { buf := new(bytes.Buffer) buf.ReadFrom(part) file = buf.Bytes() } } model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "base_data.zip")) if err != nil { return Error500(err) } defer f.Close() f.Write(file) ModelUpdateStatus(c, id, PREPARING_ZIP_FILE) go processZipFile(c, model) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { panic("Handle delete zip file json") } f, err := MyParseForm(r) if err != nil { return ErrorCode(err, 400, c.AddMap(nil)) } if !CheckId(f, "id") { return ErrorCode(err, 400, c.AddMap(nil)) } id := f.Get("id") model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { return ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } if model.Status != FAILED_PREPARING_ZIP_FILE { // TODO add message return ErrorCode(nil, 400, c.AddMap(nil)) } err = os.Remove(path.Join("savedData", id, "base_data.zip")) if err != nil { return Error500(err) } err = os.RemoveAll(path.Join("savedData", id, "data")) if err != nil { return Error500(err) } _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) if err != nil { return Error500(err) } ModelUpdateStatus(c, id, CONFIRM_PRE_TRAINING) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) }