package models import ( "archive/zip" "bytes" "fmt" "io" "net/http" "os" "path" "reflect" "sort" "strings" model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) func InsertIfNotPresent(ss []string, s string) []string { i := sort.SearchStrings(ss, s) if len(ss) > i && ss[i] == s { return ss } ss = append(ss, "") copy(ss[i+1:], ss[i:]) ss[i] = s return ss } func processZipFile(handle *Handle, id string) { reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip")) if err != nil { // TODO add msg to error modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) fmt.Printf("Faield to proccess zip file failed to open reader\n") fmt.Println(err) return } defer reader.Close() training := []string{} testing := []string{} for _, file := range reader.Reader.File { paths := strings.Split(file.Name, "/") if paths[1] == "" { continue } if paths[0] != "training" && paths[0] != "testing" { fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } if paths[0] != "training" { training = InsertIfNotPresent(training, paths[1]) } else if paths[0] != "testing" { testing = InsertIfNotPresent(testing, paths[1]) } } if !reflect.DeepEqual(testing, training) { fmt.Printf("testing and training are diferent\n") fmt.Println(testing) fmt.Println(training) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } base_path := path.Join("savedData", id, "data") ids := map[string]string{} for _, name := range training { dir_path := path.Join(base_path, "training", name) err = os.MkdirAll(dir_path, os.ModePerm) if err != nil { fmt.Printf("Failed to create dir %s\n", dir_path) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } dir_path = path.Join(base_path, "testing", name) err = os.MkdirAll(dir_path, os.ModePerm) if err != nil { fmt.Printf("Failed to create dir %s\n", dir_path) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } id, err := model_classes.CreateClass(handle.Db, id, name) if err != nil { fmt.Printf("Failed to create class '%s' on db\n", name) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } ids[name] = id } for _, file := range reader.Reader.File { if file.Name[len(file.Name) - 1] == '/' { continue } file_path := path.Join(base_path, file.Name) f, err := os.Create(file_path) if err != nil { fmt.Printf("Could not create file %s\n", file_path) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } defer f.Close() data, err := reader.Open(file.Name) if err != nil { fmt.Printf("Could not create file %s\n", file_path) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } defer data.Close() file_data, err := io.ReadAll(data) f.Write(file_data) parts := strings.Split(file.Name, "/") mode := model_classes.DATA_POINT_MODE_TRAINING if parts[0] == "testing" { mode = model_classes.DATA_POINT_MODE_TESTING } _, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode) if err != nil { fmt.Printf("Failed to add data point for %s\n", id) fmt.Println(err) modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) return } } fmt.Printf("Added data to model '%s'!\n", id) modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING) } func handleDataUpload(handle *Handle) { handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { // TODO improve message return ErrorCode(nil, 400, nil) } read_form, err := r.MultipartReader() if err != nil { LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) return nil } var id string var file []byte for { part, err_part := read_form.NextPart() if err_part == io.EOF { break } else if err_part != nil { return &Error{Code: http.StatusBadRequest} } if part.FormName() == "id" { buf := new(bytes.Buffer) buf.ReadFrom(part) id = buf.String() } if part.FormName() == "file" { buf := new(bytes.Buffer) buf.ReadFrom(part) file = buf.Bytes() } } _, err = getBaseModel(handle.Db, id) if err == ModelNotFoundError { return ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } // TODO mk this path configurable dir_path := path.Join("savedData", id) f, err := os.Create(path.Join(dir_path, "base_data.zip")) if err != nil { return Error500(err) } defer f.Close() f.Write(file) modelUpdateStatus(handle, id, PREPARING_ZIP_FILE) go processZipFile(handle, id) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { if !CheckAuthLevel(1, w, r, c) { return nil } if c.Mode == JSON { panic("Handle delete zip file json") } f, err := MyParseForm(r) if err != nil { return ErrorCode(err, 400, c.AddMap(nil)) } if !CheckId(f, "id") { return ErrorCode(err, 400, c.AddMap(nil)) } id := f.Get("id") model, err := getBaseModel(handle.Db, id) if err == ModelNotFoundError { return ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", }) } else if err != nil { return Error500(err) } if model.Status != FAILED_PREPARING_ZIP_FILE { // TODO add message return ErrorCode(nil, 400, c.AddMap(nil)) } err = os.Remove(path.Join("savedData", id, "base_data.zip")); if err != nil { return Error500(err) } err = os.RemoveAll(path.Join("savedData", id, "data")); if err != nil { return Error500(err) } _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) if err != nil { return Error500(err) } modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING) Redirect("/models/edit?id="+id, c.Mode, w, r) return nil }) }