feat: closes #11 and fixed database cons remaining open
This commit is contained in:
265
logic/models/data.go
Normal file
265
logic/models/data.go
Normal file
@@ -0,0 +1,265 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
|
||||
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
|
||||
)
|
||||
|
||||
func InsertIfNotPresent(ss []string, s string) []string {
|
||||
i := sort.SearchStrings(ss, s)
|
||||
if len(ss) > i && ss[i] == s {
|
||||
return ss
|
||||
}
|
||||
ss = append(ss, "")
|
||||
copy(ss[i+1:], ss[i:])
|
||||
ss[i] = s
|
||||
return ss
|
||||
}
|
||||
|
||||
func processZipFile(handle *Handle, id string) {
|
||||
reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
|
||||
if err != nil {
|
||||
// TODO add msg to error
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
fmt.Printf("Faield to proccess zip file failed to open reader\n")
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
training := []string{}
|
||||
testing := []string{}
|
||||
|
||||
for _, file := range reader.Reader.File {
|
||||
|
||||
paths := strings.Split(file.Name, "/")
|
||||
|
||||
if paths[1] == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if paths[0] != "training" && paths[0] != "testing" {
|
||||
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
|
||||
if paths[0] != "training" {
|
||||
training = InsertIfNotPresent(training, paths[1])
|
||||
} else if paths[0] != "testing" {
|
||||
testing = InsertIfNotPresent(testing, paths[1])
|
||||
}
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(testing, training) {
|
||||
fmt.Printf("testing and training are diferent\n")
|
||||
fmt.Println(testing)
|
||||
fmt.Println(training)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
|
||||
base_path := path.Join("savedData", id, "data")
|
||||
|
||||
ids := map[string]string{}
|
||||
|
||||
for _, name := range training {
|
||||
dir_path := path.Join(base_path, "training", name)
|
||||
err = os.MkdirAll(dir_path, os.ModePerm)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create dir %s\n", dir_path)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
dir_path = path.Join(base_path, "testing", name)
|
||||
err = os.MkdirAll(dir_path, os.ModePerm)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create dir %s\n", dir_path)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
|
||||
id, err := model_classes.CreateClass(handle.Db, id, name)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to create class '%s' on db\n", name)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
ids[name] = id
|
||||
}
|
||||
|
||||
for _, file := range reader.Reader.File {
|
||||
if file.Name[len(file.Name) - 1] == '/' {
|
||||
continue
|
||||
}
|
||||
|
||||
file_path := path.Join(base_path, file.Name)
|
||||
f, err := os.Create(file_path)
|
||||
if err != nil {
|
||||
fmt.Printf("Could not create file %s\n", file_path)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
data, err := reader.Open(file.Name)
|
||||
if err != nil {
|
||||
fmt.Printf("Could not create file %s\n", file_path)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
defer data.Close()
|
||||
file_data, err := io.ReadAll(data)
|
||||
f.Write(file_data)
|
||||
|
||||
parts := strings.Split(file.Name, "/")
|
||||
|
||||
mode := model_classes.DATA_POINT_MODE_TRAINING
|
||||
if parts[0] == "testing" {
|
||||
mode = model_classes.DATA_POINT_MODE_TESTING
|
||||
}
|
||||
|
||||
_, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
|
||||
if err != nil {
|
||||
fmt.Printf("Failed to add data point for %s\n", id)
|
||||
fmt.Println(err)
|
||||
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Added data to model '%s'!\n", id)
|
||||
modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
|
||||
}
|
||||
|
||||
func handleDataUpload(handle *Handle) {
|
||||
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
||||
if !CheckAuthLevel(1, w, r, c) {
|
||||
return nil
|
||||
}
|
||||
if c.Mode == JSON {
|
||||
// TODO improve message
|
||||
return ErrorCode(nil, 400, nil)
|
||||
}
|
||||
|
||||
read_form, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
|
||||
return nil
|
||||
}
|
||||
|
||||
var id string
|
||||
var file []byte
|
||||
|
||||
for {
|
||||
part, err_part := read_form.NextPart()
|
||||
if err_part == io.EOF {
|
||||
break
|
||||
} else if err_part != nil {
|
||||
return &Error{Code: http.StatusBadRequest}
|
||||
}
|
||||
if part.FormName() == "id" {
|
||||
buf := new(bytes.Buffer)
|
||||
buf.ReadFrom(part)
|
||||
id = buf.String()
|
||||
}
|
||||
if part.FormName() == "file" {
|
||||
buf := new(bytes.Buffer)
|
||||
buf.ReadFrom(part)
|
||||
file = buf.Bytes()
|
||||
}
|
||||
}
|
||||
|
||||
_, err = getBaseModel(handle.Db, id)
|
||||
if err == ModelNotFoundError {
|
||||
return ErrorCode(nil, http.StatusNotFound, AnyMap{
|
||||
"NotFoundMessage": "Model not found",
|
||||
"GoBackLink": "/models",
|
||||
})
|
||||
} else if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
|
||||
// TODO mk this path configurable
|
||||
dir_path := path.Join("savedData", id)
|
||||
|
||||
f, err := os.Create(path.Join(dir_path, "base_data.zip"))
|
||||
if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
f.Write(file)
|
||||
|
||||
modelUpdateStatus(handle, id, PREPARING_ZIP_FILE)
|
||||
|
||||
go processZipFile(handle, id)
|
||||
|
||||
Redirect("/models/edit?id="+id, c.Mode, w, r)
|
||||
return nil
|
||||
})
|
||||
|
||||
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
|
||||
if !CheckAuthLevel(1, w, r, c) {
|
||||
return nil
|
||||
}
|
||||
if c.Mode == JSON {
|
||||
panic("Handle delete zip file json")
|
||||
}
|
||||
|
||||
f, err := MyParseForm(r)
|
||||
if err != nil {
|
||||
return ErrorCode(err, 400, c.AddMap(nil))
|
||||
}
|
||||
|
||||
if !CheckId(f, "id") {
|
||||
return ErrorCode(err, 400, c.AddMap(nil))
|
||||
}
|
||||
|
||||
id := f.Get("id")
|
||||
|
||||
model, err := getBaseModel(handle.Db, id)
|
||||
if err == ModelNotFoundError {
|
||||
return ErrorCode(nil, http.StatusNotFound, AnyMap{
|
||||
"NotFoundMessage": "Model not found",
|
||||
"GoBackLink": "/models",
|
||||
})
|
||||
} else if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
|
||||
if model.Status != FAILED_PREPARING_ZIP_FILE {
|
||||
// TODO add message
|
||||
return ErrorCode(nil, 400, c.AddMap(nil))
|
||||
}
|
||||
|
||||
err = os.Remove(path.Join("savedData", id, "base_data.zip"));
|
||||
if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
|
||||
err = os.RemoveAll(path.Join("savedData", id, "data"));
|
||||
if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
|
||||
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
|
||||
if err != nil {
|
||||
return Error500(err)
|
||||
}
|
||||
|
||||
modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
|
||||
Redirect("/models/edit?id="+id, c.Mode, w, r)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user