feat: closes #11 and fixed database cons remaining open

logic/models/classes/data_point.go

@@ -0,0 +1,27 @@
+package model_classes
+
+import (
+	"database/sql"
+	"errors"
+)
+
+func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
+    id = ""
+    _, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode)
+    if err != nil {
+        return
+    }
+
+    rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if !rows.Next() {
+        return id, errors.New("Something worng")
+    }
+
+    err = rows.Scan(&id)
+    return
+}

logic/models/classes/main.go

@@ -0,0 +1,71 @@
+package model_classes
+
+import (
+	"database/sql"
+	"errors"
+	// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+type ModelClass struct {
+	Id       string
+	ModelId string
+	Name     string
+}
+
+func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
+    
+    rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    for rows.Next() {
+        var model ModelClass
+        err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
+        if err != nil {
+            return
+        }
+
+        cls = append(cls, model)
+    }
+
+    return
+}
+
+var ClassAlreadyExists = errors.New("Class aready exists")
+
+func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) {
+    id = ""
+    rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if rows.Next() {
+        return id, ClassAlreadyExists
+    }
+
+    _, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name)
+
+    if err != nil {
+        return
+    }
+
+    rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if !rows.Next() {
+        return id, errors.New("Something wrong")
+    }
+
+    if err = rows.Scan(&id); err != nil {
+        return
+    }
+
+    return 
+}

logic/models/classes/types.go

@@ -0,0 +1,8 @@
+package model_classes
+
+type DATA_POINT_MODE int
+
+const (
+    DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1 
+    DATA_POINT_MODE_TESTING = 2
+)

logic/models/data.go

@@ -0,0 +1,265 @@
+package models
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"strings"
+
+	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func InsertIfNotPresent(ss []string, s string) []string {
+    i := sort.SearchStrings(ss, s)
+    if len(ss) > i && ss[i] == s {
+        return ss
+    }
+    ss = append(ss, "")
+    copy(ss[i+1:], ss[i:])
+    ss[i] = s
+    return ss
+}
+
+func processZipFile(handle *Handle, id string) {
+    reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
+    if err != nil {
+        // TODO add msg to error
+        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+        fmt.Printf("Faield to proccess zip file failed to open reader\n")
+        fmt.Println(err)
+        return
+    }
+    defer reader.Close()
+
+    training := []string{}
+    testing := []string{}
+
+    for _, file := range reader.Reader.File {
+
+        paths := strings.Split(file.Name, "/")
+
+        if paths[1] == "" {
+            continue
+        }
+
+        if paths[0] != "training" && paths[0] != "testing" {
+            fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+
+        if paths[0] != "training" {
+            training = InsertIfNotPresent(training, paths[1])
+        } else if paths[0] != "testing" {
+            testing = InsertIfNotPresent(testing, paths[1])
+        }
+    }
+
+    if !reflect.DeepEqual(testing, training) {
+        fmt.Printf("testing and training are diferent\n")
+        fmt.Println(testing)
+        fmt.Println(training)
+        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+        return
+    }
+
+    base_path := path.Join("savedData", id, "data")
+
+    ids := map[string]string{}
+
+    for _, name := range training {
+        dir_path := path.Join(base_path, "training", name)
+		err = os.MkdirAll(dir_path, os.ModePerm)
+        if err != nil {
+            fmt.Printf("Failed to create dir %s\n", dir_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+        dir_path = path.Join(base_path, "testing", name)
+		err = os.MkdirAll(dir_path, os.ModePerm)
+        if err != nil {
+            fmt.Printf("Failed to create dir %s\n", dir_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+
+        id, err := model_classes.CreateClass(handle.Db, id, name)
+        if err != nil {
+            fmt.Printf("Failed to create class '%s' on db\n", name)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+        ids[name] = id
+    }
+
+    for _, file := range reader.Reader.File {
+        if file.Name[len(file.Name) - 1] == '/' {
+            continue
+        }
+
+        file_path := path.Join(base_path, file.Name)
+		f, err := os.Create(file_path)
+		if err != nil {
+            fmt.Printf("Could not create file %s\n", file_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+		}
+		defer f.Close()
+        data, err := reader.Open(file.Name)
+		if err != nil {
+            fmt.Printf("Could not create file %s\n", file_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+		}
+        defer data.Close()
+        file_data, err := io.ReadAll(data)
+		f.Write(file_data)
+
+        parts := strings.Split(file.Name, "/")
+
+        mode := model_classes.DATA_POINT_MODE_TRAINING
+        if parts[0] == "testing" {
+            mode = model_classes.DATA_POINT_MODE_TESTING
+        }
+
+        _, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
+        if err != nil {
+            fmt.Printf("Failed to add data point for %s\n", id)
+            fmt.Println(err)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+    }
+    
+    fmt.Printf("Added data to model '%s'!\n", id)
+    modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
+}
+
+func handleDataUpload(handle *Handle) {
+	handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
+        if !CheckAuthLevel(1, w, r, c) {
+            return nil
+        }
+		if c.Mode == JSON {
+			// TODO improve message
+			return ErrorCode(nil, 400, nil)
+		}
+
+		read_form, err := r.MultipartReader()
+		if err != nil {
+			LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
+			return nil
+		}
+
+		var id string
+		var file []byte
+
+		for {
+			part, err_part := read_form.NextPart()
+			if err_part == io.EOF {
+				break
+			} else if err_part != nil {
+				return &Error{Code: http.StatusBadRequest}
+			}
+			if part.FormName() == "id" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				id = buf.String()
+			}
+			if part.FormName() == "file" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				file = buf.Bytes()
+			}
+		}
+
+        _, err = getBaseModel(handle.Db, id)
+        if err == ModelNotFoundError {
+			return ErrorCode(nil, http.StatusNotFound, AnyMap{
+				"NotFoundMessage": "Model not found",
+				"GoBackLink":      "/models",
+			})
+        } else if err != nil {
+            return Error500(err)
+        }
+
+		// TODO mk this path configurable
+		dir_path := path.Join("savedData", id)
+
+		f, err := os.Create(path.Join(dir_path, "base_data.zip"))
+		if err != nil {
+			return Error500(err)
+		}
+		defer f.Close()
+
+		f.Write(file)
+
+        modelUpdateStatus(handle, id, PREPARING_ZIP_FILE)
+
+        go processZipFile(handle, id)
+
+        Redirect("/models/edit?id="+id, c.Mode, w, r)
+		return nil
+	})
+
+    handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
+        if !CheckAuthLevel(1, w, r, c) {
+            return nil
+        }
+        if c.Mode == JSON {
+            panic("Handle delete zip file json")
+        }
+
+        f, err := MyParseForm(r)
+        if err != nil {
+            return ErrorCode(err, 400, c.AddMap(nil))
+        }
+
+        if !CheckId(f, "id") {
+            return ErrorCode(err, 400, c.AddMap(nil))
+        }
+
+        id := f.Get("id")
+
+        model, err := getBaseModel(handle.Db, id)
+        if err == ModelNotFoundError {
+			return ErrorCode(nil, http.StatusNotFound, AnyMap{
+				"NotFoundMessage": "Model not found",
+				"GoBackLink":      "/models",
+			})
+        } else if err != nil {
+            return Error500(err)
+        }
+
+        if model.Status != FAILED_PREPARING_ZIP_FILE {
+            // TODO add message
+            return ErrorCode(nil, 400, c.AddMap(nil))
+        }
+
+        err = os.Remove(path.Join("savedData", id, "base_data.zip"));
+        if err != nil {
+            return Error500(err)
+        }
+
+        err = os.RemoveAll(path.Join("savedData", id, "data"));
+        if err != nil {
+            return Error500(err)
+        }
+
+        _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
+        if err != nil {
+            return Error500(err)
+        }
+
+        modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
+        Redirect("/models/edit?id="+id, c.Mode, w, r)
+        return nil
+    })
+}

logic/models/delete.go

@@ -62,6 +62,7 @@ func handleDelete(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		if !rows.Next() {
 			return ErrorCode(nil, http.StatusNotFound, AnyMap{

logic/models/edit.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 
+	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
 )
 
@@ -26,6 +27,7 @@ func handleEdit(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		if !rows.Next() {
 			return ErrorCode(nil, http.StatusNotFound, AnyMap{
@@ -66,10 +68,20 @@ func handleEdit(handle *Handle) {
 				"Model": model,
 			}))
 		case CONFIRM_PRE_TRAINING:
+            
+            cls, err := model_classes.ListClasses(handle.Db, id)
+            if err != nil {
+                return Error500(err)
+            }
+
 			LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
 				"Model": model,
+                "Classes": cls,
 			}))
-
+        case PREPARING_ZIP_FILE:
+            LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
+                "Model": model,
+            }))
 		default:
 			fmt.Printf("Unkown Status: %d\n", model.Status)
 			return Error500(nil)

logic/models/index.go

@@ -9,4 +9,8 @@ func HandleModels (handle *Handle) {
     handleEdit(handle)
     handleDelete(handle)
     handleList(handle)
+    
+    // Data endpoints
+    handleDataUpload(handle)
 }
+

logic/models/list.go

@@ -21,6 +21,7 @@ func handleList(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		type row struct {
 			Name string

logic/models/types.go

@@ -1,5 +1,10 @@
 package models
 
+import (
+	"database/sql"
+	"errors"
+)
+
 type BaseModel struct {
 	Name   string
 	Status int
@@ -7,8 +12,32 @@ type BaseModel struct {
 }
 
 const (
+	FAILED_PREPARING_ZIP_FILE = -2
 	FAILED_PREPARING = -1
 
-	PREPARING = iota
-	CONFIRM_PRE_TRAINING
+	PREPARING = 1
+	CONFIRM_PRE_TRAINING = 2
+	PREPARING_ZIP_FILE = 3
 )
+
+var ModelNotFoundError = errors.New("Model not found error")
+
+func getBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
+	rows, err := db.Query("select name, status, id from models where id=$1;", id)
+	if err != nil {
+		return
+	}
+    defer rows.Close()
+
+	if !rows.Next() {
+		return nil, ModelNotFoundError
+	}
+
+    base = &BaseModel{}
+	err = rows.Scan(&base.Name, &base.Status, &base.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}