feat: closes #11 and fixed database cons remaining open

logic/models/classes/data_point.go

@@ -0,0 +1,27 @@
+package model_classes
+
+import (
+	"database/sql"
+	"errors"
+)
+
+func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
+    id = ""
+    _, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode)
+    if err != nil {
+        return
+    }
+
+    rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if !rows.Next() {
+        return id, errors.New("Something worng")
+    }
+
+    err = rows.Scan(&id)
+    return
+}

logic/models/classes/main.go

@@ -0,0 +1,71 @@
+package model_classes
+
+import (
+	"database/sql"
+	"errors"
+	// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+type ModelClass struct {
+	Id       string
+	ModelId string
+	Name     string
+}
+
+func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
+    
+    rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    for rows.Next() {
+        var model ModelClass
+        err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
+        if err != nil {
+            return
+        }
+
+        cls = append(cls, model)
+    }
+
+    return
+}
+
+var ClassAlreadyExists = errors.New("Class aready exists")
+
+func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) {
+    id = ""
+    rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if rows.Next() {
+        return id, ClassAlreadyExists
+    }
+
+    _, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name)
+
+    if err != nil {
+        return
+    }
+
+    rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
+    if err != nil {
+        return
+    }
+    defer rows.Close()
+
+    if !rows.Next() {
+        return id, errors.New("Something wrong")
+    }
+
+    if err = rows.Scan(&id); err != nil {
+        return
+    }
+
+    return 
+}

logic/models/classes/types.go

@@ -0,0 +1,8 @@
+package model_classes
+
+type DATA_POINT_MODE int
+
+const (
+    DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1 
+    DATA_POINT_MODE_TESTING = 2
+)

logic/models/data.go

@@ -0,0 +1,265 @@
+package models
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"reflect"
+	"sort"
+	"strings"
+
+	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
+	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
+)
+
+func InsertIfNotPresent(ss []string, s string) []string {
+    i := sort.SearchStrings(ss, s)
+    if len(ss) > i && ss[i] == s {
+        return ss
+    }
+    ss = append(ss, "")
+    copy(ss[i+1:], ss[i:])
+    ss[i] = s
+    return ss
+}
+
+func processZipFile(handle *Handle, id string) {
+    reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
+    if err != nil {
+        // TODO add msg to error
+        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+        fmt.Printf("Faield to proccess zip file failed to open reader\n")
+        fmt.Println(err)
+        return
+    }
+    defer reader.Close()
+
+    training := []string{}
+    testing := []string{}
+
+    for _, file := range reader.Reader.File {
+
+        paths := strings.Split(file.Name, "/")
+
+        if paths[1] == "" {
+            continue
+        }
+
+        if paths[0] != "training" && paths[0] != "testing" {
+            fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+
+        if paths[0] != "training" {
+            training = InsertIfNotPresent(training, paths[1])
+        } else if paths[0] != "testing" {
+            testing = InsertIfNotPresent(testing, paths[1])
+        }
+    }
+
+    if !reflect.DeepEqual(testing, training) {
+        fmt.Printf("testing and training are diferent\n")
+        fmt.Println(testing)
+        fmt.Println(training)
+        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+        return
+    }
+
+    base_path := path.Join("savedData", id, "data")
+
+    ids := map[string]string{}
+
+    for _, name := range training {
+        dir_path := path.Join(base_path, "training", name)
+		err = os.MkdirAll(dir_path, os.ModePerm)
+        if err != nil {
+            fmt.Printf("Failed to create dir %s\n", dir_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+        dir_path = path.Join(base_path, "testing", name)
+		err = os.MkdirAll(dir_path, os.ModePerm)
+        if err != nil {
+            fmt.Printf("Failed to create dir %s\n", dir_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+
+        id, err := model_classes.CreateClass(handle.Db, id, name)
+        if err != nil {
+            fmt.Printf("Failed to create class '%s' on db\n", name)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+        ids[name] = id
+    }
+
+    for _, file := range reader.Reader.File {
+        if file.Name[len(file.Name) - 1] == '/' {
+            continue
+        }
+
+        file_path := path.Join(base_path, file.Name)
+		f, err := os.Create(file_path)
+		if err != nil {
+            fmt.Printf("Could not create file %s\n", file_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+		}
+		defer f.Close()
+        data, err := reader.Open(file.Name)
+		if err != nil {
+            fmt.Printf("Could not create file %s\n", file_path)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+		}
+        defer data.Close()
+        file_data, err := io.ReadAll(data)
+		f.Write(file_data)
+
+        parts := strings.Split(file.Name, "/")
+
+        mode := model_classes.DATA_POINT_MODE_TRAINING
+        if parts[0] == "testing" {
+            mode = model_classes.DATA_POINT_MODE_TESTING
+        }
+
+        _, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
+        if err != nil {
+            fmt.Printf("Failed to add data point for %s\n", id)
+            fmt.Println(err)
+            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
+            return
+        }
+    }
+    
+    fmt.Printf("Added data to model '%s'!\n", id)
+    modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
+}
+
+func handleDataUpload(handle *Handle) {
+	handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
+        if !CheckAuthLevel(1, w, r, c) {
+            return nil
+        }
+		if c.Mode == JSON {
+			// TODO improve message
+			return ErrorCode(nil, 400, nil)
+		}
+
+		read_form, err := r.MultipartReader()
+		if err != nil {
+			LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
+			return nil
+		}
+
+		var id string
+		var file []byte
+
+		for {
+			part, err_part := read_form.NextPart()
+			if err_part == io.EOF {
+				break
+			} else if err_part != nil {
+				return &Error{Code: http.StatusBadRequest}
+			}
+			if part.FormName() == "id" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				id = buf.String()
+			}
+			if part.FormName() == "file" {
+				buf := new(bytes.Buffer)
+				buf.ReadFrom(part)
+				file = buf.Bytes()
+			}
+		}
+
+        _, err = getBaseModel(handle.Db, id)
+        if err == ModelNotFoundError {
+			return ErrorCode(nil, http.StatusNotFound, AnyMap{
+				"NotFoundMessage": "Model not found",
+				"GoBackLink":      "/models",
+			})
+        } else if err != nil {
+            return Error500(err)
+        }
+
+		// TODO mk this path configurable
+		dir_path := path.Join("savedData", id)
+
+		f, err := os.Create(path.Join(dir_path, "base_data.zip"))
+		if err != nil {
+			return Error500(err)
+		}
+		defer f.Close()
+
+		f.Write(file)
+
+        modelUpdateStatus(handle, id, PREPARING_ZIP_FILE)
+
+        go processZipFile(handle, id)
+
+        Redirect("/models/edit?id="+id, c.Mode, w, r)
+		return nil
+	})
+
+    handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
+        if !CheckAuthLevel(1, w, r, c) {
+            return nil
+        }
+        if c.Mode == JSON {
+            panic("Handle delete zip file json")
+        }
+
+        f, err := MyParseForm(r)
+        if err != nil {
+            return ErrorCode(err, 400, c.AddMap(nil))
+        }
+
+        if !CheckId(f, "id") {
+            return ErrorCode(err, 400, c.AddMap(nil))
+        }
+
+        id := f.Get("id")
+
+        model, err := getBaseModel(handle.Db, id)
+        if err == ModelNotFoundError {
+			return ErrorCode(nil, http.StatusNotFound, AnyMap{
+				"NotFoundMessage": "Model not found",
+				"GoBackLink":      "/models",
+			})
+        } else if err != nil {
+            return Error500(err)
+        }
+
+        if model.Status != FAILED_PREPARING_ZIP_FILE {
+            // TODO add message
+            return ErrorCode(nil, 400, c.AddMap(nil))
+        }
+
+        err = os.Remove(path.Join("savedData", id, "base_data.zip"));
+        if err != nil {
+            return Error500(err)
+        }
+
+        err = os.RemoveAll(path.Join("savedData", id, "data"));
+        if err != nil {
+            return Error500(err)
+        }
+
+        _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
+        if err != nil {
+            return Error500(err)
+        }
+
+        modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
+        Redirect("/models/edit?id="+id, c.Mode, w, r)
+        return nil
+    })
+}

logic/models/delete.go

@@ -62,6 +62,7 @@ func handleDelete(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		if !rows.Next() {
 			return ErrorCode(nil, http.StatusNotFound, AnyMap{

logic/models/edit.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/http"
 
+	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
 	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
 )
 
@@ -26,6 +27,7 @@ func handleEdit(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		if !rows.Next() {
 			return ErrorCode(nil, http.StatusNotFound, AnyMap{
@@ -66,10 +68,20 @@ func handleEdit(handle *Handle) {
 				"Model": model,
 			}))
 		case CONFIRM_PRE_TRAINING:
+            
+            cls, err := model_classes.ListClasses(handle.Db, id)
+            if err != nil {
+                return Error500(err)
+            }
+
+			LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
+				"Model": model,
+                "Classes": cls,
+			}))
+        case PREPARING_ZIP_FILE:
             LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
                 "Model": model,
             }))
-
 		default:
 			fmt.Printf("Unkown Status: %d\n", model.Status)
 			return Error500(nil)

logic/models/index.go

@@ -9,4 +9,8 @@ func HandleModels (handle *Handle) {
     handleEdit(handle)
     handleDelete(handle)
     handleList(handle)
+    
+    // Data endpoints
+    handleDataUpload(handle)
 }
+

logic/models/list.go

@@ -21,6 +21,7 @@ func handleList(handle *Handle) {
 		if err != nil {
 			return Error500(err)
 		}
+        defer rows.Close()
 
 		type row struct {
 			Name string

logic/models/types.go

@@ -1,5 +1,10 @@
 package models
 
+import (
+	"database/sql"
+	"errors"
+)
+
 type BaseModel struct {
 	Name   string
 	Status int
@@ -7,8 +12,32 @@ type BaseModel struct {
 }
 
 const (
+	FAILED_PREPARING_ZIP_FILE = -2
 	FAILED_PREPARING = -1
 
-	PREPARING = iota
+	PREPARING = 1
-	CONFIRM_PRE_TRAINING
+	CONFIRM_PRE_TRAINING = 2
+	PREPARING_ZIP_FILE = 3
 )
+
+var ModelNotFoundError = errors.New("Model not found error")
+
+func getBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
+	rows, err := db.Query("select name, status, id from models where id=$1;", id)
+	if err != nil {
+		return
+	}
+    defer rows.Close()
+
+	if !rows.Next() {
+		return nil, ModelNotFoundError
+	}
+
+    base = &BaseModel{}
+	err = rows.Scan(&base.Name, &base.Status, &base.Id)
+	if err != nil {
+		return nil, err
+	}
+
+	return
+}

logic/utils/handler.go

@@ -375,8 +375,8 @@ func Redirect(path string, mode AnswerType, w http.ResponseWriter, r *http.Reque
 		return
 	}
 	if mode&(HTMLFULL|HTML) != 0 {
-		w.WriteHeader(http.StatusSeeOther)
+        w.Header().Add("HX-Redirect", path)
-		w.Write([]byte(path))
+		w.WriteHeader(204)
 	} else {
 		w.WriteHeader(http.StatusSeeOther)
 	}

sql/models.sql

@@ -1,3 +1,4 @@
+-- drop table if exists model_data_point;
 -- drop table if exists model_defenitions;
 -- drop table if exists models;
 create table if not exists models (
@@ -14,9 +15,20 @@ create table if not exists models (
     color_mode varchar (20)
 );
 
+-- drop table if exists model_data_point;
 -- drop table if exists model_classes;
 create table if not exists model_classes (
     id uuid primary key default gen_random_uuid(),
-    model_id uuid references models (id) not null,
+    model_id uuid references models (id) on delete cascade,
     name varchar (70) not null
 );
+
+-- drop table if exists model_data_point;
+create table if not exists model_data_point (
+    id uuid primary key default gen_random_uuid(),
+    class_id uuid references model_classes (id) on delete cascade,
+    file_path text not null,
+    -- 1 training
+    -- 2 testing
+    model_mode integer default 1
+);

sql/user.sql

@@ -1,5 +1,9 @@
 -- drop table if exists tokens;
+
+-- drop table if exists model_data_point;
+-- drop table if exists model_defenitions;
 -- drop table if exists models;
+
 -- drop table if exists users;
 create table if not exists users (
     id uuid primary key default gen_random_uuid(),

views/js/main.js

@@ -1,3 +1,47 @@
+function tabs() {
+    for (const elm of document.querySelectorAll(".tabs")) {
+        let count = 0;
+        let selected = 0;
+        for (const child of elm.children) {
+            if (child.tagName == "BUTTON")  {
+                count++;
+                if (child.classList.contains("selected")) {
+                    selected++;
+                }
+                if (!child.getAttribute("data-armed")) {
+                    child.addEventListener("click", () => {
+                        if (!child.classList.contains("selected")) {
+                            for (const elm of child.parentElement.children) {
+                                elm.classList.remove("selected");
+                            }
+                            child.classList.add("selected");
+                            document.querySelector(`.tabs .content[data-tab="${
+                                child.getAttribute("data-tab")
+                            }"]`)?.classList.add("selected");
+                        }
+                    });
+                    child.setAttribute("data-armed", "true")
+                }
+            }
+        }
+
+        if (selected > 1 || selected == 0) {
+            for (const child of elm.children) {
+                child.classList.remove("selected");
+            }
+            for (const child of elm.children) {
+                if (child.tagName == "BUTTON") {
+                    child.classList.add("selected");
+                    document.querySelector(`.tabs .content[data-tab="${
+                        child.getAttribute("data-tab")
+                    }"]`)?.classList.add("selected");
+                    break;
+                }
+            }
+
+        }
+    }
+}
 function load() {
     for (const elm of document.querySelectorAll("form > button")) {
         elm.addEventListener('click', (e) => {
@@ -35,6 +79,7 @@ function load() {
             }
         });
     }
+    tabs();
 }
 window.onload = load;
 htmx.on('htmx:afterSwap', load);

views/models/edit.html

@@ -43,9 +43,103 @@
 {{ end }}
 
 {{ define "data-model-card" }}
-    <form hx-delete="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
+    <div class="card">
-        data menu
+        <h3>
+            Training data
+        </h3>
+        {{ if eq (len .Classes) 0 }}
+            <p>
+                You need to upload data so the model can train.
+            </p>
+            <div class="tabs">
+                <button class="tab" data-tab="upload">
+                    Upload
+                </button>
+                <button class="tab" data-tab="create_class">
+                    Create Class
+                </button>
+                <button class="tab" data-tab="api">
+                    Api
+                </button>
+                <div class="content" data-tab="upload">
+                    <form hx-headers='{"REQUEST-TYPE": "htmlfull"}' enctype="multipart/form-data" hx-post="/models/data/upload">
+                        <input type="hidden" name="id" value={{.Model.Id}} />
+                        <fieldset class="file-upload" >
+                            <label for="file">Data file</label>
+                            <div class="form-msg">
+                                Please provide a file that has the training and testing data<br/>
+                                The file must have 2 folders one with testing images and one with training images. <br/>
+                                Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
+                                The class folders must have the images for the classes.
+<pre>
+    training\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+    testing\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+</pre>
+                            </div>
+                            <div class="icon-holder">
+                                <button class="icon">
+                                    <img replace="icon" src="/imgs/upload-icon.png" />
+                                    <span replace="File Selected">
+                                        Upload Zip File
+                                    </span>
+                                </button>
+                                <input id="file" name="file" type="file" required accept="application/zip" />
+                            </div>
+                        </fieldset>
+                        <button>
+                            Add
+                        </button>
                     </form>
+                </div>
+                <div class="content" data-tab="create_class">
+                    TODO
+                </div>
+                <div class="content" data-tab="api">
+                    TODO
+                </div>
+            </div>
+        {{ else }}
+            <p>
+                You need to upload data so the model can train.
+            </p>
+            <div class="tabs">
+                <button class="tab" data-tab="create_class">
+                    Create Class
+                </button>
+                <button class="tab" data-tab="api">
+                    Api
+                </button>
+                <div class="content" data-tab="create_class">
+                    TODO
+                </div>
+                <div class="content" data-tab="api">
+                    TODO
+                </div>
+            </div>
+        {{ end }}
+    </div>
 {{ end }}
 
 {{ define "train-model-card" }}
@@ -83,11 +177,33 @@
                   </button>
               </form>
           </div>
+      {{/* PRE TRAINING STATUS */}}
       {{ else if (eq .Model.Status 2) }}
         {{ template "base-model-card" . }}
         {{ template "data-model-card" . }}
         {{ template "train-model-card" . }}
         {{ template "delete-model-card" . }}
+      {{/* FAILED TO PROCCESS THE ZIPFILE */}}
+      {{ else if (eq .Model.Status -2)}}
+        {{ template "base-model-card" . }}
+        <form hx-delete="/models/data/delete-zip-file" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML">
+            Failed to proccess the zip file.<br/>
+            Delete file and proccess afain.<br/>
+            <br/>
+            <div class="spacer" ></div>
+            <input type="hidden" name="id" value="{{ .Model.Id }}" />
+            <button class="danger">
+                Delete Zip File
+            </button>
+        </form>
+        {{ template "delete-model-card" . }}
+      {{/* PROCCESS THE ZIPFILE */}}
+      {{ else if (eq .Model.Status 3)}}
+        {{ template "base-model-card" . }}
+        <div class="card" hx-get="/models/edit?id={{ .Model.Id }}" hx-headers='{"REQUEST-TYPE": "htmlfull"}' hx-push="true" hx-swap="outerHTML" hx-target=".app" hx-trigger="load delay:2s" >
+            {{/* TODO improve this */}}
+            Processing zip file...
+        </div>
       {{ else }}
         <h1>
             Unknown Status of the model.

views/styles/main.css

@@ -2,7 +2,8 @@
 
 :root {
     --white: #ffffff;
-    --grey: #ffffff;
+    --grey: #dbdcde;
+    --light-grey: #fafafa;
     --main: #fca311;
     --sec: #14213d;
     --black: #000000;
@@ -145,6 +146,12 @@ form {
     box-shadow: 2px 5px 8px 2px #66666655;
 }
 
+.card form {
+    padding: 0;
+    border-radius: none;
+    box-shadow: none;
+}
+
 form label {
     display: block;
     padding-bottom: 5px;
@@ -315,6 +322,11 @@ table tr td .button {
     box-shadow: 0 2px 5px 1px #66666655;
     padding: 20px;
     border-radius: 10px;
+    margin: 20px 0;
+}
+
+.card h3 {
+    margin-top: 0;
 }
 
 /* Model stuff */
@@ -333,3 +345,36 @@ table tr td .button {
     display: flex;
     gap: 20px;
 }
+
+/* Tabs code */
+.tabs {
+    border-radius: 5px;
+    overflow: hidden;
+    display: flex;
+    flex-wrap: wrap;
+    box-shadow: 0 2px 8px 1px #66666655;
+    gap: 0 5px;
+}
+
+.tabs .content {
+    display: none;
+    padding: 5px;
+    width: 100%;
+}
+
+.tabs .tab {
+    padding: 5px;
+    background: var(--light-grey);
+    border-radius: 5px 5px 0 0;
+    box-shadow: none;
+    font-size: 1.1rem;
+}
+
+.tabs .tab.selected {
+    box-shadow: inset 0 2px 8px 1px #66666655;
+}
+
+.tabs .content.selected {
+    display: block;
+    box-shadow: 0 2px 2px 1px #66666655;
+}