From 1964303dcec99d6eea99a5fa19af87744498c2e3 Mon Sep 17 00:00:00 2001 From: Andre Henriques Date: Fri, 22 Sep 2023 19:22:36 +0100 Subject: [PATCH] feat: closes #11 and fixed database cons remaining open --- logic/models/classes/data_point.go | 27 +++ logic/models/classes/main.go | 71 ++++++++ logic/models/classes/types.go | 8 + logic/models/data.go | 265 +++++++++++++++++++++++++++++ logic/models/delete.go | 1 + logic/models/edit.go | 14 +- logic/models/index.go | 4 + logic/models/list.go | 1 + logic/models/types.go | 33 +++- logic/utils/handler.go | 4 +- sql/models.sql | 14 +- sql/user.sql | 4 + views/js/main.js | 45 +++++ views/models/edit.html | 122 ++++++++++++- views/styles/main.css | 47 ++++- 15 files changed, 650 insertions(+), 10 deletions(-) create mode 100644 logic/models/classes/data_point.go create mode 100644 logic/models/classes/main.go create mode 100644 logic/models/classes/types.go create mode 100644 logic/models/data.go diff --git a/logic/models/classes/data_point.go b/logic/models/classes/data_point.go new file mode 100644 index 0000000..c7263b4 --- /dev/null +++ b/logic/models/classes/data_point.go @@ -0,0 +1,27 @@ +package model_classes + +import ( + "database/sql" + "errors" +) + +func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) { + id = "" + _, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode) + if err != nil { + return + } + + rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode) + if err != nil { + return + } + defer rows.Close() + + if !rows.Next() { + return id, errors.New("Something worng") + } + + err = rows.Scan(&id) + return +} diff --git a/logic/models/classes/main.go b/logic/models/classes/main.go new file mode 100644 index 0000000..2e24078 --- /dev/null +++ b/logic/models/classes/main.go @@ -0,0 +1,71 @@ +package model_classes + +import ( + "database/sql" + "errors" + // . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" +) + +type ModelClass struct { + Id string + ModelId string + Name string +} + +func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) { + + rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + var model ModelClass + err = rows.Scan(&model.Id, &model.ModelId, &model.Name) + if err != nil { + return + } + + cls = append(cls, model) + } + + return +} + +var ClassAlreadyExists = errors.New("Class aready exists") + +func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) { + id = "" + rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name) + if err != nil { + return + } + defer rows.Close() + + if rows.Next() { + return id, ClassAlreadyExists + } + + _, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name) + + if err != nil { + return + } + + rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name) + if err != nil { + return + } + defer rows.Close() + + if !rows.Next() { + return id, errors.New("Something wrong") + } + + if err = rows.Scan(&id); err != nil { + return + } + + return +} diff --git a/logic/models/classes/types.go b/logic/models/classes/types.go new file mode 100644 index 0000000..b1d96a8 --- /dev/null +++ b/logic/models/classes/types.go @@ -0,0 +1,8 @@ +package model_classes + +type DATA_POINT_MODE int + +const ( + DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1 + DATA_POINT_MODE_TESTING = 2 +) diff --git a/logic/models/data.go b/logic/models/data.go new file mode 100644 index 0000000..eee71d4 --- /dev/null +++ b/logic/models/data.go @@ -0,0 +1,265 @@ +package models + +import ( + "archive/zip" + "bytes" + "fmt" + "io" + "net/http" + "os" + "path" + "reflect" + "sort" + "strings" + + model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" + . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" +) + +func InsertIfNotPresent(ss []string, s string) []string { + i := sort.SearchStrings(ss, s) + if len(ss) > i && ss[i] == s { + return ss + } + ss = append(ss, "") + copy(ss[i+1:], ss[i:]) + ss[i] = s + return ss +} + +func processZipFile(handle *Handle, id string) { + reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip")) + if err != nil { + // TODO add msg to error + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + fmt.Printf("Faield to proccess zip file failed to open reader\n") + fmt.Println(err) + return + } + defer reader.Close() + + training := []string{} + testing := []string{} + + for _, file := range reader.Reader.File { + + paths := strings.Split(file.Name, "/") + + if paths[1] == "" { + continue + } + + if paths[0] != "training" && paths[0] != "testing" { + fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + + if paths[0] != "training" { + training = InsertIfNotPresent(training, paths[1]) + } else if paths[0] != "testing" { + testing = InsertIfNotPresent(testing, paths[1]) + } + } + + if !reflect.DeepEqual(testing, training) { + fmt.Printf("testing and training are diferent\n") + fmt.Println(testing) + fmt.Println(training) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + + base_path := path.Join("savedData", id, "data") + + ids := map[string]string{} + + for _, name := range training { + dir_path := path.Join(base_path, "training", name) + err = os.MkdirAll(dir_path, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create dir %s\n", dir_path) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + dir_path = path.Join(base_path, "testing", name) + err = os.MkdirAll(dir_path, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create dir %s\n", dir_path) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + + id, err := model_classes.CreateClass(handle.Db, id, name) + if err != nil { + fmt.Printf("Failed to create class '%s' on db\n", name) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + ids[name] = id + } + + for _, file := range reader.Reader.File { + if file.Name[len(file.Name) - 1] == '/' { + continue + } + + file_path := path.Join(base_path, file.Name) + f, err := os.Create(file_path) + if err != nil { + fmt.Printf("Could not create file %s\n", file_path) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + defer f.Close() + data, err := reader.Open(file.Name) + if err != nil { + fmt.Printf("Could not create file %s\n", file_path) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + defer data.Close() + file_data, err := io.ReadAll(data) + f.Write(file_data) + + parts := strings.Split(file.Name, "/") + + mode := model_classes.DATA_POINT_MODE_TRAINING + if parts[0] == "testing" { + mode = model_classes.DATA_POINT_MODE_TESTING + } + + _, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode) + if err != nil { + fmt.Printf("Failed to add data point for %s\n", id) + fmt.Println(err) + modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE) + return + } + } + + fmt.Printf("Added data to model '%s'!\n", id) + modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING) +} + +func handleDataUpload(handle *Handle) { + handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + if c.Mode == JSON { + // TODO improve message + return ErrorCode(nil, 400, nil) + } + + read_form, err := r.MultipartReader() + if err != nil { + LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil)) + return nil + } + + var id string + var file []byte + + for { + part, err_part := read_form.NextPart() + if err_part == io.EOF { + break + } else if err_part != nil { + return &Error{Code: http.StatusBadRequest} + } + if part.FormName() == "id" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + id = buf.String() + } + if part.FormName() == "file" { + buf := new(bytes.Buffer) + buf.ReadFrom(part) + file = buf.Bytes() + } + } + + _, err = getBaseModel(handle.Db, id) + if err == ModelNotFoundError { + return ErrorCode(nil, http.StatusNotFound, AnyMap{ + "NotFoundMessage": "Model not found", + "GoBackLink": "/models", + }) + } else if err != nil { + return Error500(err) + } + + // TODO mk this path configurable + dir_path := path.Join("savedData", id) + + f, err := os.Create(path.Join(dir_path, "base_data.zip")) + if err != nil { + return Error500(err) + } + defer f.Close() + + f.Write(file) + + modelUpdateStatus(handle, id, PREPARING_ZIP_FILE) + + go processZipFile(handle, id) + + Redirect("/models/edit?id="+id, c.Mode, w, r) + return nil + }) + + handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if !CheckAuthLevel(1, w, r, c) { + return nil + } + if c.Mode == JSON { + panic("Handle delete zip file json") + } + + f, err := MyParseForm(r) + if err != nil { + return ErrorCode(err, 400, c.AddMap(nil)) + } + + if !CheckId(f, "id") { + return ErrorCode(err, 400, c.AddMap(nil)) + } + + id := f.Get("id") + + model, err := getBaseModel(handle.Db, id) + if err == ModelNotFoundError { + return ErrorCode(nil, http.StatusNotFound, AnyMap{ + "NotFoundMessage": "Model not found", + "GoBackLink": "/models", + }) + } else if err != nil { + return Error500(err) + } + + if model.Status != FAILED_PREPARING_ZIP_FILE { + // TODO add message + return ErrorCode(nil, 400, c.AddMap(nil)) + } + + err = os.Remove(path.Join("savedData", id, "base_data.zip")); + if err != nil { + return Error500(err) + } + + err = os.RemoveAll(path.Join("savedData", id, "data")); + if err != nil { + return Error500(err) + } + + _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id) + if err != nil { + return Error500(err) + } + + modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING) + Redirect("/models/edit?id="+id, c.Mode, w, r) + return nil + }) +} diff --git a/logic/models/delete.go b/logic/models/delete.go index d861212..4d72673 100644 --- a/logic/models/delete.go +++ b/logic/models/delete.go @@ -62,6 +62,7 @@ func handleDelete(handle *Handle) { if err != nil { return Error500(err) } + defer rows.Close() if !rows.Next() { return ErrorCode(nil, http.StatusNotFound, AnyMap{ diff --git a/logic/models/edit.go b/logic/models/edit.go index 83179ed..957cbc7 100644 --- a/logic/models/edit.go +++ b/logic/models/edit.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" + model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) @@ -26,6 +27,7 @@ func handleEdit(handle *Handle) { if err != nil { return Error500(err) } + defer rows.Close() if !rows.Next() { return ErrorCode(nil, http.StatusNotFound, AnyMap{ @@ -66,10 +68,20 @@ func handleEdit(handle *Handle) { "Model": model, })) case CONFIRM_PRE_TRAINING: + + cls, err := model_classes.ListClasses(handle.Db, id) + if err != nil { + return Error500(err) + } + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ "Model": model, + "Classes": cls, })) - + case PREPARING_ZIP_FILE: + LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{ + "Model": model, + })) default: fmt.Printf("Unkown Status: %d\n", model.Status) return Error500(nil) diff --git a/logic/models/index.go b/logic/models/index.go index 014c863..728a626 100644 --- a/logic/models/index.go +++ b/logic/models/index.go @@ -9,4 +9,8 @@ func HandleModels (handle *Handle) { handleEdit(handle) handleDelete(handle) handleList(handle) + + // Data endpoints + handleDataUpload(handle) } + diff --git a/logic/models/list.go b/logic/models/list.go index d5a6cda..10b10dd 100644 --- a/logic/models/list.go +++ b/logic/models/list.go @@ -21,6 +21,7 @@ func handleList(handle *Handle) { if err != nil { return Error500(err) } + defer rows.Close() type row struct { Name string diff --git a/logic/models/types.go b/logic/models/types.go index 4b1c9b3..9e27214 100644 --- a/logic/models/types.go +++ b/logic/models/types.go @@ -1,5 +1,10 @@ package models +import ( + "database/sql" + "errors" +) + type BaseModel struct { Name string Status int @@ -7,8 +12,32 @@ type BaseModel struct { } const ( + FAILED_PREPARING_ZIP_FILE = -2 FAILED_PREPARING = -1 - PREPARING = iota - CONFIRM_PRE_TRAINING + PREPARING = 1 + CONFIRM_PRE_TRAINING = 2 + PREPARING_ZIP_FILE = 3 ) + +var ModelNotFoundError = errors.New("Model not found error") + +func getBaseModel(db *sql.DB, id string) (base *BaseModel, err error) { + rows, err := db.Query("select name, status, id from models where id=$1;", id) + if err != nil { + return + } + defer rows.Close() + + if !rows.Next() { + return nil, ModelNotFoundError + } + + base = &BaseModel{} + err = rows.Scan(&base.Name, &base.Status, &base.Id) + if err != nil { + return nil, err + } + + return +} diff --git a/logic/utils/handler.go b/logic/utils/handler.go index 6bfe694..2e0bd75 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -375,8 +375,8 @@ func Redirect(path string, mode AnswerType, w http.ResponseWriter, r *http.Reque return } if mode&(HTMLFULL|HTML) != 0 { - w.WriteHeader(http.StatusSeeOther) - w.Write([]byte(path)) + w.Header().Add("HX-Redirect", path) + w.WriteHeader(204) } else { w.WriteHeader(http.StatusSeeOther) } diff --git a/sql/models.sql b/sql/models.sql index 4d8fbd2..eb15296 100644 --- a/sql/models.sql +++ b/sql/models.sql @@ -1,3 +1,4 @@ +-- drop table if exists model_data_point; -- drop table if exists model_defenitions; -- drop table if exists models; create table if not exists models ( @@ -14,9 +15,20 @@ create table if not exists models ( color_mode varchar (20) ); +-- drop table if exists model_data_point; -- drop table if exists model_classes; create table if not exists model_classes ( id uuid primary key default gen_random_uuid(), - model_id uuid references models (id) not null, + model_id uuid references models (id) on delete cascade, name varchar (70) not null ); + +-- drop table if exists model_data_point; +create table if not exists model_data_point ( + id uuid primary key default gen_random_uuid(), + class_id uuid references model_classes (id) on delete cascade, + file_path text not null, + -- 1 training + -- 2 testing + model_mode integer default 1 +); diff --git a/sql/user.sql b/sql/user.sql index 61f69f7..a048963 100644 --- a/sql/user.sql +++ b/sql/user.sql @@ -1,5 +1,9 @@ -- drop table if exists tokens; + +-- drop table if exists model_data_point; +-- drop table if exists model_defenitions; -- drop table if exists models; + -- drop table if exists users; create table if not exists users ( id uuid primary key default gen_random_uuid(), diff --git a/views/js/main.js b/views/js/main.js index 0e3ab0f..b0cb09c 100644 --- a/views/js/main.js +++ b/views/js/main.js @@ -1,3 +1,47 @@ +function tabs() { + for (const elm of document.querySelectorAll(".tabs")) { + let count = 0; + let selected = 0; + for (const child of elm.children) { + if (child.tagName == "BUTTON") { + count++; + if (child.classList.contains("selected")) { + selected++; + } + if (!child.getAttribute("data-armed")) { + child.addEventListener("click", () => { + if (!child.classList.contains("selected")) { + for (const elm of child.parentElement.children) { + elm.classList.remove("selected"); + } + child.classList.add("selected"); + document.querySelector(`.tabs .content[data-tab="${ + child.getAttribute("data-tab") + }"]`)?.classList.add("selected"); + } + }); + child.setAttribute("data-armed", "true") + } + } + } + + if (selected > 1 || selected == 0) { + for (const child of elm.children) { + child.classList.remove("selected"); + } + for (const child of elm.children) { + if (child.tagName == "BUTTON") { + child.classList.add("selected"); + document.querySelector(`.tabs .content[data-tab="${ + child.getAttribute("data-tab") + }"]`)?.classList.add("selected"); + break; + } + } + + } + } +} function load() { for (const elm of document.querySelectorAll("form > button")) { elm.addEventListener('click', (e) => { @@ -35,6 +79,7 @@ function load() { } }); } + tabs(); } window.onload = load; htmx.on('htmx:afterSwap', load); diff --git a/views/models/edit.html b/views/models/edit.html index 7f63dbf..836fbb7 100644 --- a/views/models/edit.html +++ b/views/models/edit.html @@ -43,9 +43,103 @@ {{ end }} {{ define "data-model-card" }} -
- data menu -
+
+

+ Training data +

+ {{ if eq (len .Classes) 0 }} +

+ You need to upload data so the model can train. +

+
+ + + +
+
+ +
+ +
+ Please provide a file that has the training and testing data
+ The file must have 2 folders one with testing images and one with training images.
+ Each of the folders will contain the classes of the model. The folders must be the same in testing and training. + The class folders must have the images for the classes. +
+    training\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+    testing\
+        class1\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        class2\
+            img1.png
+            img2.png
+            img2.png
+            ...
+        ...
+
+
+
+ + +
+
+ +
+
+
+ TODO +
+
+ TODO +
+
+ {{ else }} +

+ You need to upload data so the model can train. +

+
+ + +
+ TODO +
+
+ TODO +
+
+ {{ end }} +
{{ end }} {{ define "train-model-card" }} @@ -83,11 +177,33 @@ + {{/* PRE TRAINING STATUS */}} {{ else if (eq .Model.Status 2) }} {{ template "base-model-card" . }} {{ template "data-model-card" . }} {{ template "train-model-card" . }} {{ template "delete-model-card" . }} + {{/* FAILED TO PROCCESS THE ZIPFILE */}} + {{ else if (eq .Model.Status -2)}} + {{ template "base-model-card" . }} +
+ Failed to proccess the zip file.
+ Delete file and proccess afain.
+
+
+ + +
+ {{ template "delete-model-card" . }} + {{/* PROCCESS THE ZIPFILE */}} + {{ else if (eq .Model.Status 3)}} + {{ template "base-model-card" . }} +
+ {{/* TODO improve this */}} + Processing zip file... +
{{ else }}

Unknown Status of the model. diff --git a/views/styles/main.css b/views/styles/main.css index c0da433..cc85693 100644 --- a/views/styles/main.css +++ b/views/styles/main.css @@ -2,7 +2,8 @@ :root { --white: #ffffff; - --grey: #ffffff; + --grey: #dbdcde; + --light-grey: #fafafa; --main: #fca311; --sec: #14213d; --black: #000000; @@ -145,6 +146,12 @@ form { box-shadow: 2px 5px 8px 2px #66666655; } +.card form { + padding: 0; + border-radius: none; + box-shadow: none; +} + form label { display: block; padding-bottom: 5px; @@ -315,6 +322,11 @@ table tr td .button { box-shadow: 0 2px 5px 1px #66666655; padding: 20px; border-radius: 10px; + margin: 20px 0; +} + +.card h3 { + margin-top: 0; } /* Model stuff */ @@ -333,3 +345,36 @@ table tr td .button { display: flex; gap: 20px; } + +/* Tabs code */ +.tabs { + border-radius: 5px; + overflow: hidden; + display: flex; + flex-wrap: wrap; + box-shadow: 0 2px 8px 1px #66666655; + gap: 0 5px; +} + +.tabs .content { + display: none; + padding: 5px; + width: 100%; +} + +.tabs .tab { + padding: 5px; + background: var(--light-grey); + border-radius: 5px 5px 0 0; + box-shadow: none; + font-size: 1.1rem; +} + +.tabs .tab.selected { + box-shadow: inset 0 2px 8px 1px #66666655; +} + +.tabs .content.selected { + display: block; + box-shadow: 0 2px 2px 1px #66666655; +}