andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: closes #11 and fixed database cons remaining open

Browse Source
This commit is contained in:
Andre Henriques 2023-09-22 19:22:36 +01:00
parent 1986be1a84
commit 1964303dce
15 changed files with 650 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
logic/models/classes/data_point.go Normal file
View File

@ -0,0 +1,27 @@
package model_classes
import (
"database/sql"
"errors"
)
func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
id = ""
_, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode)
if err != nil {
return
}
rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something worng")
}
err = rows.Scan(&id)
return
}

71
logic/models/classes/main.go Normal file
View File

@ -0,0 +1,71 @@
package model_classes
import (
"database/sql"
"errors"
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
type ModelClass struct {
Id string
ModelId string
Name string
}
func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var model ModelClass
err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
if err != nil {
return
}
cls = append(cls, model)
}
return
}
var ClassAlreadyExists = errors.New("Class aready exists")
func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) {
id = ""
rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
if err != nil {
return
}
defer rows.Close()
if rows.Next() {
return id, ClassAlreadyExists
}
_, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name)
if err != nil {
return
}
rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something wrong")
}
if err = rows.Scan(&id); err != nil {
return
}
return
}

8
logic/models/classes/types.go Normal file
View File

@ -0,0 +1,8 @@
package model_classes
type DATA_POINT_MODE int
const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)

265
logic/models/data.go Normal file
View File

@ -0,0 +1,265 @@
package models
import (
"archive/zip"
"bytes"
"fmt"
"io"
"net/http"
"os"
"path"
"reflect"
"sort"
"strings"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func InsertIfNotPresent(ss []string, s string) []string {
i := sort.SearchStrings(ss, s)
if len(ss) > i && ss[i] == s {
return ss
}
ss = append(ss, "")
copy(ss[i+1:], ss[i:])
ss[i] = s
return ss
}
func processZipFile(handle *Handle, id string) {
reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
if err != nil {
// TODO add msg to error
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Faield to proccess zip file failed to open reader\n")
fmt.Println(err)
return
}
defer reader.Close()
training := []string{}
testing := []string{}
for _, file := range reader.Reader.File {
paths := strings.Split(file.Name, "/")
if paths[1] == "" {
continue
}
if paths[0] != "training" && paths[0] != "testing" {
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
if paths[0] != "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
fmt.Printf("testing and training are diferent\n")
fmt.Println(testing)
fmt.Println(training)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
base_path := path.Join("savedData", id, "data")
ids := map[string]string{}
for _, name := range training {
dir_path := path.Join(base_path, "training", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
dir_path = path.Join(base_path, "testing", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
id, err := model_classes.CreateClass(handle.Db, id, name)
if err != nil {
fmt.Printf("Failed to create class '%s' on db\n", name)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
ids[name] = id
}
for _, file := range reader.Reader.File {
if file.Name[len(file.Name) - 1] == '/' {
continue
}
file_path := path.Join(base_path, file.Name)
f, err := os.Create(file_path)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
defer f.Close()
data, err := reader.Open(file.Name)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
f.Write(file_data)
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
}
_, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
if err != nil {
fmt.Printf("Failed to add data point for %s\n", id)
fmt.Println(err)
modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
}
fmt.Printf("Added data to model '%s'!\n", id)
modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
}
func handleDataUpload(handle *Handle) {
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
if c.Mode == JSON {
// TODO improve message
return ErrorCode(nil, 400, nil)
}
read_form, err := r.MultipartReader()
if err != nil {
LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
return nil
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return &Error{Code: http.StatusBadRequest}
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
_, err = getBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
} else if err != nil {
return Error500(err)
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "base_data.zip"))
if err != nil {
return Error500(err)
}
defer f.Close()
f.Write(file)
modelUpdateStatus(handle, id, PREPARING_ZIP_FILE)
go processZipFile(handle, id)
Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil
})
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
if c.Mode == JSON {
panic("Handle delete zip file json")
}
f, err := MyParseForm(r)
if err != nil {
return ErrorCode(err, 400, c.AddMap(nil))
}
if !CheckId(f, "id") {
return ErrorCode(err, 400, c.AddMap(nil))
}
id := f.Get("id")
model, err := getBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
"NotFoundMessage": "Model not found",
"GoBackLink": "/models",
})
} else if err != nil {
return Error500(err)
}
if model.Status != FAILED_PREPARING_ZIP_FILE {
// TODO add message
return ErrorCode(nil, 400, c.AddMap(nil))
}
err = os.Remove(path.Join("savedData", id, "base_data.zip"));
if err != nil {
return Error500(err)
}
err = os.RemoveAll(path.Join("savedData", id, "data"));
if err != nil {
return Error500(err)
}
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
if err != nil {
return Error500(err)
}
modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
Redirect("/models/edit?id="+id, c.Mode, w, r)
return nil
})
}

1
logic/models/delete.go
View File

@ -62,6 +62,7 @@ func handleDelete(handle *Handle) {
if err != nil {
return Error500(err)
}
defer rows.Close()
if !rows.Next() {
return ErrorCode(nil, http.StatusNotFound, AnyMap{

14
logic/models/edit.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@ -26,6 +27,7 @@ func handleEdit(handle *Handle) {
if err != nil {
return Error500(err)
}
defer rows.Close()
if !rows.Next() {
return ErrorCode(nil, http.StatusNotFound, AnyMap{
@ -66,10 +68,20 @@ func handleEdit(handle *Handle) {
"Model": model,
}))
case CONFIRM_PRE_TRAINING:
cls, err := model_classes.ListClasses(handle.Db, id)
if err != nil {
return Error500(err)
}
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
"Classes": cls,
}))
case PREPARING_ZIP_FILE:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{
"Model": model,
}))
default:
fmt.Printf("Unkown Status: %d\n", model.Status)
return Error500(nil)

4
logic/models/index.go
View File

@ -9,4 +9,8 @@ func HandleModels (handle *Handle) {
handleEdit(handle)
handleDelete(handle)
handleList(handle)
// Data endpoints
handleDataUpload(handle)
}

1
logic/models/list.go
View File

@ -21,6 +21,7 @@ func handleList(handle *Handle) {
if err != nil {
return Error500(err)
}
defer rows.Close()
type row struct {
Name string

33
logic/models/types.go
View File

@ -1,5 +1,10 @@
package models
import (
"database/sql"
"errors"
)
type BaseModel struct {
Name string
Status int
@ -7,8 +12,32 @@ type BaseModel struct {
}
const (
FAILED_PREPARING_ZIP_FILE = -2
FAILED_PREPARING = -1
PREPARING = iota
CONFIRM_PRE_TRAINING
PREPARING = 1
CONFIRM_PRE_TRAINING = 2
PREPARING_ZIP_FILE = 3
)
var ModelNotFoundError = errors.New("Model not found error")
func getBaseModel(db *sql.DB, id string) (base *BaseModel, err error) {
rows, err := db.Query("select name, status, id from models where id=$1;", id)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return nil, ModelNotFoundError
}
base = &BaseModel{}
err = rows.Scan(&base.Name, &base.Status, &base.Id)
if err != nil {
return nil, err
}
return
}

4
logic/utils/handler.go
View File

@ -375,8 +375,8 @@ func Redirect(path string, mode AnswerType, w http.ResponseWriter, r *http.Reque
return
}
if mode&(HTMLFULL|HTML) != 0 {
w.WriteHeader(http.StatusSeeOther)
w.Write([]byte(path))
w.Header().Add("HX-Redirect", path)
w.WriteHeader(204)
} else {
w.WriteHeader(http.StatusSeeOther)
}

14
sql/models.sql
View File

@ -1,3 +1,4 @@
-- drop table if exists model_data_point;
-- drop table if exists model_defenitions;
-- drop table if exists models;
create table if not exists models (
@ -14,9 +15,20 @@ create table if not exists models (
color_mode varchar (20)
);
-- drop table if exists model_data_point;
-- drop table if exists model_classes;
create table if not exists model_classes (
id uuid primary key default gen_random_uuid(),
model_id uuid references models (id) not null,
model_id uuid references models (id) on delete cascade,
name varchar (70) not null
);
-- drop table if exists model_data_point;
create table if not exists model_data_point (
id uuid primary key default gen_random_uuid(),
class_id uuid references model_classes (id) on delete cascade,
file_path text not null,
-- 1 training
-- 2 testing
model_mode integer default 1
);

4
sql/user.sql
View File

@ -1,5 +1,9 @@
-- drop table if exists tokens;
-- drop table if exists model_data_point;
-- drop table if exists model_defenitions;
-- drop table if exists models;
-- drop table if exists users;
create table if not exists users (
id uuid primary key default gen_random_uuid(),

45
views/js/main.js
View File

@ -1,3 +1,47 @@
function tabs() {
for (const elm of document.querySelectorAll(".tabs")) {
let count = 0;
let selected = 0;
for (const child of elm.children) {
if (child.tagName == "BUTTON") {
count++;
if (child.classList.contains("selected")) {
selected++;
}
if (!child.getAttribute("data-armed")) {
child.addEventListener("click", () => {
if (!child.classList.contains("selected")) {
for (const elm of child.parentElement.children) {
elm.classList.remove("selected");
}
child.classList.add("selected");
document.querySelector(`.tabs .content[data-tab="${
child.getAttribute("data-tab")
}"]`)?.classList.add("selected");
}
});
child.setAttribute("data-armed", "true")
}
}
}
if (selected > 1 || selected == 0) {
for (const child of elm.children) {
child.classList.remove("selected");
}
for (const child of elm.children) {
if (child.tagName == "BUTTON") {
child.classList.add("selected");
document.querySelector(`.tabs .content[data-tab="${
child.getAttribute("data-tab")
}"]`)?.classList.add("selected");
break;
}
}
}
}
}
function load() {
for (const elm of document.querySelectorAll("form > button")) {
elm.addEventListener('click', (e) => {
@ -35,6 +79,7 @@ function load() {
}
});
}
tabs();
}
window.onload = load;
htmx.on('htmx:afterSwap', load);

120
views/models/edit.html
View File

@ -43,9 +43,103 @@
{{ end }}
{{ define "data-model-card" }}
<form hx-delete="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
data menu
<div class="card">
<h3>
Training data
</h3>
{{ if eq (len .Classes) 0 }}
<p>
You need to upload data so the model can train.
</p>
<div class="tabs">
<button class="tab" data-tab="upload">
Upload
</button>
<button class="tab" data-tab="create_class">
Create Class
</button>
<button class="tab" data-tab="api">
Api
</button>
<div class="content" data-tab="upload">
<form hx-headers='{"REQUEST-TYPE": "htmlfull"}' enctype="multipart/form-data" hx-post="/models/data/upload">
<input type="hidden" name="id" value={{.Model.Id}} />
<fieldset class="file-upload" >
<label for="file">Data file</label>
<div class="form-msg">
Please provide a file that has the training and testing data<br/>
The file must have 2 folders one with testing images and one with training images. <br/>
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
The class folders must have the images for the classes.
<pre>
training\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
testing\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
</pre>
</div>
<div class="icon-holder">
<button class="icon">
<img replace="icon" src="/imgs/upload-icon.png" />
<span replace="File Selected">
Upload Zip File
</span>
</button>
<input id="file" name="file" type="file" required accept="application/zip" />
</div>
</fieldset>
<button>
Add
</button>
</form>
</div>
<div class="content" data-tab="create_class">
TODO
</div>
<div class="content" data-tab="api">
TODO
</div>
</div>
{{ else }}
<p>
You need to upload data so the model can train.
</p>
<div class="tabs">
<button class="tab" data-tab="create_class">
Create Class
</button>
<button class="tab" data-tab="api">
Api
</button>
<div class="content" data-tab="create_class">
TODO
</div>
<div class="content" data-tab="api">
TODO
</div>
</div>
{{ end }}
</div>
{{ end }}
{{ define "train-model-card" }}
@ -83,11 +177,33 @@
</button>
</form>
</div>
{{/* PRE TRAINING STATUS */}}
{{ else if (eq .Model.Status 2) }}
{{ template "base-model-card" . }}
{{ template "data-model-card" . }}
{{ template "train-model-card" . }}
{{ template "delete-model-card" . }}
{{/* FAILED TO PROCCESS THE ZIPFILE */}}
{{ else if (eq .Model.Status -2)}}
{{ template "base-model-card" . }}
<form hx-delete="/models/data/delete-zip-file" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML">
Failed to proccess the zip file.<br/>
Delete file and proccess afain.<br/>
<br/>
<div class="spacer" ></div>
<input type="hidden" name="id" value="{{ .Model.Id }}" />
<button class="danger">
Delete Zip File
</button>
</form>
{{ template "delete-model-card" . }}
{{/* PROCCESS THE ZIPFILE */}}
{{ else if (eq .Model.Status 3)}}
{{ template "base-model-card" . }}
<div class="card" hx-get="/models/edit?id={{ .Model.Id }}" hx-headers='{"REQUEST-TYPE": "htmlfull"}' hx-push="true" hx-swap="outerHTML" hx-target=".app" hx-trigger="load delay:2s" >
{{/* TODO improve this */}}
Processing zip file...
</div>
{{ else }}
<h1>
Unknown Status of the model.

47
views/styles/main.css
View File

@ -2,7 +2,8 @@
:root {
--white: #ffffff;
--grey: #ffffff;
--grey: #dbdcde;
--light-grey: #fafafa;
--main: #fca311;
--sec: #14213d;
--black: #000000;
@ -145,6 +146,12 @@ form {
box-shadow: 2px 5px 8px 2px #66666655;
}
.card form {
padding: 0;
border-radius: none;
box-shadow: none;
}
form label {
display: block;
padding-bottom: 5px;
@ -315,6 +322,11 @@ table tr td .button {
box-shadow: 0 2px 5px 1px #66666655;
padding: 20px;
border-radius: 10px;
margin: 20px 0;
}
.card h3 {
margin-top: 0;
}
/* Model stuff */
@ -333,3 +345,36 @@ table tr td .button {
display: flex;
gap: 20px;
}
/* Tabs code */
.tabs {
border-radius: 5px;
overflow: hidden;
display: flex;
flex-wrap: wrap;
box-shadow: 0 2px 8px 1px #66666655;
gap: 0 5px;
}
.tabs .content {
display: none;
padding: 5px;
width: 100%;
}
.tabs .tab {
padding: 5px;
background: var(--light-grey);
border-radius: 5px 5px 0 0;
box-shadow: none;
font-size: 1.1rem;
}
.tabs .tab.selected {
box-shadow: inset 0 2px 8px 1px #66666655;
}
.tabs .content.selected {
display: block;
box-shadow: 0 2px 2px 1px #66666655;
}