chore: did more clean up

This commit is contained in:
Andre Henriques 2024-04-14 15:19:32 +01:00
parent 8b13afba48
commit e7eeccd09e
20 changed files with 283 additions and 352 deletions

View File

@ -0,0 +1,8 @@
package dbtypes
type DATA_POINT_MODE int
const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)

View File

@ -1,4 +1,4 @@
package models_utils
package dbtypes
import (
"database/sql"

View File

@ -11,7 +11,7 @@ import (
"os"
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

View File

@ -3,6 +3,8 @@ package model_classes
import (
"database/sql"
"errors"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
)
var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")

View File

@ -1,43 +1,26 @@
package model_classes
import (
"strconv"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func HandleList(handle *Handle) {
handle.Get("/models/data/list", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
type DataList struct {
Id string `json:"id" validate:"required"`
Page int `json:"page"`
}
id, err := GetIdFromUrl(c, "id")
if err != nil {
return c.JsonBadRequest("Model Class not found!")
}
page := 0
if c.R.URL.Query().Has("page") {
page_url := c.R.URL.Query().Get("page")
page_url_number, err := strconv.Atoi(page_url)
if err != nil {
return c.JsonBadRequest("Page is not a number")
}
page = page_url_number
}
PostAuthJson(handle, "/models/data/list", User_Normal, func(c *Context, dat *DataList) *Error {
var class_row struct {
Name string
Model_id string
}
err = GetDBOnce(c, &class_row, "model_classes where id=$1", id)
err := GetDBOnce(c, &class_row, "model_classes where id=$1", dat.Id)
if err == NotFoundError {
return c.JsonBadRequest("Model Class not found!")
} else if err != nil {
return c.Error500(err)
return c.E500M("Failed to get classes", err)
}
type baserow struct {
@ -47,23 +30,21 @@ func HandleList(handle *Handle) {
Status int `json:"status"`
}
rows, err := GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", id, page*10)
rows, err := GetDbMultitple[baserow](c, "model_data_point where class_id=$1 limit 11 offset $2", dat.Id, dat.Page*10)
if err != nil {
return c.Error500(err)
}
type ReturnType struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
return c.E500M("Failed to get classes", err)
}
max_len := min(11, len(rows))
c.ShowMessage = false
return c.SendJSON(ReturnType{
return c.SendJSON(struct {
ImageList []*baserow `json:"image_list"`
Page int `json:"page"`
ShowNext bool `json:"showNext"`
}{
ImageList: rows[0:max_len],
Page: page,
Page: dat.Page,
ShowNext: len(rows) == 11,
})
})

View File

@ -1,8 +0,0 @@
package model_classes
type DATA_POINT_MODE int
const (
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
DATA_POINT_MODE_TESTING = 2
)

View File

@ -14,7 +14,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@ -68,9 +67,9 @@ func fileProcessor(
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
mode := DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
mode = DATA_POINT_MODE_TESTING
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)

View File

@ -6,7 +6,6 @@ import (
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@ -14,44 +13,35 @@ func deleteModelJSON(c *Context, id string) *Error {
c.Logger.Warnf("Removing model with id: %s", id)
_, err := c.Db.Exec("delete from models where id=$1;", id)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to delete models", err)
}
model_path := path.Join("./savedData", id)
c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path)
err = os.RemoveAll(model_path)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to remove data", err)
}
return c.SendJSON(id)
}
func handleDelete(handle *Handle) {
handle.Delete("/models/delete", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
var dat struct {
type DeleteModel struct {
Id string `json:"id" validate:"required"`
Name *string `json:"name,omitempty"`
}
if err_ := c.ToJSON(&dat); err_ != nil {
return err_
}
DeleteAuthJson(handle, "/models/delete", User_Normal, func(c *Context, dat *DeleteModel) *Error {
var model struct {
Id string
Name string
Status int
}
err := GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id)
if err == NotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found!")
} else if err != nil {
return c.Error500(err)
return c.E500M("Faield to get model", err)
}
switch model.Status {

View File

@ -5,7 +5,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

View File

@ -6,15 +6,7 @@ import (
)
// Auth level set when path is definied as 1
func handleStats(c *Context) *Error {
var b struct {
Id string `json:"id" validate:"required"`
}
if _err := c.ToJSON(&b); _err != nil {
return _err
}
func handleStats(c *Context, b *JustId) *Error {
type Row struct {
Name string `db:"mc.name" json:"name"`
Training string `db:"count(mdp.id) filter (where mdp.model_mode=1)" json:"training"`
@ -23,7 +15,7 @@ func handleStats(c *Context) *Error {
rows, err := GetDbMultitple[Row](c, "model_data_point as mdp inner join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 group by mc.name order by mc.name asc;", b.Id)
if err != nil {
return c.Error500(err)
return c.E500M("Failed to get stats", err)
}
c.ShowMessage = false
@ -50,5 +42,5 @@ func handleList(handle *Handle) {
return c.SendJSON(got)
})
handle.PostAuth("/models/class/stats", 1, handleStats)
PostAuthJson(handle, "/models/class/stats", User_Normal, handleStats)
}

View File

@ -6,7 +6,6 @@ import (
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
tf "github.com/galeone/tensorflow/tensorflow/go"

View File

@ -6,7 +6,7 @@ import (
_ "image/png"
"os"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

View File

@ -4,29 +4,17 @@ import (
"os"
"path"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
func handleRest(handle *Handle) {
handle.Delete("/models/train/reset", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {
return nil
}
var dat struct {
Id string `json:"id"`
}
if err := c.ToJSON(&dat); err != nil {
return err
}
DeleteAuthJson(handle, "/models/train/reset", User_Normal, func(c *Context, dat *JustId) *Error {
model, err := GetBaseModel(c.Db, dat.Id)
if err == ModelNotFoundError {
return c.JsonBadRequest("Model not found")
} else if err != nil {
// TODO improve response
return c.Error500(err)
return c.E500M("Failed to get model", err)
}
if model.Status != FAILED_PREPARING_TRAINING && model.Status != FAILED_TRAINING {
@ -37,8 +25,7 @@ func handleRest(handle *Handle) {
_, err = c.Db.Exec("delete from model_definition where model_id=$1", model.Id)
if err != nil {
// TODO improve response
return c.Error500(err)
return c.E500M("Failed to delete model", err)
}
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)

View File

@ -16,7 +16,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
"github.com/charmbracelet/log"
)
@ -80,7 +79,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e
}
count = co.Count
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2;", model_id, model_classes.DATA_POINT_MODE_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2;", model_id, DATA_POINT_MODE_TRAINING)
if err != nil {
return
}
@ -139,7 +138,7 @@ func generateCvsExp(c *Context, run_path string, model_id string, doPanic bool)
return generateCvsExp(c, run_path, model_id, true)
}
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
return
}
@ -303,7 +302,7 @@ func generateCvsExpandExp(c *Context, run_path string, model_id string, offset i
return generateCvsExpandExp(c, run_path, model_id, offset, true)
}
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
return
}
@ -337,7 +336,7 @@ func generateCvsExpandExp(c *Context, run_path string, model_id string, offset i
// This is to load some extra data so that the model has more things to train on
//
data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
data_other, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3 limit $4;", model_id, DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINED, count*10)
if err != nil {
return
}

View File

@ -9,7 +9,6 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
@ -119,6 +118,8 @@ func handleUpload(handler *Handle) {
UpdateStatus(c, "tasks", id, 1)
return c.SendJSON(struct {Id string `json:"id"`}{id})
return c.SendJSON(struct {
Id string `json:"id"`
}{id})
})
}

View File

@ -3,7 +3,6 @@ package tasks
import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

View File

@ -11,7 +11,6 @@ import (
"time"
dbtypes "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
"github.com/charmbracelet/log"
"github.com/go-playground/validator/v10"
"github.com/goccy/go-json"
@ -67,7 +66,7 @@ func (x *Handle) Get(path string, fn func(c *Context) *Error) {
x.gets = append(x.gets, HandleFunc{path, fn})
}
func (x *Handle) GetAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) GetAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@ -80,7 +79,7 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
x.posts = append(x.posts, HandleFunc{path, fn})
}
func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) PostAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@ -92,7 +91,7 @@ func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error
func PostAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(int(authLevel)) {
if !c.CheckAuthLevel(authLevel) {
return nil
}
@ -112,7 +111,7 @@ func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
x.deletes = append(x.deletes, HandleFunc{path, fn})
}
func (x *Handle) DeleteAuth(path string, authLevel int, fn func(c *Context) *Error) {
func (x *Handle) DeleteAuth(path string, authLevel dbtypes.UserType, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@ -122,7 +121,7 @@ func (x *Handle) DeleteAuth(path string, authLevel int, fn func(c *Context) *Err
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel int, fn func(c *Context, obj *T) *Error) {
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel dbtypes.UserType, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
@ -160,13 +159,13 @@ func handleLoop(array []HandleFunc, context *Context) {
handleError(&Error{404, "Endpoint not found"}, context)
}
func (c *Context) CheckAuthLevel(authLevel int) bool {
func (c *Context) CheckAuthLevel(authLevel dbtypes.UserType) bool {
if authLevel > 0 {
if c.User == nil {
contextlessLogoff(c.Writer)
return false
}
if c.User.UserType < authLevel {
if c.User.UserType < int(authLevel) {
c.Writer.WriteHeader(http.StatusUnauthorized)
e := c.SendJSON("Not Authorized")
if e != nil {
@ -321,15 +320,15 @@ func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
return c.SendJSONStatus(http.StatusBadRequest, dat)
}
func (c *Context) GetModelFromId(id_path string) (*BaseModel, *Error) {
func (c *Context) GetModelFromId(id_path string) (*dbtypes.BaseModel, *Error) {
id, err := GetIdFromUrl(c, id_path)
if err != nil {
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
}
model, err := GetBaseModel(c.Db, id)
if err == ModelNotFoundError {
model, err := dbtypes.GetBaseModel(c.Db, id)
if err == dbtypes.ModelNotFoundError {
return nil, c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return nil, c.Error500(err)

View File

@ -7,8 +7,8 @@ import (
"github.com/charmbracelet/log"
_ "github.com/lib/pq"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/db_types"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
models_utils "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/tasks/runner"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
@ -44,7 +44,7 @@ func main() {
handle := NewHandler(db, config)
// TODO remove this before commiting
_, err = db.Exec("update models set status=$1 where status=$2", models_utils.FAILED_TRAINING, models_utils.TRAINING)
_, err = db.Exec("update models set status=$1 where status=$2", FAILED_TRAINING, TRAINING)
if err != nil && clear_db {
log.Warn("Database might not be on")
panic(err)

View File

@ -188,7 +188,7 @@ func usersEndpints(db *sql.DB, handle *Handle) {
})
// TODO allow admin users to update this data
handle.GetAuth("/user/info", int(dbtypes.User_Normal), func(c *Context) *Error {
handle.GetAuth("/user/info", dbtypes.User_Normal, func(c *Context) *Error {
user, err := dbtypes.UserFromToken(c.Db, *c.Token)
if err != nil {
return c.E500M("Falied to get user data", err)

View File

@ -10,7 +10,7 @@
<script lang="ts">
import Tabs from 'src/lib/Tabs.svelte';
import type { Class } from './ModelData.svelte';
import { get, postFormData, rdelete } from 'src/lib/requests.svelte';
import { post, postFormData, rdelete } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
@ -37,11 +37,10 @@
async function getList() {
try {
let url = new URLSearchParams();
url.append('id', selected_class?.id ?? '');
url.append('page', `${page}`);
let res = await get('models/data/list?' + url.toString());
let res = await post('models/data/list', {
id: selected_class?.id ?? '',
page: page
});
showNext = res.showNext;
image_list = res.image_list;
} catch (e) {
@ -91,13 +90,12 @@
function deleteDataPoint(id: string) {
try {
rdelete('models/data/point', { id })
getList()
rdelete('models/data/point', { id });
getList();
} catch (e) {
console.error('TODO notify user', e);
}
}
</script>
{#if classes.length == 0}
@ -126,26 +124,27 @@
</button>
{/each}
</div>
<button on:click={() => {
setActive("-----New Class-----")();
<button
on:click={() => {
setActive('-----New Class-----')();
selected_class = undefined;
}}>
}}
>
<span class="bi bi-plus" />
</button>
</div>
{#if selected_class == undefined && isActive('-----New Class-----')}
<div class="content selected">
<h2>
Add New Class
</h2>
<h2>Add New Class</h2>
<form on:submit|preventDefault={uploadZip}>
<fieldset class="file-upload">
<label for="file">Data file</label>
<div class="form-msg">
Please provide a file that has the training and testing data<br />
The file must have 2 folders one with testing images and one with training images. <br/>
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
The class folders must have the images for the classes.
The file must have 2 folders one with testing images and one with training images.
<br />
Each of the folders will contain the classes of the model. The folders must be the same
in testing and training. The class folders must have the images for the classes.
<pre>
training\
class1\
@ -173,29 +172,21 @@
...
</pre>
</div>
<FileUpload replace_slot bind:file={file} accept="application/zip" notExpand >
<FileUpload replace_slot bind:file accept="application/zip" notExpand>
<img src="/imgs/upload-icon.png" alt="" />
<span>
Upload Zip File
</span>
<span> Upload Zip File </span>
<div slot="replaced" style="display: inline;">
<img src="/imgs/upload-icon.png" alt="" />
<span>
File selected
</span>
<span> File selected </span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#if file}
{#await uploading}
<button disabled>
Uploading
</button>
<button disabled> Uploading </button>
{:then}
<button>
Add
</button>
<button> Add </button>
{/await}
{/if}
</form>
@ -205,17 +196,11 @@
<div class="content selected">
{#if model.model_type == 2}
{#if selected_class?.status == 1}
<h2>
Class to train
</h2>
<h2>Class to train</h2>
{:else if selected_class?.status == 2}
<h2>
Class training
</h2>
<h2>Class training</h2>
{:else if selected_class?.status == 3}
<h2>
Class trained
</h2>
<h2>Class trained</h2>
{/if}
{/if}
<table>
@ -303,7 +288,6 @@
{/if}
<style lang="scss">
.buttons {
width: 100%;
display: flex;