andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: added way to add new images

Browse Source
This commit is contained in:
Andre Henriques 2024-03-09 09:41:16 +00:00
parent 4a95f0211d
commit 0d37ba8d59
12 changed files with 510 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
logic/models/classes/main.go
View File

@ -3,36 +3,19 @@ package model_classes
import (
"database/sql"
"errors"
// . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
type ModelClass struct {
Id string `json:"id"`
ModelId string `json:"model_id"`
ModelId string `json:"model_id" db:"model_id"`
Name string `json:"name"`
Status int `json:"status"`
}
func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) {
rows, err := db.Query("select id, model_id, name from model_classes where model_id=$1", model_id)
if err != nil {
return
}
defer rows.Close()
cls = []ModelClass{}
for rows.Next() {
var model ModelClass
err = rows.Scan(&model.Id, &model.ModelId, &model.Name)
if err != nil {
return
}
cls = append(cls, model)
}
return
func ListClasses(c *Context, model_id string) (cls []*ModelClass, err error) {
return GetDbMultitple[ModelClass](c, "model_classes where model_id=$1", model_id)
}
func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {

211
logic/models/data.go
View File

@ -151,6 +151,122 @@ func processZipFile(c *Context, model *BaseModel) {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
}
func processZipFileExpand(c *Context, model *BaseModel) {
var err error
failed := func(msg string) {
c.Logger.Error(msg, "err", err)
ModelUpdateStatus(c, model.Id, READY_FAILED)
}
reader, err := zip.OpenReader(path.Join("savedData", model.Id, "expand_data.zip"))
if err != nil {
failed("Faield to proccess zip file failed to open reader\n")
return
}
defer reader.Close()
training := []string{}
testing := []string{}
for _, file := range reader.Reader.File {
paths := strings.Split(file.Name, "/")
if paths[1] == "" {
continue
}
if paths[0] != "training" && paths[0] != "testing" {
failed(fmt.Sprintf("Invalid file '%s' TODO add msg to response!!!", file.Name))
return
}
if paths[0] != "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
failed("testing and training are diferent")
return
}
base_path := path.Join("savedData", model.Id, "data")
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
failed("Failed to create base_path dir")
return
}
ids := map[string]string{}
for i, name := range training {
id, err := model_classes.CreateClass(c.Db, model.Id, i, name)
if err != nil {
failed(fmt.Sprintf("Failed to create class '%s' on db\n", name))
return
}
ids[name] = id
}
for _, file := range reader.Reader.File {
if file.Name[len(file.Name)-1] == '/' {
continue
}
data, err := reader.Open(file.Name)
if err != nil {
failed(fmt.Sprintf("Could not open file in zip %s\n", file.Name))
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
failed(fmt.Sprintf("Could not read file file in zip %s\n", file.Name))
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database
parts := strings.Split(file.Name, "/")
mode := model_classes.DATA_POINT_MODE_TRAINING
if parts[0] == "testing" {
mode = model_classes.DATA_POINT_MODE_TESTING
}
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
if err != nil {
failed(fmt.Sprintf("Failed to add data point for %s\n", model.Id))
return
}
file_path := path.Join(base_path, data_point_id+"."+model.Format)
f, err := os.Create(file_path)
if err != nil {
failed(fmt.Sprintf("Could not create file %s\n", file_path))
return
}
defer f.Close()
f.Write(file_data)
if !testImgForModel(c, model, file_path) {
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
failed(fmt.Sprintf("Failed to update data point status"))
return
}
}
}
c.Logger.Info("Added data to model", "id", model.Id)
ModelUpdateStatus(c, model.Id, READY)
}
func handleDataUpload(handle *Handle) {
handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
@ -267,6 +383,73 @@ func handleDataUpload(handle *Handle) {
return nil
})
// ------
// ------ CLASS DATA UPLOAD
// ------
handle.PostJSON("/models/data/class/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
}
read_form, err := r.MultipartReader()
if err != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
var id string
var file []byte
for {
part, err_part := read_form.NextPart()
if err_part == io.EOF {
break
} else if err_part != nil {
return c.JsonBadRequest("Please provide a valid form data request!")
}
if part.FormName() == "id" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
id = buf.String()
}
if part.FormName() == "file" {
buf := new(bytes.Buffer)
buf.ReadFrom(part)
file = buf.Bytes()
}
}
c.Logger.Info("Trying to expand model", "id", id)
model, err := GetBaseModel(handle.Db, id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
}
// TODO mk this path configurable
dir_path := path.Join("savedData", id)
f, err := os.Create(path.Join(dir_path, "expand_data.zip"))
if err != nil {
return Error500(err)
}
defer f.Close()
f.Write(file)
ModelUpdateStatus(c, id, READY_ALTERATION)
go processZipFileExpand(c, model)
return c.SendJSON(model.Id)
})
handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
return nil
@ -285,31 +468,51 @@ func handleDataUpload(handle *Handle) {
model, err := GetBaseModel(handle.Db, dat.Id)
if err == ModelNotFoundError {
return c.SendJSONStatus(http.StatusNotFound, "Model not found");
return c.SendJSONStatus(http.StatusNotFound, "Model not found")
} else if err != nil {
return Error500(err)
}
if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.SendJSONStatus(http.StatusNotFound, "Model not in the correct status")
delete_path := "base_data.zip"
if model.Status == READY_FAILED {
delete_path = "expand_data.zip"
} else if model.Status != FAILED_PREPARING_ZIP_FILE {
return c.JsonBadRequest("Model not in the correct status")
}
err = os.Remove(path.Join("savedData", model.Id, "base_data.zip"))
err = os.Remove(path.Join("savedData", model.Id, delete_path))
if err != nil {
return Error500(err)
}
if model.Status != READY_FAILED {
err = os.RemoveAll(path.Join("savedData", model.Id, "data"))
if err != nil {
return Error500(err)
}
} else {
c.Logger.Warn("Handle failed to remove the savedData when deleteing the zip file while expanding")
}
if model.Status != READY_FAILED {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1;", model.Id)
if err != nil {
return Error500(err)
}
} else {
_, err = handle.Db.Exec("delete from model_classes where model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TO_TRAIN)
if err != nil {
return Error500(err)
}
}
if model.Status != READY_FAILED {
ModelUpdateStatus(c, model.Id, CONFIRM_PRE_TRAINING)
} else {
ModelUpdateStatus(c, model.Id, READY)
}
return c.SendJSON(model.Id)
}

6
logic/models/edit.go
View File

@ -75,7 +75,7 @@ func handleEdit(handle *Handle) {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(handle.Db, id)
cls, err := model_classes.ListClasses(c, id)
if err != nil {
return c.Error500(err)
}
@ -86,7 +86,7 @@ func handleEdit(handle *Handle) {
}
type ReturnType struct {
Classes []model_classes.ModelClass `json:"classes"`
Classes []*model_classes.ModelClass `json:"classes"`
HasData bool `json:"has_data"`
NumberOfInvalidImages int `json:"number_of_invalid_images"`
}
@ -314,7 +314,7 @@ func handleEdit(handle *Handle) {
return c.Error500(err)
}
cls, err := model_classes.ListClasses(handle.Db, id)
cls, err := model_classes.ListClasses(c, id)
if err != nil {
return c.Error500(err)
}

1
logic/models/run.go
View File

@ -103,7 +103,6 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten
var predictions = results[0].Value().([][]float32)[0]
for i, v := range predictions {
c.Logger.Info("This is test", "v", v)
if v > vmax {
order = element.Range_start + i
vmax = v

52
logic/models/train/train.go
View File

@ -102,7 +102,7 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e
}
func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) {
_, err = c.Db.Exec("update model_classes set stauts = $1 where "+filter, args...)
_, err = c.Db.Exec(fmt.Sprintf("update model_classes set status=%d where %s", status, filter), args...)
return
}
@ -485,6 +485,7 @@ func (nf ToRemoveList) Less(i, j int) bool {
}
func trainModel(c *Context, model *BaseModel) {
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
c.Logger.Error("Failed to train Model! Err:")
@ -783,9 +784,7 @@ func trainModelExp(c *Context, model *BaseModel) {
if len_def == 0 {
break
}
if len_def == 1 {
} else if len_def == 1 {
continue
}
@ -811,43 +810,37 @@ func trainModelExp(c *Context, model *BaseModel) {
}
}
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
// Set the class status to trained
err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINED, "model_id=$1 and status=$2;", model.Id, MODEL_CLASS_STATUS_TRAINING)
if err != nil {
failed("Failed to set class status")
return
}
var dat JustId
err = GetDBOnce(c, &dat, "model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err == NotFoundError {
failed("All definitions failed to train!")
return
} else if err != nil {
failed("DB: failed to read definition")
return
}
defer rows.Close()
if !rows.Next() {
failed("All definitions failed to train!")
return
}
var id string
if err = rows.Scan(&id); err != nil {
failed("Failed to read id")
return
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, dat.Id); err != nil {
failed("Failed to update model definition")
return
}
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
to_delete, err := GetDbMultitple[JustId](c, "model_definition where status!=$1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
failed("Failed to select model_definition to delete")
return
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if to_delete.Scan(&id); err != nil {
failed("Failed to scan the id of a model_definition to delete")
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
for _, d := range(to_delete) {
os.RemoveAll(path.Join("savedData", model.Id, "defs", d.Id))
}
// TODO Check if returning also works here
@ -863,7 +856,6 @@ func trainModelExp(c *Context, model *BaseModel) {
// There should only be one def availabale
def := JustId{}
if err = GetDBOnce(c, &def, "model_definition where model_id=$1", model.Id); err != nil {
return
}
@ -1099,7 +1091,7 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
}
func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
@ -1267,7 +1259,7 @@ func generateExpandableDefinition(c *Context, model *BaseModel, target_accuracy
// TODO make this json friendy
func generateExpandableDefinitions(c *Context, model *BaseModel, target_accuracy int, number_of_models int) *Error {
cls, err := model_classes.ListClasses(c.Db, model.Id)
cls, err := model_classes.ListClasses(c, model.Id)
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response

2
logic/models/utils/types.go
View File

@ -28,6 +28,8 @@ const (
PREPARING_ZIP_FILE = 3
TRAINING = 4
READY = 5
READY_ALTERATION = 6
READY_FAILED = -6
)
type ModelDefinitionStatus int

9
logic/models/utils/utils.go
View File

@ -1,17 +1,14 @@
package models_utils
import (
"fmt"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
// TODO make this return and caller handle error
func ModelUpdateStatus(c *Context, id string, status int) {
_, err := c.Db.Exec("update models set status = $1 where id = $2", status, id)
_, err := c.Db.Exec("update models set status=$1 where id=$2;", status, id)
if err != nil {
fmt.Println("Failed to update model status")
fmt.Println(err)
panic("TODO handle better")
c.Logger.Error("Failed to update model status", "err", err)
c.Logger.Warn("TODO Maybe handle better")
}
}

7
webpage/src/lib/Tabs.svelte
View File

@ -14,7 +14,7 @@
<div class="tab-buttons">
<slot name="buttons" {setActive} {isActive} />
</div>
<slot {isActive} />
<slot {isActive} {active} />
</div>
<style lang="scss">
@ -29,6 +29,11 @@
.tab-buttons {
display: flex;
overflow-x: scroll;
width: 100%;
:global(.buttons) {
width: 100%;
}
:global(.tab) {
padding: 5px;

15
webpage/src/routes/models/edit/+page.svelte
View File

@ -6,6 +6,8 @@
width: number;
height: number;
status: number;
model_type: number;
format: string;
};
export type Layer = {
@ -286,9 +288,20 @@
{/await}
<!-- TODO Add ability to stop training -->
</div>
{:else if m.status == 5}
{:else if [5, 6, -6].includes(m.status)}
<BaseModelInfo model={m} />
<RunModel model={m} />
{#if m.status == 6}
<div class="card">
Model expading... Processing ZIP file
</div>
{/if}
{#if m.status == -6}
<DeleteZip model={m} on:reload={getModel} expand />
{/if}
{#if m.model_type == 2}
<ModelData model={m} on:reload={getModel} />
{/if}
<DeleteModel model={m} />
{:else}
<h1>Unknown Status of the model.</h1>

10
webpage/src/routes/models/edit/DeleteZip.svelte
View File

@ -6,12 +6,12 @@
let message: MessageSimple;
let { model } = $props<{model: Model}>();
let { model, expand } = $props<{model: Model, expand?: boolean}>();
const dispatch = createEventDispatcher<{reload: void}>();
async function deleteZip() {
message.display("");
message.clear();
try {
await rdelete("models/data/delete-zip-file", { id: model.id });
@ -28,9 +28,15 @@
<form on:submit|preventDefault={deleteZip}>
{#if expand}
Failed to proccess the zip file.<br/>
Delete file and upload a correct version do add more classes.<br/>
<br/>
{:else}
Failed to proccess the zip file.<br/>
Delete file and proccess again.<br/>
<br/>
{/if}
<div class="spacer" ></div>
<MessageSimple bind:this={message} />
<button class="danger">

5
webpage/src/routes/models/edit/ModelData.svelte
View File

@ -2,6 +2,7 @@
export type Class = {
name: string;
id: string;
status: number;
}
</script>
<script lang="ts">
@ -153,7 +154,7 @@
</form>
</div>
<div class="content" class:selected={isActive("create-class")}>
<ModelTable {classes} {model} />
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive("api")}>
TODO
@ -180,7 +181,7 @@
</button>
</div>
<div class="content" class:selected={isActive("create-class")}>
<ModelTable {classes} {model} />
<ModelTable {classes} {model} on:reload={() => dispatch('reload')} />
</div>
<div class="content" class:selected={isActive("api")}>
TODO

143
webpage/src/routes/models/edit/ModelTable.svelte
View File

@ -3,14 +3,20 @@
file_path: string;
mode: number;
status: number;
id: string;
};
</script>
<script lang="ts">
import Tabs from 'src/lib/Tabs.svelte';
import type { Class } from './ModelData.svelte';
import { get } from 'src/lib/requests.svelte';
import { get, postFormData } from 'src/lib/requests.svelte';
import type { Model } from './+page.svelte';
import FileUpload from 'src/lib/FileUpload.svelte';
import MessageSimple from 'src/lib/MessageSimple.svelte';
import { createEventDispatcher } from 'svelte';
const dispatch = createEventDispatcher<{reload: void}>();
let selected_class: Class | undefined = $state();
@ -22,7 +28,6 @@
function setActiveClass(c: Class, tb_fn: (name: string) => () => void) {
selected_class = c;
console.log('test', c, classes, c.name);
tb_fn(c.name)();
}
@ -31,8 +36,6 @@
});
async function getList() {
console.log(selected_class);
try {
let url = new URLSearchParams();
url.append('id', selected_class?.id ?? '');
@ -59,14 +62,43 @@
getList();
}
});
let file: File | undefined = $state();
let uploadImage: MessageSimple;
let uploading = $state(Promise.resolve());
async function uploadZip() {
uploadImage.clear();
if (!file) return;
uploading = new Promise(() => {});
let form = new FormData();
form.append('id', model.id);
form.append('file', file, 'upload.zip');
try {
await postFormData('models/data/class/upload', form);
dispatch('reload');
} catch (e) {
if (e instanceof Response) {
uploadImage.display(await e.json());
} else {
uploadImage.display('');
}
}
uploading = Promise.resolve();
}
</script>
{#if classes.length == 0}
TODO CREATE TABLE
{:else}
<Tabs active={classes[0]?.name} let:isActive>
<div slot="buttons" let:setActive let:isActive>
<div class="buttons" slot="buttons" let:setActive let:isActive>
<!-- TODO Auto Load 1st -->
<div>
{#each classes as item}
<button
on:click={() => setActiveClass(item, setActive)}
@ -77,7 +109,96 @@
</button>
{/each}
</div>
<button on:click={() => {
setActive("-----New Class-----")();
selected_class = undefined;
}}>
<span class="bi bi-plus" />
</button>
</div>
{#if selected_class == undefined && isActive('-----New Class-----')}
<div class="content selected">
<h2>
Add New Class
</h2>
<form on:submit|preventDefault={uploadZip}>
<fieldset class="file-upload" >
<label for="file">Data file</label>
<div class="form-msg">
Please provide a file that has the training and testing data<br/>
The file must have 2 folders one with testing images and one with training images. <br/>
Each of the folders will contain the classes of the model. The folders must be the same in testing and training.
The class folders must have the images for the classes.
<pre>
training\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
testing\
class1\
img1.png
img2.png
img2.png
...
class2\
img1.png
img2.png
img2.png
...
...
</pre>
</div>
<FileUpload replace_slot bind:file={file} accept="application/zip" notExpand >
<img src="/imgs/upload-icon.png" alt="" />
<span>
Upload Zip File
</span>
<div slot="replaced" style="display: inline;">
<img src="/imgs/upload-icon.png" alt="" />
<span>
File selected
</span>
</div>
</FileUpload>
</fieldset>
<MessageSimple bind:this={uploadImage} />
{#await uploading}
<button disabled>
Uploading
</button>
{:then}
<button>
Add
</button>
{/await}
</form>
</div>
{/if}
{#if selected_class}
<div class="content selected">
{#if model.model_type == 2}
{#if selected_class?.status == 1}
<h2>
Class to train
</h2>
{:else if selected_class?.status == 2}
<h2>
Class training
</h2>
{:else if selected_class?.status == 3}
<h2>
Class trained
</h2>
{/if}
{/if}
<table>
<thead>
<tr>
@ -150,10 +271,22 @@
</div>
</div>
</div>
{/if}
</Tabs>
{/if}
<style lang="scss">
.buttons {
width: 100%;
display: flex;
justify-content: space-between;
&>button {
margin: 3px 5px;
}
}
table {
width: 100%;
box-shadow: 0 2px 8px 1px #66666622;