andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

chore: started working on #38

Browse Source
This commit is contained in:
Andre Henriques 2023-10-10 12:28:49 +01:00
parent 1229ad5373
commit b6afecc682
8 changed files with 225 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
logic/models/classes/data_point.go
View File

@ -8,16 +8,21 @@ import (
var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")
func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
id = ""
result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode)
if err != nil {
return
}
defer result.Close()
if !result.Next() {
err = FailedToGetIdAfterInsertError
return
}
err = result.Scan(&id)
id = ""
result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode, status) values ($1, $2, $3, 1) returning id;", class_id, file_path, mode)
if err != nil {
return
}
defer result.Close()
if !result.Next() {
err = FailedToGetIdAfterInsertError
return
}
err = result.Scan(&id)
return
}
func UpdateDataPointStatus(db *sql.DB, data_point_id string, status int, message *string) (err error) {
_, err = db.Exec("update model_data_point set status=$1, status_message=$2 where id=$3", status, message, data_point_id)
return
}

11
logic/models/classes/main.go
View File

@ -71,3 +71,14 @@ func CreateClass(db *sql.DB, model_id string, order int, name string) (id string
err = rows.Scan(&id)
return
}
func GetNumberOfWrongDataPoints(db *sql.DB, model_id string) (number int, err error) {
number = 0
rows, err := db.Query("select count(mdp.id) from model_data_point as mdp join model_classes as mc on mc.id = mdp.class_id where mc.model_id=$1 and mdp.status=-1;", model_id)
if err != nil { return }
defer rows.Close()
// TODO not an error because if there is no result means that there is no need to count
if !rows.Next() { return }
err = rows.Scan(&number)
return
}

10
logic/models/data.go
View File

@ -137,9 +137,13 @@ func processZipFile(c *Context, model *BaseModel) {
f.Write(file_data)
if !testImgForModel(c, model, file_path) {
c.Logger.Errorf("Image did not have valid format for model %s\n", file_path)
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
return
c.Logger.Errorf("Image did not have valid format for model %s (in zip: %s)!", file_path, file.Name)
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
c.Logger.Error("Failed to update data point status")
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_ZIP_FILE)
}
}
}

9
logic/models/edit.go
View File

@ -69,10 +69,12 @@ func handleEdit(handle *Handle) {
"Model": model,
}))
case CONFIRM_PRE_TRAINING:
wrong_number, err := model_classes.GetNumberOfWrongDataPoints(c.Db, model.Id)
if err != nil { return c.Error500(err) }
cls, err := model_classes.ListClasses(handle.Db, id)
if err != nil {
return Error500(err)
}
if err != nil { return c.Error500(err) }
has_data, err := model_classes.ModelHasDataPoints(handle.Db, id)
if err != nil {
@ -83,6 +85,7 @@ func handleEdit(handle *Handle) {
"Model": model,
"Classes": cls,
"HasData": has_data,
"NumberOfInvalidImages": wrong_number,
}))
case READY:
LoadBasedOnAnswer(c.Mode, w, "/models/edit.html", c.AddMap(AnyMap{

231
logic/models/train/train.go
View File

@ -19,12 +19,16 @@ import (
func MakeDefenition(db *sql.DB, model_id string, target_accuracy int) (id string, err error) {
id = ""
rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
if err != nil { return }
defer rows.Close()
if !rows.Next() { return id, errors.New("Something wrong!") }
rows, err := db.Query("insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;", model_id, target_accuracy)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something wrong!")
}
err = rows.Scan(&id)
return
return
}
type ModelDefinitionStatus int
@ -41,9 +45,9 @@ const (
type LayerType int
const (
LAYER_INPUT LayerType = 1
LAYER_DENSE = 2
LAYER_FLATTEN = 3
LAYER_INPUT LayerType = 1
LAYER_DENSE = 2
LAYER_FLATTEN = 3
)
func ModelDefinitionUpdateStatus(c *Context, id string, status ModelDefinitionStatus) (err error) {
@ -56,36 +60,48 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
return
}
func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
func generateCvs(c *Context, run_path string, model_id string) (count int, err error) {
classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
if err != nil { return }
defer classes.Close()
if !classes.Next() { return }
if err = classes.Scan(&count); err != nil { return }
classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
if err != nil {
return
}
defer classes.Close()
if !classes.Next() {
return
}
if err = classes.Scan(&count); err != nil {
return
}
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
if err != nil { return }
defer data.Close()
data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
if err != nil {
return
}
defer data.Close()
f, err := os.Create(path.Join(run_path, "train.csv"))
if err != nil { return }
defer f.Close()
f.Write([]byte("Id,Index\n"))
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil { return }
if file_path == "id://" {
f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
f, err := os.Create(path.Join(run_path, "train.csv"))
if err != nil {
return
}
defer f.Close()
f.Write([]byte("Id,Index\n"))
return
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil {
return
}
if file_path == "id://" {
f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n"))
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
return
}
func trainDefinition(c *Context, model *BaseModel, definition_id string) (accuracy float64, err error) {
@ -120,8 +136,10 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
return
}
_, err = generateCvs(c, run_path, model.Id)
if err != nil { return }
_, err = generateCvs(c, run_path, model.Id)
if err != nil {
return
}
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
@ -136,19 +154,19 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
}
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"RunPath": run_path,
"ColorMode": model.ImageMode,
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model.Id, "data"),
"RunPath": run_path,
"ColorMode": model.ImageMode,
}); err != nil {
return
}
// Run the command
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
if err != nil {
fmt.Println(string(out))
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
if err != nil {
fmt.Println(string(out))
return
}
@ -174,14 +192,14 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string) (accura
return
}
fmt.Println(string(accuracy_file_bytes))
fmt.Println(string(accuracy_file_bytes))
accuracy, err = strconv.ParseFloat(string(accuracy_file_bytes), 64)
if err != nil {
return
}
os.RemoveAll(run_path)
os.RemoveAll(run_path)
return
}
@ -205,8 +223,8 @@ func trainModel(c *Context, model *BaseModel) {
for definitionsRows.Next() {
var rowv row
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy); err != nil {
c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
c.Logger.Error(err)
c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
@ -214,7 +232,7 @@ func trainModel(c *Context, model *BaseModel) {
}
if len(definitions) == 0 {
c.Logger.Error("No Definitions defined!")
c.Logger.Error("No Definitions defined!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
@ -222,8 +240,8 @@ func trainModel(c *Context, model *BaseModel) {
for _, def := range definitions {
accuracy, err := trainDefinition(c, model, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:")
c.Logger.Error(err)
c.Logger.Error("Failed to train definition!Err:")
c.Logger.Error(err)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
continue
}
@ -247,66 +265,90 @@ func trainModel(c *Context, model *BaseModel) {
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
if err != nil {
c.Logger.Error("DB: failed to read definition")
c.Logger.Error(err)
c.Logger.Error("DB: failed to read definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
defer rows.Close()
defer rows.Close()
if !rows.Next() {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
if !rows.Next() {
// TODO Make the Model status have a message
c.Logger.Error("All definitions failed to train!")
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
var id string
if err = rows.Scan(&id); err != nil {
c.Logger.Error("Failed to read id:")
c.Logger.Error(err)
var id string
if err = rows.Scan(&id); err != nil {
c.Logger.Error("Failed to read id:")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
c.Logger.Error("Failed to update model definition")
c.Logger.Error(err)
if _, err = c.Db.Exec("update model_definition set status=$1 where id=$2;", MODEL_DEFINITION_STATUS_READY, id); err != nil {
c.Logger.Error("Failed to update model definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.Logger.Error("Failed to select model_definition to delete")
c.Logger.Error(err)
to_delete, err := c.Db.Query("select id from model_definition where status != $1 and model_id=$2", MODEL_DEFINITION_STATUS_READY, model.Id)
if err != nil {
c.Logger.Error("Failed to select model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
defer to_delete.Close()
}
defer to_delete.Close()
for to_delete.Next() {
var id string
if to_delete.Scan(&id);err != nil {
c.Logger.Error("Failed to scan the id of a model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
c.Logger.Error("Failed to delete model_definition")
c.Logger.Error(err)
for to_delete.Next() {
var id string
if to_delete.Scan(&id); err != nil {
c.Logger.Error("Failed to scan the id of a model_definition to delete")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
os.RemoveAll(path.Join("savedData", model.Id, "defs", id))
}
// TODO Check if returning also works here
if _, err = c.Db.Exec("delete from model_definition where status!=$1 and model_id=$2;", MODEL_DEFINITION_STATUS_READY, model.Id); err != nil {
c.Logger.Error("Failed to delete model_definition")
c.Logger.Error(err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
ModelUpdateStatus(c, model.Id, READY)
}
func removeFailedDataPoints(db *sql.DB, model *BaseModel) (err error) {
rows, err := db.Query("select id from model_data_point as mdp join model_classes as mc on mc.id=mpd.class_id where mc.model_id=$1 and mdp.status=-1;", model.Id)
if err != nil {
return
}
defer rows.Close()
base_path := path.Join("savedData", model.Id, "data")
for rows.Next() {
var dataPointId string
err = rows.Scan(&dataPointId)
if err != nil {
return
}
err = os.RemoveAll(path.Join(base_path, dataPointId + model.Format))
if err != nil {
return
}
}
return
}
func handleTrain(handle *Handle) {
handle.Post("/models/train", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
if !CheckAuthLevel(1, w, r, c) {
@ -376,6 +418,11 @@ func handleTrain(handle *Handle) {
return c.Error500(err)
}
err = removeFailedDataPoints(c.Db, model.Id)
if err != nil {
return c.Error500(err)
}
var fid string
for i := 0; i < number_of_models; i++ {
def_id, err := MakeDefenition(handle.Db, model.Id, accuracy)
@ -402,13 +449,13 @@ func handleTrain(handle *Handle) {
// TODO improve this response
return c.Error500(err)
}
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls) * 3))
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)*3))
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
return c.Error500(err)
}
// Using sparce
// Using sparce
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d, 1", len(cls)))
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)

6
sql/models.sql
View File

@ -33,7 +33,11 @@ create table if not exists model_data_point (
file_path text not null,
-- 1 training
-- 2 testing
model_mode integer default 1
model_mode integer default 1,
-- -1 Error on creation
-- 1 OK
status integer not null,
status_message text
);
-- drop table if exists model_definition;

64
views/models/edit.html
View File

@ -247,6 +247,11 @@
<p>
You need to upload data so the model can train.
</p>
{{ if gt .NumberOfInvalidImages 0 }}
<p class="danger">
There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
</p>
{{ end }}
<div class="tabs">
<button class="tab" data-tab="create_class">
Create Class
@ -268,35 +273,40 @@
{{ define "train-model-card" }}
<form hx-post="/models/train" hx-headers='{"REQUEST-TYPE": "html"}' hx-swap="outerHTML" {{ if .Error }} class="submitted" {{end}} >
{{ if .HasData }}
{{/* TODO expading mode */}}
<input type="hidden" value="{{ .Model.Id }}" name="id" />
<fieldset>
<legend>
Model Type
</legend>
<div class="input-radial">
<input id="model_type_simple" value="simple" name="model_type" type="radio" checked />
<label for="model_type_simple">Simple</label>
</div>
</fieldset>
{{/* TODO allow more models to be created */}}
<fieldset>
<label for="number_of_models">Number of Models</label>
<input id="number_of_models" type="number" name="number_of_models" value="1" />
</fieldset>
{{/* TODO to Change the acc */}}
<fieldset>
<label for="accuracy">Target accuracy</label>
<input id="accuracy" type="number" name="accuracy" value="95" />
</fieldset>
{{/* TODO allow to chose the base of the model */}}
{{/* TODO allow to change the shape of the model */}}
<button>
Train
</button>
{{ if gt .NumberOfInvalidImages 0 }}
<p class="danger">
There are images that were loaded that do not have the correct format. These images will be delete when the model trains.
</p>
{{ end }}
{{/* TODO expading mode */}}
<input type="hidden" value="{{ .Model.Id }}" name="id" />
<fieldset>
<legend>
Model Type
</legend>
<div class="input-radial">
<input id="model_type_simple" value="simple" name="model_type" type="radio" checked />
<label for="model_type_simple">Simple</label>
</div>
</fieldset>
{{/* TODO allow more models to be created */}}
<fieldset>
<label for="number_of_models">Number of Models</label>
<input id="number_of_models" type="number" name="number_of_models" value="1" />
</fieldset>
{{/* TODO to Change the acc */}}
<fieldset>
<label for="accuracy">Target accuracy</label>
<input id="accuracy" type="number" name="accuracy" value="95" />
</fieldset>
{{/* TODO allow to chose the base of the model */}}
{{/* TODO allow to change the shape of the model */}}
<button>
Train
</button>
{{ else }}
<h2>
Please provide data to the model first
To train the model please provide data to the model first
</h2>
{{ end }}
</form>

4
views/styles/main.css
View File

@ -66,6 +66,10 @@ main {
font-size: 1.1rem;
}
.danger {
color: red;
}
/* Generic */
.button,
button {