andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

closes #49 and possible done #46

Browse Source
This commit is contained in:
Andre Henriques 2023-10-22 23:02:39 +01:00
parent 90bc3f6acf
commit c844aeabe4
9 changed files with 256 additions and 135 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.air.toml
View File

@ -14,7 +14,7 @@ tmp_dir = "tmp"
follow_symlink = false
full_bin = ""
include_dir = []
include_ext = ["go", "tpl", "tmpl", "html"]
include_ext = ["go", "tpl", "tmpl"] # , "html"
include_file = []
kill_delay = "0s"
log = "build-errors.log"

37
logic/models/add.go
View File

@ -2,7 +2,6 @@ package models
import (
"bytes"
"fmt"
"image"
"image/color"
_ "image/jpeg"
@ -38,8 +37,8 @@ func loadBaseImage(c *Context, id string) {
case "png":
case "jpeg":
default:
// TODO better logging
fmt.Printf("Found unkown format '%s'\n", format)
ModelUpdateStatus(c, id, FAILED_PREPARING)
c.Logger.Error("Found unkown format '%s'\n", "format", format)
panic("Handle diferent files than .png")
}
@ -53,24 +52,26 @@ func loadBaseImage(c *Context, id string) {
fallthrough
case color.GrayModel:
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:
fmt.Println("Do not know how to handle this color model")
c.Logger.Error("Do not know how to handle this color model")
if src.ColorModel() == color.RGBA64Model {
fmt.Println("Color is rgb")
} else if src.ColorModel() == color.NRGBAModel {
fmt.Println("Color is nrgb")
c.Logger.Error("Color is rgb")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Error("Color is nrgb 64")
} else if src.ColorModel() == color.AlphaModel {
fmt.Println("Color is alpha")
c.Logger.Error("Color is alpha")
} else if src.ColorModel() == color.CMYKModel {
fmt.Println("Color is cmyk")
c.Logger.Error("Color is cmyk")
} else {
fmt.Println("Other so assuming color")
c.Logger.Error("Other so assuming color")
}
ModelUpdateStatus(c, id, -1)
ModelUpdateStatus(c, id, FAILED_PREPARING)
return
}
@ -130,7 +131,7 @@ func handleAdd(handle *Handle) {
row, err := handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id)
if err != nil {
return Error500(err)
return c.Error500(err)
}
if row.Next() {
@ -143,12 +144,12 @@ func handleAdd(handle *Handle) {
_, err = handle.Db.Exec("insert into models (user_id, name) values ($1, $2)", c.User.Id, name)
if err != nil {
return Error500(err)
return c.Error500(err)
}
row, err = handle.Db.Query("select id from models where name=$1 and user_id=$2;", name, c.User.Id)
if err != nil {
return Error500(err)
return c.Error500(err)
}
if !row.Next() {
@ -158,7 +159,7 @@ func handleAdd(handle *Handle) {
var id string
err = row.Scan(&id)
if err != nil {
return Error500(err)
return c.Error500(err)
}
// TODO mk this path configurable
@ -166,17 +167,17 @@ func handleAdd(handle *Handle) {
err = os.Mkdir(dir_path, os.ModePerm)
if err != nil {
return Error500(err)
return c.Error500(err)
}
f, err := os.Create(path.Join(dir_path, "baseimage.png"))
if err != nil {
return Error500(err)
return c.Error500(err)
}
defer f.Close()
f.Write(file)
fmt.Printf("Created model with id %s! Started to proccess image!\n", id)
c.Logger.Warn("Created model with id %s! Started to proccess image!\n", "id", id)
go loadBaseImage(c, id)
Redirect("/models/edit?id="+id, c.Mode, w, r)

11
logic/models/test.go
View File

@ -32,9 +32,12 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
width, height := bounds.Max.X, bounds.Max.Y
switch src.ColorModel() {
case color.Gray16Model: fallthrough
case color.Gray16Model:
fallthrough
case color.GrayModel:
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:
@ -42,8 +45,8 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
if src.ColorModel() == color.RGBA64Model {
c.Logger.Info("Color is rgb")
} else if src.ColorModel() == color.NRGBAModel {
c.Logger.Info("Color is nrgb")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Info("Color is nrgb 64")
} else if src.ColorModel() == color.AlphaModel {
c.Logger.Info("Color is alpha")
} else if src.ColorModel() == color.CMYKModel {
@ -54,7 +57,7 @@ func testImgForModel(c *Context, model *BaseModel, path string) (result bool) {
return
}
if (StringToImageMode(model_color) != model.ImageMode) {
if StringToImageMode(model_color) != model.ImageMode {
c.Logger.Warn("Color Mode does not match with model color mode", model_color, model.ImageMode)
return
}

205
logic/models/train/train.go
View File

@ -10,6 +10,7 @@ import (
"os"
"os/exec"
"path"
"sort"
"strconv"
"text/template"
@ -43,6 +44,7 @@ const (
MODEL_DEFINITION_STATUS_PRE_INIT ModelDefinitionStatus = 1
MODEL_DEFINITION_STATUS_INIT = 2
MODEL_DEFINITION_STATUS_TRAINING = 3
MODEL_DEFINITION_STATUS_PAUSED_TRAINING = 6
MODEL_DEFINITION_STATUS_TRANIED = 4
MODEL_DEFINITION_STATUS_READY = 5
)
@ -142,6 +144,7 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
if err != nil {
return
}
defer os.RemoveAll(run_path)
_, err = generateCvs(c, run_path, model.Id)
if err != nil {
@ -174,29 +177,24 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
"DefId": definition_id,
"LoadPrev": load_prev,
"LastModelRunPath": path.Join(getDir(), result_path, "model.keras"),
"SaveModelPath": path.Join(getDir(), result_path),
}); err != nil {
return
}
// Run the command
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).CombinedOutput()
if err != nil {
c.Logger.Debug(string(out))
return
}
c.Logger.Info("Python finished running")
if err = os.MkdirAll(result_path, os.ModePerm); err != nil {
return
}
if err = exec.Command("cp", "-r", path.Join(run_path, "model"), path.Join(result_path, "model")).Run(); err != nil {
return
}
if err = exec.Command("cp", "-r", path.Join(run_path, "model.keras"), path.Join(result_path, "model.keras")).Run(); err != nil {
return
}
accuracy_file, err := os.Open(path.Join(run_path, "accuracy.val"))
if err != nil {
return
@ -214,8 +212,6 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr
}
c.Logger.Info("Model finished training!", "accuracy", accuracy)
os.RemoveAll(run_path)
return
}
@ -236,6 +232,29 @@ func remove[T interface{}](lst []T, i int) []T {
return append(lst[:i], lst[i+1:]...)
}
type TrainModelRow struct {
id string
target_accuracy int
epoch int
acuracy float64
}
type TraingModelRowDefinitions []TrainModelRow
func (nf TraingModelRowDefinitions) Len() int { return len(nf) }
func (nf TraingModelRowDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf TraingModelRowDefinitions) Less(i, j int) bool {
return nf[i].acuracy < nf[j].acuracy
}
type ToRemoveList []int
func (nf ToRemoveList) Len() int { return len(nf) }
func (nf ToRemoveList) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
func (nf ToRemoveList) Less(i, j int) bool {
return nf[i] < nf[j]
}
func trainModel(c *Context, model *BaseModel) {
definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id)
if err != nil {
@ -246,16 +265,11 @@ func trainModel(c *Context, model *BaseModel) {
}
defer definitionsRows.Close()
type row struct {
id string
target_accuracy int
epoch int
}
definitions := []row{}
var definitions TraingModelRowDefinitions = []TrainModelRow{}
for definitionsRows.Next() {
var rowv row
var rowv TrainModelRow
rowv.acuracy = 0
if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil {
c.Logger.Error("Failed to train Model Could not read definition from db!Err:")
c.Logger.Error(err)
@ -271,23 +285,23 @@ func trainModel(c *Context, model *BaseModel) {
return
}
toTrain := len(definitions)
firstRound := true
var newDefinitions = []row{}
copy(newDefinitions, definitions)
finished := false
for {
var toRemove ToRemoveList = []int{}
for i, def := range definitions {
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING)
accuracy, err := trainDefinition(c, model, def.id, !firstRound)
if err != nil {
c.Logger.Error("Failed to train definition!Err:", "err", err)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toTrain = toTrain - 1
newDefinitions = remove(newDefinitions, i)
toRemove = append(toRemove, i)
continue
}
def.epoch += EPOCH_PER_RUN
accuracy = accuracy * 100
def.acuracy = accuracy
if accuracy >= float64(def.target_accuracy) {
c.Logger.Info("Found a definition that reaches target_accuracy!")
@ -305,30 +319,68 @@ func trainModel(c *Context, model *BaseModel) {
return
}
toTrain = 0
finished = true
break
}
if def.epoch > MAX_EPOCH {
fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.target_accuracy)
ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING)
toTrain = toTrain - 1
newDefinitions = remove(newDefinitions, i)
toRemove = append(toRemove, i)
continue
}
_, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2 where id=$3", accuracy, def.epoch, def.id)
_, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id)
if err != nil {
c.Logger.Error("Failed to train definition!Err:\n", "err", err)
ModelUpdateStatus(c, model.Id, FAILED_TRAINING)
return
}
}
copy(definitions, newDefinitions)
firstRound = false
if toTrain == 0 {
if finished {
break
}
sort.Reverse(toRemove)
c.Logger.Info("Round done", "toRemove", toRemove)
for _, n := range toRemove {
definitions = remove(definitions, n)
}
len_def := len(definitions)
if len_def == 0 {
break
}
if len_def == 1 {
continue
}
sort.Sort(definitions)
acc := definitions[0].acuracy - 20
c.Logger.Info("Training models, Highest acc", "acc", acc)
toRemove = []int{}
for i, def := range definitions {
if def.acuracy < acc {
toRemove = append(toRemove, i)
}
}
c.Logger.Info("Removing due to accuracy", "toRemove", toRemove)
sort.Reverse(toRemove)
for _, n := range toRemove {
c.Logger.Warn("Removing definition not fast enough learning", "n", n)
definitions = remove(definitions, n)
}
}
rows, err := c.Db.Query("select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;", model.Id, MODEL_DEFINITION_STATUS_TRANIED)
@ -437,14 +489,26 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
return failed()
}
order := 1;
order := 1
// Note the shape of the first layer defines the import size
if complexity == 2 {
// Note the shape for now is no used
width := int(math.Pow(2, math.Floor(math.Log(float64(model.Width))/math.Log(2.0))))
height := int(math.Pow(2, math.Floor(math.Log(float64(model.Height))/math.Log(2.0))))
c.Logger.Warn("Complexity 2 creating model with smaller size", "width", width, "height", height)
err = MakeLayer(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", width, height))
if err != nil {
return failed()
}
order++
} else {
err = MakeLayer(c.Db, def_id, order, LAYER_INPUT, fmt.Sprintf("%d,%d,1", model.Width, model.Height))
if err != nil {
return failed()
}
order++;
order++
}
if complexity == 0 {
@ -452,12 +516,12 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
if err != nil {
return failed()
}
order++;
order++
loop := int(math.Log2(float64(number_of_classes)))
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++;
order++
if err != nil {
ModelUpdateStatus(c, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
@ -465,17 +529,17 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
}
}
} else if (complexity == 1) {
} else if complexity == 1 {
loop := int((math.Log(float64(model.Width))/math.Log(float64(10))))
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if loop == 0 {
loop = 1;
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "")
order++;
order++
if err != nil {
return failed();
return failed()
}
}
@ -483,17 +547,49 @@ func generateDefinition(c *Context, model *BaseModel, target_accuracy int, numbe
if err != nil {
return failed()
}
order++;
order++
loop = int((math.Log(float64(number_of_classes))/math.Log(float64(10)))/2)
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1;
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++;
order++
if err != nil {
return failed();
return failed()
}
}
} else if complexity == 2 {
loop := int((math.Log(float64(model.Width)) / math.Log(float64(10))))
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_SIMPLE_BLOCK, "")
order++
if err != nil {
return failed()
}
}
err = MakeLayer(c.Db, def_id, order, LAYER_FLATTEN, "")
if err != nil {
return failed()
}
order++
loop = int((math.Log(float64(number_of_classes)) / math.Log(float64(10))) / 2)
if loop == 0 {
loop = 1
}
for i := 0; i < loop; i++ {
err = MakeLayer(c.Db, def_id, order, LAYER_DENSE, fmt.Sprintf("%d,1", number_of_classes*(loop-i)))
order++
if err != nil {
return failed()
}
}
@ -523,20 +619,27 @@ func generateDefinitions(c *Context, model *BaseModel, target_accuracy int, numb
return c.Error500(err)
}
if (number_of_models == 1) {
if (model.Width < 100 && model.Height < 100 && len(cls) < 30) {
generateDefinition(c, model, target_accuracy, len(cls), 0)
cls_len := len(cls)
if number_of_models == 1 {
if model.Width < 100 && model.Height < 100 && cls_len < 30 {
generateDefinition(c, model, target_accuracy, cls_len, 0)
} else if model.Width > 100 && model.Height > 100 {
generateDefinition(c, model, target_accuracy, cls_len, 2)
} else {
generateDefinition(c, model, target_accuracy, len(cls), 1)
generateDefinition(c, model, target_accuracy, cls_len, 1)
}
} else if number_of_models == 3 {
for i := 0; i < number_of_models; i++ {
generateDefinition(c, model, target_accuracy, cls_len, i)
}
} else {
// TODO handle incrisea the complexity
for i := 0; i < number_of_models; i++ {
generateDefinition(c, model, target_accuracy, len(cls), 0)
generateDefinition(c, model, target_accuracy, cls_len, 0)
}
}
return nil
}
@ -626,7 +729,7 @@ func handleTrain(handle *Handle) {
accuracy := 0.0
if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy){
if !CheckId(f, "model_id") || !CheckId(f, "definition") || CheckEmpty(f, "epoch") || !CheckFloat64(f, "accuracy", &accuracy) {
c.Logger.Warn("Invalid: model_id or definition or epoch or accuracy")
return c.UnsafeErrorCode(nil, 400, nil)
}

2
main.go
View File

@ -4,6 +4,7 @@ import (
"database/sql"
"fmt"
"github.com/charmbracelet/log"
_ "github.com/lib/pq"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models"
@ -36,6 +37,7 @@ func main() {
_, err = db.Exec("update models set status=$1 where status=$2", models_utils.FAILED_TRAINING, models_utils.TRAINING)
if err != nil {
log.Warn("Database might not be on")
panic(err)
}

2
sql/models.sql
View File

@ -14,7 +14,7 @@ create table if not exists models (
width integer,
height integer,
color_mode varchar (20),
format varchar (20)
format varchar (20) default ''
);
-- drop table if exists model_data_point;

30
views/models/edit.html
View File

@ -438,27 +438,37 @@
<thead>
<tr>
<th>
Status
</th>
<th>
EpochProgress
Training Round Progress
</th>
<th>
Accuracy
</th>
<th>
Status
</th>
</tr>
</thead>
<tbody>
{{ range .Defs}}
<tr>
<td>
{{.EpochProgress}}/20
</td>
<td>
{{.Accuracy}}%
</td>
<td style="text-align: center;">
{{ if (eq .Status 2) }}
<span class="bi bi-book" style="color: green;"></span>
{{ else if (eq .Status 3) }}
<span class="bi bi-book-half" style="color: green;"></span>
{{ else if (eq .Status 6) }}
<span class="bi bi-book-half" style="color: orange;"></span>
{{ else if (eq .Status -3) }}
<span class="bi bi-book-half" style="color: red;"></span>
{{ else }}
{{.Status}}
</td>
<td>
{{.EpochProgress}}
</td>
<td>
{{.Accuracy}}
{{ end }}
</td>
</tr>
{{ end }}

10
views/py/python_model_template.py
View File

@ -8,7 +8,7 @@ import requests
class NotifyServerCallback(tf.keras.callbacks.Callback):
def on_epoch_end(self, epoch, log, *args, **kwargs):
requests.get(f'http://localhost:8000/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch}&accuracy={log["accuracy"]}&definition={{.DefId}}')
requests.get(f'http://localhost:8000/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}')
DATA_DIR = "{{ .DataDir }}"
@ -160,7 +160,9 @@ model.compile(
optimizer=tf.keras.optimizers.Adam(),
metrics=['accuracy'])
his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[NotifyServerCallback()], use_multiprocessing = True)
his = model.fit(dataset, validation_data= dataset_validation, epochs={{.EPOCH_PER_RUN}}, callbacks=[
NotifyServerCallback(),
tf.keras.callbacks.EarlyStopping("loss", mode="min", patience=5)], use_multiprocessing = True)
acc = his.history["accuracy"]
@ -169,5 +171,5 @@ f.write(str(acc[-1]))
f.close()
tf.saved_model.save(model, "model")
model.save("model.keras")
tf.saved_model.save(model, "{{ .SaveModelPath }}/model")
model.save("{{ .SaveModelPath }}/model.keras")