andr3/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: started working on #23

Browse Source
This commit is contained in:
Andre Henriques 2023-10-02 21:15:31 +01:00
parent bc948d4796
commit a1d1a81ec5
9 changed files with 155 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
a.out
View File

Binary file not shown.

18
logic/models/classes/data_point.go
View File

@ -5,23 +5,19 @@ import (
"errors"
)
var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")
func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
id = ""
_, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode)
result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode)
if err != nil {
return
}
rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode)
if err != nil {
defer result.Close()
if !result.Next() {
err = FailedToGetIdAfterInsertError
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something worng")
}
err = rows.Scan(&id)
err = result.Scan(&id)
return
}

19
logic/models/classes/main.go
View File

@ -46,7 +46,7 @@ func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {
var ClassAlreadyExists = errors.New("Class aready exists")
func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) {
func CreateClass(db *sql.DB, model_id string, order int, name string) (id string, err error) {
id = ""
rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
if err != nil {
@ -58,25 +58,16 @@ func CreateClass(db *sql.DB, model_id string, name string) (id string, err error
return id, ClassAlreadyExists
}
_, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name)
if err != nil {
return
}
rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
rows, err = db.Query("insert into model_classes (model_id, name, class_order) values ($1, $2, $3) returning id;", model_id, name, order)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something wrong")
return id, errors.New("Insert did not return anything")
}
if err = rows.Scan(&id); err != nil {
return
}
return
err = rows.Scan(&id)
return
}

74
logic/models/data.go
View File

@ -28,11 +28,11 @@ func InsertIfNotPresent(ss []string, s string) []string {
return ss
}
func processZipFile(handle *Handle, id string) {
reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
func processZipFile(handle *Handle, model_id string) {
reader, err := zip.OpenReader(path.Join("savedData", model_id, "base_data.zip"))
if err != nil {
// TODO add msg to error
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Faield to proccess zip file failed to open reader\n")
fmt.Println(err)
return
@ -52,7 +52,7 @@ func processZipFile(handle *Handle, id string) {
if paths[0] != "training" && paths[0] != "testing" {
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
@ -67,31 +67,21 @@ func processZipFile(handle *Handle, id string) {
fmt.Printf("testing and training are diferent\n")
fmt.Println(testing)
fmt.Println(training)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
base_path := path.Join("savedData", id, "data")
base_path := path.Join("savedData", model_id, "data")
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
fmt.Printf("Failed to create base_path dir\n")
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
ids := map[string]string{}
for _, name := range training {
dir_path := path.Join(base_path, "training", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
dir_path = path.Join(base_path, "testing", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
id, err := model_classes.CreateClass(handle.Db, id, name)
for i, name := range training {
id, err := model_classes.CreateClass(handle.Db, model_id, i, name)
if err != nil {
fmt.Printf("Failed to create class '%s' on db\n", name)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
@ -105,23 +95,19 @@ func processZipFile(handle *Handle, id string) {
continue
}
file_path := path.Join(base_path, file.Name)
f, err := os.Create(file_path)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
defer f.Close()
data, err := reader.Open(file.Name)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Could not open file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
f.Write(file_data)
if err != nil {
fmt.Printf("Could not read file file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database
@ -132,17 +118,27 @@ func processZipFile(handle *Handle, id string) {
mode = model_classes.DATA_POINT_MODE_TESTING
}
_, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
data_point_id, err := model_classes.AddDataPoint(handle.Db, ids[parts[1]], "id://", mode)
if err != nil {
fmt.Printf("Failed to add data point for %s\n", id)
fmt.Printf("Failed to add data point for %s\n", model_id)
fmt.Println(err)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
file_path := path.Join(base_path, data_point_id + ".png")
f, err := os.Create(file_path)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
defer f.Close()
f.Write(file_data)
}
fmt.Printf("Added data to model '%s'!\n", id)
ModelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
fmt.Printf("Added data to model '%s'!\n", model_id)
ModelUpdateStatus(handle, model_id, CONFIRM_PRE_TRAINING)
}
func handleDataUpload(handle *Handle) {

2
logic/models/delete.go
View File

@ -81,6 +81,8 @@ func handleDelete(handle *Handle) {
}
switch model.Status {
case FAILED_TRAINING:
fallthrough
case FAILED_PREPARING_TRAINING:
fallthrough
case FAILED_PREPARING:

49
logic/models/train/train.go
View File

@ -71,6 +71,40 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
return
}
func generateCvs(handle *Handle, run_path string, model_id string) (count int, err error) {
classes, err := handle.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
if err != nil { return }
defer classes.Close()
if !classes.Next() { return }
if err = classes.Scan(&count); err != nil { return }
data, err := handle.Db.Query("select mpd.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
if err != nil { return }
defer data.Close()
type row struct {
path string
class_order int
}
got := []row{}
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil { return }
if file_path == "id://" {
got = append(got, row{id, class_order})
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
return
}
func trainDefinition(handle *Handle, model_id string, definition_id string) (accuracy float64, err error) {
accuracy = 0
layers, err := handle.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
@ -103,6 +137,10 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
return
}
if err = generateCvs(handle, run_path); err != nil {
return
}
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
@ -110,6 +148,7 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
}
defer f.Close()
tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
if err != nil {
return
@ -118,13 +157,15 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model_id, "data", "training"),
"DataDir": path.Join(getDir(), "savedData", model_id, "data"),
}); err != nil {
return
}
// Run the command
if err = exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Run(); err != nil {
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
if err != nil {
fmt.Println(string(out))
return
}
@ -384,7 +425,8 @@ func handleTrain(handle *Handle) {
// TODO improve this response
return Error500(err)
}
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)))
// Using sparce
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("1,1", len(cls)))
if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
@ -400,7 +442,6 @@ func handleTrain(handle *Handle) {
}
// TODO start training with id fid
go trainModel(handle, model)
ModelUpdateStatus(handle, model.Id, TRAINING)

3
sql/models.sql
View File

@ -21,7 +21,8 @@ create table if not exists models (
create table if not exists model_classes (
id uuid primary key default gen_random_uuid(),
model_id uuid references models (id) on delete cascade,
name varchar (70) not null
name varchar (70) not null,
class_order integer
);
-- drop table if exists model_data_point;

10
views/models/edit.html
View File

@ -63,7 +63,11 @@
{{range .List}}
<tr>
<td>
{{.FilePath}}
{{ if eq .FilePath "id://" }}
Managed
{{ else }}
{{.FilePath}}
{{ end }}
</td>
<td>
{{ if (eq .Mode 2) }}
@ -73,8 +77,8 @@
{{ end }}
</td>
<td class="text-center">
{{ if startsWith .FilePath "file://" }}
<img src="/savedData/{{ $.ModelId }}/data/{{ if (eq .Mode 2) }}testing{{ else }}training{{ end }}/{{ $.Name }}/{{ replace .FilePath "file://" "" 1 }}" height="30px" width="30px" style="object-fit: contain;" />
{{ if startsWith .FilePath "id://" }}
<img src="/savedData/{{ $.ModelId }}/data/{{ .Id }}.png" height="30px" width="30px" style="object-fit: contain;" />
{{ else }}
TODO
img {{ .FilePath }}

73
views/py/python_model_template.py
View File

@ -1,33 +1,64 @@
import tensorflow as tf
import random
from tensorflow import keras
from tensorflow.data import AUTOTUNE
from keras import layers, losses, optimizers
DATA_DIR = "{{ .DataDir }}"
image_size = ({{ .Size }})
#based on https://www.tensorflow.org/tutorials/load_data/images
def pathToLabel(path):
path = tf.strings.regex_replace(path, DATA_DIR, "")
path = tf.strings.regex_replace(path, ".jpg", "")
return train_labels[tf.strings.to_number(path, out_type=tf.int32)]
def decode_image(img):
# channels were reduced to 1 since image is grayscale
# TODO chnage channel number based if grayscale
img = tf.io.decode_png(img, channels=1)
return tf.image.resize(img, image_size)
def process_path(path):
label = pathToLabel(path)
img = tf.io.read_file(path)
img = decode_image(img)
return img, label
def configure_for_performance(ds: tf.data.Dataset) -> tf.data.Dataset:
#ds = ds.cache()
ds = ds.shuffle(buffer_size= 1000)
ds = ds.batch(batch_size)
ds = ds.prefetch(AUTOTUNE)
return ds
def prepare_dataset(ds: tf.data.Dataset) -> tf.data.Dataset:
ds = ds.map(process_path, num_parallel_calls=AUTOTUNE)
ds = configure_for_performance(ds)
return ds
seed = random.randint(0, 100000000)
batch_size = 100
dataset = keras.utils.image_dataset_from_directory(
"{{ .DataDir }}",
color_mode="rgb",
validation_split=0.2,
label_mode='categorical',
seed=seed,
shuffle=True,
subset="training",
image_size=({{ .Size }}),
batch_size=batch_size)
# Read all the files from the direcotry
list_ds = tf.data.Dataset.list_files(str(f'{DATA_DIR}/*'), shuffle=False)
dataset_validation = keras.utils.image_dataset_from_directory(
"{{ .DataDir }}",
color_mode="rgb",
validation_split=0.2,
label_mode='categorical',
seed=seed,
shuffle=True,
subset="validation",
image_size=({{ .Size }}),
batch_size=batch_size)
image_count = len(list_ds)
list_ds = list_ds.shuffle(image_count, seed=seed)
val_size = int(image_count * 0.3)
train_ds = list_ds.skip(val_size)
val_ds = list_ds.take(val_size)
dataset = prepare_dataset(train_ds)
dataset_validation = prepare_dataset(val_ds)
model = keras.Sequential([
{{- range .Layers }}
@ -44,7 +75,7 @@ model = keras.Sequential([
])
model.compile(
loss=losses.CategoricalCrossentropy(),
loss=losses.SparceCategoricalCrossentropy(),
optimizer=tf.keras.optimizers.Adam(),
metrics=['accuracy'])