sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: started working on #23

Browse Source
This commit is contained in:
Andre Henriques
2023-10-02 21:15:31 +01:00
parent bc948d4796
commit a1d1a81ec5
9 changed files with 155 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
logic/models/classes/data_point.go
View File

@@ -5,23 +5,19 @@ import (
"errors"
)
var FailedToGetIdAfterInsertError = errors.New("Failed to Get Id After Insert Error")
func AddDataPoint(db *sql.DB, class_id string, file_path string, mode DATA_POINT_MODE) (id string, err error) {
id = ""
_, err = db.Exec("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3);", class_id, file_path, mode)
result, err := db.Query("insert into model_data_point (class_id, file_path, model_mode) values ($1, $2, $3) returning id;", class_id, file_path, mode)
if err != nil {
return
}
rows, err := db.Query("select id from model_data_point where class_id=$1 and file_path=$2 and model_mode=$3", class_id, file_path, mode)
if err != nil {
defer result.Close()
if !result.Next() {
err = FailedToGetIdAfterInsertError
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something worng")
}
err = rows.Scan(&id)
err = result.Scan(&id)
return
}

19
logic/models/classes/main.go
View File

@@ -46,7 +46,7 @@ func ModelHasDataPoints(db *sql.DB, model_id string) (result bool, err error) {
var ClassAlreadyExists = errors.New("Class aready exists")
func CreateClass(db *sql.DB, model_id string, name string) (id string, err error) {
func CreateClass(db *sql.DB, model_id string, order int, name string) (id string, err error) {
id = ""
rows, err := db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
if err != nil {
@@ -58,25 +58,16 @@ func CreateClass(db *sql.DB, model_id string, name string) (id string, err error
return id, ClassAlreadyExists
}
_, err = db.Exec("insert into model_classes (model_id, name) values ($1, $2)", model_id, name)
if err != nil {
return
}
rows, err = db.Query("select id from model_classes where model_id=$1 and name=$2;", model_id, name)
rows, err = db.Query("insert into model_classes (model_id, name, class_order) values ($1, $2, $3) returning id;", model_id, name, order)
if err != nil {
return
}
defer rows.Close()
if !rows.Next() {
return id, errors.New("Something wrong")
return id, errors.New("Insert did not return anything")
}
if err = rows.Scan(&id); err != nil {
return
}
return
err = rows.Scan(&id)
return
}

74
logic/models/data.go
View File

@@ -28,11 +28,11 @@ func InsertIfNotPresent(ss []string, s string) []string {
return ss
}
func processZipFile(handle *Handle, id string) {
reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
func processZipFile(handle *Handle, model_id string) {
reader, err := zip.OpenReader(path.Join("savedData", model_id, "base_data.zip"))
if err != nil {
// TODO add msg to error
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Faield to proccess zip file failed to open reader\n")
fmt.Println(err)
return
@@ -52,7 +52,7 @@ func processZipFile(handle *Handle, id string) {
if paths[0] != "training" && paths[0] != "testing" {
fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
@@ -67,31 +67,21 @@ func processZipFile(handle *Handle, id string) {
fmt.Printf("testing and training are diferent\n")
fmt.Println(testing)
fmt.Println(training)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
base_path := path.Join("savedData", id, "data")
base_path := path.Join("savedData", model_id, "data")
if err = os.MkdirAll(base_path, os.ModePerm); err != nil {
fmt.Printf("Failed to create base_path dir\n")
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
ids := map[string]string{}
for _, name := range training {
dir_path := path.Join(base_path, "training", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
dir_path = path.Join(base_path, "testing", name)
err = os.MkdirAll(dir_path, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create dir %s\n", dir_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
id, err := model_classes.CreateClass(handle.Db, id, name)
for i, name := range training {
id, err := model_classes.CreateClass(handle.Db, model_id, i, name)
if err != nil {
fmt.Printf("Failed to create class '%s' on db\n", name)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
@@ -105,23 +95,19 @@ func processZipFile(handle *Handle, id string) {
continue
}
file_path := path.Join(base_path, file.Name)
f, err := os.Create(file_path)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
return
}
defer f.Close()
data, err := reader.Open(file.Name)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
fmt.Printf("Could not open file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
defer data.Close()
file_data, err := io.ReadAll(data)
f.Write(file_data)
if err != nil {
fmt.Printf("Could not read file file in zip %s\n", file.Name)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
// TODO check if the file is a valid photo that matched the defined photo on the database
@@ -132,17 +118,27 @@ func processZipFile(handle *Handle, id string) {
mode = model_classes.DATA_POINT_MODE_TESTING
}
_, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
data_point_id, err := model_classes.AddDataPoint(handle.Db, ids[parts[1]], "id://", mode)
if err != nil {
fmt.Printf("Failed to add data point for %s\n", id)
fmt.Printf("Failed to add data point for %s\n", model_id)
fmt.Println(err)
ModelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
file_path := path.Join(base_path, data_point_id + ".png")
f, err := os.Create(file_path)
if err != nil {
fmt.Printf("Could not create file %s\n", file_path)
ModelUpdateStatus(handle, model_id, FAILED_PREPARING_ZIP_FILE)
return
}
defer f.Close()
f.Write(file_data)
}
fmt.Printf("Added data to model '%s'!\n", id)
ModelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
fmt.Printf("Added data to model '%s'!\n", model_id)
ModelUpdateStatus(handle, model_id, CONFIRM_PRE_TRAINING)
}
func handleDataUpload(handle *Handle) {

2
logic/models/delete.go
View File

@@ -81,6 +81,8 @@ func handleDelete(handle *Handle) {
}
switch model.Status {
case FAILED_TRAINING:
fallthrough
case FAILED_PREPARING_TRAINING:
fallthrough
case FAILED_PREPARING:

49
logic/models/train/train.go
View File

@@ -71,6 +71,40 @@ func MakeLayer(db *sql.DB, def_id string, layer_order int, layer_type LayerType,
return
}
func generateCvs(handle *Handle, run_path string, model_id string) (count int, err error) {
classes, err := handle.Db.Query("select count(*) from model_classes where model_id=$1;", model_id)
if err != nil { return }
defer classes.Close()
if !classes.Next() { return }
if err = classes.Scan(&count); err != nil { return }
data, err := handle.Db.Query("select mpd.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;", model_id)
if err != nil { return }
defer data.Close()
type row struct {
path string
class_order int
}
got := []row{}
for data.Next() {
var id string
var class_order int
var file_path string
if err = data.Scan(&id, &class_order, &file_path); err != nil { return }
if file_path == "id://" {
got = append(got, row{id, class_order})
} else {
return count, errors.New("TODO generateCvs to file_path " + file_path)
}
}
return
}
func trainDefinition(handle *Handle, model_id string, definition_id string) (accuracy float64, err error) {
accuracy = 0
layers, err := handle.Db.Query("select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;", definition_id)
@@ -103,6 +137,10 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
return
}
if err = generateCvs(handle, run_path); err != nil {
return
}
// Create python script
f, err := os.Create(path.Join(run_path, "run.py"))
if err != nil {
@@ -110,6 +148,7 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
}
defer f.Close()
tmpl, err := template.New("python_model_template.py").ParseFiles("views/py/python_model_template.py")
if err != nil {
return
@@ -118,13 +157,15 @@ func trainDefinition(handle *Handle, model_id string, definition_id string) (acc
if err = tmpl.Execute(f, AnyMap{
"Layers": got,
"Size": got[0].Shape,
"DataDir": path.Join(getDir(), "savedData", model_id, "data", "training"),
"DataDir": path.Join(getDir(), "savedData", model_id, "data"),
}); err != nil {
return
}
// Run the command
if err = exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Run(); err != nil {
out, err := exec.Command("bash", "-c", fmt.Sprintf("cd %s && python run.py", run_path)).Output()
if err != nil {
fmt.Println(string(out))
return
}
@@ -384,7 +425,8 @@ func handleTrain(handle *Handle) {
// TODO improve this response
return Error500(err)
}
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("%d,1", len(cls)))
// Using sparce
err = MakeLayer(handle.Db, def_id, 5, LAYER_DENSE, fmt.Sprintf("1,1", len(cls)))
if err != nil {
ModelUpdateStatus(handle, model.Id, FAILED_PREPARING_TRAINING)
// TODO improve this response
@@ -400,7 +442,6 @@ func handleTrain(handle *Handle) {
}
// TODO start training with id fid
go trainModel(handle, model)
ModelUpdateStatus(handle, model.Id, TRAINING)