diff --git a/logic/models/classes/main.go b/logic/models/classes/main.go index 762a20d..9383f43 100644 --- a/logic/models/classes/main.go +++ b/logic/models/classes/main.go @@ -20,6 +20,8 @@ func ListClasses(db *sql.DB, model_id string) (cls []ModelClass, err error) { } defer rows.Close() + cls = []ModelClass{} + for rows.Next() { var model ModelClass err = rows.Scan(&model.Id, &model.ModelId, &model.Name) diff --git a/logic/models/delete.go b/logic/models/delete.go index 93fff20..7d3b55b 100644 --- a/logic/models/delete.go +++ b/logic/models/delete.go @@ -8,13 +8,14 @@ import ( . "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils" . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" + utils "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) func deleteModel(handle *Handle, id string, w http.ResponseWriter, c *Context, model BaseModel) { - c.Logger.Warnf("Removing model with id: %s", id) + c.Logger.Warnf("Removing model with id: %s", id) _, err := handle.Db.Exec("delete from models where id=$1;", id) if err != nil { - c.Logger.Error(err) + c.Logger.Error(err) panic("TODO handle better deleteModel failed delete database query") } @@ -22,7 +23,7 @@ func deleteModel(handle *Handle, id string, w http.ResponseWriter, c *Context, m c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path) err = os.RemoveAll(model_path) if err != nil { - c.Logger.Error(err) + c.Logger.Error(err) panic("TODO handle better deleteModel failed to delete folder") } @@ -37,10 +38,76 @@ func deleteModel(handle *Handle, id string, w http.ResponseWriter, c *Context, m })) } +func deleteModelJSON(c *Context, id string) *Error { + c.Logger.Warnf("Removing model with id: %s", id) + _, err := c.Db.Exec("delete from models where id=$1;", id) + if err != nil { + return c.Error500(err) + } + + model_path := path.Join("./savedData", id) + c.Logger.Warnf("Removing folder of model with id: %s at %s", id, model_path) + err = os.RemoveAll(model_path) + if err != nil { + return c.Error500(err) + } + + return c.SendJSON(id) +} + func handleDelete(handle *Handle) { handle.Delete("/models/delete", func(w http.ResponseWriter, r *http.Request, c *Context) *Error { + if c.Mode == JSON { - panic("TODO handle json on models/delete") + + var dat struct { + Id string `json:"id" validate:"required"` + Name *string `json:"name,omitempty"` + } + + if err_ := c.ToJSON(r, &dat); err_ != nil { + return err_ + } + + var model struct { + Id string + Name string + Status int + } + + err := utils.GetDBOnce(c, &model, "models where id=$1 and user_id=$2;", dat.Id, c.User.Id) + if err == NotFoundError { + return c.SendJSONStatus(http.StatusNotFound, "Model not found!") + } else if err != nil { + return c.Error500(err) + } + + switch model.Status { + case FAILED_TRAINING: + fallthrough + case FAILED_PREPARING_ZIP_FILE: + fallthrough + case FAILED_PREPARING_TRAINING: + fallthrough + case FAILED_PREPARING: + return deleteModelJSON(c, dat.Id) + + case READY: + fallthrough + case CONFIRM_PRE_TRAINING: + if dat.Name == nil { + return c.JsonBadRequest("Provided name does not match the model name") + } + + if *dat.Name != model.Name { + return c.JsonBadRequest("Provided name does not match the model name") + } + + return deleteModelJSON(c, dat.Id) + default: + c.Logger.Warn("Do not know how to handle model in status", "status", model.Status) + return c.JsonBadRequest("Model in invalid status") + } } // This is required to parse delete forms with bodies @@ -66,7 +133,7 @@ func handleDelete(handle *Handle) { defer rows.Close() if !rows.Next() { - c.Logger.Warn("Could not find model for", id, c.User.Id) + c.Logger.Warn("Could not find model for", id, c.User.Id) return c.ErrorCode(nil, http.StatusNotFound, AnyMap{ "NotFoundMessage": "Model not found", "GoBackLink": "/models", @@ -81,14 +148,18 @@ func handleDelete(handle *Handle) { } switch model.Status { - case FAILED_TRAINING: fallthrough - case FAILED_PREPARING_ZIP_FILE: fallthrough - case FAILED_PREPARING_TRAINING: fallthrough + case FAILED_TRAINING: + fallthrough + case FAILED_PREPARING_ZIP_FILE: + fallthrough + case FAILED_PREPARING_TRAINING: + fallthrough case FAILED_PREPARING: deleteModel(handle, id, w, c, model) return nil - case READY: fallthrough + case READY: + fallthrough case CONFIRM_PRE_TRAINING: if CheckEmpty(f, "name") { return c.Error400(nil, "Name is empty", w, "/models/edit.html", "delete-model-card", AnyMap{ diff --git a/logic/models/run.go b/logic/models/run.go index e79d470..394391d 100644 --- a/logic/models/run.go +++ b/logic/models/run.go @@ -37,7 +37,7 @@ func ReadJPG(scope *op.Scope, imagePath string, channels int64) *image.Image { return image.Scale(0, 255) } -func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) { +func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) { order = 0 err = nil @@ -59,10 +59,12 @@ func runModelNormal(c *Context, model *BaseModel, def_id string, inputImage *tf. } } + confidence = vmax + return } -func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, err error) { +func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Tensor) (order int, confidence float32, err error) { err = nil order = 0 @@ -101,6 +103,7 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten var predictions = results[0].Value().([][]float32)[0] for i, v := range predictions { + c.Logger.Info("This is test", "v", v) if v > vmax { order = element.Range_start + i vmax = v @@ -109,6 +112,7 @@ func runModelExp(c *Context, model *BaseModel, def_id string, inputImage *tf.Ten } // TODO runthe head model + confidence = vmax c.Logger.Info("Got", "heads", len(heads)) return @@ -135,7 +139,7 @@ func handleRun(handle *Handle) { if err_part == io.EOF { break } else if err_part != nil { - return c.JsonBadRequest("Invalid multipart data") + return c.JsonBadRequest("Invalid multipart data") } if part.FormName() == "id" { buf := new(bytes.Buffer) @@ -151,13 +155,13 @@ func handleRun(handle *Handle) { model, err := GetBaseModel(handle.Db, id) if err == ModelNotFoundError { - return c.JsonBadRequest("Models not found"); + return c.JsonBadRequest("Models not found") } else if err != nil { return c.Error500(err) } if model.Status != READY { - return c.JsonBadRequest("Model not ready to run images") + return c.JsonBadRequest("Model not ready to run images") } def := JustId{} @@ -183,7 +187,7 @@ func handleRun(handle *Handle) { img_file.Write(file) if !testImgForModel(c, model, img_path) { - return c.JsonBadRequest("Provided image does not match the model") + return c.JsonBadRequest("Provided image does not match the model") } root := tg.NewRoot() @@ -206,16 +210,17 @@ func handleRun(handle *Handle) { } vi := -1 + var confidence float32 = 0 if model.ModelType == 2 { c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, err = runModelExp(c, model, def_id, inputImage) + vi, confidence, err = runModelExp(c, model, def_id, inputImage) if err != nil { return c.Error500(err) } } else { c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, err = runModelNormal(c, model, def_id, inputImage) + vi, confidence, err = runModelNormal(c, model, def_id, inputImage) if err != nil { return c.Error500(err) } @@ -228,7 +233,7 @@ func handleRun(handle *Handle) { return c.Error500(err) } if !rows.Next() { - return c.SendJSON(nil) + return c.SendJSON(nil) } var name string @@ -236,7 +241,15 @@ func handleRun(handle *Handle) { return c.Error500(err) } - return c.SendJSON(name) + returnValue := struct { + Class string `json:"class"` + Confidence float32 `json:"confidence"` + }{ + Class: name, + Confidence: confidence, + } + + return c.SendJSON(returnValue) } read_form, err := r.MultipartReader() @@ -336,16 +349,17 @@ func handleRun(handle *Handle) { } vi := -1 + var confidence float32 = 0 if model.ModelType == 2 { c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, err = runModelExp(c, model, def_id, inputImage) + vi, confidence, err = runModelExp(c, model, def_id, inputImage) if err != nil { return c.Error500(err) } } else { c.Logger.Info("Running model normal", "model", model.Id, "def", def_id) - vi, err = runModelNormal(c, model, def_id, inputImage) + vi, confidence, err = runModelNormal(c, model, def_id, inputImage) if err != nil { return c.Error500(err) } @@ -362,6 +376,7 @@ func handleRun(handle *Handle) { "Model": model, "NotFound": true, "Result": nil, + "Confidence": confidence, })) return nil } diff --git a/logic/models/train/train.go b/logic/models/train/train.go index 029d3e8..a16d8ad 100644 --- a/logic/models/train/train.go +++ b/logic/models/train/train.go @@ -101,6 +101,70 @@ func generateCvs(c *Context, run_path string, model_id string) (count int, err e return } +func setModelClassStatus(c *Context, status ModelClassStatus, filter string, args ...any) (err error) { + _, err = c.Db.Exec("update model_classes set stauts = $1 where "+filter, args...) + return +} + +func generateCvsExp(c *Context, run_path string, model_id string, doPanic bool) (count int, err error) { + + classes, err := c.Db.Query("select count(*) from model_classes where model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TRAINING) + if err != nil { + return + } + defer classes.Close() + + if !classes.Next() { + return + } + + if err = classes.Scan(&count); err != nil { + return + } + + if count == 0 { + err = setModelClassStatus(c, MODEL_CLASS_STATUS_TRAINING, "model_id=$1 and status=$2;", model_id, MODEL_CLASS_STATUS_TO_TRAIN) + if err != nil { + return + } + + if doPanic { + return 0, errors.New("No model classes available") + } + + return generateCvsExp(c, run_path, model_id, true) + } + + data, err := c.Db.Query("select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1 and mdp.model_mode=$2 and mc.status=$3;", model_id, model_classes.DATA_POINT_MODE_TRAINING, MODEL_CLASS_STATUS_TRAINING) + if err != nil { + return + } + defer data.Close() + + f, err := os.Create(path.Join(run_path, "train.csv")) + if err != nil { + return + } + defer f.Close() + f.Write([]byte("Id,Index\n")) + + for data.Next() { + var id string + var class_order int + var file_path string + if err = data.Scan(&id, &class_order, &file_path); err != nil { + return + } + if file_path == "id://" { + f.Write([]byte(id + "," + strconv.Itoa(class_order) + "\n")) + } else { + return count, errors.New("TODO generateCvs to file_path " + file_path) + } + } + + return +} + func trainDefinition(c *Context, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) { c.Logger.Warn("About to start training definition") accuracy = 0 @@ -137,9 +201,9 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr if err != nil { return } - defer os.RemoveAll(run_path) + defer removeAll(run_path, err) - _, err = generateCvs(c, run_path, model.Id) + classCount, err := generateCvs(c, run_path, model.Id) if err != nil { return } @@ -171,6 +235,8 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr "LoadPrev": load_prev, "LastModelRunPath": path.Join(getDir(), result_path, "model.keras"), "SaveModelPath": path.Join(getDir(), result_path), + "Depth": classCount, + "StartPoint": 0, }); err != nil { return } @@ -208,6 +274,12 @@ func trainDefinition(c *Context, model *BaseModel, definition_id string, load_pr return } +func removeAll(path string, err error) { + if err != nil { + os.RemoveAll(path) + } +} + func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load_prev bool) (accuracy float64, err error) { accuracy = 0 @@ -295,9 +367,9 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load if err != nil { return } - defer os.RemoveAll(run_path) + defer removeAll(run_path, err) - _, err = generateCvs(c, run_path, model.Id) + classCount, err := generateCvsExp(c, run_path, model.Id, false) if err != nil { return } @@ -333,6 +405,8 @@ func trainDefinitionExp(c *Context, model *BaseModel, definition_id string, load "LastModelRunPath": path.Join(getDir(), result_path, "model.keras"), "SaveModelPath": path.Join(getDir(), result_path), "RemoveTopCount": remove_top_count, + "Depth": classCount, + "StartPoint": 0, }); err != nil { return } @@ -413,7 +487,7 @@ func (nf ToRemoveList) Less(i, j int) bool { func trainModel(c *Context, model *BaseModel) { definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) if err != nil { - c.Logger.Error("Failed to trainModel!Err:") + c.Logger.Error("Failed to train Model! Err:") c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) return @@ -584,7 +658,7 @@ func trainModel(c *Context, model *BaseModel) { for to_delete.Next() { var id string - if to_delete.Scan(&id); err != nil { + if err = to_delete.Scan(&id); err != nil { c.Logger.Error("Failed to scan the id of a model_definition to delete") c.Logger.Error(err) ModelUpdateStatus(c, model.Id, FAILED_TRAINING) @@ -604,6 +678,21 @@ func trainModel(c *Context, model *BaseModel) { ModelUpdateStatus(c, model.Id, READY) } +type TrainModelRowUsable struct { + Id string + TargetAccuracy int `db:"target_accuracy"` + Epoch int + Acuracy float64 `db:"0"` +} + +type TrainModelRowUsables []*TrainModelRowUsable + +func (nf TrainModelRowUsables) Len() int { return len(nf) } +func (nf TrainModelRowUsables) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] } +func (nf TrainModelRowUsables) Less(i, j int) bool { + return nf[i].Acuracy < nf[j].Acuracy +} + func trainModelExp(c *Context, model *BaseModel) { var err error = nil @@ -612,25 +701,13 @@ func trainModelExp(c *Context, model *BaseModel) { ModelUpdateStatus(c, model.Id, FAILED_TRAINING) } - definitionsRows, err := c.Db.Query("select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) - if err != nil { - failed("Failed to trainModel!") - return - } - defer definitionsRows.Close() - - var definitions TraingModelRowDefinitions = []TrainModelRow{} - - for definitionsRows.Next() { - var rowv TrainModelRow - rowv.acuracy = 0 - if err = definitionsRows.Scan(&rowv.id, &rowv.target_accuracy, &rowv.epoch); err != nil { - failed("Failed to train Model Could not read definition from db!") - return - } - definitions = append(definitions, rowv) - } + var definitions TrainModelRowUsables + definitions, err = GetDbMultitple[TrainModelRowUsable](c, "model_definition where status=$1 and model_id=$2", MODEL_DEFINITION_STATUS_INIT, model.Id) + if err != nil { + failed("Failed to get definitions"); + return + } if len(definitions) == 0 { failed("No Definitions defined!") return @@ -642,30 +719,30 @@ func trainModelExp(c *Context, model *BaseModel) { for { var toRemove ToRemoveList = []int{} for i, def := range definitions { - ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_TRAINING) - accuracy, err := trainDefinitionExp(c, model, def.id, !firstRound) + ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_TRAINING) + accuracy, err := trainDefinitionExp(c, model, def.Id, !firstRound) if err != nil { c.Logger.Error("Failed to train definition!Err:", "err", err) - ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) + ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) toRemove = append(toRemove, i) continue } - def.epoch += EPOCH_PER_RUN + def.Epoch += EPOCH_PER_RUN accuracy = accuracy * 100 - def.acuracy = float64(accuracy) + def.Acuracy = float64(accuracy) - definitions[i].epoch += EPOCH_PER_RUN - definitions[i].acuracy = accuracy + definitions[i].Epoch += EPOCH_PER_RUN + definitions[i].Acuracy = accuracy - if accuracy >= float64(def.target_accuracy) { + if accuracy >= float64(def.TargetAccuracy) { c.Logger.Info("Found a definition that reaches target_accuracy!") - _, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.epoch, def.id) + _, err = c.Db.Exec("update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4", accuracy, MODEL_DEFINITION_STATUS_TRANIED, def.Epoch, def.Id) if err != nil { failed("Failed to train definition!") return } - _, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) + _, err = c.Db.Exec("update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4", MODEL_DEFINITION_STATUS_CANCELD_TRAINING, def.Id, model.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) if err != nil { failed("Failed to train definition!") return @@ -675,14 +752,14 @@ func trainModelExp(c *Context, model *BaseModel) { break } - if def.epoch > MAX_EPOCH { - fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.target_accuracy) - ModelDefinitionUpdateStatus(c, def.id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) + if def.Epoch > MAX_EPOCH { + fmt.Printf("Failed to train definition! Accuracy less %f < %d\n", accuracy, def.TargetAccuracy) + ModelDefinitionUpdateStatus(c, def.Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) toRemove = append(toRemove, i) continue } - _, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.id) + _, err = c.Db.Exec("update model_definition set accuracy=$1, epoch=$2, status=$3 where id=$4", accuracy, def.Epoch, MODEL_DEFINITION_STATUS_PAUSED_TRAINING, def.Id) if err != nil { failed("Failed to train definition!") return @@ -713,14 +790,13 @@ func trainModelExp(c *Context, model *BaseModel) { } sort.Sort(sort.Reverse(definitions)) + acc := definitions[0].Acuracy - 20.0 - acc := definitions[0].acuracy - 20.0 - - c.Logger.Info("Training models, Highest acc", "acc", definitions[0].acuracy, "mod_acc", acc) + c.Logger.Info("Training models, Highest acc", "acc", definitions[0].Acuracy, "mod_acc", acc) toRemove = []int{} for i, def := range definitions { - if def.acuracy < acc { + if def.Acuracy < acc { toRemove = append(toRemove, i) } } @@ -730,7 +806,7 @@ func trainModelExp(c *Context, model *BaseModel) { sort.Sort(sort.Reverse(toRemove)) for _, n := range toRemove { c.Logger.Warn("Removing definition not fast enough learning", "n", n) - ModelDefinitionUpdateStatus(c, definitions[n].id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) + ModelDefinitionUpdateStatus(c, definitions[n].Id, MODEL_DEFINITION_STATUS_FAILED_TRAINING) definitions = remove(definitions, n) } } @@ -821,8 +897,7 @@ func splitModel(c *Context, model *BaseModel) (err error) { if err != nil { return } - // TODO reneable it - // defer os.RemoveAll(run_path) + defer removeAll(run_path, err) // Create python script f, err := os.Create(path.Join(run_path, "run.py")) @@ -1286,7 +1361,7 @@ func handle_models_train_json(w http.ResponseWriter, r *http.Request, c *Context return Error500(err) } - return c.SendJSON(model.Id) + return c.SendJSON(model.Id) } func handleTrain(handle *Handle) { diff --git a/logic/models/utils/types.go b/logic/models/utils/types.go index d212273..0831d93 100644 --- a/logic/models/utils/types.go +++ b/logic/models/utils/types.go @@ -6,9 +6,9 @@ import ( ) type BaseModel struct { - Name string - Status int - Id string + Name string + Status int + Id string ModelType int ImageMode int @@ -52,6 +52,14 @@ const ( MODEL_DEFINITION_STATUS_READY = 5 ) +type ModelClassStatus int + +const ( + MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1 + MODEL_CLASS_STATUS_TRAINING = 2 + MODEL_CLASS_STATUS_TRAINED = 3 +) + var ModelNotFoundError = errors.New("Model not found error") func GetBaseModel(db *sql.DB, id string) (base *BaseModel, err error) { diff --git a/logic/utils/handler.go b/logic/utils/handler.go index a556263..8f91479 100644 --- a/logic/utils/handler.go +++ b/logic/utils/handler.go @@ -387,7 +387,9 @@ func (c Context) SendJSONStatus(status int, dat any) *Error { } func (c Context) JsonBadRequest(dat any) *Error { + c.SetReportCaller(true) c.Logger.Warn("Request failed with a bad request", "dat", dat) + c.SetReportCaller(false) return c.SendJSONStatus(http.StatusBadRequest, dat) } diff --git a/logic/utils/utils.go b/logic/utils/utils.go index ec7e9e4..592ba0c 100644 --- a/logic/utils/utils.go +++ b/logic/utils/utils.go @@ -190,18 +190,29 @@ type Generic struct{ reflect.Type } var NotFoundError = errors.New("Not found") -func GetDbMultitple[T interface{}](c *Context, tablename string, args ...any) ([]*T, error) { - t := reflect.TypeFor[T]() - nargs := t.NumField() - - query := "" +func generateQuery(t reflect.Type) (query string, nargs int) { + nargs = t.NumField() + query = "" for i := 0; i < nargs; i += 1 { - query += strings.ToLower(t.Field(i).Name) + "," + field := t.Field(i) + name, ok := field.Tag.Lookup("db") + if !ok { + name = field.Name; + } + query += strings.ToLower(name) + "," } // Remove the last comma query = query[0 : len(query)-1] + + return +} + +func GetDbMultitple[T interface{}](c *Context, tablename string, args ...any) ([]*T, error) { + t := reflect.TypeFor[T]() + + query, nargs := generateQuery(t) rows, err := c.Db.Query(fmt.Sprintf("select %s from %s", query, tablename), args...) if err != nil { @@ -242,16 +253,8 @@ func mapRow(store interface{}, rows *sql.Rows, nargs int) (err error) { func GetDBOnce(c *Context, store interface{}, tablename string, args ...any) error { t := reflect.TypeOf(store).Elem() - nargs := t.NumField() - query := "" - - for i := 0; i < nargs; i += 1 { - query += strings.ToLower(t.Field(i).Name) + "," - } - - // Remove the last comma - query = query[0 : len(query)-1] + query, nargs := generateQuery(t) rows, err := c.Db.Query(fmt.Sprintf("select %s from %s", query, tablename), args...) if err != nil { diff --git a/nginx.dev.conf b/nginx.dev.conf index eb8d676..242a37b 100644 --- a/nginx.dev.conf +++ b/nginx.dev.conf @@ -13,6 +13,8 @@ http { server { listen 8000; + client_max_body_size 1G; + location / { proxy_http_version 1.1; proxy_pass http://127.0.0.1:5001; diff --git a/sql/models.sql b/sql/models.sql index 05bcb1c..26d9ba4 100644 --- a/sql/models.sql +++ b/sql/models.sql @@ -29,7 +29,12 @@ create table if not exists model_classes ( id uuid primary key default gen_random_uuid(), model_id uuid references models (id) on delete cascade, name varchar (70) not null, - class_order integer + class_order integer, + + -- 1: to_train + -- 2: training + -- 3: trained + status integer default 1, ); -- drop table if exists model_data_point; diff --git a/views/py/python_model_template.py b/views/py/python_model_template.py index fb82ee5..933effc 100644 --- a/views/py/python_model_template.py +++ b/views/py/python_model_template.py @@ -9,9 +9,9 @@ import requests class NotifyServerCallback(tf.keras.callbacks.Callback): def on_epoch_end(self, epoch, log, *args, **kwargs): {{ if .HeadId }} - requests.get(f'http://localhost:8000//model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}') + requests.get(f'http://localhost:8000/api/model/head/epoch/update?epoch={epoch + 1}&accuracy={log["accuracy"]}&head_id={{.HeadId}}') {{ else }} - requests.get(f'http://localhost:8000/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}') + requests.get(f'http://localhost:8000/api/model/epoch/update?model_id={{.Model.Id}}&epoch={epoch + 1}&accuracy={log["accuracy"]}&definition={{.DefId}}') {{end}} @@ -23,6 +23,9 @@ df = pd.read_csv("{{ .RunPath }}/train.csv", dtype=str) keys = tf.constant(df['Id'].dropna()) values = tf.constant(list(map(int, df['Index'].dropna()))) +depth = {{ .Depth }} +diff = {{ .StartPoint }} + table = tf.lookup.StaticHashTable( initializer=tf.lookup.KeyValueTensorInitializer( keys=keys, @@ -44,7 +47,8 @@ def pathToLabel(path): {{ else }} ERROR {{ end }} - return table.lookup(tf.strings.as_string([path])) + + return tf.one_hot(table.lookup(tf.strings.as_string([path])) - diff, depth)[0] def decode_image(img): {{ if eq .Model.Format "png" }} @@ -161,7 +165,8 @@ ERROR {{ end }} model.compile( - loss=losses.SparseCategoricalCrossentropy(), + #loss=losses.SparseCategoricalCrossentropy(), + loss=losses.BinaryCrossentropy(from_logits=False), optimizer=tf.keras.optimizers.Adam(), metrics=['accuracy']) diff --git a/webpage/src/lib/FileUpload.svelte b/webpage/src/lib/FileUpload.svelte index c3e1090..dd479b1 100644 --- a/webpage/src/lib/FileUpload.svelte +++ b/webpage/src/lib/FileUpload.svelte @@ -1,8 +1,9 @@
-
- +
@@ -101,4 +101,5 @@ height: calc(100% - 20px); margin-top: 5px; } + diff --git a/webpage/src/routes/models/edit/DeleteModel.svelte b/webpage/src/routes/models/edit/DeleteModel.svelte index f2a559b..261d98c 100644 --- a/webpage/src/routes/models/edit/DeleteModel.svelte +++ b/webpage/src/routes/models/edit/DeleteModel.svelte @@ -1,15 +1,29 @@ @@ -19,12 +33,8 @@ To delete this model please type "{model.name}": - {#if nameDoesNotMatch } - - Name does not match "{model.name}" - - {/if} + diff --git a/webpage/src/routes/models/edit/ModelData.svelte b/webpage/src/routes/models/edit/ModelData.svelte index 0a97a32..d02c403 100644 --- a/webpage/src/routes/models/edit/ModelData.svelte +++ b/webpage/src/routes/models/edit/ModelData.svelte @@ -127,12 +127,12 @@ ... - + Upload Zip File -
+
File selected diff --git a/webpage/src/routes/models/edit/RunModel.svelte b/webpage/src/routes/models/edit/RunModel.svelte index c069c95..0214028 100644 --- a/webpage/src/routes/models/edit/RunModel.svelte +++ b/webpage/src/routes/models/edit/RunModel.svelte @@ -8,7 +8,12 @@ let file: File | undefined = $state(); - let result: string | undefined = $state(); + type Result = { + class: string, + confidence: number, + } + + let _result: Promise = $state(new Promise(() => {})); let run = $state(false); let messages: MessageSimple; @@ -25,7 +30,8 @@ run = true; try { - result = await postFormData('models/run', form); + _result = await postFormData('models/run', form); + console.log(await _result); } catch (e) { if (e instanceof Response) { messages.display(await e.json()); @@ -60,19 +66,21 @@ Run {#if run} - {#if !result} -
-

- The class was not found -

-
- {:else} -
-

- Result -

- The image was classified as {result} -
- {/if} + {#await _result then result} + {#if !result} +
+

+ The class was not found +

+
+ {:else} +
+

+ Result +

+ The image was classified as {result.class} with confidence: {result.confidence} +
+ {/if} + {/await} {/if} diff --git a/webpage/src/styles/app.css b/webpage/src/styles/app.css index 26c5426..97761ca 100644 --- a/webpage/src/styles/app.css +++ b/webpage/src/styles/app.css @@ -101,3 +101,33 @@ a.button { .card h3 { margin-top: 0; } + + +.table { + width: 100%; + box-shadow: 0 2px 8px 1px #66666622; + border-radius: 10px; + border-collapse: collapse; + overflow: hidden; +} + +.table thead { + background: #60606022; +} + +.table tr td, +.table tr th { + border-left: 1px solid #22222244; + padding: 15px; +} + +.table tr td:first-child, +.table tr th:first-child { + border-left: none; +} + +.table tr td button, +.table tr td .button { + padding: 5px 10px; + box-shadow: 0 2px 5px 1px #66666655; +}