runner-go (#102)
Reviewed-on: andr3/fyp#102 Co-authored-by: Andre Henriques <andr3h3nriqu3s@gmail.com> Co-committed-by: Andre Henriques <andr3h3nriqu3s@gmail.com>
This commit was merged in pull request #102.
This commit is contained in:
@@ -6,3 +6,19 @@ const (
|
||||
DATA_POINT_MODE_TRAINING DATA_POINT_MODE = 1
|
||||
DATA_POINT_MODE_TESTING = 2
|
||||
)
|
||||
|
||||
type ModelClassStatus int
|
||||
|
||||
const (
|
||||
CLASS_STATUS_TO_TRAIN ModelClassStatus = iota + 1
|
||||
CLASS_STATUS_TRAINING
|
||||
CLASS_STATUS_TRAINED
|
||||
)
|
||||
|
||||
type ModelClass struct {
|
||||
Id string `db:"mc.id" json:"id"`
|
||||
ModelId string `db:"mc.model_id" json:"model_id"`
|
||||
Name string `db:"mc.name" json:"name"`
|
||||
ClassOrder int `db:"mc.class_order" json:"class_order"`
|
||||
Status int `db:"mc.status" json:"status"`
|
||||
}
|
||||
|
||||
95
logic/db_types/definitions.go
Normal file
95
logic/db_types/definitions.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package dbtypes
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
)
|
||||
|
||||
type DefinitionStatus int
|
||||
|
||||
const (
|
||||
DEFINITION_STATUS_CANCELD_TRAINING DefinitionStatus = -4
|
||||
DEFINITION_STATUS_FAILED_TRAINING = -3
|
||||
DEFINITION_STATUS_PRE_INIT = 1
|
||||
DEFINITION_STATUS_INIT = 2
|
||||
DEFINITION_STATUS_TRAINING = 3
|
||||
DEFINITION_STATUS_PAUSED_TRAINING = 6
|
||||
DEFINITION_STATUS_TRANIED = 4
|
||||
DEFINITION_STATUS_READY = 5
|
||||
)
|
||||
|
||||
type Definition struct {
|
||||
Id string `db:"md.id" json:"id"`
|
||||
ModelId string `db:"md.model_id" json:"model_id"`
|
||||
Accuracy float64 `db:"md.accuracy" json:"accuracy"`
|
||||
TargetAccuracy int `db:"md.target_accuracy" json:"target_accuracy"`
|
||||
Epoch int `db:"md.epoch" json:"epoch"`
|
||||
Status int `db:"md.status" json:"status"`
|
||||
CreatedOn time.Time `db:"md.created_on" json:"created"`
|
||||
EpochProgress int `db:"md.epoch_progress" json:"epoch_progress"`
|
||||
}
|
||||
|
||||
type SortByAccuracyDefinitions []*Definition
|
||||
|
||||
func (nf SortByAccuracyDefinitions) Len() int { return len(nf) }
|
||||
func (nf SortByAccuracyDefinitions) Swap(i, j int) { nf[i], nf[j] = nf[j], nf[i] }
|
||||
func (nf SortByAccuracyDefinitions) Less(i, j int) bool {
|
||||
return nf[i].Accuracy < nf[j].Accuracy
|
||||
}
|
||||
|
||||
func GetDefinition(db db.Db, definition_id string) (definition Definition, err error) {
|
||||
err = GetDBOnce(db, &definition, "model_definition as md where id=$1;", definition_id)
|
||||
return
|
||||
}
|
||||
|
||||
func MakeDefenition(db db.Db, model_id string, target_accuracy int) (definition Definition, err error) {
|
||||
var NewDefinition = struct {
|
||||
ModelId string `db:"model_id"`
|
||||
TargetAccuracy int `db:"target_accuracy"`
|
||||
}{ModelId: model_id, TargetAccuracy: target_accuracy}
|
||||
|
||||
id, err := InsertReturnId(db, &NewDefinition, "model_definition", "id")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return GetDefinition(db, id)
|
||||
}
|
||||
|
||||
func (d Definition) UpdateStatus(db db.Db, status DefinitionStatus) (err error) {
|
||||
_, err = db.Exec("update model_definition set status=$1 where id=$2", status, d.Id)
|
||||
return
|
||||
}
|
||||
|
||||
func (d Definition) MakeLayer(db db.Db, layer_order int, layer_type LayerType, shape string) (layer Layer, err error) {
|
||||
var NewLayer = struct {
|
||||
DefinitionId string `db:"def_id"`
|
||||
LayerOrder int `db:"layer_order"`
|
||||
LayerType LayerType `db:"layer_type"`
|
||||
Shape string `db:"shape"`
|
||||
}{
|
||||
DefinitionId: d.Id,
|
||||
LayerOrder: layer_order,
|
||||
LayerType: layer_type,
|
||||
Shape: shape,
|
||||
}
|
||||
|
||||
id, err := InsertReturnId(db, &NewLayer, "model_definition_layer", "id")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
return GetLayer(db, id)
|
||||
}
|
||||
|
||||
func (d Definition) GetLayers(db db.Db, filter string, args ...any) (layer []*Layer, err error) {
|
||||
args = append(args, d.Id)
|
||||
return GetDbMultitple[Layer](db, "model_definition_layer as mdl where mdl.def_id=$1 "+filter, args...)
|
||||
}
|
||||
|
||||
func (d *Definition) UpdateAfterEpoch(db db.Db, accuracy float64, epoch int) (err error) {
|
||||
d.Accuracy = accuracy
|
||||
d.Epoch += epoch
|
||||
_, err = db.Exec("update model_definition set epoch=$1, accuracy=$2 where id=$3", d.Epoch, d.Accuracy, d.Id)
|
||||
return
|
||||
}
|
||||
69
logic/db_types/layer.go
Normal file
69
logic/db_types/layer.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package dbtypes
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
"github.com/charmbracelet/log"
|
||||
)
|
||||
|
||||
type LayerType int
|
||||
|
||||
const (
|
||||
LAYER_INPUT LayerType = 1
|
||||
LAYER_DENSE = 2
|
||||
LAYER_FLATTEN = 3
|
||||
LAYER_SIMPLE_BLOCK = 4
|
||||
)
|
||||
|
||||
type Layer struct {
|
||||
Id string `db:"mdl.id" json:"id"`
|
||||
DefinitionId string `db:"mdl.def_id" json:"definition_id"`
|
||||
LayerOrder int `db:"mdl.layer_order" json:"layer_order"`
|
||||
LayerType LayerType `db:"mdl.layer_type" json:"layer_type"`
|
||||
Shape string `db:"mdl.shape" json:"shape"`
|
||||
ExpType int `db:"mdl.exp_type" json:"exp_type"`
|
||||
}
|
||||
|
||||
func (x *Layer) ShapeToSize() {
|
||||
v := x.GetShape()
|
||||
switch x.LayerType {
|
||||
case LAYER_INPUT:
|
||||
x.Shape = fmt.Sprintf("%d,%d", v[1], v[2])
|
||||
case LAYER_DENSE:
|
||||
x.Shape = fmt.Sprintf("(%d)", v[0])
|
||||
default:
|
||||
x.Shape = "ERROR"
|
||||
}
|
||||
}
|
||||
|
||||
func ShapeToString(args ...int) string {
|
||||
text, err := json.Marshal(args)
|
||||
if err != nil {
|
||||
log.Error("json err!", "err", err)
|
||||
panic("Could not generate Shape")
|
||||
}
|
||||
return string(text)
|
||||
}
|
||||
|
||||
func StringToShape(str string) (shape []int64) {
|
||||
err := json.Unmarshal([]byte(str), &shape)
|
||||
if err != nil {
|
||||
log.Error("json err!", "err", err)
|
||||
panic("Could not parse Shape")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (l Layer) GetShape() []int64 {
|
||||
if l.Shape == "" {
|
||||
return []int64{}
|
||||
}
|
||||
return StringToShape(l.Shape)
|
||||
}
|
||||
|
||||
func GetLayer(db db.Db, layer_id string) (layer Layer, err error) {
|
||||
err = GetDBOnce(db, &layer, "model_definition_layer as mdl where mdl.id=$1", layer_id)
|
||||
return
|
||||
}
|
||||
@@ -2,23 +2,25 @@ package dbtypes
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"git.andr3h3nriqu3s.com/andr3/fyp/logic/db"
|
||||
)
|
||||
|
||||
const (
|
||||
FAILED_TRAINING = -4
|
||||
FAILED_PREPARING_TRAINING = -3
|
||||
FAILED_PREPARING_ZIP_FILE = -2
|
||||
FAILED_PREPARING = -1
|
||||
type ModelStatus int
|
||||
|
||||
PREPARING = 1
|
||||
CONFIRM_PRE_TRAINING = 2
|
||||
PREPARING_ZIP_FILE = 3
|
||||
TRAINING = 4
|
||||
READY = 5
|
||||
READY_ALTERATION = 6
|
||||
READY_ALTERATION_FAILED = -6
|
||||
const (
|
||||
FAILED_TRAINING ModelStatus = -4
|
||||
FAILED_PREPARING_TRAINING = -3
|
||||
FAILED_PREPARING_ZIP_FILE = -2
|
||||
FAILED_PREPARING = -1
|
||||
PREPARING = 1
|
||||
CONFIRM_PRE_TRAINING = 2
|
||||
PREPARING_ZIP_FILE = 3
|
||||
TRAINING = 4
|
||||
READY = 5
|
||||
READY_ALTERATION = 6
|
||||
READY_ALTERATION_FAILED = -6
|
||||
|
||||
READY_RETRAIN = 7
|
||||
READY_RETRAIN_FAILED = -7
|
||||
@@ -26,15 +28,6 @@ const (
|
||||
|
||||
type ModelDefinitionStatus int
|
||||
|
||||
type LayerType int
|
||||
|
||||
const (
|
||||
LAYER_INPUT LayerType = 1
|
||||
LAYER_DENSE = 2
|
||||
LAYER_FLATTEN = 3
|
||||
LAYER_SIMPLE_BLOCK = 4
|
||||
)
|
||||
|
||||
const (
|
||||
MODEL_DEFINITION_STATUS_CANCELD_TRAINING ModelDefinitionStatus = -4
|
||||
MODEL_DEFINITION_STATUS_FAILED_TRAINING = -3
|
||||
@@ -46,14 +39,6 @@ const (
|
||||
MODEL_DEFINITION_STATUS_READY = 5
|
||||
)
|
||||
|
||||
type ModelClassStatus int
|
||||
|
||||
const (
|
||||
MODEL_CLASS_STATUS_TO_TRAIN ModelClassStatus = 1
|
||||
MODEL_CLASS_STATUS_TRAINING = 2
|
||||
MODEL_CLASS_STATUS_TRAINED = 3
|
||||
)
|
||||
|
||||
type ModelHeadStatus int
|
||||
|
||||
const (
|
||||
@@ -65,17 +50,16 @@ const (
|
||||
)
|
||||
|
||||
type BaseModel struct {
|
||||
Name string
|
||||
Status int
|
||||
Id string
|
||||
|
||||
ModelType int `db:"model_type"`
|
||||
ImageModeRaw string `db:"color_mode"`
|
||||
ImageMode int `db:"0"`
|
||||
Width int
|
||||
Height int
|
||||
Format string
|
||||
CanTrain int `db:"can_train"`
|
||||
Name string `json:"name"`
|
||||
Status int `json:"status"`
|
||||
Id string `json:"id"`
|
||||
ModelType int `db:"model_type" json:"model_type"`
|
||||
ImageModeRaw string `db:"color_mode" json:"image_more_raw"`
|
||||
ImageMode int `db:"0" json:"image_mode"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Format string `json:"format"`
|
||||
CanTrain int `db:"can_train" json:"can_train"`
|
||||
}
|
||||
|
||||
var ModelNotFoundError = errors.New("Model not found error")
|
||||
@@ -97,6 +81,59 @@ func (m BaseModel) CanEval() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// DO NOT Pass un filtered data on filters
|
||||
func (m BaseModel) GetDefinitions(db db.Db, filters string, args ...any) ([]*Definition, error) {
|
||||
n_args := []any{m.Id}
|
||||
n_args = append(n_args, args...)
|
||||
return GetDbMultitple[Definition](db, fmt.Sprintf("model_definition as md where md.model_id=$1 %s", filters), n_args...)
|
||||
}
|
||||
|
||||
func (m BaseModel) GetClasses(db db.Db, filters string, args ...any) ([]*ModelClass, error) {
|
||||
n_args := []any{m.Id}
|
||||
n_args = append(n_args, args...)
|
||||
return GetDbMultitple[ModelClass](db, fmt.Sprintf("model_classes as mc where mc.model_id=$1 %s", filters), n_args...)
|
||||
}
|
||||
|
||||
func (m *BaseModel) UpdateStatus(db db.Db, status ModelStatus) (err error) {
|
||||
_, err = db.Exec("update models set status=$1 where id=$2", status, m.Id)
|
||||
return
|
||||
}
|
||||
|
||||
type DataPoint struct {
|
||||
Id string `json:"id"`
|
||||
Class int `json:"class"`
|
||||
Path string `json:"path"`
|
||||
}
|
||||
|
||||
func (m BaseModel) DataPoints(db db.Db, mode DATA_POINT_MODE) (data []DataPoint, err error) {
|
||||
rows, err := db.Query(
|
||||
"select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner "+
|
||||
"join model_classes as mc on mc.id = mdp.class_id "+
|
||||
"where mc.model_id = $1 and mdp.model_mode=$2;",
|
||||
m.Id, mode)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
data = []DataPoint{}
|
||||
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var class_order int
|
||||
var file_path string
|
||||
if err = rows.Scan(&id, &class_order, &file_path); err != nil {
|
||||
return
|
||||
}
|
||||
data = append(data, DataPoint{
|
||||
Id: id,
|
||||
Path: file_path,
|
||||
Class: class_order,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func StringToImageMode(colorMode string) int {
|
||||
switch colorMode {
|
||||
case "greyscale":
|
||||
|
||||
@@ -14,10 +14,19 @@ const (
|
||||
)
|
||||
|
||||
type User struct {
|
||||
Id string `db:"u.id"`
|
||||
Username string `db:"u.username"`
|
||||
Email string `db:"u.email"`
|
||||
UserType int `db:"u.user_type"`
|
||||
Id string `db:"u.id" json:"id"`
|
||||
Username string `db:"u.username" json:"username"`
|
||||
Email string `db:"u.email" json:"email"`
|
||||
UserType int `db:"u.user_type" json:"user_type"`
|
||||
}
|
||||
|
||||
func UserFromId(db db.Db, id string) (*User, error) {
|
||||
var user User
|
||||
err := GetDBOnce(db, &user, "users as u where u.id=$1", id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &user, nil
|
||||
}
|
||||
|
||||
func UserFromToken(db db.Db, token string) (*User, error) {
|
||||
|
||||
Reference in New Issue
Block a user