2023-09-26 20:15:28 +01:00
package models_train
import (
"database/sql"
"errors"
"fmt"
2023-09-27 21:20:39 +01:00
"io"
2023-10-19 11:42:38 +01:00
"math"
2023-09-26 20:15:28 +01:00
"net/http"
2023-09-27 21:20:39 +01:00
"os"
"os/exec"
"path"
"strconv"
"text/template"
2023-09-26 20:15:28 +01:00
model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/utils"
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
2023-10-19 11:42:38 +01:00
const EPOCH_PER_RUN = 20
2023-10-19 10:44:13 +01:00
const MAX_EPOCH = 100
2023-09-26 20:15:28 +01:00
func MakeDefenition ( db * sql . DB , model_id string , target_accuracy int ) ( id string , err error ) {
2023-09-27 21:20:39 +01:00
id = ""
2023-10-10 12:28:49 +01:00
rows , err := db . Query ( "insert into model_definition (model_id, target_accuracy) values ($1, $2) returning id;" , model_id , target_accuracy )
if err != nil {
return
}
defer rows . Close ( )
if ! rows . Next ( ) {
return id , errors . New ( "Something wrong!" )
}
2023-09-27 21:20:39 +01:00
err = rows . Scan ( & id )
2023-10-10 12:28:49 +01:00
return
2023-09-26 20:15:28 +01:00
}
type ModelDefinitionStatus int
const (
2023-10-19 10:44:13 +01:00
MODEL_DEFINITION_STATUS_CANCELD_TRAINING = - 4
MODEL_DEFINITION_STATUS_FAILED_TRAINING = - 3
MODEL_DEFINITION_STATUS_PRE_INIT ModelDefinitionStatus = 1
MODEL_DEFINITION_STATUS_INIT = 2
MODEL_DEFINITION_STATUS_TRAINING = 3
MODEL_DEFINITION_STATUS_TRANIED = 4
MODEL_DEFINITION_STATUS_READY = 5
2023-09-27 21:20:39 +01:00
)
type LayerType int
const (
2023-10-10 12:28:49 +01:00
LAYER_INPUT LayerType = 1
LAYER_DENSE = 2
LAYER_FLATTEN = 3
2023-09-26 20:15:28 +01:00
)
2023-10-06 12:13:19 +01:00
func ModelDefinitionUpdateStatus ( c * Context , id string , status ModelDefinitionStatus ) ( err error ) {
_ , err = c . Db . Exec ( "update model_definition set status = $1 where id = $2" , status , id )
2023-09-27 21:20:39 +01:00
return
2023-09-26 20:15:28 +01:00
}
2023-09-27 21:20:39 +01:00
func MakeLayer ( db * sql . DB , def_id string , layer_order int , layer_type LayerType , shape string ) ( err error ) {
2023-09-26 20:15:28 +01:00
_ , err = db . Exec ( "insert into model_definition_layer (def_id, layer_order, layer_type, shape) values ($1, $2, $3, $4)" , def_id , layer_order , layer_type , shape )
return
}
2023-10-10 12:28:49 +01:00
func generateCvs ( c * Context , run_path string , model_id string ) ( count int , err error ) {
2023-10-02 21:15:31 +01:00
2023-10-10 12:28:49 +01:00
classes , err := c . Db . Query ( "select count(*) from model_classes where model_id=$1;" , model_id )
if err != nil {
return
}
defer classes . Close ( )
if ! classes . Next ( ) {
return
}
if err = classes . Scan ( & count ) ; err != nil {
return
}
data , err := c . Db . Query ( "select mdp.id, mc.class_order, mdp.file_path from model_data_point as mdp inner join model_classes as mc on mc.id = mdp.class_id where mc.model_id = $1;" , model_id )
if err != nil {
return
}
defer data . Close ( )
f , err := os . Create ( path . Join ( run_path , "train.csv" ) )
if err != nil {
return
}
defer f . Close ( )
f . Write ( [ ] byte ( "Id,Index\n" ) )
for data . Next ( ) {
var id string
var class_order int
var file_path string
if err = data . Scan ( & id , & class_order , & file_path ) ; err != nil {
return
}
if file_path == "id://" {
f . Write ( [ ] byte ( id + "," + strconv . Itoa ( class_order ) + "\n" ) )
} else {
return count , errors . New ( "TODO generateCvs to file_path " + file_path )
}
}
return
2023-10-02 21:15:31 +01:00
}
2023-10-19 10:44:13 +01:00
func trainDefinition ( c * Context , model * BaseModel , definition_id string , load_prev bool ) ( accuracy float64 , err error ) {
c . Logger . Warn ( "About to start training definition" )
2023-09-27 21:20:39 +01:00
accuracy = 0
2023-10-06 12:13:19 +01:00
layers , err := c . Db . Query ( "select layer_type, shape from model_definition_layer where def_id=$1 order by layer_order asc;" , definition_id )
2023-09-27 21:20:39 +01:00
if err != nil {
return
}
defer layers . Close ( )
type layerrow struct {
LayerType int
Shape string
}
got := [ ] layerrow { }
for layers . Next ( ) {
var row = layerrow { }
if err = layers . Scan ( & row . LayerType , & row . Shape ) ; err != nil {
return
}
row . Shape = shapeToSize ( row . Shape )
got = append ( got , row )
}
// Generate run folder
2023-10-03 19:02:02 +01:00
run_path := path . Join ( "/tmp" , model . Id , "defs" , definition_id )
2023-09-27 21:20:39 +01:00
err = os . MkdirAll ( run_path , os . ModePerm )
if err != nil {
return
}
2023-10-10 12:28:49 +01:00
_ , err = generateCvs ( c , run_path , model . Id )
if err != nil {
return
}
2023-10-02 21:15:31 +01:00
2023-09-27 21:20:39 +01:00
// Create python script
f , err := os . Create ( path . Join ( run_path , "run.py" ) )
if err != nil {
return
}
defer f . Close ( )
tmpl , err := template . New ( "python_model_template.py" ) . ParseFiles ( "views/py/python_model_template.py" )
if err != nil {
return
}
2023-10-19 10:44:13 +01:00
// Copy result around
result_path := path . Join ( "savedData" , model . Id , "defs" , definition_id )
2023-09-27 21:20:39 +01:00
if err = tmpl . Execute ( f , AnyMap {
2023-10-19 10:44:13 +01:00
"Layers" : got ,
"Size" : got [ 0 ] . Shape ,
"DataDir" : path . Join ( getDir ( ) , "savedData" , model . Id , "data" ) ,
"RunPath" : run_path ,
"ColorMode" : model . ImageMode ,
"Model" : model ,
"EPOCH_PER_RUN" : EPOCH_PER_RUN ,
"DefId" : definition_id ,
"LoadPrev" : load_prev ,
"LastModelRunPath" : path . Join ( getDir ( ) , result_path , "model.keras" ) ,
2023-09-27 21:20:39 +01:00
} ) ; err != nil {
return
}
// Run the command
2023-10-10 12:28:49 +01:00
out , err := exec . Command ( "bash" , "-c" , fmt . Sprintf ( "cd %s && python run.py" , run_path ) ) . Output ( )
if err != nil {
2023-10-19 10:44:13 +01:00
c . Logger . Debug ( string ( out ) )
2023-09-27 21:20:39 +01:00
return
}
if err = os . MkdirAll ( result_path , os . ModePerm ) ; err != nil {
return
}
if err = exec . Command ( "cp" , "-r" , path . Join ( run_path , "model" ) , path . Join ( result_path , "model" ) ) . Run ( ) ; err != nil {
return
}
2023-10-19 10:44:13 +01:00
if err = exec . Command ( "cp" , "-r" , path . Join ( run_path , "model.keras" ) , path . Join ( result_path , "model.keras" ) ) . Run ( ) ; err != nil {
return
}
2023-09-27 21:20:39 +01:00
accuracy_file , err := os . Open ( path . Join ( run_path , "accuracy.val" ) )
if err != nil {
return
}
defer accuracy_file . Close ( )
accuracy_file_bytes , err := io . ReadAll ( accuracy_file )
if err != nil {
return
}
2023-10-19 10:44:13 +01:00
c . Logger . Info ( "Model finished training!" , "accuracy" , accuracy )
2023-09-27 21:20:39 +01:00
accuracy , err = strconv . ParseFloat ( string ( accuracy_file_bytes ) , 64 )
if err != nil {
return
}
2023-10-10 12:28:49 +01:00
os . RemoveAll ( run_path )
2023-09-27 21:20:39 +01:00
return
}
2023-10-19 10:44:13 +01:00
func remove [ T interface { } ] ( lst [ ] T , i int ) [ ] T {
lng := len ( lst )
if i >= lng {
return [ ] T { }
}
if i + 1 >= lng {
return lst [ : lng - 1 ]
}
if i == 0 {
return lst [ 1 : ]
}
return append ( lst [ : i ] , lst [ i + 1 : ] ... )
}
2023-10-06 12:13:19 +01:00
func trainModel ( c * Context , model * BaseModel ) {
2023-10-19 10:44:13 +01:00
definitionsRows , err := c . Db . Query ( "select id, target_accuracy, epoch from model_definition where status=$1 and model_id=$2" , MODEL_DEFINITION_STATUS_INIT , model . Id )
2023-09-27 21:20:39 +01:00
if err != nil {
2023-10-06 12:13:19 +01:00
c . Logger . Error ( "Failed to trainModel!Err:" )
c . Logger . Error ( err )
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
}
defer definitionsRows . Close ( )
type row struct {
id string
target_accuracy int
2023-10-19 10:44:13 +01:00
epoch int
2023-09-27 21:20:39 +01:00
}
definitions := [ ] row { }
for definitionsRows . Next ( ) {
var rowv row
2023-10-19 10:44:13 +01:00
if err = definitionsRows . Scan ( & rowv . id , & rowv . target_accuracy , & rowv . epoch ) ; err != nil {
2023-10-10 12:28:49 +01:00
c . Logger . Error ( "Failed to train Model Could not read definition from db!Err:" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
}
definitions = append ( definitions , rowv )
}
if len ( definitions ) == 0 {
2023-10-10 12:28:49 +01:00
c . Logger . Error ( "No Definitions defined!" )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
}
2023-10-19 10:44:13 +01:00
toTrain := len ( definitions )
firstRound := true
var newDefinitions = [ ] row { }
copy ( newDefinitions , definitions )
for {
for i , def := range definitions {
ModelDefinitionUpdateStatus ( c , def . id , MODEL_DEFINITION_STATUS_TRAINING )
accuracy , err := trainDefinition ( c , model , def . id , ! firstRound )
if err != nil {
c . Logger . Error ( "Failed to train definition!Err:" , "err" , err )
ModelDefinitionUpdateStatus ( c , def . id , MODEL_DEFINITION_STATUS_FAILED_TRAINING )
toTrain = toTrain - 1
newDefinitions = remove ( newDefinitions , i )
continue
}
def . epoch += EPOCH_PER_RUN
int_accuracy := int ( accuracy * 100 )
if int_accuracy >= def . target_accuracy {
c . Logger . Info ( "Found a definition that reaches target_accuracy!" )
_ , err = c . Db . Exec ( "update model_definition set accuracy=$1, status=$2, epoch=$3 where id=$4" , int_accuracy , MODEL_DEFINITION_STATUS_TRANIED , def . epoch , def . id )
if err != nil {
c . Logger . Error ( "Failed to train definition!Err:\n" , "err" , err )
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
return
}
_ , err = c . Db . Exec ( "update model_definition set status=$1 where id!=$2 and model_id=$3 and status!=$4" , MODEL_DEFINITION_STATUS_CANCELD_TRAINING , def . id , model . Id , MODEL_DEFINITION_STATUS_FAILED_TRAINING )
if err != nil {
c . Logger . Error ( "Failed to train definition!Err:\n" , "err" , err )
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
return
}
toTrain = 0
break
}
2023-09-27 21:20:39 +01:00
2023-10-19 10:44:13 +01:00
if def . epoch > MAX_EPOCH {
fmt . Printf ( "Failed to train definition! Accuracy less %d < %d\n" , int_accuracy , def . target_accuracy )
ModelDefinitionUpdateStatus ( c , def . id , MODEL_DEFINITION_STATUS_FAILED_TRAINING )
toTrain = toTrain - 1
newDefinitions = remove ( newDefinitions , i )
continue
}
2023-09-27 21:20:39 +01:00
}
2023-10-19 10:44:13 +01:00
copy ( definitions , newDefinitions )
firstRound = false
if toTrain == 0 {
break
2023-09-27 21:20:39 +01:00
}
}
2023-10-06 12:13:19 +01:00
rows , err := c . Db . Query ( "select id from model_definition where model_id=$1 and status=$2 order by accuracy desc limit 1;" , model . Id , MODEL_DEFINITION_STATUS_TRANIED )
2023-09-27 21:20:39 +01:00
if err != nil {
2023-10-10 12:28:49 +01:00
c . Logger . Error ( "DB: failed to read definition" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
}
2023-10-10 12:28:49 +01:00
defer rows . Close ( )
2023-09-27 21:20:39 +01:00
2023-10-10 12:28:49 +01:00
if ! rows . Next ( ) {
// TODO Make the Model status have a message
c . Logger . Error ( "All definitions failed to train!" )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
2023-10-10 12:28:49 +01:00
}
2023-09-27 21:20:39 +01:00
2023-10-10 12:28:49 +01:00
var id string
if err = rows . Scan ( & id ) ; err != nil {
c . Logger . Error ( "Failed to read id:" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
2023-10-10 12:28:49 +01:00
}
2023-09-27 18:07:04 +01:00
2023-10-10 12:28:49 +01:00
if _ , err = c . Db . Exec ( "update model_definition set status=$1 where id=$2;" , MODEL_DEFINITION_STATUS_READY , id ) ; err != nil {
c . Logger . Error ( "Failed to update model definition" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
2023-10-10 12:28:49 +01:00
}
2023-09-27 21:20:39 +01:00
2023-10-10 12:28:49 +01:00
to_delete , err := c . Db . Query ( "select id from model_definition where status != $1 and model_id=$2" , MODEL_DEFINITION_STATUS_READY , model . Id )
if err != nil {
c . Logger . Error ( "Failed to select model_definition to delete" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
2023-10-10 12:28:49 +01:00
}
defer to_delete . Close ( )
for to_delete . Next ( ) {
var id string
if to_delete . Scan ( & id ) ; err != nil {
c . Logger . Error ( "Failed to scan the id of a model_definition to delete" )
c . Logger . Error ( err )
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
return
}
os . RemoveAll ( path . Join ( "savedData" , model . Id , "defs" , id ) )
}
// TODO Check if returning also works here
if _ , err = c . Db . Exec ( "delete from model_definition where status!=$1 and model_id=$2;" , MODEL_DEFINITION_STATUS_READY , model . Id ) ; err != nil {
c . Logger . Error ( "Failed to delete model_definition" )
c . Logger . Error ( err )
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , FAILED_TRAINING )
2023-09-27 21:20:39 +01:00
return
2023-10-10 12:28:49 +01:00
}
2023-09-27 18:07:04 +01:00
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , READY )
2023-09-27 18:07:04 +01:00
}
2023-10-10 12:28:49 +01:00
func removeFailedDataPoints ( db * sql . DB , model * BaseModel ) ( err error ) {
2023-10-10 14:58:18 +01:00
rows , err := db . Query ( "select mdp.id from model_data_point as mdp join model_classes as mc on mc.id=mdp.class_id where mc.model_id=$1 and mdp.status=-1;" , model . Id )
2023-10-10 12:28:49 +01:00
if err != nil {
return
}
defer rows . Close ( )
2023-10-19 10:44:13 +01:00
base_path := path . Join ( "savedData" , model . Id , "data" )
2023-10-10 12:28:49 +01:00
for rows . Next ( ) {
var dataPointId string
err = rows . Scan ( & dataPointId )
if err != nil {
return
}
2023-10-19 10:44:13 +01:00
err = os . RemoveAll ( path . Join ( base_path , dataPointId + model . Format ) )
if err != nil {
return
}
2023-10-10 12:28:49 +01:00
}
2023-10-19 10:44:13 +01:00
_ , err = db . Exec ( "delete from model_data_point as mdp using model_classes as mc where mdp.class_id = mc.id and mc.model_id=$1 and mdp.status=-1;" , model . Id )
2023-10-10 12:28:49 +01:00
return
}
2023-10-19 11:42:38 +01:00
// This generates a definition
func generateDefinition ( c * Context , model * BaseModel , number_of_classes int , complexity int ) * Error {
var err error = nil
failed := func ( ) * Error {
ModelUpdateStatus ( c , model . Id , FAILED_PREPARING_TRAINING )
// TODO improve this response
return c . Error500 ( err )
}
def_id , err := MakeDefenition ( c . Db , model . Id , 0 )
if err != nil {
return failed ( )
}
// Note the shape for now is no used
err = MakeLayer ( c . Db , def_id , 1 , LAYER_INPUT , fmt . Sprintf ( "%d,%d,1" , model . Width , model . Height ) )
if err != nil {
return failed ( )
}
if complexity == 0 {
err = MakeLayer ( c . Db , def_id , 4 , LAYER_FLATTEN , "" )
if err != nil {
return failed ( )
}
loop := int ( math . Log2 ( float64 ( number_of_classes ) ) )
for i := 0 ; i < loop ; i ++ {
err = MakeLayer ( c . Db , def_id , 5 , LAYER_DENSE , fmt . Sprintf ( "%d,1" , number_of_classes * ( loop - i ) ) )
if err != nil {
ModelUpdateStatus ( c , model . Id , FAILED_PREPARING_TRAINING )
// TODO improve this response
return c . Error500 ( err )
}
}
} else {
c . Logger . Error ( "Unkown complexity" , "complexity" , complexity )
return failed ( )
}
err = ModelDefinitionUpdateStatus ( c , def_id , MODEL_DEFINITION_STATUS_INIT )
if err != nil {
return failed ( )
}
return nil
}
func generateDefinitions ( c * Context , model * BaseModel , number_of_models int ) * Error {
cls , err := model_classes . ListClasses ( c . Db , model . Id )
if err != nil {
ModelUpdateStatus ( c , model . Id , FAILED_PREPARING_TRAINING )
// TODO improve this response
return c . Error500 ( err )
}
err = removeFailedDataPoints ( c . Db , model )
if err != nil {
return c . Error500 ( err )
}
for i := 0 ; i < number_of_models ; i ++ {
// TODO handle incrisea the complexity
generateDefinition ( c , model , len ( cls ) , 0 )
}
return nil
}
2023-09-26 20:15:28 +01:00
func handleTrain ( handle * Handle ) {
handle . Post ( "/models/train" , func ( w http . ResponseWriter , r * http . Request , c * Context ) * Error {
if ! CheckAuthLevel ( 1 , w , r , c ) {
return nil
}
if c . Mode == JSON {
panic ( "TODO /models/train JSON" )
}
r . ParseForm ( )
f := r . Form
number_of_models := 0
accuracy := 0
if ! CheckId ( f , "id" ) || CheckEmpty ( f , "model_type" ) || ! CheckNumber ( f , "number_of_models" , & number_of_models ) || ! CheckNumber ( f , "accuracy" , & accuracy ) {
fmt . Println (
! CheckId ( f , "id" ) , CheckEmpty ( f , "model_type" ) , ! CheckNumber ( f , "number_of_models" , & number_of_models ) , ! CheckNumber ( f , "accuracy" , & accuracy ) ,
)
// TODO improve this response
return ErrorCode ( nil , 400 , c . AddMap ( nil ) )
}
id := f . Get ( "id" )
model_type := f . Get ( "model_type" )
// Its not used rn
_ = model_type
2023-09-27 21:20:39 +01:00
// TODO check if the model has data
/ * rows , err := handle . Db . Query ( "select mc.name, mdp.file_path from model_classes as mc join model_data_point as mdp on mdp.class_id = mc.id where mdp.model_mode = 1 and mc.model_id = $1 limit 1;" , id )
if err != nil {
return Error500 ( err )
}
defer rows . Close ( )
if ! rows . Next ( ) {
return Error500 ( err )
}
var name string
var file_path string
err = rows . Scan ( & name , & file_path )
if err != nil {
return Error500 ( err )
} * /
2023-09-26 20:15:28 +01:00
model , err := GetBaseModel ( handle . Db , id )
if err == ModelNotFoundError {
return ErrorCode ( nil , http . StatusNotFound , c . AddMap ( AnyMap {
"NotFoundMessage" : "Model not found" ,
"GoBackLink" : "/models" ,
} ) )
} else if err != nil {
// TODO improve this response
return Error500 ( err )
}
if model . Status != CONFIRM_PRE_TRAINING {
// TODO improve this response
return ErrorCode ( nil , 400 , c . AddMap ( nil ) )
}
2023-10-19 11:42:38 +01:00
full_error := generateDefinitions ( c , model , number_of_models )
if full_error != nil {
return full_error
}
2023-09-26 20:15:28 +01:00
2023-10-06 12:13:19 +01:00
go trainModel ( c , model )
2023-09-27 18:07:04 +01:00
2023-10-06 12:13:19 +01:00
ModelUpdateStatus ( c , model . Id , TRAINING )
2023-09-27 21:20:39 +01:00
Redirect ( "/models/edit?id=" + model . Id , c . Mode , w , r )
2023-09-26 20:15:28 +01:00
return nil
} )
2023-10-12 12:08:12 +01:00
handle . Get ( "/model/epoch/update" , func ( w http . ResponseWriter , r * http . Request , c * Context ) * Error {
2023-10-19 10:44:13 +01:00
// TODO check auth level
2023-10-12 12:08:12 +01:00
if c . Mode != NORMAL {
2023-10-19 10:44:13 +01:00
// This should only handle normal requests
c . Logger . Warn ( "This function only works with normal" )
return c . UnsafeErrorCode ( nil , 400 , nil )
2023-10-12 12:08:12 +01:00
}
2023-10-19 10:44:13 +01:00
f := r . URL . Query ( )
2023-10-12 12:08:12 +01:00
2023-10-19 10:44:13 +01:00
if ! CheckId ( f , "model_id" ) || ! CheckId ( f , "definition" ) || CheckEmpty ( f , "epoch" ) {
c . Logger . Warn ( "Invalid: model_id or definition or epoch" )
return c . UnsafeErrorCode ( nil , 400 , nil )
}
2023-10-12 12:08:12 +01:00
model_id := f . Get ( "model_id" )
def_id := f . Get ( "definition" )
2023-10-19 10:44:13 +01:00
epoch , err := strconv . Atoi ( f . Get ( "epoch" ) )
if err != nil {
c . Logger . Warn ( "Epoch is not a number" )
// No need to improve message because this function is only called internaly
return c . UnsafeErrorCode ( nil , 400 , nil )
}
rows , err := c . Db . Query ( "select md.status from model_definition as md where md.model_id=$1 and md.id=$2" , model_id , def_id )
if err != nil {
return c . Error500 ( err )
}
defer rows . Close ( )
if ! rows . Next ( ) {
c . Logger . Error ( "Could not get status of model definition" )
return c . Error500 ( nil )
}
var status int
err = rows . Scan ( & status )
if err != nil {
return c . Error500 ( err )
}
if status != 3 {
c . Logger . Warn ( "Definition not on status 3(training)" , "status" , status )
// No need to improve message because this function is only called internaly
return c . UnsafeErrorCode ( nil , 400 , nil )
}
_ , err = c . Db . Exec ( "update model_definition set epoch_progress=$1 where id=$2" , epoch , def_id )
if err != nil {
return c . Error500 ( err )
}
2023-10-12 12:08:12 +01:00
return nil
} )
2023-09-26 20:15:28 +01:00
}