sylv/fyp
1
0
Fork 0
Code Issues 29 Pull Requests Packages Projects 1 Releases Wiki Activity

feat: add ability of user to manage old tokens

Browse Source
This commit is contained in:
Andre Henriques
2024-04-13 16:59:08 +01:00
parent f8bc8ad85a
commit 4862e9a79e
6 changed files with 421 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

264
logic/models/data.go
View File

@@ -32,34 +32,34 @@ func InsertIfNotPresent(ss []string, s string) []string {
This function will process a single file from the uploaded zip file
*/
func fileProcessor(
c *Context,
model *BaseModel,
reader *zip.ReadCloser,
ids map[string]string,
base_path string,
index int,
file_chan chan *zip.File,
back_channel chan int,
c *Context,
model *BaseModel,
reader *zip.ReadCloser,
ids map[string]string,
base_path string,
index int,
file_chan chan *zip.File,
back_channel chan int,
) {
defer func() {
if r := recover(); r != nil {
c.Logger.Error("Recovered in file processor", "processor id", index, "due to", r)
}
}()
defer func() {
if r := recover(); r != nil {
c.Logger.Error("Recovered in file processor", "processor id", index, "due to", r)
}
}()
for file := range file_chan {
c.Logger.Debug("Processing File", "file", file.Name)
c.Logger.Debug("Processing File", "file", file.Name)
data, err := reader.Open(file.Name)
if err != nil {
c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err)
back_channel <- index
c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err)
back_channel <- index
continue
}
defer data.Close()
file_data, err := io.ReadAll(data)
if err != nil {
c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err)
back_channel <- index
c.Logger.Error("Could not open file in zip %s\n", "file name", file.Name, "err", err)
back_channel <- index
continue
}
@@ -74,16 +74,16 @@ func fileProcessor(
data_point_id, err := model_classes.AddDataPoint(c.Db, ids[parts[1]], "id://", mode)
if err != nil {
c.Logger.Error("Failed to add datapoint", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
c.Logger.Error("Failed to add datapoint", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
return
}
file_path := path.Join(base_path, data_point_id+"."+model.Format)
f, err := os.Create(file_path)
if err != nil {
c.Logger.Error("Failed to add datapoint", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
c.Logger.Error("Failed to add datapoint", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
return
}
defer f.Close()
@@ -94,19 +94,18 @@ func fileProcessor(
c.Logger.Warn("Not failling updating data point to status -1")
message := "Image did not have valid format for the model"
if err = model_classes.UpdateDataPointStatus(c.Db, data_point_id, -1, &message); err != nil {
c.Logger.Error("Failed to update data point", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
return
c.Logger.Error("Failed to update data point", "model", model.Id, "file name", file.Name, "err", err)
back_channel <- -index - 1
return
}
}
back_channel <- index
back_channel <- index
}
}
func processZipFile(c *Context, model *BaseModel) {
var err error
failed := func(msg string) {
@@ -156,7 +155,7 @@ func processZipFile(c *Context, model *BaseModel) {
return
}
c.Logger.Info("File Structure looks good to append data", "model", model.Id)
c.Logger.Info("File Structure looks good to append data", "model", model.Id)
ids := map[string]string{}
@@ -169,28 +168,28 @@ func processZipFile(c *Context, model *BaseModel) {
ids[name] = id
}
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
file_chans[i] = make(chan *zip.File, 2)
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
}
clean_up_channels := func() {
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
close(file_chans[i])
}
for i := 0; i < c.Handle.Config.NumberOfWorkers - 1; i++ {
_ = <- back_channel
}
close(back_channel)
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
file_chans[i] = make(chan *zip.File, 2)
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
}
first_round := true
clean_up_channels := func() {
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
close(file_chans[i])
}
for i := 0; i < c.Handle.Config.NumberOfWorkers-1; i++ {
_ = <-back_channel
}
close(back_channel)
}
channel_to_send := 0
first_round := true
channel_to_send := 0
// Parelalize this
@@ -200,37 +199,36 @@ func processZipFile(c *Context, model *BaseModel) {
continue
}
file_chans[channel_to_send] <- file
file_chans[channel_to_send] <- file
if first_round {
channel_to_send += 1
if c.Handle.Config.NumberOfWorkers == channel_to_send {
first_round = false
}
}
if first_round {
channel_to_send += 1
if c.Handle.Config.NumberOfWorkers == channel_to_send {
first_round = false
}
}
// Can not do else if because need to handle the case where the value changes in
// previous if
if !first_round {
new_id, ok := <- back_channel
if !ok {
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
}
// Can not do else if because need to handle the case where the value changes in
// previous if
if !first_round {
new_id, ok := <-back_channel
if !ok {
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
}
if new_id < 0 {
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
clean_up_channels()
failed("One of the workers failed due to db error")
return
}
if new_id < 0 {
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
clean_up_channels()
failed("One of the workers failed due to db error")
return
}
channel_to_send = new_id
}
channel_to_send = new_id
}
}
clean_up_channels()
clean_up_channels()
c.Logger.Info("Added data to model", "model", model.Id)
@@ -301,7 +299,7 @@ func processZipFileExpand(c *Context, model *BaseModel) {
for i, name := range training {
id, _err := model_classes.CreateClass(c.Db, model.Id, base+i, name)
err = _err
err = _err
if err != nil {
failed(fmt.Sprintf("Failed to create class '%s' on db\n", name))
return
@@ -309,28 +307,28 @@ func processZipFileExpand(c *Context, model *BaseModel) {
ids[name] = id
}
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
back_channel := make(chan int, c.Handle.Config.NumberOfWorkers)
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
file_chans := make([]chan *zip.File, c.Handle.Config.NumberOfWorkers)
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
file_chans[i] = make(chan *zip.File, 2)
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
}
clean_up_channels := func() {
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
close(file_chans[i])
}
for i := 0; i < c.Handle.Config.NumberOfWorkers - 1; i++ {
_ = <- back_channel
}
close(back_channel)
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
file_chans[i] = make(chan *zip.File, 2)
go fileProcessor(c, model, reader, ids, base_path, i, file_chans[i], back_channel)
}
first_round := true
clean_up_channels := func() {
for i := 0; i < c.Handle.Config.NumberOfWorkers; i++ {
close(file_chans[i])
}
for i := 0; i < c.Handle.Config.NumberOfWorkers-1; i++ {
_ = <-back_channel
}
close(back_channel)
}
channel_to_send := 0
first_round := true
channel_to_send := 0
// Parelalize this
@@ -340,69 +338,67 @@ func processZipFileExpand(c *Context, model *BaseModel) {
continue
}
file_chans[channel_to_send] <- file
file_chans[channel_to_send] <- file
if first_round {
channel_to_send += 1
if c.Handle.Config.NumberOfWorkers == channel_to_send {
first_round = false
}
}
if first_round {
channel_to_send += 1
if c.Handle.Config.NumberOfWorkers == channel_to_send {
first_round = false
}
}
// Can not do else if because need to handle the case where the value changes in
// previous if
if !first_round {
new_id, ok := <- back_channel
if !ok {
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
}
// Can not do else if because need to handle the case where the value changes in
// previous if
if !first_round {
new_id, ok := <-back_channel
if !ok {
c.Logger.Fatal("Something is very wrong please check as this line should be unreachable")
}
if new_id < 0 {
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
clean_up_channels()
failed("One of the workers failed due to db error")
return
}
if new_id < 0 {
c.Logger.Error("Worker failed", "worker id", -(new_id + 1))
clean_up_channels()
failed("One of the workers failed due to db error")
return
}
channel_to_send = new_id
}
channel_to_send = new_id
}
}
clean_up_channels()
clean_up_channels()
c.Logger.Info("Added data to model", "id", model.Id)
ModelUpdateStatus(c, model.Id, READY)
}
func handleRemoveDataPoint(c *Context) *Error {
var dat JustId
if err := c.ToJSON(&dat); err != nil {
return err
}
var GetModelId struct {
Value string `db:"m.id"`
Format string `db:"m.format"`
}
var dat JustId
if err := c.ToJSON(&dat); err != nil {
return err
}
err := GetDBOnce(c, &GetModelId, "model_data_point as mdp inner join model_classes as mc on mdp.class_id=mc.id inner join models as m on m.id=mc.model_id where mdp.id=$1;", dat.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "Data point not found")
} else if err != nil {
return c.E500M("Failed to find data point", err)
}
var GetModelId struct {
Value string `db:"m.id"`
Format string `db:"m.format"`
}
os.Remove(path.Join("savedData", GetModelId.Value, "data", dat.Id+"."+GetModelId.Format))
err := GetDBOnce(c, &GetModelId, "model_data_point as mdp inner join model_classes as mc on mdp.class_id=mc.id inner join models as m on m.id=mc.model_id where mdp.id=$1;", dat.Id)
if err == NotFoundError {
return c.SendJSONStatus(404, "Data point not found")
} else if err != nil {
return c.E500M("Failed to find data point", err)
}
_, err = c.Db.Exec("delete from model_data_point where id=$1;", dat.Id)
if err != nil {
return c.E500M("Failed to remove datapoint from database", err)
}
os.Remove(path.Join("savedData", GetModelId.Value, "data", dat.Id+"."+GetModelId.Format))
return c.SendJSON("ok")
_, err = c.Db.Exec("delete from model_data_point where id=$1;", dat.Id)
if err != nil {
return c.E500M("Failed to remove datapoint from database", err)
}
return c.SendJSON("ok")
}
func handleDataUpload(handle *Handle) {
@@ -530,7 +526,7 @@ func handleDataUpload(handle *Handle) {
return c.SendJSON(model.Id)
})
handle.Delete("/models/data/point", handleRemoveDataPoint)
handle.DeleteAuth("/models/data/point", 1, handleRemoveDataPoint)
handle.Delete("/models/data/delete-zip-file", func(c *Context) *Error {
if !c.CheckAuthLevel(1) {

138
logic/utils/handler.go
View File

@@ -39,12 +39,12 @@ type Handler interface {
}
type Handle struct {
Db *sql.DB
gets []HandleFunc
posts []HandleFunc
deletes []HandleFunc
Config Config
validate *validator.Validate
Db *sql.DB
gets []HandleFunc
posts []HandleFunc
deletes []HandleFunc
Config Config
validate *validator.Validate
}
func decodeBody(r *http.Request) (string, *Error) {
@@ -83,35 +83,71 @@ func (x *Handle) Post(path string, fn func(c *Context) *Error) {
func (x *Handle) PostAuth(path string, authLevel int, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
}
return fn(c)
if !c.CheckAuthLevel(authLevel) {
return nil
}
return fn(c)
}
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func PostAuthJson[T interface{}](x *Handle, path string, authLevel int, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
}
obj := new(T)
if err := c.ToJSON(obj); err != nil {
return err
}
return fn(c, obj)
}
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func (x *Handle) Delete(path string, fn func(c *Context) *Error) {
x.deletes = append(x.deletes, HandleFunc{path, fn})
}
func (x *Handle) DeleteAuth(path string, authLevel int, fn func(c *Context) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
}
return fn(c)
if !c.CheckAuthLevel(authLevel) {
return nil
}
return fn(c)
}
x.posts = append(x.posts, HandleFunc{path, inner_fn})
}
func DeleteAuthJson[T interface{}](x *Handle, path string, authLevel int, fn func(c *Context, obj *T) *Error) {
inner_fn := func(c *Context) *Error {
if !c.CheckAuthLevel(authLevel) {
return nil
}
obj := new(T)
if err := c.ToJSON(obj); err != nil {
return err
}
return fn(c, obj)
}
x.deletes = append(x.deletes, HandleFunc{path, inner_fn})
}
func (x *Handle) handleGets(context *Context) {
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
for _, s := range x.gets {
if s.path == context.R.URL.Path {
@@ -124,12 +160,12 @@ func (x *Handle) handleGets(context *Context) {
}
func (x *Handle) handlePosts(context *Context) {
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
for _, s := range x.posts {
if s.path == context.R.URL.Path {
@@ -142,12 +178,12 @@ func (x *Handle) handlePosts(context *Context) {
}
func (x *Handle) handleDeletes(context *Context) {
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
defer func() {
if r := recover(); r != nil {
context.Logger.Error("Something went very wrong", "Error", r)
handleError(&Error{500, "500"}, context)
}
}()
for _, s := range x.deletes {
if s.path == context.R.URL.Path {
@@ -185,20 +221,18 @@ type Context struct {
Handle *Handle
}
func (c Context) GetDb() (*sql.DB) {
return c.Db
func (c Context) GetDb() *sql.DB {
return c.Db
}
func (c Context) GetLogger() (*log.Logger) {
return c.Logger
func (c Context) GetLogger() *log.Logger {
return c.Logger
}
func (c Context) Query(query string, args ...any) (*sql.Rows, error) {
return c.Db.Query(query, args...)
return c.Db.Query(query, args...)
}
func (c Context) Prepare(str string) (*sql.Stmt, error) {
if c.Tx == nil {
return c.Db.Prepare(str)
@@ -245,31 +279,31 @@ func (c *Context) RollbackTx() error {
/**
* Parse and vailidates the json
*/
*/
func (c Context) ParseJson(dat any, str string) *Error {
decoder := json.NewDecoder(strings.NewReader(str))
return c.decodeAndValidade(decoder, dat)
return c.decodeAndValidade(decoder, dat)
}
func (c Context) ToJSON(dat any) *Error {
decoder := json.NewDecoder(c.R.Body)
return c.decodeAndValidade(decoder, dat)
return c.decodeAndValidade(decoder, dat)
}
func (c Context) decodeAndValidade(decoder *json.Decoder, dat any) *Error {
err := decoder.Decode(dat)
if err != nil {
c.Logger.Error("Failed to decode json", "dat", dat, "err", err)
return c.JsonBadRequest("Bad Request! Invalid json passed!");
c.Logger.Error("Failed to decode json", "dat", dat, "err", err)
return c.JsonBadRequest("Bad Request! Invalid json passed!")
}
err = c.Handle.validate.Struct(dat)
if err != nil {
c.Logger.Error("Failed invalid json passed", "dat", dat, "err", err)
return c.JsonBadRequest("Bad Request! Invalid json passed!");
}
err = c.Handle.validate.Struct(dat)
if err != nil {
c.Logger.Error("Failed invalid json passed", "dat", dat, "err", err)
return c.JsonBadRequest("Bad Request! Invalid json passed!")
}
return nil
}
@@ -303,7 +337,7 @@ func (c Context) JsonBadRequest(dat any) *Error {
c.SetReportCaller(true)
c.Logger.Warn("Request failed with a bad request", "dat", dat)
c.SetReportCaller(false)
return c.ErrorCode(nil, 404, dat)
return c.ErrorCode(nil, 404, dat)
}
func (c Context) JsonErrorBadRequest(err error, dat any) *Error {
@@ -556,7 +590,7 @@ func NewHandler(db *sql.DB, config Config) *Handle {
var gets []HandleFunc
var posts []HandleFunc
var deletes []HandleFunc
validate := validator.New()
validate := validator.New()
x := &Handle{db, gets, posts, deletes, config, validate}
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {