fyp/logic/models/data.go

package models

import (
	"archive/zip"
	"bytes"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"reflect"
	"sort"
	"strings"

	model_classes "git.andr3h3nriqu3s.com/andr3/fyp/logic/models/classes"
	. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)

func InsertIfNotPresent(ss []string, s string) []string {
    i := sort.SearchStrings(ss, s)
    if len(ss) > i && ss[i] == s {
        return ss
    }
    ss = append(ss, "")
    copy(ss[i+1:], ss[i:])
    ss[i] = s
    return ss
}

func processZipFile(handle *Handle, id string) {
    reader, err := zip.OpenReader(path.Join("savedData", id, "base_data.zip"))
    if err != nil {
        // TODO add msg to error
        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
        fmt.Printf("Faield to proccess zip file failed to open reader\n")
        fmt.Println(err)
        return
    }
    defer reader.Close()

    training := []string{}
    testing := []string{}

    for _, file := range reader.Reader.File {

        paths := strings.Split(file.Name, "/")

        if paths[1] == "" {
            continue
        }

        if paths[0] != "training" && paths[0] != "testing" {
            fmt.Printf("Invalid file '%s' TODO add msg to response!!!\n", file.Name)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
        }

        if paths[0] != "training" {
            training = InsertIfNotPresent(training, paths[1])
        } else if paths[0] != "testing" {
            testing = InsertIfNotPresent(testing, paths[1])
        }
    }

    if !reflect.DeepEqual(testing, training) {
        fmt.Printf("testing and training are diferent\n")
        fmt.Println(testing)
        fmt.Println(training)
        modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
        return
    }

    base_path := path.Join("savedData", id, "data")

    ids := map[string]string{}

    for _, name := range training {
        dir_path := path.Join(base_path, "training", name)
		err = os.MkdirAll(dir_path, os.ModePerm)
        if err != nil {
            fmt.Printf("Failed to create dir %s\n", dir_path)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
        }
        dir_path = path.Join(base_path, "testing", name)
		err = os.MkdirAll(dir_path, os.ModePerm)
        if err != nil {
            fmt.Printf("Failed to create dir %s\n", dir_path)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
        }

        id, err := model_classes.CreateClass(handle.Db, id, name)
        if err != nil {
            fmt.Printf("Failed to create class '%s' on db\n", name)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
        }
        ids[name] = id
    }

    for _, file := range reader.Reader.File {
        if file.Name[len(file.Name) - 1] == '/' {
            continue
        }

        file_path := path.Join(base_path, file.Name)
		f, err := os.Create(file_path)
		if err != nil {
            fmt.Printf("Could not create file %s\n", file_path)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
		}
		defer f.Close()
        data, err := reader.Open(file.Name)
		if err != nil {
            fmt.Printf("Could not create file %s\n", file_path)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
		}
        defer data.Close()
        file_data, err := io.ReadAll(data)
		f.Write(file_data)

        parts := strings.Split(file.Name, "/")

        mode := model_classes.DATA_POINT_MODE_TRAINING
        if parts[0] == "testing" {
            mode = model_classes.DATA_POINT_MODE_TESTING
        }

        _, err = model_classes.AddDataPoint(handle.Db, ids[parts[1]], "file://" + parts[2], mode)
        if err != nil {
            fmt.Printf("Failed to add data point for %s\n", id)
            fmt.Println(err)
            modelUpdateStatus(handle, id, FAILED_PREPARING_ZIP_FILE)
            return
        }
    }
    
    fmt.Printf("Added data to model '%s'!\n", id)
    modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
}

func handleDataUpload(handle *Handle) {
	handle.Post("/models/data/upload", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
        if !CheckAuthLevel(1, w, r, c) {
            return nil
        }
		if c.Mode == JSON {
			// TODO improve message
			return ErrorCode(nil, 400, nil)
		}

		read_form, err := r.MultipartReader()
		if err != nil {
			LoadBasedOnAnswer(c.Mode, w, "models/add.html", c.AddMap(nil))
			return nil
		}

		var id string
		var file []byte

		for {
			part, err_part := read_form.NextPart()
			if err_part == io.EOF {
				break
			} else if err_part != nil {
				return &Error{Code: http.StatusBadRequest}
			}
			if part.FormName() == "id" {
				buf := new(bytes.Buffer)
				buf.ReadFrom(part)
				id = buf.String()
			}
			if part.FormName() == "file" {
				buf := new(bytes.Buffer)
				buf.ReadFrom(part)
				file = buf.Bytes()
			}
		}

        _, err = getBaseModel(handle.Db, id)
        if err == ModelNotFoundError {
			return ErrorCode(nil, http.StatusNotFound, AnyMap{
				"NotFoundMessage": "Model not found",
				"GoBackLink":      "/models",
			})
        } else if err != nil {
            return Error500(err)
        }

		// TODO mk this path configurable
		dir_path := path.Join("savedData", id)

		f, err := os.Create(path.Join(dir_path, "base_data.zip"))
		if err != nil {
			return Error500(err)
		}
		defer f.Close()

		f.Write(file)

        modelUpdateStatus(handle, id, PREPARING_ZIP_FILE)

        go processZipFile(handle, id)

        Redirect("/models/edit?id="+id, c.Mode, w, r)
		return nil
	})

    handle.Delete("/models/data/delete-zip-file", func(w http.ResponseWriter, r *http.Request, c *Context) *Error {
        if !CheckAuthLevel(1, w, r, c) {
            return nil
        }
        if c.Mode == JSON {
            panic("Handle delete zip file json")
        }

        f, err := MyParseForm(r)
        if err != nil {
            return ErrorCode(err, 400, c.AddMap(nil))
        }

        if !CheckId(f, "id") {
            return ErrorCode(err, 400, c.AddMap(nil))
        }

        id := f.Get("id")

        model, err := getBaseModel(handle.Db, id)
        if err == ModelNotFoundError {
			return ErrorCode(nil, http.StatusNotFound, AnyMap{
				"NotFoundMessage": "Model not found",
				"GoBackLink":      "/models",
			})
        } else if err != nil {
            return Error500(err)
        }

        if model.Status != FAILED_PREPARING_ZIP_FILE {
            // TODO add message
            return ErrorCode(nil, 400, c.AddMap(nil))
        }

        err = os.Remove(path.Join("savedData", id, "base_data.zip"));
        if err != nil {
            return Error500(err)
        }

        err = os.RemoveAll(path.Join("savedData", id, "data"));
        if err != nil {
            return Error500(err)
        }

        _, err = handle.Db.Exec("delete from model_classes where model_id=$1;", id)
        if err != nil {
            return Error500(err)
        }

        modelUpdateStatus(handle, id, CONFIRM_PRE_TRAINING)
        Redirect("/models/edit?id="+id, c.Mode, w, r)
        return nil
    })
}