From 516d1d7634d016349e6a432c7d6430e3a81e0052 Mon Sep 17 00:00:00 2001 From: Andre Henriques Date: Sun, 12 May 2024 15:29:36 +0100 Subject: [PATCH] added docker compose to run everything in one go --- DockerfileDev => DockerfileProxy | 2 +- README.md | 42 ++++++++++ config.toml | 5 ++ docker-compose.yml | 46 ++++++++--- logic/models/add.go | 12 ++- logic/models/data.go | 10 +-- logic/models/test.go | 2 + logic/utils/config.go | 7 +- main.go | 15 +--- nginx.dev.conf => nginx.proxy.conf | 4 +- sql/base.sql | 2 +- sql/models.sql | 2 +- webpage/.dockerignore | 1 + webpage/Dockerfile | 9 +++ webpage/package.json | 78 +++++++++---------- webpage/src/routes/admin/runners/+page.svelte | 32 ++++---- .../src/routes/admin/runners/CardInfo.svelte | 2 +- .../src/routes/models/edit/ModelTable.svelte | 4 +- 18 files changed, 184 insertions(+), 91 deletions(-) rename DockerfileDev => DockerfileProxy (75%) create mode 100644 README.md rename nginx.dev.conf => nginx.proxy.conf (84%) create mode 120000 webpage/.dockerignore create mode 100644 webpage/Dockerfile diff --git a/DockerfileDev b/DockerfileProxy similarity index 75% rename from DockerfileDev rename to DockerfileProxy index 6784fc5..37ba265 100644 --- a/DockerfileDev +++ b/DockerfileProxy @@ -1,6 +1,6 @@ # vi: ft=dockerfile FROM docker.io/nginx -ADD nginx.dev.conf /nginx.conf +ADD nginx.proxy.conf /nginx.conf CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..a65c7e5 --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# Configure the system + +Go to the config.toml file and setup your hostname + +# Build the containers + +Running this commands on the root of the project will setup the nessesary. + +Make sure that your docker/podman installation supports domain name resolution between containers + +```bash +docker build -t andre-fyp-proxy -f DockerfileProxy +docker build -t andre-fyp-server -f DockerfileServer +cd webpage +docker build -t andre-fyp-web-server . +cd .. +``` + +# Run the docker compose + +Running docker compose sets up the database server, the web page server, the proxy server and the main server + +```bash + docker compose up +``` + +# Setup the Database + +On another terminal instance create the database and tables. + +Note: the password can be changed in the docker-compose file + +```bash +PGPASSWORD=verysafepassword psql -h localhost -U postgres -f sql/base.sql +PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/user.sql +PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/models.sql +PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/tasks.sql +``` + +# Restart docker compose + +Now restart docker compose and the system should be available under the domain name set up on the config.toml file diff --git a/config.toml b/config.toml index 86ba94c..89d4777 100644 --- a/config.toml +++ b/config.toml @@ -16,3 +16,8 @@ NUMBER_OF_WORKERS = 1 [DB] MAX_CONNECTIONS = 600 +host = "db" +port = 5432 +user = "postgres" +password = "verysafepassword" +dbname = "fyp" diff --git a/docker-compose.yml b/docker-compose.yml index 9909955..b1cb945 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,11 +1,37 @@ -version: "3.1" - services: - db: - image: docker.andr3h3nriqu3s.com/services/postgres - command: -c 'max_connections=600' - restart: always - environment: - POSTGRES_PASSWORD: verysafepassword - ports: - - "5432:5432" + db: + image: docker.io/postgres:16.3 + command: -c 'max_connections=600' + restart: always + networks: + - fyp-network + environment: + POSTGRES_PASSWORD: verysafepassword + ports: + - "5432:5432" + web-page: + image: andre-fyp-web-server + hostname: webpage + networks: + - fyp-network + server: + image: andre-fyp-server + hostname: server + networks: + - fyp-network + depends_on: + - db + volumes: + - "./config.toml:/app/config.toml" + proxy-server: + image: andre-fyp-proxy + networks: + - fyp-network + ports: + - "8000:8000" + depends_on: + - web-page + - server + +networks: + fyp-network: {} diff --git a/logic/models/add.go b/logic/models/add.go index cb4b442..81d1350 100644 --- a/logic/models/add.go +++ b/logic/models/add.go @@ -54,21 +54,29 @@ func loadBaseImage(c *Context, id string) { model_color = "greyscale" case color.NRGBAModel: fallthrough + case color.RGBAModel: + fallthrough case color.YCbCrModel: model_color = "rgb" default: c.Logger.Error("Do not know how to handle this color model") if src.ColorModel() == color.RGBA64Model { - c.Logger.Error("Color is rgb") + c.Logger.Error("Color is rgb 64") } else if src.ColorModel() == color.NRGBA64Model { c.Logger.Error("Color is nrgb 64") } else if src.ColorModel() == color.AlphaModel { c.Logger.Error("Color is alpha") } else if src.ColorModel() == color.CMYKModel { c.Logger.Error("Color is cmyk") + } else if src.ColorModel() == color.NRGBA64Model { + c.Logger.Error("Color is cmyk") + } else if src.ColorModel() == color.NYCbCrAModel { + c.Logger.Error("Color is cmyk a") + } else if src.ColorModel() == color.Alpha16Model { + c.Logger.Error("Color is cmyk a") } else { - c.Logger.Error("Other so assuming color") + c.Logger.Error("Other so assuming color", "color mode", src.ColorModel()) } ModelUpdateStatus(c, id, FAILED_PREPARING) diff --git a/logic/models/data.go b/logic/models/data.go index cebdeab..a9af809 100644 --- a/logic/models/data.go +++ b/logic/models/data.go @@ -265,16 +265,15 @@ func processZipFileExpand(c *Context, model *BaseModel) { return } - if paths[0] != "training" { + if paths[0] == "training" { training = InsertIfNotPresent(training, paths[1]) - } else if paths[0] != "testing" { + } else if paths[0] == "testing" { testing = InsertIfNotPresent(testing, paths[1]) } } if !reflect.DeepEqual(testing, training) { - failed("testing and training are diferent") - return + c.GetLogger().Warn("testing and training differ", "testing", testing, "training", training) } base_path := path.Join("savedData", model.Id, "data") @@ -635,7 +634,8 @@ func handleDataUpload(handle *Handle) { // TODO work in allowing the model to add new in the pre ready moment if model.Status != READY { - return c.JsonBadRequest("Model not in the correct state to add a more classes") + c.GetLogger().Error("Model not in the ready status", "status", model.Status) + return c.JsonBadRequest("Model not in the correct state to add more classes") } // TODO mk this path configurable diff --git a/logic/models/test.go b/logic/models/test.go index fcb2ce5..1f63308 100644 --- a/logic/models/test.go +++ b/logic/models/test.go @@ -38,6 +38,8 @@ func TestImgForModel(c *Context, model *BaseModel, path string) (result bool) { model_color = "greyscale" case color.NRGBAModel: fallthrough + case color.RGBAModel: + fallthrough case color.YCbCrModel: model_color = "rgb" default: diff --git a/logic/utils/config.go b/logic/utils/config.go index dfb0c0b..86f6b04 100644 --- a/logic/utils/config.go +++ b/logic/utils/config.go @@ -23,7 +23,12 @@ type ServiceUser struct { } type DbInfo struct { - MaxConnections int `toml:"max_connections"` + MaxConnections int `toml:"max_connections"` + Host string `toml:"host"` + Port int `toml:"port"` + User string `toml:"user"` + Password string `toml:"password"` + Dbname string `toml:"dbname"` } type Config struct { diff --git a/main.go b/main.go index 2ba1973..740e98f 100644 --- a/main.go +++ b/main.go @@ -15,25 +15,18 @@ import ( . "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils" ) -const ( - host = "localhost" - port = 5432 - user = "postgres" - password = "verysafepassword" - dbname = "aistuff" -) - func main() { + config := LoadConfig() + log.Info("Config loaded!", "config", config) + psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+ "password=%s dbname=%s sslmode=disable", - host, port, user, password, dbname) + config.DbInfo.Host, config.DbInfo.Port, config.DbInfo.User, config.DbInfo.Password, config.DbInfo.Dbname) db := db.StartUp(psqlInfo) defer db.Close() - config := LoadConfig() - log.Info("Config loaded!", "config", config) config.GenerateToken(db) //TODO check if file structure exists to save data diff --git a/nginx.dev.conf b/nginx.proxy.conf similarity index 84% rename from nginx.dev.conf rename to nginx.proxy.conf index 96e6229..a4b9504 100644 --- a/nginx.dev.conf +++ b/nginx.proxy.conf @@ -17,7 +17,7 @@ http { location / { proxy_http_version 1.1; - proxy_pass http://127.0.0.1:5001; + proxy_pass http://webpage:5001; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; @@ -25,7 +25,7 @@ http { location /api { - proxy_pass http://127.0.0.1:5002; + proxy_pass http://server:5002; } } } diff --git a/sql/base.sql b/sql/base.sql index bc269ff..3ce9344 100644 --- a/sql/base.sql +++ b/sql/base.sql @@ -1 +1 @@ -CREATE DATABASE aistuff; +CREATE DATABASE fyp; diff --git a/sql/models.sql b/sql/models.sql index 130291e..62d7071 100644 --- a/sql/models.sql +++ b/sql/models.sql @@ -35,7 +35,7 @@ create table if not exists model_classes ( -- 1: to_train -- 2: training -- 3: trained - status integer default 1, + status integer default 1 ); -- drop table if exists model_data_point; diff --git a/webpage/.dockerignore b/webpage/.dockerignore new file mode 120000 index 0000000..3e4e48b --- /dev/null +++ b/webpage/.dockerignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/webpage/Dockerfile b/webpage/Dockerfile new file mode 100644 index 0000000..c6a2080 --- /dev/null +++ b/webpage/Dockerfile @@ -0,0 +1,9 @@ +FROM docker.io/node:22 + +ADD . . + +RUN npm install + +RUN npm run build + +CMD ["npm", "run", "preview"] diff --git a/webpage/package.json b/webpage/package.json index 262c7e6..a6b36a5 100644 --- a/webpage/package.json +++ b/webpage/package.json @@ -1,41 +1,41 @@ { - "name": "webpage", - "version": "0.0.1", - "private": true, - "scripts": { - "dev:raw": "vite dev", - "dev": "vite dev --port 5001 --host", - "build": "vite build", - "preview": "vite preview", - "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", - "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", - "lint": "prettier --check . && eslint .", - "format": "prettier --write ." - }, - "devDependencies": { - "@sveltejs/adapter-auto": "^3.2.0", - "@sveltejs/kit": "^2.5.6", - "@sveltejs/vite-plugin-svelte": "^3.0.0", - "@types/d3": "^7.4.3", - "@types/eslint": "^8.56.9", - "@typescript-eslint/eslint-plugin": "^7.7.0", - "@typescript-eslint/parser": "^7.7.0", - "eslint": "^8.57.0", - "eslint-config-prettier": "^9.1.0", - "eslint-plugin-svelte": "^2.37.0", - "prettier": "^3.2.5", - "prettier-plugin-svelte": "^3.2.3", - "sass": "^1.75.0", - "svelte": "^5.0.0-next.104", - "svelte-check": "^3.6.9", - "tslib": "^2.6.2", - "typescript": "^5.4.5", - "vite": "^5.2.8" - }, - "type": "module", - "dependencies": { - "chart.js": "^4.4.2", - "d3": "^7.9.0", - "highlight.js": "^11.9.0" - } + "name": "webpage", + "version": "0.0.1", + "private": true, + "scripts": { + "dev:raw": "vite dev", + "dev": "vite dev --port 5001 --host", + "build": "vite build", + "preview": "vite preview --port 5001 --host", + "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json", + "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch", + "lint": "prettier --check . && eslint .", + "format": "prettier --write ." + }, + "devDependencies": { + "@sveltejs/adapter-auto": "^3.2.0", + "@sveltejs/kit": "^2.5.6", + "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@types/d3": "^7.4.3", + "@types/eslint": "^8.56.9", + "@typescript-eslint/eslint-plugin": "^7.7.0", + "@typescript-eslint/parser": "^7.7.0", + "eslint": "^8.57.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-svelte": "^2.37.0", + "prettier": "^3.2.5", + "prettier-plugin-svelte": "^3.2.3", + "sass": "^1.75.0", + "svelte": "^5.0.0-next.104", + "svelte-check": "^3.6.9", + "tslib": "^2.6.2", + "typescript": "^5.4.5", + "vite": "^5.2.8" + }, + "type": "module", + "dependencies": { + "chart.js": "^4.4.2", + "d3": "^7.9.0", + "highlight.js": "^11.9.0" + } } diff --git a/webpage/src/routes/admin/runners/+page.svelte b/webpage/src/routes/admin/runners/+page.svelte index 6f4347e..be5a8b6 100644 --- a/webpage/src/routes/admin/runners/+page.svelte +++ b/webpage/src/routes/admin/runners/+page.svelte @@ -55,7 +55,7 @@ if (Object.keys(data.localRunners).length > 0) { for (const objId of Object.keys(data.localRunners)) { - localRunners.push({ name: objId, type: 'local_runner' }); + localRunners.push({ name: objId, type: 'local_runner', task: data.localRunners[objId] }); } dataObj.children.push({ @@ -65,24 +65,26 @@ }); } - for (const objId of Object.keys(data.remoteRunners)) { - let obj = data.remoteRunners[objId]; - if (remotePairs[obj.runner_info.user_id as string]) { - remotePairs[obj.runner_info.user_id as string].push({ - name: objId, - type: 'runner', - task: obj.task, - parent: data.remoteRunners[objId].runner_info.user_id - }); - } else { - remotePairs[data.remoteRunners[objId].runner_info.user_id] = [ - { + if (Object.keys(data.remoteRunners).length > 0) { + for (const objId of Object.keys(data.remoteRunners)) { + let obj = data.remoteRunners[objId]; + if (remotePairs[obj.runner_info.user_id as string]) { + remotePairs[obj.runner_info.user_id as string].push({ name: objId, type: 'runner', task: obj.task, parent: data.remoteRunners[objId].runner_info.user_id - } - ]; + }); + } else { + remotePairs[data.remoteRunners[objId].runner_info.user_id] = [ + { + name: objId, + type: 'runner', + task: obj.task, + parent: data.remoteRunners[objId].runner_info.user_id + } + ]; + } } } diff --git a/webpage/src/routes/admin/runners/CardInfo.svelte b/webpage/src/routes/admin/runners/CardInfo.svelte index 244ee6b..b225dd8 100644 --- a/webpage/src/routes/admin/runners/CardInfo.svelte +++ b/webpage/src/routes/admin/runners/CardInfo.svelte @@ -48,7 +48,7 @@ This is a local runner
{#if item.task} - test + This runner is runing a task {:else} Not running any task {/if} diff --git a/webpage/src/routes/models/edit/ModelTable.svelte b/webpage/src/routes/models/edit/ModelTable.svelte index 6bc6154..d7da405 100644 --- a/webpage/src/routes/models/edit/ModelTable.svelte +++ b/webpage/src/routes/models/edit/ModelTable.svelte @@ -32,10 +32,10 @@ }); async function getList() { - console.log(page); + if (!selected_class) return; try { let res = await post('models/data/list', { - id: selected_class?.id ?? '', + id: selected_class.id, page: page }); showNext = res.showNext;