added docker compose to run everything in one go

This commit is contained in:
Andre Henriques 2024-05-12 15:29:36 +01:00
parent 0c0d16c846
commit 516d1d7634
18 changed files with 184 additions and 91 deletions

View File

@ -1,6 +1,6 @@
# vi: ft=dockerfile
FROM docker.io/nginx
ADD nginx.dev.conf /nginx.conf
ADD nginx.proxy.conf /nginx.conf
CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"]

42
README.md Normal file
View File

@ -0,0 +1,42 @@
# Configure the system
Go to the config.toml file and setup your hostname
# Build the containers
Running this commands on the root of the project will setup the nessesary.
Make sure that your docker/podman installation supports domain name resolution between containers
```bash
docker build -t andre-fyp-proxy -f DockerfileProxy
docker build -t andre-fyp-server -f DockerfileServer
cd webpage
docker build -t andre-fyp-web-server .
cd ..
```
# Run the docker compose
Running docker compose sets up the database server, the web page server, the proxy server and the main server
```bash
docker compose up
```
# Setup the Database
On another terminal instance create the database and tables.
Note: the password can be changed in the docker-compose file
```bash
PGPASSWORD=verysafepassword psql -h localhost -U postgres -f sql/base.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/user.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/models.sql
PGPASSWORD=verysafepassword psql -h localhost -U postgres -d fyp -f sql/tasks.sql
```
# Restart docker compose
Now restart docker compose and the system should be available under the domain name set up on the config.toml file

View File

@ -16,3 +16,8 @@ NUMBER_OF_WORKERS = 1
[DB]
MAX_CONNECTIONS = 600
host = "db"
port = 5432
user = "postgres"
password = "verysafepassword"
dbname = "fyp"

View File

@ -1,11 +1,37 @@
version: "3.1"
services:
db:
image: docker.andr3h3nriqu3s.com/services/postgres
command: -c 'max_connections=600'
restart: always
environment:
POSTGRES_PASSWORD: verysafepassword
ports:
- "5432:5432"
db:
image: docker.io/postgres:16.3
command: -c 'max_connections=600'
restart: always
networks:
- fyp-network
environment:
POSTGRES_PASSWORD: verysafepassword
ports:
- "5432:5432"
web-page:
image: andre-fyp-web-server
hostname: webpage
networks:
- fyp-network
server:
image: andre-fyp-server
hostname: server
networks:
- fyp-network
depends_on:
- db
volumes:
- "./config.toml:/app/config.toml"
proxy-server:
image: andre-fyp-proxy
networks:
- fyp-network
ports:
- "8000:8000"
depends_on:
- web-page
- server
networks:
fyp-network: {}

View File

@ -54,21 +54,29 @@ func loadBaseImage(c *Context, id string) {
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.RGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:
c.Logger.Error("Do not know how to handle this color model")
if src.ColorModel() == color.RGBA64Model {
c.Logger.Error("Color is rgb")
c.Logger.Error("Color is rgb 64")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Error("Color is nrgb 64")
} else if src.ColorModel() == color.AlphaModel {
c.Logger.Error("Color is alpha")
} else if src.ColorModel() == color.CMYKModel {
c.Logger.Error("Color is cmyk")
} else if src.ColorModel() == color.NRGBA64Model {
c.Logger.Error("Color is cmyk")
} else if src.ColorModel() == color.NYCbCrAModel {
c.Logger.Error("Color is cmyk a")
} else if src.ColorModel() == color.Alpha16Model {
c.Logger.Error("Color is cmyk a")
} else {
c.Logger.Error("Other so assuming color")
c.Logger.Error("Other so assuming color", "color mode", src.ColorModel())
}
ModelUpdateStatus(c, id, FAILED_PREPARING)

View File

@ -265,16 +265,15 @@ func processZipFileExpand(c *Context, model *BaseModel) {
return
}
if paths[0] != "training" {
if paths[0] == "training" {
training = InsertIfNotPresent(training, paths[1])
} else if paths[0] != "testing" {
} else if paths[0] == "testing" {
testing = InsertIfNotPresent(testing, paths[1])
}
}
if !reflect.DeepEqual(testing, training) {
failed("testing and training are diferent")
return
c.GetLogger().Warn("testing and training differ", "testing", testing, "training", training)
}
base_path := path.Join("savedData", model.Id, "data")
@ -635,7 +634,8 @@ func handleDataUpload(handle *Handle) {
// TODO work in allowing the model to add new in the pre ready moment
if model.Status != READY {
return c.JsonBadRequest("Model not in the correct state to add a more classes")
c.GetLogger().Error("Model not in the ready status", "status", model.Status)
return c.JsonBadRequest("Model not in the correct state to add more classes")
}
// TODO mk this path configurable

View File

@ -38,6 +38,8 @@ func TestImgForModel(c *Context, model *BaseModel, path string) (result bool) {
model_color = "greyscale"
case color.NRGBAModel:
fallthrough
case color.RGBAModel:
fallthrough
case color.YCbCrModel:
model_color = "rgb"
default:

View File

@ -23,7 +23,12 @@ type ServiceUser struct {
}
type DbInfo struct {
MaxConnections int `toml:"max_connections"`
MaxConnections int `toml:"max_connections"`
Host string `toml:"host"`
Port int `toml:"port"`
User string `toml:"user"`
Password string `toml:"password"`
Dbname string `toml:"dbname"`
}
type Config struct {

15
main.go
View File

@ -15,25 +15,18 @@ import (
. "git.andr3h3nriqu3s.com/andr3/fyp/logic/utils"
)
const (
host = "localhost"
port = 5432
user = "postgres"
password = "verysafepassword"
dbname = "aistuff"
)
func main() {
config := LoadConfig()
log.Info("Config loaded!", "config", config)
psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
"password=%s dbname=%s sslmode=disable",
host, port, user, password, dbname)
config.DbInfo.Host, config.DbInfo.Port, config.DbInfo.User, config.DbInfo.Password, config.DbInfo.Dbname)
db := db.StartUp(psqlInfo)
defer db.Close()
config := LoadConfig()
log.Info("Config loaded!", "config", config)
config.GenerateToken(db)
//TODO check if file structure exists to save data

View File

@ -17,7 +17,7 @@ http {
location / {
proxy_http_version 1.1;
proxy_pass http://127.0.0.1:5001;
proxy_pass http://webpage:5001;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection $connection_upgrade;
@ -25,7 +25,7 @@ http {
location /api {
proxy_pass http://127.0.0.1:5002;
proxy_pass http://server:5002;
}
}
}

View File

@ -1 +1 @@
CREATE DATABASE aistuff;
CREATE DATABASE fyp;

View File

@ -35,7 +35,7 @@ create table if not exists model_classes (
-- 1: to_train
-- 2: training
-- 3: trained
status integer default 1,
status integer default 1
);
-- drop table if exists model_data_point;

1
webpage/.dockerignore Symbolic link
View File

@ -0,0 +1 @@
.gitignore

9
webpage/Dockerfile Normal file
View File

@ -0,0 +1,9 @@
FROM docker.io/node:22
ADD . .
RUN npm install
RUN npm run build
CMD ["npm", "run", "preview"]

View File

@ -1,41 +1,41 @@
{
"name": "webpage",
"version": "0.0.1",
"private": true,
"scripts": {
"dev:raw": "vite dev",
"dev": "vite dev --port 5001 --host",
"build": "vite build",
"preview": "vite preview",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/d3": "^7.4.3",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.37.0",
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"sass": "^1.75.0",
"svelte": "^5.0.0-next.104",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.8"
},
"type": "module",
"dependencies": {
"chart.js": "^4.4.2",
"d3": "^7.9.0",
"highlight.js": "^11.9.0"
}
"name": "webpage",
"version": "0.0.1",
"private": true,
"scripts": {
"dev:raw": "vite dev",
"dev": "vite dev --port 5001 --host",
"build": "vite build",
"preview": "vite preview --port 5001 --host",
"check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
"check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
"lint": "prettier --check . && eslint .",
"format": "prettier --write ."
},
"devDependencies": {
"@sveltejs/adapter-auto": "^3.2.0",
"@sveltejs/kit": "^2.5.6",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@types/d3": "^7.4.3",
"@types/eslint": "^8.56.9",
"@typescript-eslint/eslint-plugin": "^7.7.0",
"@typescript-eslint/parser": "^7.7.0",
"eslint": "^8.57.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-svelte": "^2.37.0",
"prettier": "^3.2.5",
"prettier-plugin-svelte": "^3.2.3",
"sass": "^1.75.0",
"svelte": "^5.0.0-next.104",
"svelte-check": "^3.6.9",
"tslib": "^2.6.2",
"typescript": "^5.4.5",
"vite": "^5.2.8"
},
"type": "module",
"dependencies": {
"chart.js": "^4.4.2",
"d3": "^7.9.0",
"highlight.js": "^11.9.0"
}
}

View File

@ -55,7 +55,7 @@
if (Object.keys(data.localRunners).length > 0) {
for (const objId of Object.keys(data.localRunners)) {
localRunners.push({ name: objId, type: 'local_runner' });
localRunners.push({ name: objId, type: 'local_runner', task: data.localRunners[objId] });
}
dataObj.children.push({
@ -65,24 +65,26 @@
});
}
for (const objId of Object.keys(data.remoteRunners)) {
let obj = data.remoteRunners[objId];
if (remotePairs[obj.runner_info.user_id as string]) {
remotePairs[obj.runner_info.user_id as string].push({
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
});
} else {
remotePairs[data.remoteRunners[objId].runner_info.user_id] = [
{
if (Object.keys(data.remoteRunners).length > 0) {
for (const objId of Object.keys(data.remoteRunners)) {
let obj = data.remoteRunners[objId];
if (remotePairs[obj.runner_info.user_id as string]) {
remotePairs[obj.runner_info.user_id as string].push({
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
}
];
});
} else {
remotePairs[data.remoteRunners[objId].runner_info.user_id] = [
{
name: objId,
type: 'runner',
task: obj.task,
parent: data.remoteRunners[objId].runner_info.user_id
}
];
}
}
}

View File

@ -48,7 +48,7 @@
This is a local runner
<div>
{#if item.task}
test
This runner is runing a <Tooltip title={item.task.id}>task</Tooltip>
{:else}
Not running any task
{/if}

View File

@ -32,10 +32,10 @@
});
async function getList() {
console.log(page);
if (!selected_class) return;
try {
let res = await post('models/data/list', {
id: selected_class?.id ?? '',
id: selected_class.id,
page: page
});
showNext = res.showNext;