diff --git a/.github/workflows/docker_build_cohorter.yml b/.github/workflows/docker_build_cohorter.yml index 0f9c756e..2d161d6b 100644 --- a/.github/workflows/docker_build_cohorter.yml +++ b/.github/workflows/docker_build_cohorter.yml @@ -26,12 +26,12 @@ jobs: - name: nl2dsl context: apps/cohorter/NL2DSL dockerfile: apps/cohorter/NL2DSL/Dockerfile - image: cogstacksystems/cogstack-cohorter-nl2dsl + image: cogstacksystems/cohorter-nl2dsl - name: webapp context: apps/cohorter/WebAPP dockerfile: apps/cohorter/WebAPP/Dockerfile - image: cogstacksystems/cogstack-cohorter-webapp + image: cogstacksystems/cohorter-webapp steps: - name: Checkout branch @@ -43,6 +43,9 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 @@ -67,6 +70,7 @@ jobs: with: context: ${{ matrix.context }} file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/docker_build_nginx.yml b/.github/workflows/docker_build_nginx.yml new file mode 100644 index 00000000..ac593dce --- /dev/null +++ b/.github/workflows/docker_build_nginx.yml @@ -0,0 +1,54 @@ +name: docker_build_nginx + +on: + push: + branches: [main] + tags: ["v*.*.*"] + paths: + - "apps/nginx/**" + - ".github/workflows/docker_build_nginx.yml" + pull_request: + paths: + - "apps/nginx/**" + - ".github/workflows/docker_build_nginx.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + docker: + runs-on: ubuntu-latest + if: github.event.pull_request.user.login != 'dependabot[bot]' && github.repository == 'CogStack/cogstack-platform' + steps: + - uses: actions/checkout@v7 + - uses: docker/setup-qemu-action@v4 + - uses: docker/setup-buildx-action@v4 + + - uses: docker/login-action@v4 + if: github.event_name != 'pull_request' + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - uses: docker/metadata-action@v6 + id: meta + with: + images: cogstacksystems/nginx + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=sha + + - uses: docker/build-push-action@v7 + with: + context: apps/nginx + file: apps/nginx/Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=cogstack-nginx + cache-to: type=gha,scope=cogstack-nginx,mode=max diff --git a/.github/workflows/docker_build_observability.yml b/.github/workflows/docker_build_observability.yml index dfe686f8..b14cad5b 100644 --- a/.github/workflows/docker_build_observability.yml +++ b/.github/workflows/docker_build_observability.yml @@ -7,11 +7,13 @@ on: - "v*.*.*" # e.g., v0.1.1 paths: - "apps/observability/**" + - "apps/nginx/**" - "deployment/examples/observability/**" - ".github/workflows/docker_build_observability.yml" pull_request: paths: - "apps/observability/**" + - "apps/nginx/**" - "deployment/examples/observability/**" - ".github/workflows/docker_build_observability.yml" @@ -37,10 +39,6 @@ jobs: context: apps/observability/grafana dockerfile: apps/observability/grafana/Dockerfile image: cogstacksystems/cogstack-observability-grafana - - name: traefik - context: apps/observability/traefik - dockerfile: apps/observability/traefik/Dockerfile - image: cogstacksystems/cogstack-observability-traefik - name: alloy context: apps/observability/grafana-alloy dockerfile: apps/observability/grafana-alloy/Dockerfile @@ -93,6 +91,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v7 + - name: Build NGINX for integration tests + run: docker build -t cogstacksystems/nginx:test apps/nginx + - name: Set up Python uses: actions/setup-python@v7 with: @@ -102,32 +103,19 @@ jobs: - name: Install Python dependencies run: pip install -r deployment/examples/observability/test/requirements.txt - - name: Run Simple quickstart test + - name: Run quickstart tests run: | - echo "πŸ§ͺ Running Simple quickstart test..." + echo "πŸ§ͺ Running quickstart tests..." IMAGE_TAG="sha-${GITHUB_SHA::7}" echo "Using images tagged ${IMAGE_TAG}" export OBSERVABILITY_BASE_URL="file://${GITHUB_WORKSPACE}/deployment/examples/observability" - export COMPOSE_FILE="${GITHUB_WORKSPACE}/deployment/examples/observability/simple/observability-simple/docker-compose.yml:${RUNNER_TEMP}/observability-images.yml" + export COMPOSE_FILE="docker-compose.yml:${RUNNER_TEMP}/observability-images.yml" { echo "services:" - for service in alloy prometheus grafana traefik; do + for service in alloy prometheus grafana; do printf ' %s:\n image: cogstacksystems/cogstack-observability-%s:%s\n' \ "$service" "$service" "$IMAGE_TAG" done + printf ' nginx:\n image: cogstacksystems/nginx:test\n' } > "${RUNNER_TEMP}/observability-images.yml" - if ! bash deployment/examples/observability/test/test-quickstart.sh; then - docker compose ps --all || true - docker compose logs --no-color --tail=100 || true - exit 1 - fi - - - name: Report test result - if: always() - run: | - if [ $? -eq 0 ]; then - echo "βœ… quickstart test PASSED" - else - echo "❌ quickstart test FAILED" - exit 1 - fi + bash deployment/examples/observability/test/test-quickstart.sh diff --git a/.github/workflows/docker_build_ocr_service.yml b/.github/workflows/docker_build_ocr_service.yml index e3e1d6f1..44386997 100644 --- a/.github/workflows/docker_build_ocr_service.yml +++ b/.github/workflows/docker_build_ocr_service.yml @@ -29,7 +29,7 @@ jobs: if: github.event_name != 'release' || startsWith(github.event.release.tag_name, 'ocr-service-v') uses: ./.github/workflows/docker_template_ocr_service.yml with: - image: cogstacksystems/cogstack-ocr-service + image: cogstacksystems/ocr-service context: apps/ocr-service dockerfile: apps/ocr-service/Dockerfile cache_scope: cogstack-ocr-service diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 1492d8b9..d4e780b0 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -4,7 +4,7 @@ "deployment/charts/ce": "0.7.1", "deployment/charts/cohorter": "0.5.0", "deployment/charts/observability": "0.4.1", - "deployment/charts/jupyterhub": "0.5.0", + "deployment/charts/jupyter-hub": "0.5.0", "deployment/charts/ocr-service": "0.1.0", "deployment/charts/nifi": "0.0.0", "deployment/charts/nginx": "0.0.0", diff --git a/apps/nginx/.dockerignore b/apps/nginx/.dockerignore new file mode 100644 index 00000000..509eed6f --- /dev/null +++ b/apps/nginx/.dockerignore @@ -0,0 +1,6 @@ +** +!Dockerfile +!LICENSE +!nginx.conf +!profiles/ +!profiles/** diff --git a/apps/nginx/Dockerfile b/apps/nginx/Dockerfile new file mode 100644 index 00000000..28b07d1e --- /dev/null +++ b/apps/nginx/Dockerfile @@ -0,0 +1,38 @@ +FROM nginx:1.31.9@sha256:84f7876d21d154885f99554d7ae325d8c2213f9169726ea7ab60642f2573a787 + +ARG NGINX_UID=1000 +ARG NGINX_GID=1000 + +COPY nginx.conf /etc/nginx/nginx.conf +COPY profiles/ /etc/nginx/templates/ +COPY LICENSE /usr/share/doc/cogstack-nginx/LICENSE + +ENV HOME=/tmp \ + NGINX_ENVSUBST_TEMPLATE_DIR=/etc/nginx/templates/nifi \ + NGINX_ENVSUBST_OUTPUT_DIR=/tmp \ + NGINX_ENVSUBST_FILTER="^(NGINX_|NIFI_|MEDCAT_)" \ + NGINX_ENTRYPOINT_LOCAL_RESOLVERS=1 \ + NGINX_HEALTH_PORT=8080 \ + NIFI_UPSTREAM_HOST=nifi \ + NIFI_UPSTREAM_PORT=8443 \ + NIFI_UPSTREAM_TLS_NAME=nifi \ + NIFI_INTERNAL_PORT_NGINX=8443 \ + NIFI_WEB_PROXY_PORT=8443 \ + NGINX_NIFI_SERVER_NAME=_ \ + NGINX_PROXY_CONTEXT_PATH="" \ + NGINX_PROXY_SCHEME=https \ + NGINX_TLS_CERTIFICATE=/certificates/nifi/nifi.pem \ + NGINX_TLS_CERTIFICATE_KEY=/certificates/nifi/nifi.key \ + NGINX_UPSTREAM_CA=/certificates/root/root-ca.pem \ + MEDCAT_LISTEN_PORT=8001 \ + MEDCAT_SERVER_NAME=_ \ + MEDCAT_UPSTREAM_HOST=127.0.0.1 \ + MEDCAT_UPSTREAM_PORT=8000 \ + MEDCAT_SOLR_HOST=solr \ + MEDCAT_SOLR_PORT=8983 \ + MEDCAT_STATIC_ROOT=/home/api/static \ + MEDCAT_MEDIA_ROOT=/home/api/media + +USER ${NGINX_UID}:${NGINX_GID} +HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ + CMD ["sh", "-c", "exec curl -fsS --noproxy '*' --max-time 5 -o /dev/null \"http://127.0.0.1:${NGINX_HEALTH_PORT}/healthz/ready\""] diff --git a/apps/nginx/LICENSE b/apps/nginx/LICENSE new file mode 100644 index 00000000..cd1d6840 --- /dev/null +++ b/apps/nginx/LICENSE @@ -0,0 +1,94 @@ + +# Elastic License 2.0 + +URL: + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/apps/nginx/README.md b/apps/nginx/README.md new file mode 100644 index 00000000..92217664 --- /dev/null +++ b/apps/nginx/README.md @@ -0,0 +1,67 @@ +# CogStack NGINX + +One image with NiFi, MedCAT-trainer and observability configurations, selected per container. +It uses the upstream image's entrypoint, environment substitution and DNS +resolver discovery. There is no custom entrypoint or configuration framework. + +Adapted from [CogStack/CogStack-NiFi services/nginx](https://github.com/CogStack/CogStack-NiFi/tree/71b3d483747a504986205addb4d2b17f58f19a0b/services/nginx) +at `71b3d483747a504986205addb4d2b17f58f19a0b`. NiFi routes are retained; +startup uses NGINX's existing implementation and the health check invokes curl. +Trainer routes come from the existing MedCAT-trainer Helm chart. These are +modified copies; see `LICENSE`. + +Build from the repository root: + +```bash +docker build -t cogstacksystems/nginx:local apps/nginx +``` + +The image is published independently by [docker_build_nginx.yml](../../.github/workflows/docker_build_nginx.yml) +as `cogstacksystems/nginx` for `linux/amd64` and `linux/arm64`. + +Select the configuration with the upstream image's variable: + +| Application | `NGINX_ENVSUBST_TEMPLATE_DIR` | +| --- | --- | +| NiFi (default) | `/etc/nginx/templates/nifi` | +| MedCAT-trainer | `/etc/nginx/templates/medcat-trainer` | +| Observability | `/etc/nginx/templates/observability` | + +All environment defaults are declared in `Dockerfile`. Treat template values +as trusted deployment configuration. The substitution filter preserves NGINX +variables such as `$host`. + +For NiFi, mount the frontend certificate/key at `/certificates/nifi/nifi.pem` +and `/certificates/nifi/nifi.key`, and the upstream CA at +`/certificates/root/root-ca.pem`. Set `NIFI_UPSTREAM_HOST`, +`NIFI_UPSTREAM_PORT`, `NIFI_UPSTREAM_TLS_NAME` and `NIFI_WEB_PROXY_PORT` for +the deployment. TLS verification stays enabled. An optional +`NGINX_PROXY_CONTEXT_PATH` must start with `/` and have no trailing slash. + +For MedCAT-trainer, mount static/media directories at `/home/api/static` and +`/home/api/media`. Set `MEDCAT_UPSTREAM_HOST` and `MEDCAT_SOLR_HOST` to the +backend service names. Defaults use ports 8000 and 8983, with NGINX listening +on 8001. In a trainer Pod, the backend host can remain `127.0.0.1`. The +configuration preserves Solr path rewriting, gzip, the `4000m` upload limit +and `3000s` read/send timeouts for model loading and transfers. Connections +use NGINX's default `60s` timeout; idle client keepalive uses `65s`. +TLS terminates at the deployment's external gateway. + +For observability, NGINX listens on 8001 and routes `/grafana/`, `/prometheus/` +and `/alloy/` to their Compose services. `/blackbox-exporter/` is available when +a standalone `blackbox-exporter` service is deployed. Grafana and blackbox +prefixes are stripped; Prometheus and Alloy prefixes are preserved. + +Access logs include request time (`rt`), upstream response time (`urt`) and +upstream status (`us`). + +The image runs as UID/GID `1000:1000`; build arguments `NGINX_UID` and +`NGINX_GID` can change them. Mount readable certificates and files. A read-only +root filesystem works with writable `/tmp`; size temporary storage for the +deployment's buffered uploads and responses. + +Health endpoints listen on loopback port 8080 (`NGINX_HEALTH_PORT`). +`/healthz/live` checks NGINX alone; `/healthz/ready` checks the application +backend. Docker's health check uses readiness; Kubernetes exec probes can +call curl directly. The health port must differ from the application port. +Existing chart image defaults are unchanged. diff --git a/apps/nginx/nginx.conf b/apps/nginx/nginx.conf new file mode 100644 index 00000000..61300366 --- /dev/null +++ b/apps/nginx/nginx.conf @@ -0,0 +1,27 @@ +worker_processes auto; +pid /tmp/nginx.pid; +error_log /dev/stderr notice; + +events { worker_connections 1024; } + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" "$http_user_agent" ' + 'rt=$request_time urt=$upstream_response_time us=$upstream_status'; + access_log /dev/stdout main; + server_tokens off; + server_name_in_redirect off; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 8192; + client_body_temp_path /tmp/client_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + include /tmp/nginx.conf; +} diff --git a/apps/nginx/profiles/medcat-trainer/nginx.conf.template b/apps/nginx/profiles/medcat-trainer/nginx.conf.template new file mode 100644 index 00000000..2d1c6bb6 --- /dev/null +++ b/apps/nginx/profiles/medcat-trainer/nginx.conf.template @@ -0,0 +1,58 @@ +# Adapted from deployment/charts/medcat-trainer/templates/nginx-configmap.yaml. +resolver ${NGINX_LOCAL_RESOLVERS} valid=10s; +resolver_timeout 3s; +upstream medcat { + zone medcat 64k; + server ${MEDCAT_UPSTREAM_HOST}:${MEDCAT_UPSTREAM_PORT} resolve; +} +upstream medcat_solr { + zone medcat_solr 64k; + server ${MEDCAT_SOLR_HOST}:${MEDCAT_SOLR_PORT} resolve; +} + +server { + listen 127.0.0.1:${NGINX_HEALTH_PORT}; + access_log off; + location = /healthz/live { return 200 'ok\n'; } + location = /healthz/ready { + proxy_set_header Host localhost; + proxy_connect_timeout 3s; + proxy_read_timeout 3s; + proxy_pass http://medcat/api/health/ready/; + } +} + +server { + listen ${MEDCAT_LISTEN_PORT}; + server_name ${MEDCAT_SERVER_NAME}; + charset utf-8; + large_client_header_buffers 4 32k; + client_max_body_size 4000m; + proxy_send_timeout 3000s; + proxy_read_timeout 3000s; + send_timeout 3000s; + + gzip on; + gzip_comp_level 5; + gzip_min_length 256; + gzip_proxied any; + gzip_vary on; + gzip_types application/atom+xml application/javascript application/json + application/ld+json application/manifest+json application/rss+xml + application/vnd.geo+json application/vnd.ms-fontobject application/x-font-ttf + application/x-web-app-manifest+json application/xhtml+xml application/xml + font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest + text/css text/plain text/vcard text/vnd.rim.location.xloc text/vtt + text/x-component text/x-cross-domain-policy; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + location = /static { return 301 /static/; } + location /static/ { alias "${MEDCAT_STATIC_ROOT}/"; } + location = /media { return 301 /media/; } + location /media/ { alias "${MEDCAT_MEDIA_ROOT}/"; } + location /api/concepts/ { proxy_pass http://medcat_solr/solr/; } + location / { proxy_pass http://medcat; } + location = /nginx/health/live { access_log off; return 200 'OK'; } +} diff --git a/apps/nginx/profiles/nifi/nginx.conf.template b/apps/nginx/profiles/nifi/nginx.conf.template new file mode 100644 index 00000000..15941bf7 --- /dev/null +++ b/apps/nginx/profiles/nifi/nginx.conf.template @@ -0,0 +1,51 @@ +# Adapted from CogStack/CogStack-NiFi services/nginx; see README.md. +resolver ${NGINX_LOCAL_RESOLVERS} valid=10s; +resolver_timeout 3s; +upstream nifi { + zone nifi 64k; + server ${NIFI_UPSTREAM_HOST}:${NIFI_UPSTREAM_PORT} resolve; +} + +proxy_ssl_server_name on; +proxy_ssl_name ${NIFI_UPSTREAM_TLS_NAME}; +proxy_ssl_trusted_certificate "${NGINX_UPSTREAM_CA}"; +proxy_ssl_verify on; +proxy_ssl_verify_depth 3; +proxy_ssl_protocols TLSv1.2 TLSv1.3; + +server { + listen 127.0.0.1:${NGINX_HEALTH_PORT}; + access_log off; + location = /healthz/live { return 200 'ok\n'; } + location = /healthz/ready { + proxy_set_header Host ${NIFI_UPSTREAM_TLS_NAME}; + proxy_connect_timeout 3s; + proxy_read_timeout 3s; + proxy_pass https://nifi/nifi-api/authentication/configuration; + } +} + +server { + listen ${NIFI_INTERNAL_PORT_NGINX} ssl; + server_name ${NGINX_NIFI_SERVER_NAME}; + ssl_certificate "${NGINX_TLS_CERTIFICATE}"; + ssl_certificate_key "${NGINX_TLS_CERTIFICATE_KEY}"; + ssl_protocols TLSv1.2 TLSv1.3; + + proxy_set_header Host ${NIFI_UPSTREAM_TLS_NAME}; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-ProxyHost $host; + proxy_set_header X-ProxyPort ${NIFI_WEB_PROXY_PORT}; + proxy_set_header X-ProxyScheme ${NGINX_PROXY_SCHEME}; + proxy_set_header X-ProxyContextPath "${NGINX_PROXY_CONTEXT_PATH}"; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto ${NGINX_PROXY_SCHEME}; + + location ${NGINX_PROXY_CONTEXT_PATH}/ { + proxy_pass https://nifi/; + } + location ${NGINX_PROXY_CONTEXT_PATH}/nifi-api/ { + proxy_hide_header X-Frame-Options; + proxy_pass https://nifi/nifi-api/; + } +} diff --git a/apps/nginx/profiles/observability/nginx.conf.template b/apps/nginx/profiles/observability/nginx.conf.template new file mode 100644 index 00000000..af835e69 --- /dev/null +++ b/apps/nginx/profiles/observability/nginx.conf.template @@ -0,0 +1,48 @@ +resolver ${NGINX_LOCAL_RESOLVERS} valid=10s; +resolver_timeout 3s; +upstream grafana { zone grafana 64k; server grafana:3000 resolve; } +upstream prometheus { zone prometheus 64k; server prometheus:9090 resolve; } +upstream alloy { zone alloy 64k; server alloy:12345 resolve; } + +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen 127.0.0.1:${NGINX_HEALTH_PORT}; + access_log off; + location = /healthz/live { return 200 'ok\n'; } + location = /healthz/ready { + proxy_connect_timeout 3s; + proxy_read_timeout 3s; + proxy_pass http://grafana/api/health; + } +} + +server { + listen 8001; + absolute_redirect off; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + + location = /grafana { return 308 /grafana/$is_args$args; } + location /grafana/ { proxy_pass http://grafana/; } + location = /prometheus { return 308 /prometheus/$is_args$args; } + location /prometheus/ { proxy_pass http://prometheus; } + location = /alloy { return 308 /alloy/$is_args$args; } + location /alloy/ { proxy_pass http://alloy; } + location = /blackbox-exporter { return 308 /blackbox-exporter/$is_args$args; } + location /blackbox-exporter/ { + # Resolve the optional exporter only when requested. + set $blackbox blackbox-exporter:9115; + rewrite ^/blackbox-exporter/(.*)$ /$1 break; + proxy_pass http://$blackbox; + } + location / { return 404; } +} diff --git a/apps/observability/README.md b/apps/observability/README.md index 78251110..7c20d16b 100644 --- a/apps/observability/README.md +++ b/apps/observability/README.md @@ -18,9 +18,19 @@ See the [full documentation](../../docs/docs/platform/observability/index.md). From the repository root, run the stack locally with ``` cd ./deployment/examples/observability/simple -docker compose up -d +docker compose up -d --remove-orphans ``` This will start the observability stack, set to probe and monitor itself. -Access Grafana on `http://localhost/grafana` \ No newline at end of file +Access Grafana on `http://localhost/grafana` + +The stack uses the [shared NGINX image](../nginx) with its observability profile. +For local rebuilds, build the proxy separately from the repository root with +`docker build -t cogstacksystems/nginx:latest apps/nginx`. +Host port 80 maps to NGINX port 8001; the existing Grafana, Prometheus and Alloy +URLs are retained. Availability probes run through NGINX. The former Traefik +metrics scrape and dashboard are removed; proxy timings are in NGINX access logs. +When upgrading an existing stack, remove its orphaned `traefik` container with +`docker compose up -d --remove-orphans`. Retained Grafana volumes may still hold +the old Traefik dashboard because the default dashboard provider disables deletion. diff --git a/apps/observability/build-docker.sh b/apps/observability/build-docker.sh index 20030500..339d7ef2 100644 --- a/apps/observability/build-docker.sh +++ b/apps/observability/build-docker.sh @@ -1,3 +1,5 @@ +#!/bin/sh +set -e docker build -t cogstacksystems/cogstack-observability-prometheus:latest -f prometheus/Dockerfile.prometheus ./prometheus @@ -5,6 +7,4 @@ docker build -t cogstacksystems/cogstack-observability-blackbox-exporter:latest docker build -t cogstacksystems/cogstack-observability-grafana:latest -f grafana/Dockerfile ./grafana -docker build -t cogstacksystems/cogstack-observability-traefik:latest -f traefik/Dockerfile ./traefik - -docker build -t cogstacksystems/cogstack-observability-alloy:latest -f grafana-alloy/Dockerfile ./grafana-alloy \ No newline at end of file +docker build -t cogstacksystems/cogstack-observability-alloy:latest -f grafana-alloy/Dockerfile ./grafana-alloy diff --git a/apps/observability/grafana-alloy/Dockerfile b/apps/observability/grafana-alloy/Dockerfile index c4353de0..c64bdfe9 100644 --- a/apps/observability/grafana-alloy/Dockerfile +++ b/apps/observability/grafana-alloy/Dockerfile @@ -1,8 +1,5 @@ FROM grafana/alloy:latest -LABEL traefik.enable="true" \ - traefik.http.routers.alloy.rule="PathPrefix(`/alloy`)" - EXPOSE 12345 RUN mkdir -p /etc/alloy/probers diff --git a/apps/observability/grafana/Dockerfile b/apps/observability/grafana/Dockerfile index bef90d9c..2e6207ce 100644 --- a/apps/observability/grafana/Dockerfile +++ b/apps/observability/grafana/Dockerfile @@ -1,10 +1,5 @@ FROM grafana/grafana-enterprise -LABEL traefik.enable="true" \ - traefik.http.routers.grafana-path-router.rule="PathPrefix(`/grafana`)" \ - traefik.http.middlewares.grafana-stripprefix.stripprefix.prefixes="/grafana" \ - traefik.http.routers.grafana-path-router.middlewares="grafana-stripprefix@docker" - RUN mkdir -p /etc/grafana/provisioning/dashboards/site COPY ./provisioning /etc/grafana/provisioning @@ -17,4 +12,4 @@ ENV GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH=/etc/grafana/provisioning/dashboar ALERTING_PAUSE_BURN_RATE=true \ ALERTING_DEFAULT_CONTACT=Slack \ SLACK_WEBHOOK_URL=http://localhost:3000/missing-SLACK_WEBHOOK_URL-env-var \ - PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus \ No newline at end of file + PROMETHEUS_URL=http://cogstack-observability-prometheus-1:9090/prometheus diff --git a/apps/observability/grafana/provisioning/dashboards/default/internals/traefik-dashboard.json b/apps/observability/grafana/provisioning/dashboards/default/internals/traefik-dashboard.json deleted file mode 100644 index 8f9aad65..00000000 --- a/apps/observability/grafana/provisioning/dashboards/default/internals/traefik-dashboard.json +++ /dev/null @@ -1,1537 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "description": "Official dashboard for Standalone Traefik. Created by Traefik Labs", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 19, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 9, - "panels": [], - "title": "General", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 2, - "w": 5, - "x": 0, - "y": 1 - }, - "id": 13, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "count(traefik_config_reloads_total)", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Traefik Instances", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 7, - "x": 5, - "y": 1 - }, - "id": 7, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\"}[$interval])) by (entrypoint)", - "legendFormat": "{{entrypoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Requests per Entrypoint", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "https://medium.com/@tristan_96324/prometheus-apdex-alerting-d17a065e39d0", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 6, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "(sum(rate(traefik_entrypoint_request_duration_seconds_bucket{le=\"0.3\",code=\"200\",entrypoint=~\"$entrypoint\"}[$interval])) by (method) + \n sum(rate(traefik_entrypoint_request_duration_seconds_bucket{le=\"1.2\",code=\"200\",entrypoint=~\"$entrypoint\"}[$interval])) by (method)) / 2 / \n sum(rate(traefik_entrypoint_request_duration_seconds_count{code=\"200\",entrypoint=~\"$entrypoint\"}[$interval])) by (method)\n", - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Apdex score", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "Mean Distribution", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 0, - "y": 3 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "multi", - "sort": "asc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "sum(rate(traefik_service_requests_total{service=~\"$service.*\",protocol=\"http\"}[$interval])) by (method, code)", - "legendFormat": "{{method}}[{{code}}]", - "range": true, - "refId": "A" - } - ], - "title": "Http Code ", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n traefik_service_request_duration_seconds_sum{service=~\"$service.*\",protocol=\"http\"} / \n traefik_service_request_duration_seconds_count{service=~\"$service.*\",protocol=\"http\"},\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)\n\n", - "legendFormat": "{{method}}[{{code}}] on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Top slow services", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.4.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,code) \n (rate(traefik_service_requests_total{service=~\"$service.*\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "[{{code}}] on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Most requested services", - "type": "timeseries" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "id": 11, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 3, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "label_replace(\n 1 - (sum by (service)\n (rate(traefik_service_request_duration_seconds_bucket{le=\"1.2\",service=~\"$service.*\"}[$interval])) / sum by (service) \n (rate(traefik_service_request_duration_seconds_count{service=~\"$service.*\"}[$interval]))\n ) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\"\n)", - "legendFormat": "{{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Services failing SLO of 1200ms", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "label_replace(\n 1 - (sum by (service)\n (rate(traefik_service_request_duration_seconds_bucket{le=\"0.3\",service=~\"$service.*\"}[$interval])) / sum by (service) \n (rate(traefik_service_request_duration_seconds_count{service=~\"$service.*\"}[$interval]))\n ) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\"\n)", - "legendFormat": "{{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Services failing SLO of 300ms", - "type": "timeseries" - } - ], - "title": "SLO", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 18 - }, - "id": 16, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 8, - "x": 0, - "y": 19 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,method,code) \n (rate(traefik_service_requests_total{service=~\"$service.*\",code=~\"2..\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "{{method}}[{{code}}] on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "2xx over $interval", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 8, - "x": 8, - "y": 19 - }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,method,code) \n (rate(traefik_service_requests_total{service=~\"$service.*\",code=~\"5..\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "{{method}}[{{code}}] on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "5xx over $interval", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 12, - "w": 8, - "x": 16, - "y": 19 - }, - "id": 19, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,method,code) \n (rate(traefik_service_requests_total{service=~\"$service.*\",code!~\"2..|5..\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "{{method}}[{{code}}] on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Other codes over $interval", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 31 - }, - "id": 20, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,method) \n (rate(traefik_service_requests_bytes_total{service=~\"$service.*\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "{{method}} on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Requests Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisGridShow": true, - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 31 - }, - "id": 24, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "topk(15,\n label_replace(\n sum by (service,method) \n (rate(traefik_service_responses_bytes_total{service=~\"$service.*\",protocol=\"http\"}[$interval])) > 0,\n \"service\", \"$1\", \"service\", \"([^-]+-[^-]+).*\")\n)", - "legendFormat": "{{method}} on {{service}}", - "range": true, - "refId": "A" - } - ], - "title": "Responses Size", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 21, - "options": { - "legend": { - "calcs": [ - "mean", - "max" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true, - "sortBy": "Max", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "editorMode": "code", - "expr": "sum(traefik_open_connections{entrypoint=~\"$entrypoint\"}) by (entrypoint)\n", - "legendFormat": "{{entrypoint}}", - "range": true, - "refId": "A" - } - ], - "title": "Connections per Entrypoint", - "type": "timeseries" - } - ], - "title": "HTTP Details", - "type": "row" - } - ], - "preload": false, - "refresh": "", - "schemaVersion": 40, - "tags": [ - "infra", - "traefik", - "metrics", - "networking" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "datasource-prometheus-local-docker" - }, - "includeAll": false, - "label": "datasource", - "name": "DS_PROMETHEUS", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - { - "auto": true, - "auto_count": 30, - "auto_min": "1m", - "current": { - "text": "$__auto", - "value": "$__auto" - }, - "name": "interval", - "options": [ - { - "selected": false, - "text": "1m", - "value": "1m" - }, - { - "selected": false, - "text": "5m", - "value": "5m" - }, - { - "selected": false, - "text": "10m", - "value": "10m" - }, - { - "selected": false, - "text": "30m", - "value": "30m" - }, - { - "selected": false, - "text": "1h", - "value": "1h" - }, - { - "selected": false, - "text": "2h", - "value": "2h" - }, - { - "selected": false, - "text": "4h", - "value": "4h" - }, - { - "selected": false, - "text": "8h", - "value": "8h" - } - ], - "query": "1m,5m,10m,30m,1h,2h,4h,8h", - "refresh": 2, - "type": "interval" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "definition": "label_values(traefik_open_connections, entrypoint)", - "includeAll": true, - "name": "entrypoint", - "options": [], - "query": { - "query": "label_values(traefik_open_connections, entrypoint)", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "", - "type": "query" - }, - { - "current": { - "text": "All", - "value": "$__all" - }, - "datasource": { - "type": "prometheus", - "uid": "datasource-prometheus-local-docker" - }, - "definition": "label_values(traefik_service_requests_total, service)", - "includeAll": true, - "name": "service", - "options": [], - "query": { - "query": "label_values(traefik_service_requests_total, service)", - "refId": "StandardVariableQuery" - }, - "refresh": 2, - "regex": "", - "sort": 1, - "type": "query" - } - ] - }, - "time": { - "from": "now-6h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Traefik Dashboard", - "uid": "traefik-dashboard", - "version": 1, - "weekStart": "" -} \ No newline at end of file diff --git a/apps/observability/prometheus/Dockerfile.blackbox b/apps/observability/prometheus/Dockerfile.blackbox index f6ee8a00..0d1a8699 100644 --- a/apps/observability/prometheus/Dockerfile.blackbox +++ b/apps/observability/prometheus/Dockerfile.blackbox @@ -1,12 +1,7 @@ FROM quay.io/prometheus/blackbox-exporter:latest -LABEL traefik.enable="true" \ - traefik.http.routers.blackbox-exporter-path-router.rule="PathPrefix(`/blackbox-exporter`)" \ - traefik.http.middlewares.blackbox-exporter-stripprefix.stripprefix.prefixes="/blackbox-exporter" \ - traefik.http.routers.blackbox-exporter-path-router.middlewares="blackbox-exporter-stripprefix@docker" - COPY ./blackbox-exporter /config CMD [ \ "--config.file=/config/blackbox.yml" \ - ] \ No newline at end of file + ] diff --git a/apps/observability/prometheus/Dockerfile.prometheus b/apps/observability/prometheus/Dockerfile.prometheus index 4dfd2d4f..1988aa3e 100644 --- a/apps/observability/prometheus/Dockerfile.prometheus +++ b/apps/observability/prometheus/Dockerfile.prometheus @@ -1,8 +1,5 @@ FROM prom/prometheus -LABEL traefik.enable="true" \ - traefik.http.routers.prometheus-path-router.rule="PathPrefix(`/prometheus`)" - RUN mkdir -p /etc/prometheus/cogstack/site/scrape-configs/probers RUN mkdir -p /etc/prometheus/cogstack/site/scrape-configs/exporters @@ -25,4 +22,4 @@ exec /bin/prometheus \ "$@" EOT -ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["/entrypoint.sh"] diff --git a/apps/observability/prometheus/defaults/scrape-configs/scrape-config.probe.yml b/apps/observability/prometheus/defaults/scrape-configs/scrape-config.probe.yml index 508bc69e..ecc68648 100644 --- a/apps/observability/prometheus/defaults/scrape-configs/scrape-config.probe.yml +++ b/apps/observability/prometheus/defaults/scrape-configs/scrape-config.probe.yml @@ -13,4 +13,4 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: cogstack-observability-traefik-1 # The blackbox exporter's real host:port. + replacement: nginx:8001 # Gateway to the standalone blackbox exporter. diff --git a/apps/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml b/apps/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml index 2e39c794..3017be57 100644 --- a/apps/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml +++ b/apps/observability/prometheus/defaults/scrape-configs/scrape-config.self.yml @@ -12,12 +12,8 @@ scrape_configs: labels: name: grafana # collect grafana metrics like dashboard usage __metrics_path__: /metrics - - targets: - - cogstack-observability-traefik-1:8080 # Collect Traefik metrics like number of success calls - labels: - name: traefik relabel_configs: - target_label: env replacement: observability-stack - target_label: host - replacement: prometheus-host \ No newline at end of file + replacement: prometheus-host diff --git a/apps/observability/traefik/Dockerfile b/apps/observability/traefik/Dockerfile deleted file mode 100644 index cf0db58c..00000000 --- a/apps/observability/traefik/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM traefik:v3.7.19 - -CMD [ \ - "--api.insecure=true", \ - "--providers.docker", \ - "--providers.docker.exposedbydefault=false", \ - "--accesslog=true", \ - "--accesslog.fields.headers.names.User-Agent=keep", \ - "--metrics.prometheus=true", \ - "--providers.file.directory=/etc/traefik/config", \ - "--providers.file.watch=true" \ -] \ No newline at end of file diff --git a/deployment/Makefile b/deployment/Makefile new file mode 100644 index 00000000..26b16018 --- /dev/null +++ b/deployment/Makefile @@ -0,0 +1,18 @@ +SHELL := /usr/bin/env bash +.SHELLFLAGS := -o pipefail -c + +.PHONY: setup-install-pre-requisites setup-k3s-cluster-init setup-verify setup-k3s-cluster-remote + +setup-install-pre-requisites: + bash scripts/installation_utils/install_os_tools_and_software.sh + +setup-k3s-cluster-init: + bash scripts/installation_utils/setup_kubernetes_cluster.sh k3s server --cluster-init + +setup-k3s-cluster-remote: + bash scripts/installation_utils/setup_k3s_cluster.sh $(NODES) + +setup-verify: + bash scripts/installation_utils/verify_docker.sh + bash scripts/installation_utils/verify_k3s.sh + diff --git a/deployment/charts/README.md b/deployment/charts/README.md index e9e4df1c..bfbbad23 100644 --- a/deployment/charts/README.md +++ b/deployment/charts/README.md @@ -2,7 +2,7 @@ Reusable deployment charts for CogStack. Each chart sits directly under this directory and declares its package name in `Chart.yaml`. -Folder names use component names, such as `medcat-service` and `jupyterhub`. +Folder names use component names, such as `medcat-service` and `jupyter-hub`. ## NiFi charts diff --git a/deployment/charts/ce/Chart.lock b/deployment/charts/ce/Chart.lock index e69373ad..4c7c0d56 100644 --- a/deployment/charts/ce/Chart.lock +++ b/deployment/charts/ce/Chart.lock @@ -15,10 +15,10 @@ dependencies: repository: https://opensearch-project.github.io/helm-charts/ version: 3.5.0 - name: cogstack-jupyterhub-helm - repository: file://../jupyterhub + repository: file://../jupyter-hub version: 0.5.0 - name: ocr-service-helm repository: file://../ocr-service version: 0.1.0 -digest: sha256:dd9c57fa23a97923003067e81ac40fbbf683d61f43b98b42f592d90e1309e74c +digest: sha256:0d274dec4f435fb9841b12f76f43191cd20799751614f179e964c15117ea64ca generated: "2026-11-19T18:15:18.209696Z" diff --git a/deployment/charts/ce/Chart.yaml b/deployment/charts/ce/Chart.yaml index c9863046..ab986569 100644 --- a/deployment/charts/ce/Chart.yaml +++ b/deployment/charts/ce/Chart.yaml @@ -57,7 +57,7 @@ dependencies: alias: cogstack-jupyterhub condition: cogstack-jupyterhub.enabled version: ">=0.0.1" - repository: "file://../jupyterhub" + repository: "file://../jupyter-hub" - name: ocr-service-helm alias: ocr-service condition: cogstack-jupyterhub.enabled diff --git a/deployment/charts/ce/README.md b/deployment/charts/ce/README.md index 91f5646c..5aeb569b 100644 --- a/deployment/charts/ce/README.md +++ b/deployment/charts/ce/README.md @@ -83,7 +83,7 @@ kubectl delete namespace cogstack | Repository | Name | Version | |------------|------|---------| -| file://../jupyterhub | cogstack-jupyterhub(cogstack-jupyterhub-helm) | >=0.0.1 | +| file://../jupyter-hub | cogstack-jupyterhub(cogstack-jupyterhub-helm) | >=0.0.1 | | file://../medcat-service | medcat-service(medcat-service-helm) | >=0.0.1 | | file://../medcat-service | anoncat-service(medcat-service-helm) | >=0.0.1 | | file://../medcat-trainer | medcat-trainer(medcat-trainer-helm) | >=0.0.1 | diff --git a/deployment/charts/cohorter/README.md b/deployment/charts/cohorter/README.md index d0d9e6e5..310731ec 100644 --- a/deployment/charts/cohorter/README.md +++ b/deployment/charts/cohorter/README.md @@ -6,8 +6,8 @@ CogStack Cohorter β€” cohort identification powered by Ollama and MedCAT | Component | Image | Description | |-----------|-------|-------------| -| **WebApp** | `cogstacksystems/cogstack-cohorter-webapp` | React + Node.js frontend and API | -| **NL2DSL** | `cogstacksystems/cogstack-cohorter-nl2dsl` | Natural language β†’ cohort DSL compiler | +| **WebApp** | `cogstacksystems/cohorter-webapp` | React + Node.js frontend and API | +| **NL2DSL** | `cogstacksystems/cohorter-nl2dsl` | Natural language β†’ cohort DSL compiler | | **MedCAT** | `cogstacksystems/medcat-service` | Clinical NER and concept normalisation (subchart) | | **Ollama** | `ollama/ollama` | LLM serving backend (subchart) | @@ -147,7 +147,7 @@ For issues and questions, please visit the [CogStack GitHub repository](https:// | medcat | object | `{"enabled":true,"env":{"APP_ENABLE_METRICS":"true","APP_MEDCAT_MODEL_PACK":"/cat/models/examples/example-medcat-v2-model-pack.zip"},"image":{"tag":"latest"},"resources":{},"service":{"port":5000}}` | ------------------------------------------------------------------------- | | medcat.enabled | bool | `true` | Enable the MedCAT subchart. | | nameOverride | string | `"cogstack-cohorter"` | ------------------------------------------------------------------------- | -| nl2dsl | object | `{"affinity":{},"enabled":true,"env":{"ALLOW_ORIGINS":"*","OLLAMA_MODEL":"llama3.2:3b"},"image":{"pullPolicy":"Always","repository":"cogstacksystems/cogstack-cohorter-nl2dsl","tag":"latest"},"livenessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":30,"periodSeconds":10},"nodeSelector":{},"readinessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":10,"periodSeconds":5},"replicaCount":1,"resources":{},"service":{"port":3002,"type":"ClusterIP"},"tolerations":[]}` | ------------------------------------------------------------------------- | +| nl2dsl | object | `{"affinity":{},"enabled":true,"env":{"ALLOW_ORIGINS":"*","OLLAMA_MODEL":"llama3.2:3b"},"image":{"pullPolicy":"Always","repository":"cogstacksystems/cohorter-nl2dsl","tag":"latest"},"livenessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":30,"periodSeconds":10},"nodeSelector":{},"readinessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":10,"periodSeconds":5},"replicaCount":1,"resources":{},"service":{"port":3002,"type":"ClusterIP"},"tolerations":[]}` | ------------------------------------------------------------------------- | | nl2dsl.enabled | bool | `true` | Enable the NL2DSL deployment and service. | | ollama | object | `{"enabled":true,"ollama":{"models":{"pull":["llama3.2:3b"]}},"persistentVolume":{"enabled":true,"size":"10Gi","storageClass":""},"resources":{},"service":{"port":11434,"type":"ClusterIP"}}` | ------------------------------------------------------------------------- | | ollama.enabled | bool | `true` | Enable the Ollama subchart. | @@ -156,7 +156,7 @@ For issues and questions, please visit the [CogStack GitHub repository](https:// | podSecurityContext | object | `{}` | | | securityContext | object | `{}` | | | serviceAccount | object | `{"annotations":{},"automount":true,"create":true,"name":""}` | ------------------------------------------------------------------------- | -| webapp | object | `{"affinity":{},"data":{"downloadUrl":""},"enabled":true,"env":{"RANDOM_DATA":"true"},"image":{"pullPolicy":"Always","repository":"cogstacksystems/cogstack-cohorter-webapp","tag":"latest"},"initContainers":[],"livenessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":60,"periodSeconds":15},"nodeSelector":{},"persistence":{"accessMode":"ReadWriteOnce","enabled":true,"existingClaim":"","size":"5Gi","storageClass":""},"readinessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":30,"periodSeconds":10},"replicaCount":1,"resources":{},"service":{"port":3000,"type":"ClusterIP"},"tolerations":[]}` | ------------------------------------------------------------------------- | +| webapp | object | `{"affinity":{},"data":{"downloadUrl":""},"enabled":true,"env":{"RANDOM_DATA":"true"},"image":{"pullPolicy":"Always","repository":"cogstacksystems/cohorter-webapp","tag":"latest"},"initContainers":[],"livenessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":60,"periodSeconds":15},"nodeSelector":{},"persistence":{"accessMode":"ReadWriteOnce","enabled":true,"existingClaim":"","size":"5Gi","storageClass":""},"readinessProbe":{"httpGet":{"path":"/","port":"http"},"initialDelaySeconds":30,"periodSeconds":10},"replicaCount":1,"resources":{},"service":{"port":3000,"type":"ClusterIP"},"tolerations":[]}` | ------------------------------------------------------------------------- | | webapp.enabled | bool | `true` | Enable the WebApp deployment and service. | ---------------------------------------------- diff --git a/deployment/charts/cohorter/README.md.gotmpl b/deployment/charts/cohorter/README.md.gotmpl index beea0c7f..273dbd58 100644 --- a/deployment/charts/cohorter/README.md.gotmpl +++ b/deployment/charts/cohorter/README.md.gotmpl @@ -8,8 +8,8 @@ | Component | Image | Description | |-----------|-------|-------------| -| **WebApp** | `cogstacksystems/cogstack-cohorter-webapp` | React + Node.js frontend and API | -| **NL2DSL** | `cogstacksystems/cogstack-cohorter-nl2dsl` | Natural language β†’ cohort DSL compiler | +| **WebApp** | `cogstacksystems/cohorter-webapp` | React + Node.js frontend and API | +| **NL2DSL** | `cogstacksystems/cohorter-nl2dsl` | Natural language β†’ cohort DSL compiler | | **MedCAT** | `cogstacksystems/medcat-service` | Clinical NER and concept normalisation (subchart) | | **Ollama** | `ollama/ollama` | LLM serving backend (subchart) | diff --git a/deployment/charts/cohorter/values.yaml b/deployment/charts/cohorter/values.yaml index e456597d..bf32dbbf 100644 --- a/deployment/charts/cohorter/values.yaml +++ b/deployment/charts/cohorter/values.yaml @@ -107,7 +107,7 @@ webapp: replicaCount: 1 image: - repository: cogstacksystems/cogstack-cohorter-webapp + repository: cogstacksystems/cohorter-webapp tag: latest pullPolicy: Always @@ -180,7 +180,7 @@ nl2dsl: replicaCount: 1 image: - repository: cogstacksystems/cogstack-cohorter-nl2dsl + repository: cogstacksystems/cohorter-nl2dsl tag: latest pullPolicy: Always diff --git a/deployment/charts/jupyterhub/CHANGELOG.md b/deployment/charts/jupyter-hub/CHANGELOG.md similarity index 100% rename from deployment/charts/jupyterhub/CHANGELOG.md rename to deployment/charts/jupyter-hub/CHANGELOG.md diff --git a/deployment/charts/jupyterhub/Chart.lock b/deployment/charts/jupyter-hub/Chart.lock similarity index 100% rename from deployment/charts/jupyterhub/Chart.lock rename to deployment/charts/jupyter-hub/Chart.lock diff --git a/deployment/charts/jupyterhub/Chart.yaml b/deployment/charts/jupyter-hub/Chart.yaml similarity index 100% rename from deployment/charts/jupyterhub/Chart.yaml rename to deployment/charts/jupyter-hub/Chart.yaml diff --git a/deployment/charts/jupyterhub/README.md b/deployment/charts/jupyter-hub/README.md similarity index 100% rename from deployment/charts/jupyterhub/README.md rename to deployment/charts/jupyter-hub/README.md diff --git a/deployment/charts/jupyterhub/README.md.gotmpl b/deployment/charts/jupyter-hub/README.md.gotmpl similarity index 100% rename from deployment/charts/jupyterhub/README.md.gotmpl rename to deployment/charts/jupyter-hub/README.md.gotmpl diff --git a/deployment/charts/jupyterhub/examples/medcat-opensearch-e2e.ipynb b/deployment/charts/jupyter-hub/examples/medcat-opensearch-e2e.ipynb similarity index 100% rename from deployment/charts/jupyterhub/examples/medcat-opensearch-e2e.ipynb rename to deployment/charts/jupyter-hub/examples/medcat-opensearch-e2e.ipynb diff --git a/deployment/charts/jupyterhub/examples/medcat-service-tutorial.ipynb b/deployment/charts/jupyter-hub/examples/medcat-service-tutorial.ipynb similarity index 100% rename from deployment/charts/jupyterhub/examples/medcat-service-tutorial.ipynb rename to deployment/charts/jupyter-hub/examples/medcat-service-tutorial.ipynb diff --git a/deployment/charts/jupyterhub/examples/ocr-service-tutorial.ipynb b/deployment/charts/jupyter-hub/examples/ocr-service-tutorial.ipynb similarity index 100% rename from deployment/charts/jupyterhub/examples/ocr-service-tutorial.ipynb rename to deployment/charts/jupyter-hub/examples/ocr-service-tutorial.ipynb diff --git a/deployment/charts/jupyterhub/templates/_helpers.tpl b/deployment/charts/jupyter-hub/templates/_helpers.tpl similarity index 100% rename from deployment/charts/jupyterhub/templates/_helpers.tpl rename to deployment/charts/jupyter-hub/templates/_helpers.tpl diff --git a/deployment/charts/jupyterhub/templates/examples-configmap.yaml b/deployment/charts/jupyter-hub/templates/examples-configmap.yaml similarity index 100% rename from deployment/charts/jupyterhub/templates/examples-configmap.yaml rename to deployment/charts/jupyter-hub/templates/examples-configmap.yaml diff --git a/deployment/charts/jupyterhub/templates/servicemonitor.yaml b/deployment/charts/jupyter-hub/templates/servicemonitor.yaml similarity index 100% rename from deployment/charts/jupyterhub/templates/servicemonitor.yaml rename to deployment/charts/jupyter-hub/templates/servicemonitor.yaml diff --git a/deployment/charts/jupyterhub/values.yaml b/deployment/charts/jupyter-hub/values.yaml similarity index 100% rename from deployment/charts/jupyterhub/values.yaml rename to deployment/charts/jupyter-hub/values.yaml diff --git a/deployment/charts/local_dev_startup.sh b/deployment/charts/local_dev_startup.sh index f6cc8805..71cef85b 100644 --- a/deployment/charts/local_dev_startup.sh +++ b/deployment/charts/local_dev_startup.sh @@ -31,7 +31,7 @@ helm test medcat-service --logs helm repo add opensearch https://opensearch-project.github.io/helm-charts/ helm dependency update ./medcat-trainer -helm dependency update ./jupyterhub +helm dependency update ./jupyter-hub helm dependency update ./ce helm upgrade cogstack ./ce --install --dependency-update --wait --timeout 30m0s diff --git a/deployment/examples/observability/full/alloy/probers/probe-observability.yml b/deployment/examples/observability/full/alloy/probers/probe-observability.yml index 974e5276..3b8425f2 100644 --- a/deployment/examples/observability/full/alloy/probers/probe-observability.yml +++ b/deployment/examples/observability/full/alloy/probers/probe-observability.yml @@ -1,12 +1,12 @@ - targets: - - cogstack-observability-traefik-1/grafana/api/health + - http://nginx:8001/grafana/api/health labels: name: grafana job: probe-observability-stack host: localhost - targets: - - cogstack-observability-traefik-1/prometheus/-/healthy + - http://nginx:8001/prometheus/-/healthy labels: name: prometheus job: probe-observability-stack - host: localhost \ No newline at end of file + host: localhost diff --git a/deployment/examples/observability/full/docker-compose.yml b/deployment/examples/observability/full/docker-compose.yml index 52d03e8f..869bcbcd 100755 --- a/deployment/examples/observability/full/docker-compose.yml +++ b/deployment/examples/observability/full/docker-compose.yml @@ -37,15 +37,15 @@ services: environment: - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - traefik: - image: cogstacksystems/cogstack-observability-traefik:latest + nginx: + image: cogstacksystems/nginx:latest + environment: + - NGINX_ENVSUBST_TEMPLATE_DIR=/etc/nginx/templates/observability networks: - observability restart: unless-stopped ports: - - "80:80" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events + - "80:8001" networks: observability: @@ -53,4 +53,4 @@ networks: volumes: prometheus-data: - grafana-data: \ No newline at end of file + grafana-data: diff --git a/deployment/examples/observability/full/full-quickstart.sh b/deployment/examples/observability/full/full-quickstart.sh index c96ea72b..75d59b96 100644 --- a/deployment/examples/observability/full/full-quickstart.sh +++ b/deployment/examples/observability/full/full-quickstart.sh @@ -1,4 +1,3 @@ - #!/bin/bash set -e OBSERVABILITY_BASE_URL="${OBSERVABILITY_BASE_URL:-https://raw.githubusercontent.com/CogStack/cogstack-platform/main/deployment/examples/observability}" @@ -31,6 +30,6 @@ echo "Setup complete in cogstack-observability/" echo "Starting the observability stack" -docker compose up -d +docker compose up -d --remove-orphans -echo "Please open http://localhost/grafana in your browser" \ No newline at end of file +echo "Please open http://localhost/grafana in your browser" diff --git a/deployment/examples/observability/simple/alloy/probers/probe-observability.yml b/deployment/examples/observability/simple/alloy/probers/probe-observability.yml index 974e5276..3b8425f2 100644 --- a/deployment/examples/observability/simple/alloy/probers/probe-observability.yml +++ b/deployment/examples/observability/simple/alloy/probers/probe-observability.yml @@ -1,12 +1,12 @@ - targets: - - cogstack-observability-traefik-1/grafana/api/health + - http://nginx:8001/grafana/api/health labels: name: grafana job: probe-observability-stack host: localhost - targets: - - cogstack-observability-traefik-1/prometheus/-/healthy + - http://nginx:8001/prometheus/-/healthy labels: name: prometheus job: probe-observability-stack - host: localhost \ No newline at end of file + host: localhost diff --git a/deployment/examples/observability/simple/docker-compose.yml b/deployment/examples/observability/simple/docker-compose.yml index 3f225734..36af0b45 100755 --- a/deployment/examples/observability/simple/docker-compose.yml +++ b/deployment/examples/observability/simple/docker-compose.yml @@ -26,15 +26,15 @@ services: environment: - GF_AUTH_ANONYMOUS_ENABLED=true # Allows use of grafana without sign in - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - traefik: - image: cogstacksystems/cogstack-observability-traefik:latest + nginx: + image: cogstacksystems/nginx:latest + environment: + - NGINX_ENVSUBST_TEMPLATE_DIR=/etc/nginx/templates/observability networks: - observability restart: unless-stopped ports: - - "80:80" - volumes: - - /var/run/docker.sock:/var/run/docker.sock:ro # So that Traefik can listen to the Docker events + - "80:8001" networks: observability: @@ -42,4 +42,4 @@ networks: volumes: prometheus-data: - grafana-data: \ No newline at end of file + grafana-data: diff --git a/deployment/examples/observability/simple/quickstart.sh b/deployment/examples/observability/simple/quickstart.sh index 29fac19c..468436f1 100644 --- a/deployment/examples/observability/simple/quickstart.sh +++ b/deployment/examples/observability/simple/quickstart.sh @@ -17,6 +17,6 @@ curl -fsSL -o alloy/probers/probe-observability.yml \ echo "Setup complete in observability-simple/" echo "Starting the observability stack" -docker compose up -d +docker compose up -d --remove-orphans - echo "Please open http://localhost/grafana in your browser" \ No newline at end of file + echo "Please open http://localhost/grafana in your browser" diff --git a/deployment/examples/observability/test/health-check.py b/deployment/examples/observability/test/health-check.py index d071e8fb..fd5225c9 100644 --- a/deployment/examples/observability/test/health-check.py +++ b/deployment/examples/observability/test/health-check.py @@ -38,6 +38,8 @@ def main(): (f"http://{localhost_url}/grafana", "Grafana"), (f"http://{localhost_url}/prometheus", "Prometheus"), (f"http://{localhost_url}/alloy", "Alloy"), + (f"http://{localhost_url}/grafana/api/health", "Grafana API"), + (f"http://{localhost_url}/prometheus/-/ready", "Prometheus readiness"), ] all_healthy = True @@ -54,4 +56,4 @@ def main(): return 1 if __name__ == "__main__": - sys.exit(main()) \ No newline at end of file + sys.exit(main()) diff --git a/deployment/examples/observability/test/test-quickstart.sh b/deployment/examples/observability/test/test-quickstart.sh index fa87621f..d73b9d28 100644 --- a/deployment/examples/observability/test/test-quickstart.sh +++ b/deployment/examples/observability/test/test-quickstart.sh @@ -2,44 +2,21 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -echo "πŸš€ Starting deployment test..." - -# Run the simple quickstart script -echo "πŸ“¦ Running quickstart..." -cd ${SCRIPT_DIR}/../simple/ -bash ./quickstart.sh - -# Run the health check -echo "πŸ” Running health check..." -python -u ../test/health-check.py - -# Check if health check was successful -if [ $? -eq 0 ]; then - echo "βœ… Success! All services are running for Simple Quickstart." - docker compose down - exit 0 -else - echo "❌ Health check failed for simple quickstart." - exit 1 -fi - - -# Run the quickstart script -echo "πŸ“¦ Running full quickstart..." -cd ${SCRIPT_DIR}/../full/ -bash ./full-quickstart.sh - -# Run the health check -echo "πŸ” Running health check..." -python -u ../test/health-check.py - -# Check if health check was successful -if [ $? -eq 0 ]; then - echo "βœ… Success! All services are running for Full Quickstart." - docker compose down - exit 0 -else - echo "❌ Health check failed for full quickstart." - exit 1 -fi - +for example in simple full; do + ( + cd "$SCRIPT_DIR/../$example" + trap 'docker compose down' EXIT + if [ "$example" = simple ]; then + bash ./quickstart.sh + cd observability-simple + else + bash ./full-quickstart.sh + cd cogstack-observability + fi + if ! python -u "$SCRIPT_DIR/health-check.py"; then + docker compose ps --all + docker compose logs --no-color --tail=100 + exit 1 + fi + ) +done diff --git a/deployment/scripts/README.md b/deployment/scripts/README.md index 301977e1..2d356ac5 100644 --- a/deployment/scripts/README.md +++ b/deployment/scripts/README.md @@ -1,137 +1,112 @@ # Deployment scripts -Shared utilities for preparing deployment hosts and maintaining this repository. -Run the examples below from the repository root. +Run commands from the repository root. Requires Make, Bash, `sudo`, systemd and network +access. Installation and cluster setup invoke `sudo` internally; run setup as the user who will +use `kubectl`. -| Directory | Purpose | -| --- | --- | -| [`installation_utils/`](installation_utils/) | Install host prerequisites and provide Docker configuration examples. | -| [`maintenance/`](maintenance/) | Audit repository Python dependencies. | +## Install, initialize and verify -Keep application-specific startup and runtime scripts alongside their applications -under `apps/`. - -## Installation utilities - -### Prerequisites - -The installers recognise Debian, Ubuntu, Red Hat Enterprise Linux and CentOS. -Package and repository availability depends on the distribution release. They -require Bash, `sudo`, internet access to package repositories and a host using -systemd. They invoke `sudo` internally, including when run as root. - -Both installers update and upgrade system packages. The Docker installer also -changes package repositories, Docker group membership and system settings; its -Red Hat/CentOS branch removes existing Docker packages, Podman and Buildah. -Review the script before running it on an existing host. - -### Docker and platform tools - -[`install_docker_and_utils.sh`](installation_utils/install_docker_and_utils.sh) -installs Docker Engine, Buildx, Compose, Azure CLI, Python utilities, filesystem -tools, monitoring tools and build dependencies. By default it also installs -Kubernetes tools, Helm and K3s. +Install on every machine before creating or joining a cluster: ```bash -bash deployment/scripts/installation_utils/install_docker_and_utils.sh +make -C deployment setup-install-pre-requisites ``` -To install Docker and the general utilities while skipping Kubernetes, Helm and -K3s: +By default, this installs host utilities, Docker, K3s and Helm. **New K3s installations +remain stopped until cluster setup.** Installers skip existing system and Python +packages, Azure CLI, K3s and Helm, and reuse existing Docker and Azure apt sources +(`.list` or `.sources`, including custom filenames). They do not run system upgrades +or autoremove. Installing missing Docker packages on RHEL may remove conflicting +container packages; review the scripts before running on an existing host. +Start a new login session to use Docker without sudo. + +On the first machine, initialize the cluster and check it: ```bash -INSTALL_KUBERNETES=false INSTALL_K3S=false \ - bash deployment/scripts/installation_utils/install_docker_and_utils.sh +make -C deployment setup-k3s-cluster-init +make -C deployment setup-verify ``` -The following environment variables control the optional components. Versions -shown are the defaults configured in the script. +Setup enables/restarts K3s and defaults to Secrets encryption at rest on servers. +Verification checks Docker and K3s without sudo. Expect the node to become `Ready`. -| Variable | Default | Effect | -| --- | --- | --- | -| `INSTALL_KUBERNETES` | `true` | Install `kubeadm`, `kubelet`, `kubectl` and Helm. Accepts `true` or `false`. | -| `INSTALL_K3S` | Value of `INSTALL_KUBERNETES` | Install K3s independently of the Kubernetes tools. Accepts `true` or `false`. | -| `KUBERNETES_MINOR_VERSION` | `v1.37` | Kubernetes package repository minor release. | -| `HELM_VERSION` | `v4.3.1` | Helm release; the installer accepts Helm 4 versions. | -| `K3S_VERSION` | `v1.37.2+k3s1` | K3s release. | +## Set up a cluster over SSH -To install the Kubernetes tools and Helm while skipping K3s: +From your workstation, run once with the first server followed by the other servers: ```bash -INSTALL_K3S=false \ - bash deployment/scripts/installation_utils/install_docker_and_utils.sh +make -C deployment setup-k3s-cluster-remote \ + NODES="admin@10.0.0.11 admin@10.0.0.12 admin@10.0.0.13" ``` -The installer enables and starts Docker, adds root and the invoking user to the -Docker group, and sets `vm.max_map_count=262144` immediately and in -`/etc/sysctl.conf`. Start a new login session to pick up Docker group membership. - -Kubernetes packages are installed for later cluster setup; the installer does -not initialise or join a cluster. It enables the kubelet service and holds the -Kubernetes packages on Debian/Ubuntu. K3s installation skips enabling and starting -its service, and preserves the standalone `kubectl` command. +Requires SSH access (key or password), passwordless sudo, Bash and tar on each +machine. Password authentication requires a terminal and may prompt on each +connection. Use unique +hostnames and addresses reachable between nodes. The script installs prerequisites, +initializes the first server, joins the others and verifies Docker and K3s as the +SSH user. Existing members are retained; nodes from another cluster are rejected. +Use one server initially, or three or more servers (an odd total) for HA. -### Database clients +## Join servers manually -[`install_db_utils.sh`](installation_utils/install_db_utils.sh) installs the -PostgreSQL client tools: +On the first machine, read the join token: ```bash -bash deployment/scripts/installation_utils/install_db_utils.sh +sudo cat /var/lib/rancher/k3s/server/token ``` -It uses `postgresql-client` on Debian/Ubuntu and the PostgreSQL 12 module and -`postgresql` package on Red Hat/CentOS. - -### OS detection and Docker settings - -Both installers use [`detect_os.sh`](installation_utils/detect_os.sh), which -reads `/etc/os-release` and rejects unsupported distributions. To check the -detected distribution without installing anything: +After installation on machines 2 and 3, run this on each, replacing +`FIRST_SERVER_IP` and `SERVER_TOKEN`: ```bash -bash deployment/scripts/installation_utils/detect_os.sh +bash deployment/scripts/installation_utils/setup_kubernetes_cluster.sh k3s server \ + --server https://FIRST_SERVER_IP:6443 --token 'SERVER_TOKEN' ``` -[`docker_installation_settings/`](installation_utils/docker_installation_settings/) -contains optional Docker proxy and DNS configuration examples. See its -[README](installation_utils/docker_installation_settings/README.md) for details. -Adapt these settings to the host; the installers do not apply them automatically. +Run the verification commands above again after joining both servers. For +worker-only nodes, use `agent` instead of `server`. See the +[K3s HA guide](https://docs.k3s.io/datastore/ha-embedded) for availability and +networking requirements. -## Maintenance +## kubectl access -[`audit_pip_dependency.sh`](maintenance/audit_pip_dependency.sh) exports locked -Python dependencies with `uv` and checks them with `pip-audit`. +Server setup creates `~/.kube/config` with mode `0600` if absent and preserves an +existing file. K3s's bundled kubectl defaults to `/etc/rancher/k3s/k3s.yaml`; use +the printed `--kubeconfig` command to select the intended configuration. -### Prerequisites +To configure access to an existing cluster without restarting K3s: -- Bash, `uv` and `pip-audit` available on `PATH`. -- The checked-in `docs/pyproject.toml` and `docs/uv.lock` files. -- Network access for dependency resolution and vulnerability lookups. +```bash +bash deployment/scripts/installation_utils/setup_kubernetes_cluster.sh kubeconfig /etc/rancher/k3s/k3s.yaml +``` -No sudo privileges are required. The script uses temporary requirements files, -removes them on exit and leaves the project lockfiles unchanged. +## Options and individual utilities -### Usage +| Variable | Default | Purpose | +| --- | --- | --- | +| `INSTALL_K3S` | `true` | Install K3s and Helm; `false` skips both. | -Audit documentation dependencies: +Individual installers can also be run with `bash`: +[host utilities](installation_utils/install_host_utils.sh), +[Docker](installation_utils/install_docker.sh), +[K3s](installation_utils/install_k3s.sh), +[Helm](installation_utils/install_helm.sh) and +[PostgreSQL clients](installation_utils/install_db_utils.sh). +On a new host, install host utilities first. +[Docker configuration examples](installation_utils/docker_installation_settings/README.md) +are optional and are not applied automatically. -```bash -bash deployment/scripts/maintenance/audit_pip_dependency.sh -``` +## Dependency audits -Show available options: +Requires `uv`, `pip-audit` and network access. Audits locked dependencies without +changing lockfiles; failures return a nonzero exit status. ```bash -bash deployment/scripts/maintenance/audit_pip_dependency.sh --help -``` - -To audit the documentation followed by NiFi runtime and development dependencies -from `apps/nifi/uv.lock`: +# Documentation dependencies +bash deployment/scripts/maintenance/audit_pip_dependency.sh -```bash +# Include NiFi runtime and development dependencies bash deployment/scripts/maintenance/audit_pip_dependency.sh --include-nifi ``` -The script exits with a nonzero status when an export or audit fails, including -when `pip-audit` reports vulnerabilities. +Use `--help` for available options. diff --git a/deployment/scripts/installation_utils/detect_os.sh b/deployment/scripts/installation_utils/detect_os.sh index deb80bc0..a40f8753 100755 --- a/deployment/scripts/installation_utils/detect_os.sh +++ b/deployment/scripts/installation_utils/detect_os.sh @@ -1,7 +1,5 @@ #!/usr/bin/env bash -# Read one canonical ID instead of matching every release file, which can emit -# duplicate distribution names and filename prefixes on the same machine. if [ ! -r /etc/os-release ]; then echo "Distribution version could not be found. Stopping script..." >&2 exit 1 @@ -11,7 +9,7 @@ fi . /etc/os-release case "${ID:-}" in - debian|ubuntu|centos) echo "$ID" ;; + debian|ubuntu) echo "$ID" ;; rhel) echo "redhat" ;; *) echo "Unsupported distribution: ${ID:-unknown}" >&2; exit 1 ;; esac diff --git a/deployment/scripts/installation_utils/install_db_utils.sh b/deployment/scripts/installation_utils/install_db_utils.sh index a4c3cec8..08c9725a 100755 --- a/deployment/scripts/installation_utils/install_db_utils.sh +++ b/deployment/scripts/installation_utils/install_db_utils.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=installation_helpers.sh +source "${script_dir}/installation_helpers.sh" if [ "$(id -u)" -eq 0 ]; then echo "Running with root privileges." @@ -14,17 +16,10 @@ echo "Found distribution: $os_distribution " if [ "$os_distribution" == "debian" ] || [ "$os_distribution" == "ubuntu" ]; then - sudo apt-get update -y && sudo apt-get upgrade -y - sudo apt-get install -y postgresql-client -elif [ "$os_distribution" == "redhat" ] || [ "$os_distribution" == "red hat" ] || [ "$os_distribution" == "centos" ]; + install_missing_packages "$os_distribution" postgresql-client +elif [ "$os_distribution" == "redhat" ]; then - sudo yum -y update && sudo yum -y upgrade - - # install postgresql client - sudo dnf -y module enable postgresql:12 - sudo dnf -y install postgresql - - sudo yum -y autoremove + install_missing_packages "$os_distribution" postgresql else echo "No instructions given for distribution: $os_distribution" exit 1 diff --git a/deployment/scripts/installation_utils/install_docker.sh b/deployment/scripts/installation_utils/install_docker.sh new file mode 100755 index 00000000..4f16fe1e --- /dev/null +++ b/deployment/scripts/installation_utils/install_docker.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=installation_helpers.sh +source "${script_dir}/installation_helpers.sh" +target_user="${SUDO_USER:-${USER:-$(id -un)}}" +os_distribution="$(bash "${script_dir}/detect_os.sh")" +missing_docker_packages=() +for package in docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin; do + if ! package_is_installed "$os_distribution" "$package"; then + missing_docker_packages+=("$package") + fi +done + +if [ "$os_distribution" == "debian" ] || [ "$os_distribution" == "ubuntu" ]; +then + if [ "${#missing_docker_packages[@]}" -gt 0 ]; then + if ! apt_repository_exists docker "https://download.docker.com/linux/${os_distribution}"; then + sudo install -m 0755 -d /etc/apt/keyrings + if [ ! -s /etc/apt/keyrings/docker.gpg ]; then + curl -fsSL "https://download.docker.com/linux/${os_distribution}/gpg" | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg + fi + sudo chmod a+r /etc/apt/keyrings/docker.gpg + + # shellcheck disable=SC1091 # Standard OS metadata file provided by Debian/Ubuntu. + docker_codename="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")" + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${os_distribution} \ + ${docker_codename} stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + else + echo "Docker apt repository already exists." + fi + + install_missing_packages "$os_distribution" "${missing_docker_packages[@]}" + else + echo "Docker packages are already installed." + fi + +elif [ "$os_distribution" == "redhat" ]; +then + if [ "${#missing_docker_packages[@]}" -gt 0 ]; then + sudo yum remove -y docker docker-client docker-client-latest docker-common \ + docker-latest docker-latest-logrotate docker-logrotate docker-engine buildah podman + + if [ ! -e /etc/yum.repos.d/docker-ce.repo ]; then + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + sudo yum-config-manager --enable docker-ce-stable + sudo yum-config-manager --enable docker-ce-stable-source + fi + + install_missing_packages "$os_distribution" "${missing_docker_packages[@]}" + else + echo "Docker packages are already installed." + fi + +else + echo "No instructions given for distribution: $os_distribution" >&2 + exit 1 +fi + +sudo groupadd -f docker +sudo usermod -aG docker root +if [ -n "$target_user" ] && [ "$target_user" != "root" ] && id "$target_user" >/dev/null 2>&1; then + sudo usermod -aG docker "$target_user" +fi + +sudo systemctl enable --now docker.service + +sudo sysctl -w vm.max_map_count=262144 + +sudo sh -c "echo 'vm.max_map_count=262144' >> /etc/sysctl.conf" +sudo sysctl -p diff --git a/deployment/scripts/installation_utils/install_docker_and_utils.sh b/deployment/scripts/installation_utils/install_docker_and_utils.sh deleted file mode 100755 index 3c31eb64..00000000 --- a/deployment/scripts/installation_utils/install_docker_and_utils.sh +++ /dev/null @@ -1,221 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -target_user="${SUDO_USER:-${USER:-$(id -un)}}" -install_kubernetes="${INSTALL_KUBERNETES:-true}" -# INSTALL_K3S=true installs K3s; false skips it. Unset follows INSTALL_KUBERNETES. -install_k3s="${INSTALL_K3S:-$install_kubernetes}" -kubernetes_minor_version="${KUBERNETES_MINOR_VERSION:-v1.37}" -helm_version="${HELM_VERSION:-v4.3.1}" -k3s_version="${K3S_VERSION:-v1.37.2+k3s1}" - -case "$install_kubernetes" in - true|false) ;; - *) echo "INSTALL_KUBERNETES must be true or false." >&2; exit 1 ;; -esac - -case "$install_k3s" in - true|false) ;; - *) echo "INSTALL_K3S must be true or false." >&2; exit 1 ;; -esac - -if [ "$install_kubernetes" = "true" ]; then - if [[ ! "$kubernetes_minor_version" =~ ^v1\.[0-9]+$ ]]; then - echo "KUBERNETES_MINOR_VERSION must be a minor release such as v1.37." >&2 - exit 1 - fi - if [[ ! "$helm_version" =~ ^v4\.[0-9]+\.[0-9]+$ ]]; then - echo "HELM_VERSION must be a Helm 4 release such as v4.3.1." >&2 - exit 1 - fi -fi - -if [ "$install_k3s" = "true" ] && [[ ! "$k3s_version" =~ ^v1\.[0-9]+\.[0-9]+\+k3s[0-9]+$ ]]; then - echo "K3S_VERSION must be a K3s release such as v1.37.2+k3s1." >&2 - exit 1 -fi - -if [ "$(id -u)" -eq 0 ]; then - echo "Running with root privileges." -else - echo "This script installs system packages and may prompt for sudo privileges." - sudo -v -fi - -os_distribution="$(bash "${script_dir}/detect_os.sh")" - -echo "Found distribution: $os_distribution " - -if [ "$os_distribution" == "debian" ] || [ "$os_distribution" == "ubuntu" ]; -then - sudo apt-get update -y - sudo apt-get upgrade -y - - # NFS STUFF - sudo apt-get install -y samba cifs-utils nfs-common nfs-kernel-server archivemount - - # monitoring - sudo apt-get install -y htop iotop sysstat - - sudo apt-get install -y jq wget curl gnupg gnupg-agent git ca-certificates apt-transport-https python3 python3-pip python3-full openssl libssl-dev zip unzip tar nano gcc make python3-dev build-essential software-properties-common - - sudo install -m 0755 -d /etc/apt/keyrings - curl -fsSL "https://download.docker.com/linux/${os_distribution}/gpg" | sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/docker.gpg - sudo chmod a+r /etc/apt/keyrings/docker.gpg - - echo "Installing Azure CLI..." - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - - # shellcheck disable=SC1091 # Standard OS metadata file provided by Debian/Ubuntu. - docker_codename="$(. /etc/os-release && printf '%s' "$VERSION_CODENAME")" - echo \ - "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${os_distribution} \ - ${docker_codename} stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - sudo apt -y update - sudo apt -y upgrade - sudo apt -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - # create docker group and add the root user to it, as root will be used to run the docker - sudo groupadd -f docker - sudo usermod -aG docker root - if [ -n "$target_user" ] && [ "$target_user" != "root" ] && id "$target_user" >/dev/null 2>&1; then - sudo usermod -aG docker "$target_user" - fi - - # start the service - sudo systemctl enable docker.service - sudo systemctl start docker - - sudo apt-get -y autoremove - -elif [ "$os_distribution" == "redhat" ] || [ "$os_distribution" == "red hat" ] || [ "$os_distribution" == "centos" ]; -then - sudo yum -y update - sudo yum -y upgrade - - # NFS STUFF - sudo yum install -y samba samba-client cifs-utils nfs-utils rpcbind archivemount - - # monitoring - sudo yum install -y htop iotop sysstat - - sudo yum remove -y docker \ - docker-client \ - docker-client-latest \ - docker-common \ - docker-latest \ - docker-latest-logrotate \ - docker-logrotate \ - docker-engine - - sudo yum remove -y buildah podman - - # install necessary prerequisites - sudo yum install -y jq yum-utils wget curl git device-mapper-persistent-data lvm2 python3 python3-pip libffi-devel openssl openssl-devel zip unzip tar nano gcc gcc-c++ make python3-devel libevent-devel - - echo "Installing Azure CLI..." - sudo rpm --import https://packages.microsoft.com/keys/microsoft-2025.asc - sudo dnf install -y azure-cli - - sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - sudo yum-config-manager --enable docker-ce-stable - sudo yum-config-manager --enable docker-ce-stable-source - sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - - # create docker group and add the root user to it, as root will be used to run the docker process - sudo groupadd -f docker - sudo usermod -aG docker root - if [ -n "$target_user" ] && [ "$target_user" != "root" ] && id "$target_user" >/dev/null 2>&1; then - sudo usermod -aG docker "$target_user" - fi - - # start the service - sudo systemctl enable docker.service - sudo systemctl start docker - - sudo yum -y autoremove -else - echo "No instructions given for distribution: $os_distribution" >&2 - exit 1 -fi; - -if [ "$install_kubernetes" = "true" ] || [ "$install_k3s" = "true" ]; then - installer_temp_dir="$(mktemp -d)" - trap 'rm -rf "$installer_temp_dir"' EXIT -fi - -if [ "$install_kubernetes" = "true" ]; then - echo "Installing Kubernetes ${kubernetes_minor_version} (kubeadm, kubelet, kubectl)..." - kubernetes_repository="https://pkgs.k8s.io/core:/stable:/${kubernetes_minor_version}" - - if [ "$os_distribution" = "debian" ] || [ "$os_distribution" = "ubuntu" ]; then - curl -fsSL "${kubernetes_repository}/deb/Release.key" | \ - sudo gpg --batch --yes --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - sudo chmod a+r /etc/apt/keyrings/kubernetes-apt-keyring.gpg - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] ${kubernetes_repository}/deb/ /" | \ - sudo tee /etc/apt/sources.list.d/kubernetes.list > /dev/null - sudo apt-get update -y - sudo apt-get install -y kubelet kubeadm kubectl - # Cluster upgrades must be coordinated with kubeadm, not unattended upgrades. - sudo apt-mark hold kubelet kubeadm kubectl - else - sudo tee /etc/yum.repos.d/kubernetes.repo > /dev/null < /dev/null; then - pip_options+=(--break-system-packages) -fi -sudo -H pip3 "${pip_options[@]}" - -sudo sysctl -w vm.max_map_count=262144 - -sudo sh -c "echo 'vm.max_map_count=262144' >> /etc/sysctl.conf" -sudo sysctl -p - -echo "Finished installing Docker and utilities." diff --git a/deployment/scripts/installation_utils/install_helm.sh b/deployment/scripts/installation_utils/install_helm.sh new file mode 100755 index 00000000..abd12d33 --- /dev/null +++ b/deployment/scripts/installation_utils/install_helm.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bash "${script_dir}/detect_os.sh" > /dev/null + +if command -v helm >/dev/null 2>&1; then + echo "Helm is already installed." + exit 0 +fi + +installer_temp_dir="$(mktemp -d)" +trap 'rm -rf "$installer_temp_dir"' EXIT + +echo "Installing the latest Helm release..." +helm_installer="${installer_temp_dir}/get-helm-4" +curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 -o "$helm_installer" +sudo env -u DESIRED_VERSION bash "$helm_installer" + +helm version --short diff --git a/deployment/scripts/installation_utils/install_host_utils.sh b/deployment/scripts/installation_utils/install_host_utils.sh new file mode 100755 index 00000000..d3a1ec73 --- /dev/null +++ b/deployment/scripts/installation_utils/install_host_utils.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=installation_helpers.sh +source "${script_dir}/installation_helpers.sh" +os_distribution="$(bash "${script_dir}/detect_os.sh")" + +if [ "$os_distribution" == "debian" ] || [ "$os_distribution" == "ubuntu" ]; +then + install_missing_packages "$os_distribution" \ + samba cifs-utils nfs-common nfs-kernel-server archivemount \ + htop iotop sysstat \ + jq gettext-base wget curl gnupg gnupg-agent git ca-certificates apt-transport-https python3 python3-pip python3-full openssl libssl-dev zip unzip tar nano gcc make python3-dev build-essential software-properties-common + + if command -v az >/dev/null 2>&1 || package_is_installed "$os_distribution" azure-cli; then + echo "Azure CLI is already installed." + elif apt_repository_exists azure-cli https://packages.microsoft.com/repos/azure-cli; then + install_missing_packages "$os_distribution" azure-cli + else + echo "Installing Azure CLI..." + curl -fsSL https://aka.ms/InstallAzureCLIDeb | sudo bash + fi + +elif [ "$os_distribution" == "redhat" ]; +then + install_missing_packages "$os_distribution" \ + samba samba-client cifs-utils nfs-utils rpcbind archivemount \ + htop iotop sysstat \ + jq gettext yum-utils wget curl git device-mapper-persistent-data lvm2 python3 python3-pip libffi-devel openssl openssl-devel zip unzip tar nano gcc gcc-c++ make python3-devel libevent-devel + + if command -v az >/dev/null 2>&1 || package_is_installed "$os_distribution" azure-cli; then + echo "Azure CLI is already installed." + else + echo "Installing Azure CLI..." + sudo rpm --import https://packages.microsoft.com/keys/microsoft-2025.asc + install_missing_packages "$os_distribution" azure-cli + fi +else + echo "No instructions given for distribution: $os_distribution" >&2 + exit 1 +fi + +missing_python_packages=() +for package in html2text jsoncsv detect; do + if ! sudo -H pip3 show "$package" >/dev/null 2>&1; then + missing_python_packages+=("$package") + fi +done + +if [ "${#missing_python_packages[@]}" -gt 0 ]; then + echo "Installing missing Python packages: ${missing_python_packages[*]}" + + # Keep the distribution's pip; newer releases require an explicit override + # when installing into an externally managed system Python environment. + pip_options=(install "${missing_python_packages[@]}") + if pip3 install --help | grep -- '--break-system-packages' > /dev/null; then + pip_options+=(--break-system-packages) + fi + sudo -H pip3 "${pip_options[@]}" +else + echo "All required Python packages are already installed." +fi diff --git a/deployment/scripts/installation_utils/install_k3s.sh b/deployment/scripts/installation_utils/install_k3s.sh new file mode 100755 index 00000000..1f061846 --- /dev/null +++ b/deployment/scripts/installation_utils/install_k3s.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +bash "${script_dir}/detect_os.sh" > /dev/null + +if command -v k3s >/dev/null 2>&1; then + echo "K3s is already installed." + exit 0 +fi + +installer_temp_dir="$(mktemp -d)" +trap 'rm -rf "$installer_temp_dir"' EXIT + +echo "Installing the latest K3s release..." +k3s_installer="${installer_temp_dir}/install-k3s.sh" +curl -fsSL https://get.k3s.io -o "$k3s_installer" +# Install the binary and service without starting a cluster. K3s provides kubectl. +sudo env -u INSTALL_K3S_VERSION INSTALL_K3S_CHANNEL=latest \ + INSTALL_K3S_SKIP_ENABLE=true INSTALL_K3S_SKIP_START=true \ + sh "$k3s_installer" server --secrets-encryption +k3s --version +echo "K3s installed. Its service is not automatically enabled or started." diff --git a/deployment/scripts/installation_utils/install_os_tools_and_software.sh b/deployment/scripts/installation_utils/install_os_tools_and_software.sh new file mode 100755 index 00000000..54c3076a --- /dev/null +++ b/deployment/scripts/installation_utils/install_os_tools_and_software.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +install_k3s="${INSTALL_K3S:-true}" + +case "$install_k3s" in + true|false) ;; + *) echo "INSTALL_K3S must be true or false." >&2; exit 1 ;; +esac + +if [ "$(id -u)" -eq 0 ]; then + echo "Running with root privileges." +else + echo "This script installs system packages and may prompt for sudo privileges." + sudo -v +fi + +os_distribution="$(bash "${script_dir}/detect_os.sh")" + +echo "Found distribution: $os_distribution " + +bash "${script_dir}/install_host_utils.sh" +bash "${script_dir}/install_docker.sh" + +if [ "$install_k3s" = "true" ]; then + bash "${script_dir}/install_k3s.sh" + bash "${script_dir}/install_helm.sh" +fi + +echo "Finished installing Docker and utilities." diff --git a/deployment/scripts/installation_utils/installation_helpers.sh b/deployment/scripts/installation_utils/installation_helpers.sh new file mode 100644 index 00000000..3de2ffa4 --- /dev/null +++ b/deployment/scripts/installation_utils/installation_helpers.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +package_is_installed() { + local distribution="$1" package="$2" + + case "$distribution" in + debian|ubuntu) + [ "$(dpkg-query -W -f='${Status}' "$package" 2>/dev/null)" = "install ok installed" ] + ;; + redhat) rpm -q "$package" >/dev/null 2>&1 ;; + *) return 1 ;; + esac +} + +install_missing_packages() { + local distribution="$1" package + local missing_packages=() + shift + + for package in "$@"; do + if ! package_is_installed "$distribution" "$package"; then + missing_packages+=("$package") + fi + done + + if [ "${#missing_packages[@]}" -eq 0 ]; then + echo "All requested system packages are already installed." + return 0 + fi + + echo "Installing missing system packages: ${missing_packages[*]}" + case "$distribution" in + debian|ubuntu) + sudo apt-get update -y + sudo apt-get install -y "${missing_packages[@]}" + ;; + redhat) sudo yum install -y "${missing_packages[@]}" ;; + *) echo "No instructions given for distribution: $distribution" >&2; return 1 ;; + esac +} + +apt_repository_exists() { + local name="$1" uri="$2" source_file + + # Preserve existing files, including the newer deb822 .sources format. + if [ -e "/etc/apt/sources.list.d/${name}.list" ] || [ -e "/etc/apt/sources.list.d/${name}.sources" ]; then + return 0 + fi + + # The same repository may already be configured under a different filename. + for source_file in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [ -f "$source_file" ] && \ + grep -E '^[[:space:]]*(deb(-src)?[[:space:]]|URIs:)' "$source_file" | grep -F -- "$uri" >/dev/null; then + return 0 + fi + done + return 1 +} diff --git a/deployment/scripts/installation_utils/setup_k3s_cluster.sh b/deployment/scripts/installation_utils/setup_k3s_cluster.sh new file mode 100644 index 00000000..41170eda --- /dev/null +++ b/deployment/scripts/installation_utils/setup_k3s_cluster.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 [user@]FIRST_SERVER [[user@]SERVER ...]" >&2 + exit 1 +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +nodes=("$@") +remote_dirs=() + +remote() { + ssh -o BatchMode=no -S none -- "$@" +} + +cleanup() { + for index in "${!remote_dirs[@]}"; do + remote "${nodes[index]}" "rm -rf -- ${remote_dirs[index]}" || true + done +} +trap cleanup EXIT + +server_token() { + remote "$1" 'sudo -n bash -s' <<'REMOTE' +set -euo pipefail +if [ -s /var/lib/rancher/k3s/server/token ]; then + cat /var/lib/rancher/k3s/server/token +elif [ -e /var/lib/rancher/k3s/server ] || [ -e /var/lib/rancher/k3s/agent ]; then + echo "Existing K3s state has no server token; stopping." >&2 + exit 1 +fi +REMOTE +} + +for node in "${nodes[@]}"; do + echo "Installing prerequisites on $node..." + remote_dir="$(remote "$node" 'sudo -n true && mktemp -d')" + printf -v remote_dir '%q' "$remote_dir" + remote_dirs+=("$remote_dir") + tar -C "$script_dir" -cf - . | remote "$node" "tar -xf - -C $remote_dir" + remote "$node" "bash $remote_dir/install_os_tools_and_software.sh" +done + +token="$(server_token "${nodes[0]}")" +if [ -z "$token" ]; then + remote "${nodes[0]}" "bash ${remote_dirs[0]}/setup_kubernetes_cluster.sh k3s server --cluster-init" + token="$(server_token "${nodes[0]}")" +else + remote "${nodes[0]}" 'sudo -n systemctl start k3s' +fi +if [ -z "$token" ]; then + echo "K3s server token is unavailable." >&2 + exit 1 +fi +if [ "${#nodes[@]}" -gt 1 ]; then + remote "${nodes[0]}" 'sudo -n test -d /var/lib/rancher/k3s/server/db/etcd' || { + echo "The first server must use embedded etcd before other servers can join." >&2 + exit 1 + } +fi + +server_host="${nodes[0]#*@}" +if [[ "$server_host" == *:* && "$server_host" != \[*\] ]]; then + server_host="[$server_host]" +fi +printf -v server_url '%q' "https://${server_host}:6443" +for ((index = 1; index < ${#nodes[@]}; index++)); do + node="${nodes[index]}" + existing_token="$(server_token "$node")" + if [ -n "$existing_token" ]; then + if [ "$existing_token" != "$token" ]; then + echo "$node belongs to another cluster; stopping." >&2 + exit 1 + fi + remote "$node" 'sudo -n systemctl start k3s' + else + printf '%s\n' "$token" | remote "$node" \ + "IFS= read -r K3S_TOKEN && export K3S_TOKEN && bash ${remote_dirs[index]}/setup_kubernetes_cluster.sh k3s server --server $server_url" + fi +done + +for index in "${!nodes[@]}"; do + echo "Verifying ${nodes[index]}..." + remote "${nodes[index]}" \ + "sudo -n install -m 0600 -o \"\$(id -u)\" -g \"\$(id -g)\" /etc/rancher/k3s/k3s.yaml ${remote_dirs[index]}/kubeconfig && + bash ${remote_dirs[index]}/verify_docker.sh && + KUBECONFIG=${remote_dirs[index]}/kubeconfig bash ${remote_dirs[index]}/verify_k3s.sh" +done diff --git a/deployment/scripts/installation_utils/setup_kubernetes_cluster.sh b/deployment/scripts/installation_utils/setup_kubernetes_cluster.sh new file mode 100755 index 00000000..474d6d22 --- /dev/null +++ b/deployment/scripts/installation_utils/setup_kubernetes_cluster.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail + +case "${1:-}:${2:-}" in + k3s:server|k3s:agent|kubeconfig:?*) ;; + *) + echo "Usage: $0 k3s server|agent [options] | kubeconfig FILE" >&2 + exit 1 + ;; +esac + +backend="$1" +role="$2" +shift 2 + +case "$backend" in + k3s) + if [ "$role" = "server" ]; then + set -- --secrets-encryption "$@" + fi + k3s --version + installer="$(mktemp)" + trap 'rm -f "$installer"' EXIT + curl -fsSL https://get.k3s.io -o "$installer" + sudo --preserve-env=K3S_TOKEN env INSTALL_K3S_SKIP_DOWNLOAD=true \ + INSTALL_K3S_SKIP_ENABLE=false INSTALL_K3S_SKIP_START=false INSTALL_K3S_FORCE_RESTART=true \ + sh "$installer" "$role" "$@" + if [ "$role" = "agent" ]; then + exit 0 + fi + kubeconfig="/etc/rancher/k3s/k3s.yaml" + ;; + kubeconfig) + if [ "$#" -ne 0 ]; then + echo "Usage: $0 kubeconfig FILE" >&2 + exit 1 + fi + kubeconfig="$role" + ;; +esac + +if ! sudo test -r "$kubeconfig"; then + echo "Kubeconfig unavailable: $kubeconfig. Initialize or start the control plane first." >&2 + exit 1 +fi + +config_dir="${HOME}/.kube" +config_file="${config_dir}/config" +if [ -e "$config_file" ] || [ -L "$config_file" ]; then + echo "Existing kubeconfig retained: $config_file" + echo "Use: sudo kubectl --kubeconfig \"$kubeconfig\" get nodes" +else + mkdir -p "$config_dir" + sudo install -m 0600 -o "$(id -u)" -g "$(id -g)" "$kubeconfig" "$config_file" + echo "Kubeconfig installed: $config_file" + echo "Use: kubectl --kubeconfig \"$config_file\" get nodes" +fi diff --git a/deployment/scripts/installation_utils/verify_docker.sh b/deployment/scripts/installation_utils/verify_docker.sh new file mode 100644 index 00000000..236a892d --- /dev/null +++ b/deployment/scripts/installation_utils/verify_docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +docker version +docker compose version +docker ps diff --git a/deployment/scripts/installation_utils/verify_k3s.sh b/deployment/scripts/installation_utils/verify_k3s.sh new file mode 100644 index 00000000..2ec62ede --- /dev/null +++ b/deployment/scripts/installation_utils/verify_k3s.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export KUBECONFIG="${KUBECONFIG:-${HOME}/.kube/config}" +kubectl get nodes +kubectl get pods -A diff --git a/deployment/terraform/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl b/deployment/terraform/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl index a718dc63..c2807628 100644 --- a/deployment/terraform/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl +++ b/deployment/terraform/cogstack-docker-services/resources/config/observability/alloy/probers/probe.observability.yml.tmpl @@ -1,14 +1,14 @@ - targets: - - cogstack-observability-traefik-1/grafana/api/health + - http://nginx:8001/grafana/api/health labels: name: grafana job: probe-observability-stack ip_address: "{{ OBSERVABILITY_SERVICE_IP }}" host: "{{ OBSERVABILITY_SERVICE_HOSTNAME }}" - targets: - - cogstack-observability-traefik-1/prometheus/-/healthy + - http://nginx:8001/prometheus/-/healthy labels: name: prometheus job: probe-observability-stack ip_address: "{{ OBSERVABILITY_SERVICE_IP }}" - host: "{{ OBSERVABILITY_SERVICE_HOSTNAME }}" \ No newline at end of file + host: "{{ OBSERVABILITY_SERVICE_HOSTNAME }}" diff --git a/deployment/terraform/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml b/deployment/terraform/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml index ced2c3ba..916452b9 100644 --- a/deployment/terraform/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml +++ b/deployment/terraform/cogstack-docker-services/resources/config/observability/observability.docker-compose.yml @@ -21,19 +21,15 @@ services: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - CONFIG_HASH=${CONFIG_HASH} - traefik: - image: cogstacksystems/cogstack-observability-traefik:latest + nginx: + image: cogstacksystems/nginx:latest + environment: + - NGINX_ENVSUBST_TEMPLATE_DIR=/etc/nginx/templates/observability networks: - observability restart: unless-stopped ports: - # The HTTP port - - "80:80" - # The Web UI (enabled by --api.insecure=true) - - "8080:8080" - volumes: - # So that Traefik can listen to the Docker events - - /var/run/docker.sock:/var/run/docker.sock:ro + - "80:8001" alloy: image: cogstacksystems/cogstack-observability-alloy:latest networks: diff --git a/docs/docs/cogstack-ce/reference.md b/docs/docs/cogstack-ce/reference.md index 9120dc32..5dbb8613 100644 --- a/docs/docs/cogstack-ce/reference.md +++ b/docs/docs/cogstack-ce/reference.md @@ -87,7 +87,7 @@ Everything in CogStack CE is deployed as Helm subcharts of the [CogStack CE Helm - [MedCAT Service Helm](../platform/deployment/helm/charts/medcat-service.md) (used for both MedCAT and AnonCAT) - [OCR Service Helm](../platform/deployment/helm/charts/ocr-service.md) - [MedCAT Trainer Helm](../platform/deployment/helm/charts/medcat-trainer.md) -- [CogStack JupyterHub Helm](../platform/deployment/helm/charts/jupyterhub.md) +- [CogStack JupyterHub Helm](../platform/deployment/helm/charts/jupyter-hub.md) JupyterHub is based on the [official JupyterHub Helm chart](https://hub.jupyter.org/helm-chart/). OpenSearch and OpenSearch Dashboards also come from the [official OpenSearch Helm charts](https://opensearch-project.github.io/helm-charts/). The CogStack wrappers add defaults, images, and example notebooks on top of those upstream charts. diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 662333f0..73518e22 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -78,7 +78,7 @@ nav: - OCR Service Helm: platform/deployment/helm/charts/ocr-service.md - Cogstack Cohorter Helm: platform/deployment/helm/charts/cohorter.md - Cogstack Observability Helm: platform/deployment/helm/charts/observability.md - - Cogstack JupyterHub Helm: platform/deployment/helm/charts/jupyterhub.md + - Cogstack JupyterHub Helm: platform/deployment/helm/charts/jupyter-hub.md - Examples: - Examples: platform/deployment/examples/index.md - AWS Kubernetes EKS: platform/deployment/examples/aws-kubernetes-eks.md diff --git a/docs/scripts/copy_files_from_repo.py b/docs/scripts/copy_files_from_repo.py index e4c91f11..d904f42a 100644 --- a/docs/scripts/copy_files_from_repo.py +++ b/docs/scripts/copy_files_from_repo.py @@ -22,15 +22,15 @@ # Add more entries here to copy additional static files into documentation. COPY_SPECS = [ { - "sourceFilePath": "deployment/charts/jupyterhub/examples/medcat-service-tutorial.ipynb", + "sourceFilePath": "deployment/charts/jupyter-hub/examples/medcat-service-tutorial.ipynb", "outputFilePath": "platform/cogstack-ai/medcat-service-tutorial.ipynb", }, { - "sourceFilePath": "deployment/charts/jupyterhub/examples/medcat-opensearch-e2e.ipynb", + "sourceFilePath": "deployment/charts/jupyter-hub/examples/medcat-opensearch-e2e.ipynb", "outputFilePath": "cogstack-ce/tutorial/medcat-opensearch-e2e.ipynb", }, { - "sourceFilePath": "deployment/charts/jupyterhub/examples/ocr-service-tutorial.ipynb", + "sourceFilePath": "deployment/charts/jupyter-hub/examples/ocr-service-tutorial.ipynb", "outputFilePath": "data-engineering/ocr-service-tutorial.ipynb", }, ] diff --git a/release-please-config.json b/release-please-config.json index 3c72baed..ac58c00a 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -98,7 +98,7 @@ { "type": "revert", "section": "Reverts" } ] }, - "deployment/charts/jupyterhub": { + "deployment/charts/jupyter-hub": { "release-type": "helm", "component": "helm-charts/cogstack-jupyterhub-helm", "changelog-path": "CHANGELOG.md",