Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/scripts/helm-recursive-dependency-update.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
medcat-trainer-helm depends on postgresql.
When we publish this into dockerhub, it means that inside medcat-trainer-helm.tgz, there's a charts/postgresql.tgz file.

cogstack-ce-helm depends on medcat-trainer-helm
When we publish this into dockerhub, inside cogstack-ce-helm.tgz, we want to have a charts/medcat-trainer-helm.tgz file, and inside that file we should have charts/solr.tgz.
ce-helm depends on medcat-trainer-helm
When we publish this into dockerhub, inside ce-helm.tgz, we want to have a charts/medcat-trainer-helm.tgz file, and inside that file we should have charts/solr.tgz.

The complex part is that we are using the file:// path to depend on the subchart. This means we need to run `helm dependency update` in that folder specifically before we can reference it in the parent chart.
If we weren't using file:// but referencing the tar in dockerhub, there wouldnt be any issue.
Expand Down
89 changes: 89 additions & 0 deletions .github/workflows/docker_build_nifi.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
name: docker_build_nifi

on:
push:
branches: ['**']
tags: ['v*.*.*']
paths: &paths
- 'apps/nifi/**'
- '.github/workflows/docker_build_nifi.yml'
- '!**/*.md'
- '!**/README*'
pull_request:
paths: *paths
release:
types: [published]
workflow_dispatch:

permissions:
contents: read

concurrency:
group: docker-nifi-${{ github.ref }}
cancel-in-progress: true

jobs:
build:
if: github.event_name != 'release' || startsWith(github.event.release.tag_name, 'v')
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- {arch: amd64, runner: ubuntu-24.04}
- {arch: arm64, runner: ubuntu-24.04-arm}
steps:
- uses: actions/checkout@v7
- uses: docker/setup-buildx-action@v4
- &login
uses: docker/login-action@v4
if: github.event_name != 'pull_request'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- uses: docker/metadata-action@v6
id: meta
with:
images: &image cogstacksystems/nifi
flavor: suffix=-${{ matrix.arch }},onlatest=true
tags: &tags |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
type=ref,event=branch,enable=${{ github.event_name != 'pull_request' }}
type=sha,format=short
- uses: docker/build-push-action@v7
with:
context: apps/nifi
file: apps/nifi/Dockerfile
platforms: linux/${{ matrix.arch }}
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=nifi-${{ matrix.arch }}
cache-to: type=gha,scope=nifi-${{ matrix.arch }},mode=max
provenance: false

manifest:
needs: build
if: github.event_name != 'pull_request'
runs-on: ubuntu-24.04
steps:
- uses: docker/setup-buildx-action@v4
- *login
- uses: docker/metadata-action@v6
id: meta
with:
images: *image
tags: *tags
- name: Publish multi-arch manifests
env:
TAGS: ${{ steps.meta.outputs.tags }}
run: |
while IFS= read -r tag; do
[ -n "$tag" ] || continue
for attempt in {1..12}; do
docker buildx imagetools create --tag "$tag" "$tag-amd64" "$tag-arm64" && break
[ "$attempt" -lt 12 ] || exit 1
sleep 5
done
done <<< "$TAGS"
24 changes: 16 additions & 8 deletions .github/workflows/docker_build_observability.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,24 +30,25 @@ jobs:
- name: prometheus
context: apps/observability/prometheus
dockerfile: apps/observability/prometheus/Dockerfile.prometheus
image: cogstacksystems/cogstack-observability-prometheus
image: cogstacksystems/observability-prometheus
- name: blackbox
context: apps/observability/prometheus
dockerfile: apps/observability/prometheus/Dockerfile.blackbox
image: cogstacksystems/cogstack-observability-blackbox-exporter
image: cogstacksystems/observability-blackbox-exporter
- name: grafana
context: apps/observability/grafana
dockerfile: apps/observability/grafana/Dockerfile
image: cogstacksystems/cogstack-observability-grafana
image: cogstacksystems/observability-grafana
- name: alloy
context: apps/observability/grafana-alloy
dockerfile: apps/observability/grafana-alloy/Dockerfile
image: cogstacksystems/cogstack-observability-alloy
image: cogstacksystems/observability-alloy
steps:
- name: Checkout branch
uses: actions/checkout@v7

- name: Login to DockerHub
if: github.event_name != 'pull_request'
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
Expand Down Expand Up @@ -76,11 +77,11 @@ jobs:
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
push: true
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ matrix.image}}:buildcache
cache-to: type=registry,ref=${{ matrix.image}}:buildcache,mode=max
cache-from: type=gha,scope=observability-${{ matrix.name }}
cache-to: type=gha,scope=observability-${{ matrix.name }},mode=max

test-observability-quickstart-scripts:
needs: docker
Expand All @@ -91,6 +92,13 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v7

- name: Build images for PR integration tests
if: github.event_name == 'pull_request'
run: |
docker build -t cogstacksystems/observability-alloy:sha-${GITHUB_SHA::7} apps/observability/grafana-alloy
docker build -t cogstacksystems/observability-prometheus:sha-${GITHUB_SHA::7} -f apps/observability/prometheus/Dockerfile.prometheus apps/observability/prometheus
docker build -t cogstacksystems/observability-grafana:sha-${GITHUB_SHA::7} apps/observability/grafana

- name: Build NGINX for integration tests
run: docker build -t cogstacksystems/nginx:test apps/nginx

Expand All @@ -113,7 +121,7 @@ jobs:
{
echo "services:"
for service in alloy prometheus grafana; do
printf ' %s:\n image: cogstacksystems/cogstack-observability-%s:%s\n' \
printf ' %s:\n image: cogstacksystems/observability-%s:%s\n' \
"$service" "$service" "$IMAGE_TAG"
done
printf ' nginx:\n image: cogstacksystems/nginx:test\n'
Expand Down
86 changes: 71 additions & 15 deletions .github/workflows/docker_build_ocr_service.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,33 +4,89 @@ on:
push:
branches: [main]
tags: ['ocr-service-v*.*.*']
paths:
paths: &paths
- "apps/ocr-service/**"
- ".github/workflows/docker_build_ocr_service.yml"
- ".github/workflows/test_ocr_service.yml"
- ".github/workflows/docker_template_ocr_service.yml"
- ".github/workflows/docker_smoke_ocr_service.yml"
pull_request:
paths:
- "apps/ocr-service/**"
- ".github/workflows/docker_build_ocr_service.yml"
- ".github/workflows/test_ocr_service.yml"
- ".github/workflows/docker_template_ocr_service.yml"
- ".github/workflows/docker_smoke_ocr_service.yml"
paths: *paths
release:
types: [published]
workflow_dispatch:

permissions:
contents: read

concurrency:
group: docker-cogstacksystems/ocr-service-${{ github.ref }}
cancel-in-progress: true

jobs:
build:
if: github.event_name != 'release' || startsWith(github.event.release.tag_name, 'ocr-service-v')
uses: ./.github/workflows/docker_template_ocr_service.yml
with:
image: cogstacksystems/ocr-service
context: apps/ocr-service
dockerfile: apps/ocr-service/Dockerfile
cache_scope: cogstack-ocr-service
secrets: inherit
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- {platform: amd64, runner: ubuntu-24.04}
- {platform: arm64, runner: ubuntu-24.04-arm}
steps:
- uses: actions/checkout@v7
- uses: docker/setup-buildx-action@v4
- &login
uses: docker/login-action@v4
if: github.event_name != 'pull_request'
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- uses: docker/metadata-action@v6
id: meta
with:
images: &image cogstacksystems/ocr-service
flavor: |
latest=false
suffix=-${{ matrix.platform }},onlatest=true
tags: &tags |
type=raw,value=latest,enable={{is_default_branch}}
type=schedule
type=ref,event=branch
type=ref,event=tag
type=ref,event=pr
type=sha
type=match,pattern=ocr-service-v(\d+\.\d+\.\d+),group=1
- uses: docker/build-push-action@v7
with:
context: apps/ocr-service
file: apps/ocr-service/Dockerfile
platforms: linux/${{ matrix.platform }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha,scope=cogstack-ocr-service-${{ matrix.platform }}
cache-to: type=gha,mode=max,scope=cogstack-ocr-service-${{ matrix.platform }}
push: ${{ github.event_name != 'pull_request' }}
provenance: false

manifest:
needs: build
if: github.event_name != 'pull_request'
runs-on: ubuntu-24.04
steps:
- uses: docker/setup-buildx-action@v4
- *login
- uses: docker/metadata-action@v6
id: meta
with:
images: *image
tags: *tags
flavor: latest=false
- name: Create multi-arch manifests
env:
TAGS: ${{ steps.meta.outputs.tags }}
run: |
while IFS= read -r ref; do
tag="${ref##*:}"
[[ "$tag" == latest || "$tag" =~ ^v?[0-9]+\.[0-9]+(\.[0-9]+)?$ ]] || continue
ref="${ref%:*}:${tag#v}"
docker buildx imagetools create --tag "$ref" "$ref-amd64" "$ref-arm64"
done <<< "$TAGS"
4 changes: 2 additions & 2 deletions .github/workflows/docker_smoke_ocr_service.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,14 @@ on:
paths:
- "apps/ocr-service/**"
- ".github/workflows/test_ocr_service.yml"
- ".github/workflows/docker_template_ocr_service.yml"
- ".github/workflows/docker_build_ocr_service.yml"
- ".github/workflows/docker_smoke_ocr_service.yml"
pull_request:
branches: [ "*" ]
paths:
- "apps/ocr-service/**"
- ".github/workflows/test_ocr_service.yml"
- ".github/workflows/docker_template_ocr_service.yml"
- ".github/workflows/docker_build_ocr_service.yml"
- ".github/workflows/docker_smoke_ocr_service.yml"
workflow_dispatch:

Expand Down
Loading
Loading