feat: deploy gubbins images

.github/workflows/deployment.yml

@@ -219,3 +219,109 @@ jobs:
           platforms: linux/amd64,linux/arm64
           build-args: |
             EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
+
+# -------------------------------------------------------------------------------------------------
+# Gubbins deployment
+# -------------------------------------------------------------------------------------------------
+
+  build-gubbins-wheels:
+    name: Build wheels
+    runs-on: "ubuntu-latest"
+    needs: deploy-pypi
+    defaults:
+      run:
+        # We need extglob for REFERENCE_BRANCH substitution
+        shell: bash -l -O extglob {0}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Installing dependencies
+        run: |
+          python -m pip install \
+              build \
+              python-dateutil \
+              pytz \
+              readme_renderer[md] \
+              requests \
+              setuptools_scm
+      - name: Build distributions
+        run: |
+          # build the gubbins package
+          for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
+            # Skip the testing package
+            if [[ "${pkg_dir}" =~ .*testing.* ]];
+            then
+              echo "Do not build ${pkg_dir}";
+              continue;
+            fi
+            echo "Building $pkg_dir"
+            python -m build --outdir $PWD/dist $pkg_dir
+          done
+      - name: 'Upload Artifact'
+        uses: actions/upload-artifact@v4
+        with:
+          name: gubbins-whl
+          path: dist/*.whl
+          retention-days: 5
+
+  # Build to docker image with the code in it
+  docker-gubbins:
+    needs:
+      - deploy-pypi
+      - build-gubbins-wheels
+    timeout-minutes:  30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download diracx wheels
+        uses: actions/download-artifact@v4
+        with:
+            name: diracx-whl
+      - name: Download gubbins wheels
+        uses: actions/download-artifact@v4
+        with:
+            name: gubbins-whl
+      - name: "Find wheels"
+        id: find_wheel
+        run: |
+          # We need to copy them there to be able to access them in the RUN --mount
+          cp diracx*.whl gubbins*.whl extensions/containers/client/
+          cp diracx*.whl gubbins*.whl extensions/containers/services/
+          for wheel_fn in *.whl; do
+            pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
+            echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
+          done
+
+      - name: Build and push client (dev)
+        uses: docker/build-push-action@v6
+        with:
+          context: extensions/containers/client
+          push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
+          tags: ghcr.io/diracgrid/gubbins/client:dev
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
+      - name: Build and push service (dev)
+        uses: docker/build-push-action@v6
+        with:
+          context: extensions/containers/services
+          push: ${{ github.event_name != 'pull_request' && github.repository == 'DIRACGrid/diracx' && github.ref_name == 'main' }}
+          tags: ghcr.io/diracgrid/gubbins/services:dev
+          platforms: linux/amd64,linux/arm64
+          build-args: |
+            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
+            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl

.github/workflows/main.yml

@@ -113,15 +113,19 @@ jobs:
         with:
           cache: false
           environments: ${{ matrix.extension == 'diracx' && 'default' || 'default-gubbins' }}
-      - name: Build gubbins wheels
-        if: ${{ matrix.extension == 'gubbins' }}
+      - name: Build diracx wheels
         run: |
           for pkg_dir in $PWD/diracx-*; do
             echo "Building $pkg_dir"
-            pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
+            pixi exec python-build --outdir $PWD/containers/services/ $pkg_dir
           done
           # Also build the diracx metapackage
-          pixi exec python-build --outdir $PWD/extensions/containers/services/ .
+          pixi exec python-build --outdir $PWD/containers/services/ .
+      - name: Build gubbins wheels
+        if: ${{ matrix.extension == 'gubbins' }}
+        run: |
+          cp $PWD/containers/services/*.whl $PWD/extensions/containers/services/
+          cp $PWD/containers/services/*.tar.gz $PWD/extensions/containers/services/
           # And build the gubbins package
           for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
             # Skip the testing package
@@ -134,32 +138,38 @@ jobs:
             pixi exec python-build --outdir $PWD/extensions/containers/services/ $pkg_dir
           done
       - name: Set up Docker Buildx
-        if: ${{ matrix.extension == 'gubbins' }}
         uses: docker/setup-buildx-action@v3
+      - name: Build container for diracx
+        if: ${{ matrix.extension == 'diracx' }}
+        uses: docker/build-push-action@v6
+        with:
+          context: containers/services
+          tags: diracx/services:dev
+          load: true
+          build-args: |
+            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
       - name: Build container for gubbins
         if: ${{ matrix.extension == 'gubbins' }}
         uses: docker/build-push-action@v6
         with:
           context: extensions/containers/services
           tags: gubbins/services:dev
-          outputs: type=docker,dest=/tmp/gubbins_services_image.tar
+          load: true
           build-args: |
+            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
             EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_logic*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
-      - name: Load image
-        if: ${{ matrix.extension == 'gubbins' }}
-        run: |
-          docker load --input /tmp/gubbins_services_image.tar
-          docker image ls -a
+
       - name: Start demo
         run: |
-          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
+          git clone --branch main_FEAT_remove-loaded-images https://github.com/aldbr/diracx-charts.git ../diracx-charts
 
           declare -a demo_args=()
           demo_args+=("--enable-open-telemetry")
           demo_args+=("--enable-coverage")
           demo_args+=("--exit-when-done")
           demo_args+=("--set-value" "developer.autoReload=false")
           demo_args+=("--ci-values" "../diracx-charts/demo/ci_values.yaml")
+          demo_args+=("--prune-loaded-images")
 
           declare -a demo_source_dirs=("$PWD")
 
@@ -176,7 +186,10 @@ jobs:
             demo_args+=("--ci-values" "./extensions/gubbins_values.yaml")
             demo_args+=("--load-docker-image" "gubbins/services:dev")
             demo_source_dirs+=("/tmp/gubbins/")
-          elif [ ${{ matrix.extension }} != 'diracx' ]; then
+          elif [ ${{ matrix.extension }} == 'diracx' ]; then
+            demo_args+=("--ci-values" "./diracx_values.yaml")
+            demo_args+=("--load-docker-image" "diracx/services:dev")
+          else
             echo "Unknown extension: ${{ matrix.extension }}"
             exit 1
           fi

diracx_values.yaml

@@ -0,0 +1,11 @@
+# values specific to testing gubbins
+global:
+  # Needed to be able to force pre-load the image in kind
+  # see https://kind.sigs.k8s.io/docs/user/quick-start/#loading-an-image-into-your-cluster
+  # and https://iximiuz.com/en/posts/kubernetes-kind-load-docker-image/
+  imagePullPolicy: IfNotPresent
+  images:
+    services: diracx/services
+
+developer:
+  enabled: true