name: 'VM Integration Tests'

on:
  workflow_dispatch:
    inputs:
      git_ref:
        description: git ref, branch or tag to test against
        required: false
        type: string
  workflow_call:
    inputs:
      git_ref:
        description: git ref, branch or tag to test against
        required: false
        type: string
  push:
    branches:
      - master

jobs:
  installation-test:
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:bookworm
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
        UID: "${{ github.run_id }}-install"
    outputs:
      server_address: ${{ steps.create-test-instance.outputs.server_address }}
      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
      version: ${{ env.VERSION }}
      test_result: ${{ steps.final_test.outputs.test_result }}
      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
      HOME: /root
      SSH_ARTIFACT_NAME: "${{ github.run_id }}-install-ssh"
      UID: "${{ github.run_id }}-install"
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v3
        with:
          path: /__w/nextcloudpi/nextcloudpi
      - name: Generate ssh keypair
        working-directory: /__w/nextcloudpi/nextcloudpi
        run: |
          set -e
          mkdir -p .ssh
          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
          . /ncp-test-automation/bin/entrypoint.sh
      - name: upload ssh private key to artifact store
        uses: actions/upload-artifact@v4
        with:
          name: "${{ env.SSH_ARTIFACT_NAME }}"
          path: /__w/nextcloudpi/nextcloudpi/.ssh
          if-no-files-found: error
          include-hidden-files: true
      - id: create-test-instance
        uses: ./.github/actions/create-test-instance
        with:
          version: ${{ env.VERSION }}
          uid: "${{ env.UID }}"
          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
          server_type: "cx22"
      - name: set instance variables
        run: |
          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
      - name: Test postinstall VM
        id: final_test
        working-directory: /ncp-test-automation/bin
        run: |
          set -e
          echo "Setup ssh"
          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          
          echo "Run integration tests"
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
          set -x
          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
          
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "and ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            exit 1
          }
          echo "test_result=success" >> "$GITHUB_OUTPUT";

#  update-test:
#    runs-on: ubuntu-latest
#    container:
#      image: thecalcaholic/ncp-test-automation:bullseye
#      env:
#        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
#        UID: "${{ github.run_id }}-update"
#    defaults:
#      run:
#        shell: bash
#    outputs:
#      server_address: ${{ steps.create-test-instance.outputs.server_address }}
#      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
#      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
#      previous_version: ${{ steps.find-version.outputs.previous_version }}
#      version: ${{ env.VERSION }}
#      test_result: ${{ steps.final_test.outputs.test_result }}
#      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
#    env:
#      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
#      HOME: /root
#      SSH_ARTIFACT_NAME: "${{ github.run_id }}-update-ssh"
#      UID: "${{ github.run_id }}-update"
#    steps:
#      - uses: actions/checkout@v3
#        with:
#          fetch-depth: 0
#          path: /__w/nextcloudpi/nextcloudpi
#      - name: find reference version
#        working-directory: /__w/nextcloudpi/nextcloudpi
#        shell: bash
#        id: find-version
#        run: |
#          chown -R "$(id -u):$(id -g)" .
#          set -e
#          if [[ -n "${{ github.base_ref }}" ]]
#          then
#            version="${{ github.base_ref }}"
#          elif [[ "${{ github.ref }}" == "refs/heads/devel" ]]
#          then
#            version="master"
#          else
#            git fetch -fu --tags origin ${{ github.ref }}:${{ github.ref }}
#            version="$(git describe --tags)"
#            [[ "$version" =~ .*-.*-.* ]] || {
#              git checkout HEAD~1
#              version="$(git describe --tags)"
#            }
#            version="${version%-*-*}"
#          fi
#          echo "Previous version is '$version'"
#          #TODO: Revert to dynamically found version
#          #echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV"
#          echo "PREVIOUS_VERSION=v1.53.3" >> "$GITHUB_ENV"
#      - name: Generate ssh key
#        run: |
#          set -x
#          mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
#          ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
#          . /ncp-test-automation/bin/entrypoint.sh
#      - name: upload ssh private key to artifact store
#        uses: actions/upload-artifact@v3
#        with:
#          name: "${{ env.SSH_ARTIFACT_NAME }}"
#          path: /__w/nextcloudpi/nextcloudpi/.ssh
#          if-no-files-found: error
#      - id: create-test-instance
#        uses: ./.github/actions/create-test-instance-bullseye
#        with:
#          version: "${{ env.PREVIOUS_VERSION }}"
#          uid: "${{ env.UID }}"
#          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
#          server_type: "cx11"
#      - name: Set instance variables
#        run: |
#          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
#          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
#      - uses: actions/checkout@v3
#        with:
#          repository: 'theCalcaholic/ncp-test-automation'
#          ref: "bullseye"
#          path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
#      - name: Activate and Test postinstall VM
#        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
#        run: |
#          set -e
#          echo "Setup ssh"
#          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
#          eval "$(ssh-agent)"
#          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
#
#          source ./library.sh
#
#          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
#
#          setup-ssh-port-forwarding "$SERVER_ADDRESS"
#
#          echo "Run integration tests"
#          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
#          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
#
#            echo "Integration tests failed"
#            echo "Here are the last lines of ncp-install.log:"
#            echo "==========================================="
#            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
#            echo "==========================================="
#            echo "and ncp.log:"
#            echo "==========================================="
#            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
#            echo "==========================================="
#            exit 1
#          }
#      - name: perform update
#        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
#        run: |
#          set -e
#
#          echo "Setup ssh"
#          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
#          eval "$(ssh-agent)"
#          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
#
#          source ./library.sh
#
#          echo "Updating from $PREVIOUS_VERSION to $VERSION"
#          ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
#          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
#      - name: Run integration tests after update
#        id: final_test
#        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
#        run: |
#          set -e
#
#          echo "Setup ssh"
#          eval "$(ssh-agent)"
#          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
#
#          source ./library.sh
#
#          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
#
#          echo "Run integration tests"
#          setup-ssh-port-forwarding "$SERVER_ADDRESS"
#          NC_TEST_ARGS=()
#          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
#          set -x
#          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
#
#            echo "Integration tests failed"
#            echo "Here are the last lines of ncp-install.log:"
#            echo "==========================================="
#            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
#            echo "==========================================="
#            echo "and ncp.log:"
#            echo "==========================================="
#            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
#            echo "==========================================="
#            exit 1
#          }
#
#          echo "test_result=success" >> "$GITHUB_OUTPUT";

  dist-upgrade-test:
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:bullseye
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
        UID: "${{ github.run_id }}-distupgrade"
    defaults:
      run:
        shell: bash
    outputs:
      test_result: ${{ steps.final_test.outputs.test_result }}
      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
      server_address: ${{ steps.create-test-instance.outputs.server_address }}
      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
      previous_version: ${{ env.PREVIOUS_VERSION }}
      version: ${{ env.VERSION }}
    env:
      PREVIOUS_VERSION: "v1.53.3"
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
      HOME: /root
      SSH_ARTIFACT_NAME: "${{ github.run_id }}-distupgrade-ssh"
      UID: "${{ github.run_id }}-distupgrade"
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
          path: /__w/nextcloudpi/nextcloudpi
      - name: Generate ssh key
        run: |
          set -x
          chown -R "$(id -u):$(id -g)" /__w/nextcloudpi/nextcloudpi
          mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
          ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
          . /ncp-test-automation/bin/entrypoint.sh
      - name: upload ssh private key to artifact store
        uses: actions/upload-artifact@v4
        with:
          name: "${{ env.SSH_ARTIFACT_NAME }}"
          path: /__w/nextcloudpi/nextcloudpi/.ssh
          if-no-files-found: error
          include-hidden-files: true
      - id: create-test-instance
        uses: ./.github/actions/create-test-instance-bullseye
        with:
          version: "${{ env.PREVIOUS_VERSION }}"
          uid: "${{ env.UID }}"
          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
          server_type: "cx22"
      - name: Set instance variables
        run: |
          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
      - uses: actions/checkout@v3
        with:
          repository: 'theCalcaholic/ncp-test-automation'
          ref: "bullseye"
          path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
      - name: Activate and Test postinstall VM
        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
        run: |
          set -e
          echo "Setup ssh"
          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          
          echo "Run integration tests"
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "v1.54.3" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
          
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "and ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            exit 1
          }
      - name: perform update
        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
        run: |
          set -e
          
          echo "Setup ssh"
          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh

          echo "Updating from $PREVIOUS_VERSION to $VERSION"
          ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
      - name: Run integration tests after update
        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
        run: |
          set -e
          
          echo "Setup ssh"
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          echo "Run integration tests"
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          NC_TEST_ARGS=()
          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
          set -x
          test-ncp-instance -f "$SNAPSHOT_ID" -b "v1.54.3" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
          
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "and ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            exit 1
          }

      - name: NCP distupgrade
        id: distupgrade
        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
        run: |
          set -e
          
          echo "Setup ssh"
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep 'VERSION_ID="11"' || {
            echo "Can't upgrade from Debian $(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" grep 'VERSION_ID=' /etc/os-release)"
            echo "skipped=yes" | tee -a $GITHUB_OUTPUT
            exit 1
          }
          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade
          echo "skipped=no" | tee -a $GITHUB_OUTPUT
      - name: Update Nextcloud
        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
        run: |
          set -ex
          apk add jq
          
          echo "Setup ssh"
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          current_nc_version="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')"
          latest_nc_version="$(cat ../../etc/ncp.cfg | jq -r '.nextcloud_version')"
          
          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
          then
            echo "Nextcloud is up to date - skipping NC update test."
          else
            for i in {1..10};
            do
              echo "running nc update ($i/10)..."
              ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "bash -c 'DBG=x ncp-update-nc ${latest_nc_version?}'"
          
              ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" /usr/local/bin/ncc status
          
              current_nc_version_new="$(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncc status" | grep "version:" | awk '{ print $3 }')"
              if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]]
              then
                break
              fi
              if [[ "$current_nc_version" == "$current_nc_version_new" ]]
              then
                echo "failed to update to $latest_nc_version"
                exit 1
              fi
              current_nc_version="$current_nc_version_new"
            done
          fi

      - name: Run integration tests after dist-upgrade
        id: final_test
        working-directory: /ncp-test-automation/bin
        run: |
          set -e
          
          echo "Setup ssh"
          eval "$(ssh-agent)"
          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
          
          source ./library.sh
          
          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          echo "Run integration tests"
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          
          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
          
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            echo "nextcloud.log:"
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /opt/ncdata/data/nextcloud.log;
            exit 1
          }
          echo "test_result=success" >> "$GITHUB_OUTPUT"

  install-postactivation-snapshot:
    if: ${{ always() }}
    needs:
      - installation-test
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"

    env:
      TEST_TYPE: install
      SERVER_ADDRESS: ${{ needs.installation-test.outputs.server_address }}
      TEST_RESULT: ${{ needs.installation-test.test_result }}
      TEST_SERVER_ID: ${{ needs.installation-test.outputs.test_server_id }}
      VERSION: ${{ needs.installation-test.outputs.version }}
      SSH_ARTIFACT_NAME: "${{ needs.installation-test.outputs.ssh_artifact_name }}"
      UID: ${{ github.run_id }}-install
    steps:
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v4
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        with:
          name: ${{ env.SSH_ARTIFACT_NAME }}
          path: /github/workspace/.ssh
      - name: Shutdown server
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        run: |
          chmod 0600  /github/workspace/.ssh/automation_ssh_key
          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
          bash /ncp-test-automation/bin/entrypoint.sh
          eval "$(ssh-agent)"
          ssh-add  /github/workspace/.ssh/automation_ssh_key
          
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
          systemctl stop mariadb
          systemctl poweroff
          EOF
      - name: Create Snapshot
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        shell: bash
        working-directory: /ncp-test-automation/bin
        run: |
          set -x
          echo "${TEST_SERVER_ID?}"
          
          . ./library.sh
          
          tf-init "$TF_SNAPSHOT"
          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"

#  update-postactivation-snapshot:
#    if: ${{ always() }}
#    needs:
#      - update-test
#    runs-on: ubuntu-latest
#    container:
#      image: thecalcaholic/ncp-test-automation:latest
#      env:
#        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
#
#    env:
#      TEST_TYPE: update
#      SERVER_ADDRESS: ${{ needs.update-test.outputs.server_address }}
#      TEST_RESULT: ${{ needs.update-test.test_result }}
#      TEST_SERVER_ID: ${{ needs.update-test.outputs.test_server_id }}
#      VERSION: ${{ needs.update-test.outputs.version }}
#      UID: ${{ github.run_id }}-update
#      SSH_ARTIFACT_NAME: "${{ needs.update-test.outputs.ssh_artifact_name }}"
#    steps:
#      - name: download ssh private key from artifact store
#        uses: actions/download-artifact@v3
#        if: ${{ contains('success|failure', env.TEST_RESULT) }}
#        with:
#          name: ${{ env.SSH_ARTIFACT_NAME }}
#          path: /github/workspace/.ssh
#      - name: Shutdown server
#        if: ${{ contains('success|failure', env.TEST_RESULT) }}
#        run: |
#          chmod 0600  /github/workspace/.ssh/automation_ssh_key
#          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
#          bash /ncp-test-automation/bin/entrypoint.sh
#          eval "$(ssh-agent)"
#          ssh-add  /github/workspace/.ssh/automation_ssh_key
#
#          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
#          systemctl stop mariadb
#          systemctl poweroff
#          EOF
#      - name: Create Snapshot
#        if: ${{ contains('success|failure', env.TEST_RESULT) }}
#        shell: bash
#        working-directory: /ncp-test-automation/bin
#        run: |
#          set -x
#          echo "${{ needs.update-test.outputs.test_server_id }}"
#          echo "${TEST_SERVER_ID?}"
#
#          . ./library.sh
#
#          tf-init "$TF_SNAPSHOT"
#          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
#          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
#          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"

  dist-upgrade-postactivation-snapshot:
    if: ${{ always() }}
    needs:
      - dist-upgrade-test
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"

    env:
      TEST_TYPE: distupgrade
      SERVER_ADDRESS: ${{ needs.dist-upgrade-test.outputs.server_address }}
      TEST_RESULT: ${{ needs.dist-upgrade-test.outputs.test_result }}
      TEST_SERVER_ID: ${{ needs.dist-upgrade-test.outputs.test_server_id }}
      VERSION: ${{ needs.dist-upgrade-test.outputs.version }}
      UID: ${{ github.run_id }}-distupgrade
      SSH_ARTIFACT_NAME: "${{ needs.dist-upgrade-test.outputs.ssh_artifact_name }}"
    steps:
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v4
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        with:
          name: ${{ env.SSH_ARTIFACT_NAME }}
          path: /github/workspace/.ssh
      - name: Shutdown server
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        run: |
          chmod 0600  /github/workspace/.ssh/automation_ssh_key
          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
          bash /ncp-test-automation/bin/entrypoint.sh
          eval "$(ssh-agent)"
          ssh-add  /github/workspace/.ssh/automation_ssh_key
          
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
          systemctl stop mariadb
          systemctl poweroff
          EOF
      - name: Create Snapshot
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        shell: bash
        working-directory: /ncp-test-automation/bin
        run: |
          set -x
          echo "${{ needs.dist-upgrade-test.outputs.test_server_id }}"
          echo "${TEST_SERVER_ID?}"
          
          . ./library.sh
          
          tf-init "$TF_SNAPSHOT"
          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"
  
  cleanup:
    if: ${{ always() }}
    needs:
      - install-postactivation-snapshot
      #- update-postactivation-snapshot
      - dist-upgrade-postactivation-snapshot
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
    strategy:
      matrix:
        uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update", "${{ github.run_id }}-distupgrade"]
      fail-fast: false
    env:
      HOME: '/root'
      UID: ${{ matrix.uid }}
    defaults:
      run:
        shell: bash
        working-directory: /ncp-test-automation/bin
    steps:
      - name: Teardown VMs
        run: |
          for server in $(hcloud server list -o noheader -o columns=id -l "ci=${UID?}")
          do
            echo "Deleting server '$server'..."
            hcloud server delete "$server"
            echo "done."
          done
      - name: Delete ssh key
        run: |
          source ./library.sh
          hcloud-clear-root-key
  cleanup-snapshots:
    if: ${{ always() }}
    needs:
      - install-postactivation-snapshot
      #- update-postactivation-snapshot
      - dist-upgrade-postactivation-snapshot
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
    env:
      HOME: '/root'
    steps:
      - name: Delete old snapshots
        run: |
          for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -12)
          do
            echo "Deleting snapshot '$snapshot'..."
            hcloud image delete "$snapshot"
            echo "done."
          done