name: "Build and test LXD image"
on:
  workflow_call:
    inputs:
      git_ref:
        required: true
        type: string
      arch:
        required: false
        type: string
        default: 'x86'
    outputs:
      lxd_artifact_name:
        value: "${{ jobs.build-current.outputs.artifact_name }}"
      lxd_artifact_file:
        value: "${{ jobs.build-current.outputs.artifact_file }}"
      lxc_artifact_name:
        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_name }}"
      lxc_artifact_file:
        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_file }}"
  push:
    branches:
      - "**"
  pull_request:
    branches:
      - master
      - devel
      - release/*

jobs:
  determine-runner:
    env:
      LXD_ARCH: "${{ inputs.arch || 'x86' }}"
    runs-on: ubuntu-latest
    outputs:
      runner_label: ${{ steps.script.outputs.runner_label }}
      lxc_cmd: ${{ steps.script.outputs.lxc_cmd }}
      lxd_extra_profile: ${{ steps.script.outputs.lxd_extra_profile }}
    steps:
      - name: script
        id: script
        run: |
          if [[ "${LXD_ARCH}" == "arm64" ]]
          then
            LXC_CMD="incus"
            RUNNER_LABEL="ubuntu-20.04-arm64"
            echo "lxd_extra_profile=network" | tee -a "$GITHUB_OUTPUT"
          else
            LXC_CMD="incus"
            RUNNER_LABEL="ubuntu-20.04"
          fi

          echo "runner_label=$RUNNER_LABEL" | tee -a $GITHUB_OUTPUT
          echo "lxc_cmd=$LXC_CMD" | tee -a $GITHUB_OUTPUT
  build-current:
    needs:
      - determine-runner
    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
    outputs:
      artifact_name: "${{ env.ARTIFACT_NAME }}"
      artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}"
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
      ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image"
      LXD_ARCH: "${{ inputs.arch || 'x86' }}"
      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
      LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
        with:
          ref: "${{ env.VERSION }}"
      - name: Setup incus
        if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
        run: |
          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
          sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
          sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
          sudo incus admin init --auto
      - name: Build LXD image
        env:
          USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
        run: |
          export CI
          LXD_EXTRA_PROFILE="$LXD_EXTRA_PROFILE" BRANCH="${VERSION##refs/heads/}" ./build/build-LXD.sh
      - name: Pack LXD image
        id: pack-lxd
        run: |
          . ./build/buildlib.sh
          ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
          sudo "$LXC" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
          echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
      - name: upload LXD image to artifact store
        uses: actions/upload-artifact@v4
        with:
          name: "${{ env.ARTIFACT_NAME }}"
          path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
          if-no-files-found: error

#  build-previous:
#    needs:
#      - determine-runner
#    runs-on: [ ubuntu-20.04 ]
#    if: ${{ inputs.arch == 'x86' || inputs.arch == '' }}
#    outputs:
#      artifact_name: "${{ env.ARTIFACT_NAME }}"
#      artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}"
#      previous_version: "${{ steps.checkout_previous_version.outputs.previous_version }}"
#    env:
#      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
#      ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image-previous"
#      LXD_ARCH: "${{ inputs.arch || 'x86' }}"
#      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
#      LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}"
#    steps:
#      - name: Checkout code
#        uses: actions/checkout@v3
#        with:
#          ref: "${{ env.VERSION }}"
#          fetch-depth: 0
#      - name: Checkout previous release
#        id: checkout_previous_version
#        shell: bash
#        run: |
#          set -e
#          if [[ -n "${{ github.base_ref }}" ]]
#          then
#            version="${{ github.base_ref }}"
#          elif [[ "${{ env.VERSION }}" == "refs/heads/devel" ]]
#          then
#            version="master"
#          else
#            if [[ "${{ github.ref_type }}" != "tag" ]] || ! git describe --tags > /dev/null
#            then
#              git fetch -fu --tags origin ${{ env.VERSION }}:${{ env.VERSION }}
#            fi
#            version="$(git describe --tags)"
#            [[ "$version" =~ .*-.*-.* ]] || {
#              git checkout HEAD~1
#              version="$(git describe --tags)"
#            }
#            version="${version%-*-*}"
#          fi
#          echo "Previous version is '$version'"
#          git checkout "$version"
#          echo "VERSION=$version" >> "$GITHUB_ENV"
#          echo "previous_version=${version}" >> $GITHUB_OUTPUT
#      - name: Setup incus
#        if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
#        run: |
#          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
#          sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
#          sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
#          sudo incus admin init --auto
#
#      - name: Build LXD image
#        env:
#          USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
#        run: |
#          export CI
#          LXD_EXTRA_PROFILE="$LXD_EXTRA_PROFILE" BRANCH="${VERSION##refs/heads/}" ./build/build-LXD.sh
#      - name: Pack LXD image
#        id: pack-lxd
#        run: |
#          . ./build/buildlib.sh
#          ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
#          sudo "${LXC}" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
#          echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
#      - name: upload LXD image to artifact store
#        uses: actions/upload-artifact@v3
#        with:
#          name: "${{ env.ARTIFACT_NAME }}"
#          path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
#          if-no-files-found: error
#
#  test-update:
#    needs:
#      - determine-runner
#      - build-previous
#    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
#    outputs:
#      artifact_name: "${{ env.ARTIFACT_NAME }}"
#      artifact_file: "${{ steps.pack-lxd.outputs.artifact_file }}"
#    env:
#      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
#      ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}"
#      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
#      USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
#    steps:
#      - name: Checkout code
#        uses: actions/checkout@v3
#        with:
#          ref: "${{ env.VERSION }}"
#      - name: Setup incus
#        if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
#        run: |
#          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
#          sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
#          sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
#          sudo incus admin init --auto
#      - name: Setup Firefox
#        continue-on-error: true
#        id: setup-firefox-browser-action
#        uses: browser-actions/setup-firefox@latest
#      - name: Setup Firefox from packages
#        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
#        run: |
#          sudo apt-get install -y --no-install-recommends firefox
#      - name: Setup GeckoDriver
#        env:
#          GH_TOKEN: ${{ github.token }}
#        run: |
#          arch=linux64
#          if [[ "$RUNNER_ARCH" == "ARM64" ]]
#          then
#            arch="linux-aarch64"
#          fi
#          gh release download -R mozilla/geckodriver -p 'geckodriver-*-'"$arch.tar.gz"
#          tar xf "geckodriver-"*"-$arch.tar.gz"
#          sudo mv geckodriver /usr/local/bin/
#      - name: Setup Selenium
#        run: pip install selenium
#      - name: download LXD image from artifact store
#        uses: actions/download-artifact@v3
#        with:
#          name: ${{ needs.build-previous.outputs.artifact_name }}
#      - name: Remove old lxd images
#        if: ${{ inputs.arch == 'arm64' }}
#        run: |
#          for img in $("$LXC" image list -c f -f csv)
#          do
#            sudo "$LXC" image delete "$img"
#          done
#      - name: Launch ncp container
#        run: |
#          set -x
#          sudo "$LXC" delete -q -f ncp || true
#          sudo "$LXC" image import -q "./${{ needs.build-previous.outputs.artifact_file }}" --alias "ncp/update"
#          systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp || \
#          sudo systemd-run --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp
#          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
#          sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
#          sleep 30
#          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
#          ip="${ip/% *}"
#          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
#          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
#      - name: Activate and Test LXD Image
#        working-directory: ./tests
#        run: |
#          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
#          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
#            echo "Activation test failed!"
#            echo "Geckodriver logs:"
#            tail -n 20 geckodriver.log >&2 || true
#            echo "================"
#            echo "ncp.log: "
#            "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
#            exit 1
#          }
#          python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
#            echo "Nextcloud test failed!"
#            echo "Geckodriver logs:"
#            tail -n 20 geckodriver.log >&2 || true
#            echo "================"
#            echo "ncp.log: "
#            "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
#            echo "================"
#            echo "nextcloud log: "
#            datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
#            "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
#            exit 1
#          }
#          USE_INCUS="$USE_INCUS" python system_tests.py --non-interactive --skip-update-test || {
#            echo "System test failed!"
#            exit 1
#          }
#      - name: Update ncp
#        env:
#          HEAD_REF: "${{ github.head_ref }}"
#        run: |
#          set -ex
#          BRANCH="${VERSION/refs\/heads\//}"
#          BRANCH="${BRANCH/refs\/tags\//}"
#          if [[ "$BRANCH" =~ "refs/pull/"* ]]
#          then
#            UPDATE_ARGS=("${HEAD_REF}" "$VERSION")
#          else
#            UPDATE_ARGS=("$BRANCH")
#          fi
#          echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
#          echo "Running update ${{ needs.build-previous.outputs.previous_version }} -> ${VERSION}"
#
#          current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
#          latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
#
#          sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
#          sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
#
#          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
#          then
#            echo "Nextcloud is up to date - skipping NC update test."
#          else
#            sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
#          fi
#
#          sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
#
#          sudo "$LXC" stop ncp
#      - name: Relaunch container
#        run: |
#          set -x
#          systemd-run --user --scope -p "Delegate=yes" "$LXC" start -q ncp || \
#          sudo systemd-run --scope -p "Delegate=yes" "$LXC" start -q ncp
#          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
#          sleep 30
#          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
#          ip="${ip/% *}"
#          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
#          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
#      - name: Test LXD Image
#        working-directory: ./tests
#        run: |
#          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
#          python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
#            echo "Nextcloud test failed!"
#            echo "Geckodriver logs:"
#            tail -n 20 geckodriver.log >&2 || true
#            echo "================"
#            echo "ncp.log: "
#            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
#            echo "================"
#            echo "nextcloud log: "
#            datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
#            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
#            exit 1
#          }
#          USE_INCUS="$USE_INCUS" python system_tests.py --non-interactive || {
#            echo "System test failed!"
#            exit 1
#          }

  test-dist-upgrade:
    needs:
      - determine-runner
    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
    env:
      VERSION: "${{ inputs.git_ref || github.ref }}"
      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
      LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}"
      PREVIOUS_IMAGE_URL_ARM64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextcloudPi_LXD_arm64_v1.53.0.tar.gz"
      PREVIOUS_IMAGE_URL_AMD64: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.0/NextCloudPi_LXD_x86_v1.53.0.tar.gz"
      USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
    steps:
      - name: Setup incus
        if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
        run: |
          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
          sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
          sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
          sudo incus admin init --auto
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: "v1.54.3"
      - name: Setup Firefox
        continue-on-error: true
        id: setup-firefox-browser-action
        uses: browser-actions/setup-firefox@latest
      - name: Setup Firefox from packages
        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
        run: |
          sudo apt-get install -y --no-install-recommends firefox
      - name: Setup GeckoDriver
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          arch=linux64
          if [[ "$RUNNER_ARCH" == "ARM64" ]]
          then
            arch="linux-aarch64"
          fi
          gh release download -R mozilla/geckodriver -p 'geckodriver-*-'"$arch.tar.gz"
          tar xf "geckodriver-"*"-$arch.tar.gz"
          sudo mv geckodriver /usr/local/bin/
      - name: Setup Selenium
        run: pip install selenium
      - name: download LXD image
        run: |
          image_url="${PREVIOUS_IMAGE_URL_AMD64?}"
          if [[ "$RUNNER_ARCH" == "ARM64" ]]
          then
            image_url="${PREVIOUS_IMAGE_URL_ARM64?}"
          fi
          wget -qO ./ncp.tar.gz "${image_url?}"
      - name: Launch ncp container
        run: |
          set -x
          sudo "$LXC" delete -q -f ncp || true
          sudo "$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update"
          LXC_ARGS=(-p default)
          [[ -z "$LXD_EXTRA_PROFILE" ]] || LXC_ARGS+=(-p "$LXD_EXTRA_PROFILE")
          systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "${LXC_ARGS[@]}" "ncp/update" ncp || \
          sudo systemd-run --scope -p "Delegate=yes" "$LXC" launch -q "${LXC_ARGS[@]}" "ncp/update" ncp
          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
          sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
          sleep 30
          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
          ip="${ip/% *}"
          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
      - name: Activate and Test LXD Image
        working-directory: ./tests
        run: |
          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
            echo "Activation test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            exit 1
          }
          python nextcloud_tests.py --skip-release-check --no-gui "nextcloudpi.local" 443 4443 || {
            echo "Nextcloud test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            echo "================"
            echo "nextcloud log: "
            datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)"
            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
            exit 1
          }
          USE_INCUS="$USE_INCUS"  python system_tests.py --non-interactive --skip-update-test || {
            echo "System test failed!"
            exit 1
          }

      - name: Update ncp
        env:
          HEAD_REF: "${{ github.head_ref }}"
        run: |
          set -ex
          BRANCH="${VERSION/refs\/heads\//}"
          BRANCH="${BRANCH/refs\/tags\//}"
          if [[ "$BRANCH" =~ "refs/pull/"* ]]
          then
            UPDATE_ARGS=("${HEAD_REF}" "$VERSION") 
          else
            UPDATE_ARGS=("$BRANCH")
          fi
          echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
          echo "Running update to ${VERSION}"
          
          current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
          latest_nc_version="27.1.6"
          
          sudo "$LXC" exec ncp -- apt-get update ||:
          sudo "$LXC" exec ncp -- apt-get install --no-install-recommends -y gnupg2
          sudo "$LXC" exec ncp -- apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg
          
          sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
          sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
          
          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
          then
            echo "Nextcloud is up to date - skipping NC update test."
          else
            sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
            sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
          fi
  
          sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
          
          sudo "$LXC" stop ncp
      - name: Relaunch container
        run: |
          set -x
          LXC_ARGS=()
          [[ -z "$LXD_EXTRA_PROFILE" ]] || LXC_ARGS+=(-p "$LXD_EXTRA_PROFILE")
          systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp || \
          sudo systemd-run --scope -p "Delegate=yes" "$LXC" start ncp
          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
          sleep 30
          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
          ip="${ip/% *}"
          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
      - name: Test LXD Image
        working-directory: ./tests
        run: |
          python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
            echo "Nextcloud test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            echo "================"
            echo "nextcloud log: "
            datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)"
            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
            exit 1
          }
          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
          USE_INCUS="$USE_INCUS" python system_tests.py --non-interactive --skip-update-test || {
            echo "System test failed!"
            exit 1
          }
      - name: Checkout current version
        run: |
          git fetch origin
          git checkout "${{ env.VERSION }}"
      - name: NCP distupgrade
        id: distupgrade
        run: |
          set -x
          sudo "$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || {
            echo "can't upgrade from Debian $(sudo "$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)"
            exit 1
          }
          current_nc_version="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
          latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
          
          sudo "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade"
          sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
          
          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
          then
            echo "Nextcloud is up to date - skipping NC update test."
          else
            
            for i in {1..10};
            do
              echo "running nc update ($i/10)..."
              sudo "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
            
              sudo "$LXC" exec ncp -- /usr/local/bin/ncc status
              current_nc_version_new="$(sudo "$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
              if [[ "$current_nc_version_new" =~ "$latest_nc_version".* ]]
              then
                break
              fi
              if [[ "$current_nc_version" == "$current_nc_version_new" ]]
              then
                echo "failed to update to $latest_nc_version"
                exit 1
              fi
              current_nc_version="$current_nc_version_new"
            done
          fi
          
          sudo "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
          sudo "$LXC" stop ncp
      - name: Relaunch container
        run: |
          set -x
          LXC_ARGS=()
          [[ -z "$LXD_EXTRA_PROFILE" ]] || LXC_ARGS+=(-p "$LXD_EXTRA_PROFILE")
          systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp || \
          sudo systemd-run --scope -p "Delegate=yes" "$LXC" start ncp
          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
          sleep 30
          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
          ip="${ip/% *}"
          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
      - name: Test LXD Image
        working-directory: ./tests
        run: |
          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
          python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
            echo "Nextcloud test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            echo "================"
            echo "nextcloud log: "
            datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)"
            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
            exit 1
          }
          USE_INCUS="$USE_INCUS" python system_tests.py --non-interactive || {
            echo "System test failed!"
            exit 1
          }
          sudo "$LXC" stop ncp

  test-fresh-install:
    needs:
      - determine-runner
      - build-current
    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
      ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
      ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
      USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
      LXD_EXTRA_PROFILE: "${{ needs.determine-runner.outputs.lxd_extra_profile }}"
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: "${{ env.VERSION }}"
      - name: Setup incus
        if: ${{ needs.determine-runner.outputs.runner_label != 'ubuntu-20.04-arm64' }}
        run: |
          curl https://pkgs.zabbly.com/get/incus-stable | sudo sh -x
          sudo iptables -I DOCKER-USER -i incusbr0 -j ACCEPT
          sudo iptables -I DOCKER-USER -o incusbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
          sudo incus admin init --auto
      - name: Setup Firefox
        continue-on-error: true
        id: setup-firefox-browser-action
        uses: browser-actions/setup-firefox@latest
      - name: Setup Firefox from packages
        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
        run: |
          sudo apt-get install -y --no-install-recommends firefox
      - name: Setup GeckoDriver
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          arch=linux64
          if [[ "$RUNNER_ARCH" == "ARM64" ]]
          then
            arch="linux-aarch64"
          fi
          gh release download -R mozilla/geckodriver -p 'geckodriver-*-'"$arch.tar.gz"
          tar xf "geckodriver-"*"-$arch.tar.gz"
          sudo mv geckodriver /usr/local/bin/
      - name: Setup Selenium
        run: pip install selenium
      - name: download LXD image from artifact store
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.ARTIFACT_NAME }}
      - name: Remove old lxd images
        if: ${{ inputs.arch == 'arm64' }}
        run: |
          for img in $("$LXC" image list -c f -f csv)
          do
            sudo "$LXC" image delete "$img"
          done
      - name: Launch ncp container
        run: |
          set -x
          . ./build/buildlib.sh
          sudo "$LXC" delete -q -f ncp || true
          sudo "$LXC" image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true
          LXC_ARGS=(-p default)
          [[ -z "$LXD_EXTRA_PROFILE" ]] || LXC_ARGS+=(-p "$LXD_EXTRA_PROFILE")
          systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "${LXC_ARGS[@]}" "ncp/test" ncp || \
          sudo systemd-run --scope -p "Delegate=yes" "$LXC" launch -q "${LXC_ARGS[@]}" "ncp/test" ncp
          sudo "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
          sleep 30
          ip="$(sudo "$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
          ip="${ip/% *}"
          sudo sed -i 's/^.*nextcloudpi.local.*$//g' /etc/hosts
          echo "${ip} nextcloudpi.local" | sudo tee -a /etc/hosts
      - name: Test LXD Image
        working-directory: ./tests
        run: |
          sudo "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
            echo "Activation test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            echo "================"
            echo "nextcloud log: "
            datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
            exit 1
          }
          python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
            echo "Nextcloud test failed!"
            echo "Geckodriver logs:"
            tail -n 20 geckodriver.log >&2 || true
            echo "================"
            echo "ncp.log: "
            sudo "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
            echo "================"
            echo "nextcloud log: "
            datadir="$(sudo "$LXC" exec ncp -- ncc config:system:get datadirectory)"
            sudo "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
            exit 1
          }
          USE_INCUS="$USE_INCUS" python system_tests.py --non-interactive || {
            echo "System test failed!"
            exit 1
          }
          sudo "$LXC" stop ncp

  convert-to-lxc-image:
    needs:
      - build-current
    runs-on: ubuntu-latest
    outputs:
      artifact_name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
      artifact_file: "${{ steps.lxd-to-lxc.outputs.artifact_file }}"
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
      LXD_ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
      LXD_ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
    steps:
      - name: download LXD image from artifact store
        uses: actions/download-artifact@v4
        with:
          name: ${{ env.LXD_ARTIFACT_NAME }}
      - name: convert to LXC image
        id: lxd-to-lxc
        run: |
          mkdir repackage output;
          cd repackage; 
          sudo su - -c "cd '$(pwd)'; tar xpf '../${LXD_ARTIFACT_FILE?}';"
          sudo rm -rf ./rootfs/dev;
          LXC_ARTIFACT_FILE="${LXD_ARTIFACT_FILE//LXD/LXC}";
          sudo tar cpzf "../output/${LXC_ARTIFACT_FILE?}" -C rootfs/ .;
          cd ..;
          sudo chown "$(id -un):" "./output/${LXC_ARTIFACT_FILE}";  
          ls -l ./output
          echo "artifact_file=${LXC_ARTIFACT_FILE?}" >> $GITHUB_OUTPUT;
          echo "artifact_name=${LXD_ARTIFACT_NAME//lxd/lxc}" >> $GITHUB_OUTPUT;
      - name: upload LXD image to artifact store
        uses: actions/upload-artifact@v4
        with:
          name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
          path: "output/${{ steps.lxd-to-lxc.outputs.artifact_file }}"
          if-no-files-found: error
