/etc/ncp-dist-upgrade*: Implement dist-upgrade for bullseye->bookworm and add integration tests for it

Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com>
Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com>

.github/actions/create-test-instance-bullseye/action.yml

@@ -0,0 +1,35 @@
+name: Create Test VM
+description: Create NCP instance for testing in the Hetzner cloud
+inputs:
+  version:
+    description: version (git rev / tag / branch) to install
+    required: true
+  uid:
+    description: A unique ID for labeling/naming generated resources
+    required: true
+  hcloud_token:
+    description: A auth token for Hetzner cloud
+    required: true
+  server_type:
+    description: Server type to use for hetzner servers
+    required: true
+    default: "cx11"
+
+outputs:
+  server_address:
+    description: Adress of the test instance
+  snapshot_id:
+    description: ID of the generated postinstall snapshot
+  test_server_id:
+    description: ID of the created test server
+runs:
+  using: docker
+  image: docker://thecalcaholic/ncp-test-automation:bullseye
+
+  env:
+    HCLOUD_TOKEN: ${{ inputs.hcloud_token }}
+    UID: ${{ inputs.uid }}
+    SERVER_TYPE: ${{ inputs.server_type }}
+  args:
+    - /ncp-test-automation/bin/actions/create-test-instance.sh
+    - ${{ inputs.version }}

.github/actions/create-test-instance/action.yml

@@ -24,7 +24,7 @@ outputs:
     description: ID of the created test server
 runs:
   using: docker
-  image: docker://thecalcaholic/ncp-test-automation
+  image: docker://thecalcaholic/ncp-test-automation:bookworm
 
   env:
     HCLOUD_TOKEN: ${{ inputs.hcloud_token }}

.github/workflows/build-lxd.yml

@@ -321,6 +321,175 @@ jobs:
             exit 1
           }
 
+  test-dist-upgrade:
+    needs:
+      - determine-runner
+    runs-on: [ubuntu-20.04]
+    env:
+      VERSION: "${{ inputs.git_ref || github.ref }}"
+      ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}"
+      LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
+      PREVIOUS_IMAGE_URL: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz"
+    steps:
+      - uses: whywaita/setup-lxd@v1
+        if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
+        continue-on-error: true
+        with:
+          lxd_version: latest/stable
+      - name: Fix LXD
+        run: |
+          sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT
+          sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: "${{ env.VERSION }}"
+      - name: Setup Firefox
+        continue-on-error: true
+        id: setup-firefox-browser-action
+        uses: browser-actions/setup-firefox@latest
+      - name: Setup Firefox from packages
+        if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
+        run: |
+          sudo apt-get install -y --no-install-recommends firefox
+      - name: Setup GeckoDriver
+        uses: ChlodAlejandro/setup-geckodriver@latest
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Selenium
+        run: pip install selenium
+      - name: download LXD image
+        run: |
+          wget -qO ./ncp.tar.gz "${PREVIOUS_IMAGE_URL?}"
+      - name: Launch ncp container
+        run: |
+          set -x
+          "$LXC" delete -q -f ncp || true
+          "$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update"
+          systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp
+          "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
+          "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
+          sleep 30
+          ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
+          ip="${ip/% *}"
+          echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
+      - name: Activate and Test LXD Image
+        working-directory: ./tests
+        run: |
+          "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
+          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
+            echo "Activation test failed!"
+            echo "Geckodriver logs:"
+            tail -n 20 geckodriver.log >&2 || true
+            echo "================"
+            echo "ncp.log: "
+            "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
+            exit 1
+          }
+          python system_tests.py --non-interactive --skip-update-test || {
+            echo "System test failed!"
+            exit 1
+          }
+          python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
+            echo "Nextcloud test failed!"
+            echo "Geckodriver logs:"
+            tail -n 20 geckodriver.log >&2 || true
+            echo "================"
+            echo "ncp.log: "
+            "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
+            echo "================"
+            echo "nextcloud log: "
+            datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
+            "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
+            exit 1
+          }
+
+      - name: Update ncp
+        run: |
+          set -ex
+          BRANCH="${VERSION/refs\/heads\//}"
+          BRANCH="${BRANCH/refs\/tags\//}"
+          if [[ "$BRANCH" =~ "refs/pull/"* ]]
+          then
+            UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION") 
+          else
+            UPDATE_ARGS=("$BRANCH")
+          fi
+          echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
+          echo "Running update to ${VERSION}"
+          
+          current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
+          latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
+          
+          "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
+          "$LXC" exec ncp -- /usr/local/bin/ncc status
+          
+          if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
+          then
+            echo "Nextcloud is up to date - skipping NC update test."
+          else
+            "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
+          fi
+  
+          "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
+          
+          "$LXC" stop ncp
+      - name: Relaunch container
+        run: |
+          set -x
+          systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp
+          "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
+          sleep 30
+          ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
+          ip="${ip/% *}"
+          echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
+      - name: Test LXD Image
+        working-directory: ./tests
+        run: |
+          "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
+          python system_tests.py --non-interactive --skip-update-test || {
+            echo "System test failed!"
+            exit 1
+          }
+          python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
+            echo "Nextcloud test failed!"
+            echo "Geckodriver logs:"
+            tail -n 20 geckodriver.log >&2 || true
+            echo "================"
+            echo "ncp.log: "
+            "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
+            echo "================"
+            echo "nextcloud log: "
+            datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
+            "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
+            exit 1
+          }
+      - name: NCP distupgrade
+        id: distupgrade
+        run: |
+          set -x
+          "$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || {
+            echo "can't upgrade from Debian $("$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)"
+            exit 1
+          }
+          "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade"
+          
+          "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
+          
+          "$LXC" stop ncp
+      - name: Relaunch container
+        run: |
+          set -x
+          systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp
+          "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
+          sleep 30
+          ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
+          ip="${ip/% *}"
+          echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
+      - name: Test LXD Image
+        working-directory: ./tests
+        run: |
+          "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
           python system_tests.py --non-interactive || {
             echo "System test failed!"
             exit 1

.github/workflows/build-sd-images.yml

@@ -186,6 +186,14 @@ jobs:
           sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static
           sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static
           echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf
+          echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf
+          sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d
+          echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
+          echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
+          
+          sudo mkdir -p raspbian_root/etc/systemd/system/php8.1-fpm.service.d
+          echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf
+          echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf
       - name: Test image
         id: test
         run: |
@@ -308,8 +316,9 @@ jobs:
               echo -e "${LOG_DIAG} ncp.log: "
               "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
               echo "================"
-              echo "Nextcloud log: "
-              "${CONTAINER_CMD[@]}" -q ncp cat /opt/ncdata/data/nextcloud.log
+              echo "${LOG_DIAG} Nextcloud log: "
+              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
               sleep 12
               continue
             }
@@ -323,3 +332,515 @@ jobs:
             echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
             exit 1
           }
+
+#  test-distupgrade:
+#    runs-on: ubuntu-20.04
+#    env:
+#      VERSION: "${{ inputs.git_ref || github.ref }}"
+#      LOG_GUEST: "\\033[1;34mGUEST::\\033[0m"
+#      LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
+#      LOG_CICD: "\\033[1;35mCICD::\\033[0m"
+#      LOG_TEST: "\\033[1;33mTEST::\\033[0m"
+#      LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
+#      PREVIOUS_VERSION: "v1.53.2"
+#    defaults:
+#      run:
+#        shell: bash
+#    steps:
+#      - name: Set up QEMU
+#        uses: docker/setup-qemu-action@v3
+#      - name: Checkout code
+#        uses: actions/checkout@v3
+#        with:
+#          ref: "${{ env.VERSION }}"
+#      - name: Download previous image
+#        id: download-previous-image
+#        run: |
+#          set -x
+#          mkdir -p output
+#          cd output
+#          wgetrc=0
+#          wgeterr="$(wget -O ./ncp.zip "https://github.com/nextcloud/nextcloudpi/releases/download/${PREVIOUS_VERSION?}/NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.zip" 2>&1)" || wgetrc=$?
+#          if [[ $wgetrc -ne 0 ]]
+#          then
+#            if echo "$wgeterr" | grep '404 Not Found'
+#            then
+#              echo "Board not found in previous release - skipping."
+#              echo "skipped=true" >> "$GITHUB_OUTPUT"
+#              exit 0
+#            else
+#              echo "$wgeterr"
+#              exit $wgetrc
+#            fi
+#          fi
+#          echo "skipped=false" >> "$GITHUB_OUTPUT"
+#          unzip ncp.zip
+#          rm ncp.zip
+#          mv NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.img ./ncp.img
+#          echo "ARTIFACT_FILE=ncp.img" >> "$GITHUB_ENV"
+#      - name: Prepare test
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        run: |
+#          set -x
+#          mv output/${ARTIFACT_FILE?} ncp.img
+#          sudo apt-get install -y systemd-container
+#          sudo pip install selenium
+#          sudo rm -rf raspbian_root
+#          . ./build/buildlib.sh
+#          mount_raspbian "ncp.img"
+#          sudo cat raspbian_root/etc/machine-id
+#          sudo systemd-id128 new | sudo tee ./raspbian_root/etc/machine-id
+#          sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O raspbian_root/usr/bin/qemu-aarch64-static
+#          sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static
+#          sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static
+#          echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf
+#          echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf
+#          sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d
+#          echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
+#          echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
+#      - name: Test and activate image
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        id: test
+#        run: |
+#
+#          log_err() {
+#            rc="${1?}"
+#            msg="${2?}"
+#            echo -e "${LOG_DIAG} $msg" >&2
+#            return $rc
+#          }
+#
+#          sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
+#          sleep 60
+#
+#          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
+#
+#          success=false
+#          for attempt in {1..30}
+#          do
+#            echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
+#            ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
+#            [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
+#            success=true
+#            break
+#          done
+#          sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
+#          sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Could not reach container. Aborting..."
+#            exit 1
+#          }
+#
+#          attempt=0
+#          success=false
+#          for attempt in {1..150}
+#          do
+#            echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
+#            "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
+#            "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
+#            redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
+#            && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
+#            || log_err $? "Error retrieving redis credentials" || true
+#            if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
+#              && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
+#            then
+#              echo -e "${LOG_CICD} Startup successful"
+#              success=true
+#              break
+#            fi
+#            attempt=$((attempt + 1))
+#            sleep 5
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#              echo -e "${LOG_CICD} Timeout reached."
+#              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              exit 1
+#          }
+#
+#          set -x
+#          set +e
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) =="
+#            python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} Activation test failed!"
+#              echo -e "${LOG_DIAG} Geckodriver logs:"
+#              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo "================"
+#              echo -e "${LOG_DIAG} mysql: "
+#              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Activation test failed in all attempts!"
+#            exit 1
+#          }
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
+#            sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} System test failed!"
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} System test failed in all attempts!"
+#            exit 1
+#          }
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
+#            python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} Nextcloud test failed!"
+#              echo -e "{$LOG_DIAG} Geckodriver logs:"
+#              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo -e "${LOG_CICD} ================"
+#              echo -e "${LOG_DIAG} ncp.log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo "================"
+#              echo "${LOG_DIAG} Nextcloud log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
+#            exit 1
+#          }
+#      - name: Update NCP
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        run: |
+#          set -ex
+#
+#          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
+#
+#          BRANCH="${VERSION/refs\/heads\//}"
+#          BRANCH="${BRANCH/refs\/tags\//}"
+#          if [[ "$BRANCH" =~ "refs/pull/"* ]]
+#          then
+#            UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION")
+#          else
+#            UPDATE_ARGS=("$BRANCH")
+#          fi
+#          current_nc_version="$("${CONTAINER_CMD[@]}" ncc status | grep "version:" | awk '{ print $3 }')"
+#          latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
+#
+#          echo "Updating from $PREVIOUS_VERSION to $VERSION"
+#
+#          "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
+#          "${CONTAINER_CMD[@]}" /usr/local/bin/ncc status
+#          # "${CONTAINER_CMD[@]}" bash -c 'curl https://download.nextcloud.com/server/releases/nextcloud-28.0.4.tar.bz2 > /var/www/nextcloud-28.0.4.tar.bz2'
+#          k0nKat1Nation
+#
+#          # if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
+#          # then
+#          #   echo "Nextcloud is up to date - skipping NC update test."
+#          # else
+#          #   "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
+#          # fi
+#
+#          sudo machinectl terminate ncp
+#          sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log
+#      - name: Test image after update
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        run: |
+#
+#          log_err() {
+#            rc="${1?}"
+#            msg="${2?}"
+#            echo -e "${LOG_DIAG} $msg" >&2
+#            return $rc
+#          }
+#
+#          sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
+#          sleep 60
+#
+#          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
+#
+#          success=false
+#          for attempt in {1..30}
+#          do
+#            echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
+#            ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
+#            [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
+#            success=true
+#            break
+#          done
+#          sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
+#          sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Could not reach container. Aborting..."
+#            exit 1
+#          }
+#
+#          attempt=0
+#          success=false
+#          for attempt in {1..150}
+#          do
+#            echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
+#            "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
+#            "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
+#            redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
+#            && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
+#            || log_err $? "Error retrieving redis credentials" || true
+#            if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
+#              && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
+#            then
+#              echo -e "${LOG_CICD} Startup successful"
+#              success=true
+#              break
+#            fi
+#            attempt=$((attempt + 1))
+#            sleep 5
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#              echo -e "${LOG_CICD} Timeout reached."
+#              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              exit 1
+#          }
+#
+#          set -x
+#          set +e
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
+#            sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} System test failed!"
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} System test failed in all attempts!"
+#            exit 1
+#          }
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
+#            python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} Nextcloud test failed!"
+#              echo -e "{$LOG_DIAG} Geckodriver logs:"
+#              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo -e "${LOG_CICD} ================"
+#              echo -e "${LOG_DIAG} ncp.log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo "================"
+#              echo "${LOG_DIAG} Nextcloud log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
+#            exit 1
+#          }
+#
+#      - name: Run dist-upgrade
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        run: |
+#          set -ex
+#
+#          source ./library.sh
+#
+#          echo "Updating from $PREVIOUS_VERSION to $VERSION
+#
+#          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
+#
+#          sudo grep 'VERSION="11"' ./raspbian_root/etc/os-release || {
+#            echo "Can't dist-upgrade from debian version $(sudo grep 'Version=' ./raspbian_root/etc/os-release)"
+#            exit 1
+#          }
+#          "${CONTAINER_CMD[@]}" DBG=x ncp-dist-upgrade "$VERSION"
+#
+#          sudo machinectl terminate ncp
+#          sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log
+#
+#      - name: Test image after dist-upgrade
+#        if:  ${{ steps.download-previous-image.outputs.skipped == 'false' }}
+#        run: |
+#
+#          log_err() {
+#            rc="${1?}"
+#            msg="${2?}"
+#            echo -e "${LOG_DIAG} $msg" >&2
+#            return $rc
+#          }
+#
+#          sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
+#          sleep 60
+#
+#          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
+#
+#          success=false
+#          for attempt in {1..30}
+#          do
+#            echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
+#            ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
+#            [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
+#            success=true
+#            break
+#          done
+#          sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
+#          sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Could not reach container. Aborting..."
+#            exit 1
+#          }
+#
+#          attempt=0
+#          success=false
+#          for attempt in {1..150}
+#          do
+#            echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
+#            "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
+#            "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
+#            redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
+#            && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
+#            || log_err $? "Error retrieving redis credentials" || true
+#            if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
+#              && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
+#              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
+#            then
+#              echo -e "${LOG_CICD} Startup successful"
+#              success=true
+#              break
+#            fi
+#            attempt=$((attempt + 1))
+#            sleep 5
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#              echo -e "${LOG_CICD} Timeout reached."
+#              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
+#              exit 1
+#          }
+#
+#          set -x
+#          set +e
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) =="
+#            python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} Activation test failed!"
+#              echo -e "${LOG_DIAG} Geckodriver logs:"
+#              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo "================"
+#              echo -e "${LOG_DIAG} mysql: "
+#              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Activation test failed in all attempts!"
+#            exit 1
+#          }
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
+#            sudo python tests/system_tests.py --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} System test failed!"
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} System test failed in all attempts!"
+#            exit 1
+#          }
+#
+#          success=false
+#          for attempt in {1..5}
+#          do
+#            echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
+#            python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
+#            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
+#              echo -e "${LOG_CICD} Nextcloud test failed!"
+#              echo -e "{$LOG_DIAG} Geckodriver logs:"
+#              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo -e "${LOG_CICD} ================"
+#              echo -e "${LOG_DIAG} ncp.log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              echo "================"
+#              echo "${LOG_DIAG} Nextcloud log: "
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
+#              sleep 12
+#              continue
+#            }
+#            success=true
+#            break
+#          done
+#
+#          [[ "$success" == "true" ]] || {
+#            echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
+#            exit 1
+#          }

.github/workflows/release.yml

@@ -89,13 +89,22 @@ jobs:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
     secrets: inherit
 
-  raspberrypi:
+  raspberrypi-4:
     if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
     uses: ./.github/workflows/build-sd-images.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      board_id: raspberrypi
-      board_name: RaspberryPi
+      board_id: rpi4b
+      board_name: RaspberryPi4
+    secrets: inherit
+
+  raspberrypi-5:
+    if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
+    uses: ./.github/workflows/build-sd-images.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      board_id: rpi5b
+      board_name: RaspberryPi5
     secrets: inherit
 
   # TODO: Fix 32bit armbian images
@@ -207,19 +216,31 @@ jobs:
       artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
 
-  raspberrypi-release:
+  raspberrypi-4-release:
     needs:
-      - raspberrypi
+      - raspberrypi-4
       - github-release
     if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
     uses: ./.github/workflows/publish-image.yml
     with:
       git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.raspberrypi.outputs.artifact_name }}"
-      artifact_file: "${{ needs.raspberrypi.outputs.artifact_file }}"
+      artifact_id: "${{ needs.raspberrypi-4.outputs.artifact_name }}"
+      artifact_file: "${{ needs.raspberrypi-4.outputs.artifact_file }}"
       dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
     secrets: inherit
 
+  raspberrypi-5-release:
+    needs:
+      - raspberrypi-5
+      - github-release
+    if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
+    uses: ./.github/workflows/publish-image.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      artifact_id: "${{ needs.raspberrypi-5.outputs.artifact_name }}"
+      artifact_file: "${{ needs.raspberrypi-5.outputs.artifact_file }}"
+      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+    secrets: inherit
   odroidxu4-release:
     needs:
       - odroidxu4

.github/workflows/vm-tests.yml

@@ -18,53 +18,124 @@ on:
       - master
 
 jobs:
-  setup-installation-test-instance:
+  installation-test:
     runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:bookworm
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+        UID: "${{ github.run_id }}-install"
     outputs:
       server_address: ${{ steps.create-test-instance.outputs.server_address }}
       snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
       test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
       version: ${{ env.VERSION }}
+      test_result: ${{ steps.final_test.outputs.test_result }}
+      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
     env:
       VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      HOME: /root
+      SSH_ARTIFACT_NAME: "${{ github.run_id }}-install-ssh"
+      UID: "${{ github.run_id }}-install"
+    defaults:
+      run:
+        shell: bash
     steps:
       - uses: actions/checkout@v3
-      - run: |
+        with:
+          path: /home/runner/actions-runner/_work/nextcloudpi/nextcloudpi
+      - name: Generate ssh keypair
+        working-directory: /__w/nextcloudpi/nextcloudpi
+        run: |
           set -e
-          mkdir -p ./.ssh
+          mkdir -p .ssh
           ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
+          . /ncp-test-automation/bin/entrypoint.sh
       - name: upload ssh private key to artifact store
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ github.run_id }}-install-ssh-privkey
-          path: .ssh
+          name: "${{ env.SSH_ARTIFACT_NAME }}"
+          path: /__w/nextcloudpi/nextcloudpi/.ssh
           if-no-files-found: error
       - id: create-test-instance
         uses: ./.github/actions/create-test-instance
         with:
           version: ${{ env.VERSION }}
-          uid: "${{ github.run_id }}-install"
+          uid: "${{ env.UID }}"
           hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
           server_type: "cx11"
+      - name: set instance variables
+        run: |
+          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
+          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
+      - name: Test postinstall VM
+        id: final_test
+        working-directory: /ncp-test-automation/bin
+        run: |
+          set -e
+          echo "Setup ssh"
+          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+          
+          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          
+          echo "Run integration tests"
+          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
+          set -x
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "and ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            exit 1
+          }
+          echo "test_result=success" >> "$GITHUB_OUTPUT";
 
-  setup-update-test-instance:
+  update-test:
     runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:bullseye
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+        UID: "${{ github.run_id }}-update"
+    defaults:
+      run:
+        shell: bash
     outputs:
       server_address: ${{ steps.create-test-instance.outputs.server_address }}
       snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
       test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
       previous_version: ${{ steps.find-version.outputs.previous_version }}
       version: ${{ env.VERSION }}
+      test_result: ${{ steps.final_test.outputs.test_result }}
+      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
     env:
       VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      HOME: /root
+      SSH_ARTIFACT_NAME: "${{ github.run_id }}-update-ssh"
+      UID: "${{ github.run_id }}-update"
     steps:
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
+          path: /__w/nextcloudpi/nextcloudpi
       - name: find reference version
+        working-directory: /__w/nextcloudpi/nextcloudpi
         shell: bash
         id: find-version
         run: |
+          chown -R "$(id -u):$(id -g)" .
           set -e
           if [[ -n "${{ github.base_ref }}" ]]
           then
@@ -82,121 +153,44 @@ jobs:
             version="${version%-*-*}"
           fi
           echo "Previous version is '$version'"
-          echo "previous_version=${version}" >> $GITHUB_OUTPUT
+          echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV"
       - name: Generate ssh key
         run: |
           set -x
-          mkdir -p ./.ssh
-          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
+          mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
+          ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
+          . /ncp-test-automation/bin/entrypoint.sh
       - name: upload ssh private key to artifact store
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ github.run_id }}-update-ssh-privkey
-          path: .ssh
+          name: "${{ env.SSH_ARTIFACT_NAME }}"
+          path: /__w/nextcloudpi/nextcloudpi/.ssh
           if-no-files-found: error
       - id: create-test-instance
         uses: ./.github/actions/create-test-instance
         with:
-          version: "${{ steps.find-version.outputs.previous_version }}"
-          uid: "${{ github.run_id }}-update"
+          version: "${{ env.PREVIOUS_VERSION }}"
+          uid: "${{ env.UID }}"
           hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
           server_type: "cx11"
-
-  run-installation-test:
-    needs:
-      - setup-installation-test-instance
-    runs-on: ubuntu-latest
-
-    container:
-      image: thecalcaholic/ncp-test-automation:latest
-      env:
-        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
-        UID: "${{ github.run_id }}-install"
-    env:
-      VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
-      SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}"
-      SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}"
-      HOME: /root
-    defaults:
-      run:
-        shell: bash
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: 'theCalcaholic/ncp-test-automation'
-      - name: download ssh private key from artifact store
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ github.run_id }}-install-ssh-privkey
-          path: .ssh
-      - name: Test postinstall VM
+      - name: Set instance variables
         run: |
-          set -e
-          echo "Setup ssh"
-          chmod 0600 ./.ssh/automation_ssh_key
-          eval "$(ssh-agent)"
-          ssh-add ./.ssh/automation_ssh_key
-          
-          cd bin
-          source ./library.sh
-          
-          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
-          
-          setup-ssh-port-forwarding "$SERVER_ADDRESS"
-          
-          echo "Run integration tests"
-          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
-          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
-    
-            echo "Integration tests failed"
-            echo "Here are the last lines of ncp-install.log:"
-            echo "==========================================="
-            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
-            echo "==========================================="
-            echo "and ncp.log:"
-            echo "==========================================="
-            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
-            echo "==========================================="
-            exit 1
-          }
-
-  run-update-test:
-    needs:
-      - setup-update-test-instance
-    runs-on: ubuntu-latest
-
-    container:
-      image: thecalcaholic/ncp-test-automation:latest
-      env:
-        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
-        UID: "${{ github.run_id }}-update"
-    env:
-      PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }}
-      VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
-      SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}"
-      SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}"
-      HOME: /root
-    defaults:
-      run:
-        shell: bash
-    steps:
+          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
+          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
       - uses: actions/checkout@v3
         with:
           repository: 'theCalcaholic/ncp-test-automation'
-      - name: download ssh private key from artifact store
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ github.run_id }}-update-ssh-privkey
-          path: .ssh
+          ref: "bullseye"
+          path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
       - name: Activate and Test postinstall VM
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
         run: |
           set -e
           echo "Setup ssh"
-          chmod 0600 ./.ssh/automation_ssh_key
+          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
           eval "$(ssh-agent)"
-          ssh-add ./.ssh/automation_ssh_key
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
           
-          cd bin
           source ./library.sh
           
           trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
@@ -205,7 +199,7 @@ jobs:
           
           echo "Run integration tests"
           ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
-          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
           
             echo "Integration tests failed"
             echo "Here are the last lines of ncp-install.log:"
@@ -219,36 +213,177 @@ jobs:
             exit 1
           }
       - name: perform update
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
         run: |
           set -e
           
           echo "Setup ssh"
-          chmod 0600 ./.ssh/automation_ssh_key
+          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
           eval "$(ssh-agent)"
-          ssh-add ./.ssh/automation_ssh_key
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
           
-          . ./bin/library.sh
+          source ./library.sh
 
           echo "Updating from $PREVIOUS_VERSION to $VERSION"
           ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
           ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
-      - name: Run integration tests
+      - name: Run integration tests after update
+        id: final_test
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
         run: |
           set -e
           
           echo "Setup ssh"
           eval "$(ssh-agent)"
-          ssh-add ./.ssh/automation_ssh_key
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+          
+          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          echo "Run integration tests"
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          NC_TEST_ARGS=()
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
+          set -x
+          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "and ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            exit 1
+          }
+          
+          echo "test_result=success" >> "$GITHUB_OUTPUT";
+
+  dist-upgrade-test:
+    runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:bullseye
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+        UID: "${{ github.run_id }}-distupgrade"
+    defaults:
+      run:
+        shell: bash
+    outputs:
+      test_result: ${{ steps.final_test.outputs.test_result }}
+      ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
+      server_address: ${{ steps.create-test-instance.outputs.server_address }}
+      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
+      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
+      previous_version: ${{ env.PREVIOUS_VERSION }}
+      version: ${{ env.VERSION }}
+    env:
+      PREVIOUS_VERSION: "v1.53.3"
+      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      HOME: /root
+      SSH_ARTIFACT_NAME: "${{ github.run_id }}-distupgrade-ssh"
+      UID: "${{ github.run_id }}-distupgrade"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          path: /__w/nextcloudpi/nextcloudpi
+      - name: Generate ssh key
+        run: |
+          set -x
+          chown -R "$(id -u):$(id -g)" /__w/nextcloudpi/nextcloudpi
+          mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
+          ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
+          . /ncp-test-automation/bin/entrypoint.sh
+      - name: upload ssh private key to artifact store
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.SSH_ARTIFACT_NAME }}"
+          path: /__w/nextcloudpi/nextcloudpi/.ssh
+          if-no-files-found: error
+      - id: create-test-instance
+        uses: ./.github/actions/create-test-instance-bullseye
+        with:
+          version: "${{ env.PREVIOUS_VERSION }}"
+          uid: "${{ env.UID }}"
+          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
+          server_type: "cx11"
+      - name: Set instance variables
+        run: |
+          echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
+          echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
+      - uses: actions/checkout@v3
+        with:
+          repository: 'theCalcaholic/ncp-test-automation'
+          ref: "bullseye"
+          path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
+      - name: Activate and Test postinstall VM
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
+        run: |
+          set -e
+          echo "Setup ssh"
+          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+          
+          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          
+          echo "Run integration tests"
+          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "and ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            exit 1
+          }
+      - name: perform update
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+
+          echo "Updating from $PREVIOUS_VERSION to $VERSION"
+          ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
+      - name: Run integration tests after update
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
           
-          cd bin
           source ./library.sh
           
           trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
           
           echo "Run integration tests"
           setup-ssh-port-forwarding "$SERVER_ADDRESS"
-          
-          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          NC_TEST_ARGS=()
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
+          set -x
+          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
           
             echo "Integration tests failed"
             echo "Here are the last lines of ncp-install.log:"
@@ -262,11 +397,64 @@ jobs:
             exit 1
           }
 
+      - name: NCP distupgrade
+        id: distupgrade
+        working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+          
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep 'VERSION_ID="11"' || {
+            echo "Can't upgrade from Debian $(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" grep 'VERSION_ID=' /etc/os-release)"
+            echo "skipped=yes" | tee -a $GITHUB_OUTPUT
+            exit 1
+          }
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade
+          echo "skipped=no" | tee -a $GITHUB_OUTPUT
+
+      - name: Run integration tests after dist-upgrade
+        id: final_test
+        working-directory: /ncp-test-automation/bin
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          eval "$(ssh-agent)"
+          ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
+          
+          source ./library.sh
+          
+          trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          echo "Run integration tests"
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          
+          test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            echo "nextcloud.log:"
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /opt/ncdata/data/nextcloud.log;
+            exit 1
+          }
+          echo "test_result=success" >> "$GITHUB_OUTPUT"
+
   install-postactivation-snapshot:
     if: ${{ always() }}
     needs:
-      - setup-installation-test-instance
-      - run-installation-test
+      - installation-test
     runs-on: ubuntu-latest
     container:
       image: thecalcaholic/ncp-test-automation:latest
@@ -275,28 +463,28 @@ jobs:
 
     env:
       TEST_TYPE: install
-      SERVER_ADDRESS: ${{ needs.setup-installation-test-instance.outputs.server_address }}
-      TEST_RESULT: ${{ needs.setup-installation-test-instance.result }}
-      TEST_SERVER_ID: ${{ needs.setup-installation-test-instance.outputs.test_server_id }}
-      VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
-
+      SERVER_ADDRESS: ${{ needs.installation-test.outputs.server_address }}
+      TEST_RESULT: ${{ needs.installation-test.test_result }}
+      TEST_SERVER_ID: ${{ needs.installation-test.outputs.test_server_id }}
+      VERSION: ${{ needs.installation-test.outputs.version }}
+      SSH_ARTIFACT_NAME: "${{ needs.installation-test.outputs.ssh_artifact_name }}"
       UID: ${{ github.run_id }}-install
     steps:
       - name: download ssh private key from artifact store
         uses: actions/download-artifact@v3
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         with:
-          name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
+          name: ${{ env.SSH_ARTIFACT_NAME }}
           path: /github/workspace/.ssh
       - name: Shutdown server
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         run: |
-          chmod 0600 /github/workspace/.ssh/automation_ssh_key
-          export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
+          chmod 0600  /github/workspace/.ssh/automation_ssh_key
+          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
           bash /ncp-test-automation/bin/entrypoint.sh
           eval "$(ssh-agent)"
-          ssh-add /github/workspace/.ssh/automation_ssh_key
-  
+          ssh-add  /github/workspace/.ssh/automation_ssh_key
+          
           ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
           systemctl stop mariadb
           systemctl poweroff
@@ -304,11 +492,10 @@ jobs:
       - name: Create Snapshot
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         shell: bash
+        working-directory: /ncp-test-automation/bin
         run: |
           set -x
-          echo "${{ needs.setup-installation-test-instance.outputs.test_server_id }}"
           echo "${TEST_SERVER_ID?}"
-          cd /ncp-test-automation/bin
           
           . ./library.sh
           
@@ -320,8 +507,7 @@ jobs:
   update-postactivation-snapshot:
     if: ${{ always() }}
     needs:
-      - setup-update-test-instance
-      - run-update-test
+      - update-test
     runs-on: ubuntu-latest
     container:
       image: thecalcaholic/ncp-test-automation:latest
@@ -330,26 +516,27 @@ jobs:
 
     env:
       TEST_TYPE: update
-      SERVER_ADDRESS: ${{ needs.setup-update-test-instance.outputs.server_address }}
-      TEST_RESULT: ${{ needs.setup-update-test-instance.result }}
-      TEST_SERVER_ID: ${{ needs.setup-update-test-instance.outputs.test_server_id }}
-      VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
+      SERVER_ADDRESS: ${{ needs.update-test.outputs.server_address }}
+      TEST_RESULT: ${{ needs.update-test.test_result }}
+      TEST_SERVER_ID: ${{ needs.update-test.outputs.test_server_id }}
+      VERSION: ${{ needs.update-test.outputs.version }}
       UID: ${{ github.run_id }}-update
+      SSH_ARTIFACT_NAME: "${{ needs.update-test.outputs.ssh_artifact_name }}"
     steps:
       - name: download ssh private key from artifact store
         uses: actions/download-artifact@v3
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         with:
-          name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
+          name: ${{ env.SSH_ARTIFACT_NAME }}
           path: /github/workspace/.ssh
       - name: Shutdown server
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         run: |
-          chmod 0600 /github/workspace/.ssh/automation_ssh_key
-          export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
+          chmod 0600  /github/workspace/.ssh/automation_ssh_key
+          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
           bash /ncp-test-automation/bin/entrypoint.sh
           eval "$(ssh-agent)"
-          ssh-add /github/workspace/.ssh/automation_ssh_key
+          ssh-add  /github/workspace/.ssh/automation_ssh_key
           
           ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
           systemctl stop mariadb
@@ -358,11 +545,11 @@ jobs:
       - name: Create Snapshot
         if: ${{ contains('success|failure', env.TEST_RESULT) }}
         shell: bash
+        working-directory: /ncp-test-automation/bin
         run: |
           set -x
-          echo "${{ needs.setup-update-test-instance.outputs.test_server_id }}"
+          echo "${{ needs.update-test.outputs.test_server_id }}"
           echo "${TEST_SERVER_ID?}"
-          cd /ncp-test-automation/bin
           
           . ./library.sh
           
@@ -371,11 +558,66 @@ jobs:
           snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
           hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"
 
+  dist-upgrade-postactivation-snapshot:
+    if: ${{ always() }}
+    needs:
+      - dist-upgrade-test
+    runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+
+    env:
+      TEST_TYPE: distupgrade
+      SERVER_ADDRESS: ${{ needs.dist-upgrade-test.outputs.server_address }}
+      TEST_RESULT: ${{ needs.dist-upgrade-test.outputs.test_result }}
+      TEST_SERVER_ID: ${{ needs.dist-upgrade-test.outputs.test_server_id }}
+      VERSION: ${{ needs.dist-upgrade-test.outputs.version }}
+      UID: ${{ github.run_id }}-distupgrade
+      SSH_ARTIFACT_NAME: "${{ needs.dist-upgrade-test.outputs.ssh_artifact_name }}"
+    steps:
+      - name: download ssh private key from artifact store
+        uses: actions/download-artifact@v3
+        if: ${{ contains('success|failure', env.TEST_RESULT) }}
+        with:
+          name: ${{ env.SSH_ARTIFACT_NAME }}
+          path: /github/workspace/.ssh
+      - name: Shutdown server
+        if: ${{ contains('success|failure', env.TEST_RESULT) }}
+        run: |
+          chmod 0600  /github/workspace/.ssh/automation_ssh_key
+          export SSH_PUBLIC_KEY="$(cat  /github/workspace/.ssh/automation_ssh_key.pub)"
+          bash /ncp-test-automation/bin/entrypoint.sh
+          eval "$(ssh-agent)"
+          ssh-add  /github/workspace/.ssh/automation_ssh_key
+          
+          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
+          systemctl stop mariadb
+          systemctl poweroff
+          EOF
+      - name: Create Snapshot
+        if: ${{ contains('success|failure', env.TEST_RESULT) }}
+        shell: bash
+        working-directory: /ncp-test-automation/bin
+        run: |
+          set -x
+          echo "${{ needs.dist-upgrade-test.outputs.test_server_id }}"
+          echo "${TEST_SERVER_ID?}"
+          
+          . ./library.sh
+          
+          tf-init "$TF_SNAPSHOT"
+          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
+          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
+          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"
+  
   cleanup:
     if: ${{ always() }}
     needs:
       - install-postactivation-snapshot
       - update-postactivation-snapshot
+      - dist-upgrade-postactivation-snapshot
     runs-on: ubuntu-latest
     container:
       image: thecalcaholic/ncp-test-automation:latest
@@ -383,7 +625,7 @@ jobs:
         HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
     strategy:
       matrix:
-        uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update"]
+        uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update", "${{ github.run_id }}-distupgrade"]
       fail-fast: false
     env:
       HOME: '/root'
@@ -410,6 +652,7 @@ jobs:
     needs:
       - install-postactivation-snapshot
       - update-postactivation-snapshot
+      - dist-upgrade-postactivation-snapshot
     runs-on: ubuntu-latest
     container:
       image: thecalcaholic/ncp-test-automation:latest
@@ -420,7 +663,7 @@ jobs:
     steps:
       - name: Delete old snapshots
         run: |
-          for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -16)
+          for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -12)
           do
             echo "Deleting snapshot '$snapshot'..."
             hcloud image delete "$snapshot"

bin/ncp-dist-upgrade

@@ -1,137 +1,35 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
-set -eu -o pipefail
-
-[[ -f /.dockerenv ]] && { echo "Not supported in Docker. Upgrade the container instead"; exit 0; }
-
-new_cfg=/usr/local/etc/ncp-recommended.cfg
-[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
-
-APTINSTALL="apt-get install -y --no-install-recommends"
-export DEBIAN_FRONTEND=noninteractive
-
-echo "
->>> ATTENTION <<<
-This is a dangerous process that is only guaranteed to work properly if you
-have not made manual changes in the system. Backup the SD card first and
-proceed at your own risk.
-
-Note that this is not a requirement for NCP to continue working properly.
-The current distribution will keep receiving updates for some time.
-
-Do you want to continue? [y/N]"
-
-read key
-[[ "$key" == y ]] || exit 0
-
-source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER
-old_cfg="${NCPCFG}"
-
-trap "echo 'Something went wrong. Fix it and try again'" EXIT
-
-save_maintenance_mode
-
-# Fix grub-pc issue in VM
-if apt show grub-pc-bin &>/dev/null; then
-  $APTINSTALL grub
-fi
-
-apt-get update
-apt-get upgrade -y
-
-# remove old PHP version
-set +e
-apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \
-	php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \
-	php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json
-apt-get purge -y php${PHPVER}-mysql
-apt-get purge -y php${PHPVER}-redis
-apt-get purge -y php${PHPVER}-exif
-apt-get purge -y php${PHPVER}-bcmath
-apt-get purge -y php${PHPVER}-gmp
-apt-get purge -y php${PHPVER}-imagick
 set -e
 
-# update sources
-sed -i 's/buster/bullseye/g' /etc/apt/sources.list
-sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true
-sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list
-rm -f /etc/apt/sources.list.d/php.list
+[[ "$UID" -eq 0 ]] || {
+  echo "ERROR: Must be run as root (try sudo ncp-dist-upgrade)"
+  exit 1
+}
 
-# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian
-if [[ -f /usr/bin/raspi-config ]]; then
-  sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf
+. /etc/os-release
+
+if [[ "$VERSION_ID" -eq 10 ]]
+then
+  UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-10.sh)
+elif [[ "$VERSION_ID" -eq 11 ]]
+then
+  UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-11.sh)
+else
+  echo "No dist-upgrade available for OS version: Debian ${VERSION}."
+  exit 0
 fi
 
-# install latest distro
-apt-get update
-apt-get dist-upgrade -y
+if { [ "$TERM" = "screen" ] && [ -n "$TMUX" ]; } || [[ "${DEBIAN_FRONTEND:-}" == "noninteractive" ]] || ! [[ -t 0 ]]
+then
+  "${UPGRADE_CMD[@]}"
+else
+  tmux list-sessions | grep ncp-distupgrade && {
+    echo "Existing distupgrade process detected. Connecting..."
+    sleep 5
+    tmux attach -t ncp-distupgrade
+    exit 0
+  }
+  tmux new-session -s ncp-distupgrade bash -c "${UPGRADE_CMD[*]}; bash"
+fi
 
-# install latest PHP version
-release_new=$(jq -r '.release'     < "${new_cfg}")
-# the default repo in bullseye is bullseye-security - use bullseye if it is not available
-grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security"
-php_ver_new=$(jq -r '.php_version' < "${new_cfg}")
-# PHP 8.1 is only supported via the
-[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4
-
-$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \
-	php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \
-	php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json
-
-$APTINSTALL php${php_ver_new}-mysql
-$APTINSTALL -t ${release_new} php${php_ver_new}-redis
-
-$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils
-sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
-$APTINSTALL -t ${release_new} php${php_ver_new}-exif
-sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
-$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath
-sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
-$APTINSTALL -t ${release_new} php${php_ver_new}-gmp
-#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript
-
-# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later
-{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true
-
-apt-get autoremove -y
-apt-get clean
-
-cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <<EOF
-; disable .user.ini files for performance and workaround NC update bugs
-user_ini.filename =
-
-; from Nextcloud .user.ini
-upload_max_filesize=10G
-post_max_size=10G
-memory_limit=768M
-mbstring.func_overload=0
-always_populate_raw_post_data=-1
-default_charset='UTF-8'
-output_buffering=0
-
-; slow transfers will be killed after this time
-max_execution_time=3600
-max_input_time=3600
-EOF
-
-# restart services
-service php${php_ver_new}-fpm restart
-a2enconf php${php_ver_new}-fpm
-service apache2 restart
-
-is_active_app unattended-upgrades && run_app unattended-upgrades || true
-
-# mark as successful
-mv "${new_cfg}" "${old_cfg}"
-install_template "php/opcache.ini.sh" "/etc/php/${php_ver_new}/mods-available/opcache.ini" --defaults
-clear_opcache
-
-source /usr/local/etc/library.sh # refresh NCPCFG RELEASE PHPVER
-run_app nc-limits
-restore_maintenance_mode
-
-rm -f /etc/update-motd.d/30ncp-dist-upgrade
-
-echo "Upgrade to ${release_new} successful"
-trap '' EXIT

bin/ncp-dist-upgrade.d/debian-10.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+[[ -f /.dockerenv ]] && { echo "Not supported in Docker. Upgrade the container instead"; exit 0; }
+
+new_cfg=/usr/local/etc/ncp-recommended.cfg
+[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
+
+APTINSTALL="apt-get install -y --no-install-recommends"
+export DEBIAN_FRONTEND=noninteractive
+
+echo "
+>>> ATTENTION <<<
+This is a dangerous process that is only guaranteed to work properly if you
+have not made manual changes in the system. Backup the SD card first and
+proceed at your own risk.
+
+Note that this is not a requirement for NCP to continue working properly.
+The current distribution will keep receiving updates for some time.
+
+Do you want to continue? [y/N]"
+
+read key
+[[ "$key" == y ]] || exit 0
+
+source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER
+old_cfg="${NCPCFG}"
+
+trap "echo 'Something went wrong. Fix it and try again'" EXIT
+
+save_maintenance_mode
+
+# Fix grub-pc issue in VM
+if apt show grub-pc-bin &>/dev/null; then
+  $APTINSTALL grub
+fi
+
+apt-get update
+apt-get upgrade -y
+
+# remove old PHP version
+set +e
+apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \
+	php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \
+	php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json
+apt-get purge -y php${PHPVER}-mysql
+apt-get purge -y php${PHPVER}-redis
+apt-get purge -y php${PHPVER}-exif
+apt-get purge -y php${PHPVER}-bcmath
+apt-get purge -y php${PHPVER}-gmp
+apt-get purge -y php${PHPVER}-imagick
+set -e
+
+# update sources
+sed -i 's/buster/bullseye/g' /etc/apt/sources.list
+sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true
+sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list
+rm -f /etc/apt/sources.list.d/php.list
+
+# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian
+if [[ -f /usr/bin/raspi-config ]]; then
+  sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf
+fi
+
+# install latest distro
+apt-get update
+apt-get dist-upgrade -y
+
+# install latest PHP version
+release_new=$(jq -r '.release'     < "${new_cfg}")
+# the default repo in bullseye is bullseye-security - use bullseye if it is not available
+grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security"
+php_ver_new=$(jq -r '.php_version' < "${new_cfg}")
+# PHP 8.1 is only supported via the
+[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4
+
+$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \
+	php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \
+	php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json
+
+$APTINSTALL php${php_ver_new}-mysql
+$APTINSTALL -t ${release_new} php${php_ver_new}-redis
+
+$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils
+sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
+$APTINSTALL -t ${release_new} php${php_ver_new}-exif
+sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
+$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath
+sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
+$APTINSTALL -t ${release_new} php${php_ver_new}-gmp
+#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript
+
+# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later
+{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true
+
+apt-get autoremove -y
+apt-get clean
+
+cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <<EOF
+; disable .user.ini files for performance and workaround NC update bugs
+user_ini.filename =
+
+; from Nextcloud .user.ini
+upload_max_filesize=10G
+post_max_size=10G
+memory_limit=768M
+mbstring.func_overload=0
+always_populate_raw_post_data=-1
+default_charset='UTF-8'
+output_buffering=0
+
+; slow transfers will be killed after this time
+max_execution_time=3600
+max_input_time=3600
+EOF
+
+# restart services
+service php${php_ver_new}-fpm restart
+a2enconf php${php_ver_new}-fpm
+service apache2 restart
+
+is_active_app unattended-upgrades && run_app unattended-upgrades || true
+
+# mark as successful
+mv "${new_cfg}" "${old_cfg}"
+install_template "php/opcache.ini.sh" "/etc/php/${php_ver_new}/mods-available/opcache.ini" --defaults
+clear_opcache
+
+source /usr/local/etc/library.sh # refresh NCPCFG RELEASE PHPVER
+run_app nc-limits
+restore_maintenance_mode
+
+rm -f /etc/update-motd.d/30ncp-dist-upgrade
+
+echo "Upgrade to ${release_new} successful"
+trap '' EXIT

bin/ncp-dist-upgrade.d/debian-11.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+new_cfg=/usr/local/etc/ncp-recommended.cfg
+[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
+
+echo "
+>>> ATTENTION <<<
+This is a dangerous process that is only guaranteed to work properly if you
+have not made manual changes in the system. Backup the SD card first and
+proceed at your own risk.
+
+Note that this is not a requirement for NCP to continue working properly.
+The current distribution will keep receiving updates for some time.
+
+Do you want to continue? [y/N]"
+
+if [[ "${DEBIAN_FRONTEND:-}" == "noninteractive" ]] || ! [[ -t 0 ]]
+then
+  echo "Noninteractive environment detected. Automatically proceeding in 30 seconds..."
+  sleep 30
+else
+  read -n1 -r key
+  [[ "${key,,}" == y ]] || exit 0
+fi
+
+export DEBIAN_FRONTEND=noninteractive
+
+source /usr/local/etc/library.sh
+is_more_recent_than "${PHPVER}.0" "8.0.0" || {
+  echo "You still have PHP version ${PHPVER} installed. Please update to the latest supported version of nextcloud (which will also update your PHP version) before proceeding with the distribution upgrade."
+  echo "Exiting."
+  exit 1
+}
+save_maintenance_mode
+
+# Perform dist-upgrade
+
+apt-get update && apt-get upgrade -y
+for aptlist in /etc/apt/sources.list /etc/apt/sources.list.d/php.list /etc/apt/sources.list.d/armbian.list
+do
+  [ -f "$aptlist" ] && sed -i -e "s/bullseye/bookworm/g" "$aptlist"
+done
+for aptlist in /etc/apt/sources.list.d/*.list
+do
+	[[ "$aptlist" =~ "/etc/apt/sources.list.d/"(php|armbian)".list" ]] || continue
+    echo "Disabling repositories from \"$aptlist\""
+    sed -i -e "s/deb/#deb/g" "$aptlist"
+done
+apt-get update && apt-get upgrade -y --without-new-pkgs
+if is_lxc
+then
+  # Required to avoid breakage of /etc/resolv.conf
+  apt-get install -y --no-install-recommends systemd-resolved && systemctl enable --now systemd-resolved
+fi
+apt-get full-upgrade -y
+sudo apt-get --purge  autoremove -y
+
+apt-get install -y --no-install-recommends exfatprogs
+
+#mkdir -p /etc/systemd/system/php8.1-fpm.service.d
+#echo '[Service]' > /etc/systemd/system/php8.1-fpm.service.d/ncp.conf
+#echo 'ExecStartPre=mkdir -p /var/run/php' >> /etc/systemd/system/php8.1-fpm.service.d/ncp.conf
+#[[ "$INIT_SYSTEM" != "systemd" ]] || { systemctl daemon-reload && systemctl restart php8.1-fpm; }
+
+restore_maintenance_mode
+cfg="$(jq "." "$NCPCFG")"
+cfg="$(jq ".release = \"bookworm\"" <<<"$cfg")"
+echo "$cfg" > "$NCPCFG"
+rm -f /etc/update-motd.d/30ncp-dist-upgrade
+
+echo "Update to Debian 12 (bookworm) successful."
+
+is_active_app unattended-upgrades && {
+  echo "Setting up unattended upgrades..."
+  run_app unattended-upgrades || true
+  echo "done."
+}

ncp.sh

@@ -22,7 +22,7 @@ install()
 {
   # NCP-CONFIG
   apt-get update
-  $APTINSTALL git dialog whiptail jq file lsb-release
+  $APTINSTALL git dialog whiptail jq file lsb-release tmux
   mkdir -p "$CONFDIR" "$BINDIR"
 
   # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager

tests/nextcloud_tests.py

@@ -140,7 +140,7 @@ def close_first_run_wizard(driver: WebDriver):
         time.sleep(3)
 
 
-def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
+def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool):
     """ Login and assert admin page checks"""
     test = Test()
     test.new("nextcloud page")
@@ -263,8 +263,11 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
                     expected['ncp_version'] = True
                 elif 'php version' in divs[0].text.lower() and divs[1].text == ncp_cfg['php_version']:
                     expected['php_version'] = True
-                elif 'debian release' in divs[0].text.lower() and divs[1].text == ncp_cfg['release']:
-                    expected['debian_release'] = True
+                elif 'debian release' in divs[0].text.lower():
+                    if divs[1].text == ncp_cfg['release'] or skip_release_check:
+                        expected['debian_release'] = True
+                    else:
+                        print(f"{tc.yellow}{divs[1].text} != {ncp_cfg['release']}")
         failed = list(map(lambda item: item[0], filter(lambda item: not item[1], expected.items())))
         test.check(len(failed) == 0, f"checks failed for admin section: [{', '.join(failed)}]")
     except Exception as e:
@@ -302,11 +305,12 @@ if __name__ == "__main__":
 
     # parse options
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui'])
+        opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check'])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
 
+    skip_release_check = False
     options = webdriver.FirefoxOptions()
     for opt, arg in opts:
         if opt in ('-h', '--help'):
@@ -317,6 +321,8 @@ if __name__ == "__main__":
                 os.unlink(test_cfg)
         elif opt == '--no-gui':
             options.add_argument("-headless")
+        elif opt == '--skip-release-check':
+            skip_release_check = True
         else:
             usage()
             sys.exit(2)
@@ -355,7 +361,7 @@ if __name__ == "__main__":
     driver = webdriver.Firefox(options=options)
     failed=False
     try:
-        test_nextcloud(IP, nc_port, driver)
+        test_nextcloud(IP, nc_port, driver, skip_release_check)
     except Exception as e:
         print(e)
         print(traceback.format_exc())