nextcloudpi/.github/workflows/build-sd-images.yml

name: "Build Armbian Images"
on:
  workflow_call:
    inputs:
      git_ref:
        required: true
        type: string
      board_id:
        required: true
        type: string
      board_name:
        required: true
        type: string
      require_test:
        required: false
        default: true
        type: boolean
    outputs:
      artifact_name:
        value: "${{ jobs.build.outputs.artifact_name }}"
      artifact_file:
        value: "${{ jobs.build.outputs.artifact_file }}"

jobs:
  build:
    runs-on: ubuntu-latest
    env:
      VERSION: "${{ inputs.git_ref }}"
      LOG_GUEST: "\\033[1;34mGUEST::\\033[0m"
      LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
      LOG_CICD: "\\033[1;35mCICD::\\033[0m"
      LOG_TEST: "\\033[1;33mTEST::\\033[0m"
      LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
    defaults:
      run:
        shell: bash
    outputs:
      artifact_file: ${{ env.ARTIFACT_FILE }}
      artifact_name: ${{ github.run_id }}-${{ inputs.board_id }}-image
    steps:
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Setup qemu-user-static
        run: |
          sudo apt-get update
#          docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
#          sudo mkdir -p /etc/binfmt
#          for conf in qemu-{aarch64,arm}-static.conf
#          do
#            sed 's/:F$/:OC/' /usr/lib/binfmt.d/$conf | sudo tee /etc/binfmt/$conf
#          done
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: "${{ env.VERSION }}"
#      - name: Debug
#        run: |
#          which qemu-aarch64-static
#          update-binfmts --display qemu-aarch64
#          update-binfmts --display qemu-arm
      - name: "Build Armbian"
        if: ${{ inputs.board_id != 'raspberrypi' }}
        id: build-armbian
        continue-on-error: true
        run: |
          set -ex
          export LIB_TAG=master
          export IMG="NextCloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img"
          [[ "${{ github.ref_protected }}" == true ]] || export DBG=x
          
          ./build/build-SD-armbian.sh "${{ inputs.board_id }}" "${{ inputs.board_name }}"
          
          artifacts=("armbian/output/images/Armbian"*.img)
          mkdir -p output
          mv "${artifacts[0]}" "output/$IMG"
          echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT
          echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV
      - name: "Build Armbian (2nd attempt)"
        if: ${{ inputs.board_id != 'raspberrypi' && steps.build-armbian.outcome == 'failure' }}
        id: build-armbian-2nd
        run: |
          set -ex
          echo -e "${LOG_CICD} Cleanup armbian build leftovers..."
          sudo rm -rf armbian/ tmp/ output/
          
          export LIB_TAG=master
          export IMG="NextCloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img"
          [[ "${{ github.ref_protected }}" == true ]] || export DBG=x

          ./build/build-SD-armbian.sh "${{ inputs.board_id }}" "${{ inputs.board_name }}"

          artifacts=("armbian/output/images/Armbian"*.img)
          mkdir -p output
          mv "${artifacts[0]}" "output/$IMG"
          echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT
          echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV
      - name: "Upload Armbian logs"
        if: ${{ inputs.board_id != 'raspberrypi' && failure() }}
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.run_id }}-${{ inputs.board_id }}-logs
          path: armbian/output
      - name: Build RPI SD Image
        if: ${{ inputs.board_id == 'raspberrypi' }}
        id: build-rpi
        run: |
          set -ex
          echo -e "${LOG_CICD} Protected? ${{ github.ref_protected }}"
          export IMG="NextCloudPi_${{ inputs.board_name }}_${VERSION//\//_}.img"
          [[ "${{ github.ref_protected }}" == true ]] || export DBG=x
          wget -q https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O ./qemu-aarch64-static
          ./build/build-SD-rpi.sh
          mkdir -p output
          mv "tmp/$IMG" ./output/

          for i in {1..10}
          do
            sudo losetup | grep "${IMG}" || break;
            [[ "$i" -lt 10 ]] || { echo -e "${LOG_CICD} Timeout while waiting for image to unmount"; exit 1; }
            sleep 6
            echo -e "${LOG_CICD} Retrying ($i out of 10)"
          done

          echo "artifact_file=${IMG}" >> $GITHUB_OUTPUT
          echo "ARTIFACT_FILE=${IMG}" >> $GITHUB_ENV
      - name: upload image to artifact store
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.run_id }}-${{ inputs.board_id }}-image
          path: output/${{ env.ARTIFACT_FILE }}
          if-no-files-found: error

  test:
    needs: build
    runs-on: ubuntu-latest
    env:
      VERSION: "${{ inputs.git_ref }}"
      ARTIFACT_ID: ${{ needs.build.outputs.artifact_name }}
      ARTIFACT_FILE: ${{ needs.build.outputs.artifact_file }}
      LOG_GUEST: "\\033[1;34mGUEST::\\033[0m"
      LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
      LOG_CICD: "\\033[1;35mCICD::\\033[0m"
      LOG_TEST: "\\033[1;33mTEST::\\033[0m"
      LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
    defaults:
      run:
        shell: bash
    steps:
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      - name: Apt update
        run: |
          sudo apt-get update
#          sudo apt-get -y --no-install-recommends install qemu-user-static
#      - name: Apply workaround for sudo bug (https://github.com/multiarch/qemu-user-static/issues/17)
#        run: |
#          sudo apt-get update
#          sudo apt-get -y --no-install-recommends install binfmt-support qemu-user-static
#          docker run --rm --privileged multiarch/qemu-user-static:register --reset --credential yes
#          sudo mkdir -p /etc/binfmt
#          for conf in qemu-{aarch64,arm}-static.conf
#          do
#            sed 's/:F$/:OC/' /usr/lib/binfmt.d/$conf | sudo tee /etc/binfmt/$conf
#          done
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          ref: "${{ env.VERSION }}"
      - uses: actions/download-artifact@v3
        with:
          name: ${{ env.ARTIFACT_ID }}
          path: output
      - name: Prepare test
        run: |
          set -x
          mv output/${ARTIFACT_FILE?} ncp.img
          sudo apt-get install -y systemd-container
          sudo pip install selenium
          sudo rm -rf raspbian_root
          . ./build/buildlib.sh
          mount_raspbian "ncp.img"
          sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O raspbian_root/usr/bin/qemu-aarch64-static
          sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static
          sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static
          echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf
      - name: Test image
        id: test
        run: |
          
          log_err() {
            rc="${1?}"
            msg="${2?}"
            echo -e "${LOG_DIAG} $msg" >&2
            return $rc
          }
          
          sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
          sleep 60
          
          CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
          
          success=false
          for attempt in {1..30}
          do
            echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
            ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
            [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
            success=true
            break
          done
          sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
          sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
          
          [[ "$success" == "true" ]] || {
            echo -e "${LOG_CICD} Could not reach container. Aborting..."
            exit 1
          }

          attempt=0
          success=false
          for attempt in {1..150}
          do
            echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
            redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
            && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
            || log_err $? "Error retrieving redis credentials" || true
            if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
              && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
              && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
            then
              echo -e "${LOG_CICD} Startup successful"
              success=true
              break
            fi
            attempt=$((attempt + 1))
            sleep 5
          done
          
          [[ "$success" == "true" ]] || {
              echo -e "${LOG_CICD} Timeout reached."
              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
              "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
              "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
              "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
              exit 1
          }
          
          set -x
          set +e
          
          success=false
          for attempt in {1..5}
          do
            echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) =="
            python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
              echo -e "${LOG_CICD} Activation test failed!"
              echo -e "${LOG_DIAG} Geckodriver logs:"
              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
              echo "================"
              echo -e "${LOG_DIAG} mysql: "
              "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
              "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
              sleep 12
              continue
            }
            success=true
            break
          done
          [[ "$success" == "true" ]] || {
            echo -e "${LOG_CICD} Activation test failed in all attempts!"
            exit 1
          }
          
          success=false
          for attempt in {1..5}
          do
            echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
            sudo python tests/system_tests.py --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }"
            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
              echo -e "${LOG_CICD} System test failed!"
              sleep 12
              continue
            }
            success=true
            break
          done
          [[ "$success" == "true" ]] || {
            echo -e "${LOG_CICD} System test failed in all attempts!"
            exit 1
          }
          
          success=false
          for attempt in {1..5}
          do
            echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
            python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
            [[ ${PIPESTATUS[0]} -eq 0 ]] || {
              echo -e "${LOG_CICD} Nextcloud test failed!"
              echo -e "{$LOG_DIAG} Geckodriver logs:"
              tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
              echo -e "${LOG_CICD} ================"
              echo -e "${LOG_DIAG} ncp.log: "
              "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
              sleep 12
              continue
            }
            success=true
            break
          done
          
          sudo machinectl terminate ncp
          
          [[ "$success" == "true" ]] || {
            echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
            exit 1
          }