Merge pull request #1763 from nextcloud/feature/cicd/build-lxc

Build LXC images in CICD pipeline
2026-01-10 15:12:01 -03:30 · 2023-04-15 02:45:00 +02:00 · 2023-04-15 02:45:00 +02:00 · 62859363ae
commit 62859363ae
parent d2c480ac1a f72b11d938
9 changed files with 198 additions and 120 deletions
--- a/.github/workflows/build-lxd.yml
+++ b/.github/workflows/build-lxd.yml
@ -10,10 +10,14 @@ on:
        type: string
        default: 'x86'
    outputs:
-      artifact_name:
+      lxd_artifact_name:
        value: "${{ jobs.build-current.outputs.artifact_name }}"
-      artifact_file:
+      lxd_artifact_file:
        value: "${{ jobs.build-current.outputs.artifact_file }}"
+      lxc_artifact_name:
+        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_name }}"
+      lxc_artifact_file:
+        value: "${{ jobs.convert-to-lxc-image.outputs.artifact_file }}"
  push:
    branches:
      - "**"
@ -168,7 +172,7 @@ jobs:
          path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
          if-no-files-found: error

-  update-previous:
+  test-update:
    needs:
      - build-previous
    runs-on: ubuntu-20.04
@ -193,6 +197,14 @@ jobs:
              nictype: bridged
              type: nic
          EOF
+      - name: Setup Firefox
+        uses: browser-actions/setup-firefox@latest
+      - name: Setup GeckoDriver
+        uses: ChlodAlejandro/setup-geckodriver@latest
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Setup Selenium
+        run: pip install selenium
      - name: Checkout code
        uses: actions/checkout@v3
        with:
@ -241,22 +253,44 @@ jobs:
          fi
          
          lxc stop ncp
-      - name: Pack LXD image
-        id: pack-lxd
+          lxc publish -q ncp -f --alias "ncp/updated"
+      - name: Relaunch container
        run: |
          set -x
-          . ./build/buildlib.sh
-          ARTIFACT_FILE="NextCloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
-          lxc publish -q ncp -f --alias "ncp/updated"
-          mkdir -p output
-          lxc image export -q "ncp/updated" "output/${ARTIFACT_FILE}"
-          echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
-      - name: upload LXD image to artifact store
-        uses: actions/upload-artifact@v3
-        with:
-          name: "${{ env.ARTIFACT_NAME }}"
-          path: "output/${{ steps.pack-lxd.outputs.artifact_file }}"
-          if-no-files-found: error
+          lxc delete -q -f ncp || true
+          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/updated" ncp
+          lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
+          sleep 30
+          ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
+          ip="${ip/% *}"
+          echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
+      - name: Test LXD Image
+        working-directory: ./tests
+        run: |
+          lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
+          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
+            echo "Activation test failed!"
+            echo "Geckodriver logs:"
+            tail -n 20 geckodriver.log >&2 || true
+            echo "================"
+            echo "ncp.log: "
+            lxc exec ncp -- "tail -n20 /var/log/ncp.log"
+            exit 1
+          }
+          python system_tests.py --non-interactive || {
+            echo "System test failed!"
+            exit 1
+          }
+          python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
+            echo "Nextcloud test failed!"
+            echo "Geckodriver logs:"
+            tail -n 20 geckodriver.log >&2 || true
+            echo "================"
+            echo "ncp.log: "
+            lxc exec ncp -- "tail -n20 /var/log/ncp.log"
+            exit 1
+          }
+          lxc stop ncp

  test-fresh-install:
    needs:
@ -350,79 +384,39 @@ jobs:
          }
          lxc stop ncp

-  test-update:
+  convert-to-lxc-image:
    needs:
-      - update-previous
-    runs-on: ubuntu-20.04
+      - determine-runner
+      - build-current
+    runs-on: ${{ needs.determine-runner.outputs.runner_label }}
+    outputs:
+      artifact_name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
+      artifact_file: "${{ steps.lxd-to-lxc.outputs.artifact_file }}"
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      ARTIFACT_NAME: ${{ needs.update-previous.outputs.artifact_name }}
-      ARTIFACT_FILE: ${{ needs.update-previous.outputs.artifact_file }}
+      LXD_ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
+      LXD_ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
    steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-        with:
-          ref: "${{ env.VERSION }}"
-      - uses: whywaita/setup-lxd@v1
-        with:
-          lxd_version: latest/stable
-      - name: Fix LXD
-        run: |
-          lxc profile create network
-          cat <<EOF | lxc profile edit network
-          devices:
-            eth0:
-              name: eth0
-              network: lxdbr0
-              type: nic
-          EOF
-      - name: Setup Firefox
-        uses: browser-actions/setup-firefox@latest
-      - name: Setup GeckoDriver
-        uses: ChlodAlejandro/setup-geckodriver@latest
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Setup Selenium
-        run: pip install selenium
      - name: download LXD image from artifact store
        uses: actions/download-artifact@v3
        with:
-          name: ${{ env.ARTIFACT_NAME }}
-      - name: Launch ncp container
+          name: ${{ env.LXD_ARTIFACT_NAME }}
+      - name: convert to LXC image
+        id: lxd-to-lxc
        run: |
-          set -x
-          lxc delete -q -f ncp || true
-          lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test"
-          systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp
-          lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
-          sleep 30
-          ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
-          ip="${ip/% *}"
-          echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
-      - name: Test LXD Image
-        working-directory: ./tests
-        run: |
-          lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
-          python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
-            echo "Activation test failed!"
-            echo "Geckodriver logs:"
-            tail -n 20 geckodriver.log >&2 || true
-            echo "================"
-            echo "ncp.log: "
-            lxc exec ncp -- "tail -n20 /var/log/ncp.log"
-            exit 1
-          }
-          python system_tests.py --non-interactive || {
-            echo "System test failed!"
-            exit 1
-          }
-          python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
-            echo "Nextcloud test failed!"
-            echo "Geckodriver logs:"
-            tail -n 20 geckodriver.log >&2 || true
-            echo "================"
-            echo "ncp.log: "
-            lxc exec ncp -- "tail -n20 /var/log/ncp.log"
-            exit 1
-          }
-          lxc stop ncp
+          mkdir repackage output
+          cd repackage
+          sudo tar xpf "../${LXD_ARTIFACT_FILE?}"
+          sudo rm -rf ./rootfs/dev
+          LXC_ARTIFACT_FILE="${LXD_ARTIFACT_FILE//LXD/LXC_EXPERIMENTAL}"
+          sudo tar cpzf "../output/${LXC_ARTIFACT_FILE?}" -C rootfs/ .
+          cd ..
+          sudo chown "$(id -un):" "./output/${LXC_ARTIFACT_FILE}"
+          echo "artifact_file=${LXC_ARTIFACT_FILE?}" >> $GITHUB_OUTPUT
+          echo "artifact_name=${LXD_ARTIFACT_NAME//lxd/lxc}" >> $GITHUB_OUTPUT
+      - name: upload LXD image to artifact store
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ steps.lxd-to-lxc.outputs.artifact_name }}"
+          path: "output/${{ steps.lxd-to-lxc.outputs.artifact_file }}"
+          if-no-files-found: error
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -164,9 +164,10 @@ jobs:
    uses: ./.github/workflows/publish-image.yml
    with:
      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.lxd-x86.outputs.artifact_name }}"
-      artifact_file: "${{ needs.lxd-x86.outputs.artifact_file }}"
+      artifact_id: "${{ needs.lxd-x86.outputs.lxd_artifact_name }}"
+      artifact_file: "${{ needs.lxd-x86.outputs.lxd_artifact_file }}"
      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
  lxd-arm64-release:
    needs:
      - github-release
@ -175,8 +176,32 @@ jobs:
    uses: ./.github/workflows/publish-image.yml
    with:
      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
-      artifact_id: "${{ needs.lxd-arm64.outputs.artifact_name }}"
-      artifact_file: "${{ needs.lxd-arm64.outputs.artifact_file }}"
+      artifact_id: "${{ needs.lxd-arm64.outputs.lxd_artifact_name }}"
+      artifact_file: "${{ needs.lxd-arm64.outputs.lxd_artifact_file }}"
+      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
+  lxc-x86-release:
+    needs:
+      - github-release
+      - lxd-x86
+    if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }}
+    uses: ./.github/workflows/publish-image.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      artifact_id: "${{ needs.lxd-x86.outputs.lxc_artifact_name }}"
+      artifact_file: "${{ needs.lxd-x86.outputs.lxc_artifact_file }}"
+      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}
+
+  lxc-arm64-release:
+    needs:
+      - github-release
+      - lxd-arm64
+    if: ${{ inputs.lxd || github.event_name != 'workflow_dispatch' }}
+    uses: ./.github/workflows/publish-image.yml
+    with:
+      git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
+      artifact_id: "${{ needs.lxd-arm64.outputs.lxc_artifact_name }}"
+      artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}"
      dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v'))  }}

  raspberrypi-release:
--- a/bin/nc-broadcast
+++ b/bin/nc-broadcast
@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+if [[ ${EUID} -ne 0 ]]; then
+  printf "Must be run as root. Try 'sudo %s'\n" "$( basename "$0" )"
+  exit 1
+fi
+
+if [[ " $* " =~ " "(--help|-h)" " ]]
+then
+  echo 'Description:
+  Generate notifications for all Nextcloud users
+
+Usage:
+  nc-broadcast <short-message> [options]
+
+Arguments:
+  short-message                    Short message to be sent to the user (max. 255 characters)
+
+Options:
+  -l, --long-message=LONG-MESSAGE  Long message to be sent to the users (max. 4000 characters) [default: ""]
+  -h, --help                       Display this help message
+  -q, --quiet                      Do not output any message
+  -V, --version                    Display ncc/occ version
+      --ansi|--no-ansi             Force (or disable --no-ansi) ANSI output
+  -n, --no-interaction             Do not ask any interactive question
+      --no-warnings                Skip global warnings, show command output only
+  -v|vv|vvv, --verbose             Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug'
+  exit 0
+fi
+
+ncc user:list --output=json | jq -r 'keys[]' | while read -r user
+do
+  echo "Sending notification to $user"
+  ncc notification:generate "${@:2}" "$user" "$1"
+done
+echo "All users have been notified."
--- a/bin/ncp/NETWORKING/SSH.sh
+++ b/bin/ncp/NETWORKING/SSH.sh
@ -8,7 +8,12 @@
 #


-install() { :; }
+install() {
+  apt-get update
+  apt-get install -y --no-install-recommends openssh-server
+  systemctl stop    ssh
+  systemctl disable ssh
+ }

 is_active()
 {
--- a/bin/ncp/SECURITY/fail2ban.sh
+++ b/bin/ncp/SECURITY/fail2ban.sh
@ -24,6 +24,7 @@
 install()
 {
  apt-get update
+  apt-get install --no-install-recommends -y python3-systemd
  apt-get install --no-install-recommends -y fail2ban whois
  update-rc.d fail2ban disable
  rm -f /etc/fail2ban/jail.d/defaults-debian.conf
@ -123,6 +124,7 @@ action = %($ACTION)s
 enabled  = true
 port     = ssh
 filter   = sshd
+backend  = systemd
 logpath  = /var/log/auth.log
 maxretry = $MAXRETRY

@ -135,16 +137,18 @@ port     = http,https
 filter   = nextcloud
 logpath  = $NCLOG
 maxretry = $MAXRETRY
+backend  = auto

 #
 # UFW
 #
 [ufwban]
 enabled = true
-port = ssh, http, https
-filter = ufwban
+port    = ssh, http, https
+filter  = ufwban
 logpath = /var/log/ufw.log
-action = ufw
+action  = ufw
+backend = auto
 EOF

  cp /etc/fail2ban/jail.conf /etc/fail2ban/jail.local
--- a/build/docker/debian-ncp/run-parts.sh
+++ b/build/docker/debian-ncp/run-parts.sh
@ -26,6 +26,16 @@ CURRENT_NC_MAJ="${CURRENT_NC_VERSION%%.*}"
  exit 1
 }

+DOCKER_DISCONTINUATION_ALERT="ATTENTION: NextcloudPi docker is being discontinued after Nextcloud 25! Minor NC updates and security updates and fixes will be provided until 2023/11.
+Learn more in the official announcement: https://help.nextcloud.com/t/nextcloudpi-planning-to-discontinue-its-docker-version-with-nc-25"
+
+echo -e "
+\033[1;31m$DOCKER_DISCONTINUATION_ALERT\033[0m
+
+Continue in 5 seconds..."
+
+sleep 5
+

 # wrapper to simulate update-rc.d
 cat > /usr/local/sbin/update-rc.d <<'EOF'
@ -75,6 +85,9 @@ fi

 # wait for trap from 'docker stop'
 echo "Init done"
+
+[[ -f /data/docker_discontinuation_alert_sent ]] || \
+  ( . /usr/local/etc/library.sh; notify_admin "$DOCKER_DISCONTINUATION_ALERT" && touch /data/docker_discontinuation_alert_sent )
 while true; do sleep 0.5; done


--- a/etc/ncp-templates/ncp-metrics.cfg.sh
+++ b/etc/ncp-templates/ncp-metrics.cfg.sh
@ -51,7 +51,7 @@ EOF
 EOF
 }

-is_docker || {
+is_docker || is_lxc || {

  DATADIR=$( get_nc_config_value datadirectory ) || {
    echo "ERROR: Could not get data directory. Is NextCloud running?" >&2
--- a/ncp.sh
+++ b/ncp.sh
@ -132,8 +132,10 @@ EOF
  ## NCP USER FOR AUTHENTICATION
  id -u "$WEBADMIN" &>/dev/null || useradd --home-dir /nonexistent "$WEBADMIN"
  echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN"
-  chsh -s /usr/sbin/nologin "$WEBADMIN"
-  chsh -s /usr/sbin/nologin root
+  is_docker || is_lxc || {
+    chsh -s /usr/sbin/nologin "$WEBADMIN"
+    chsh -s /usr/sbin/nologin root
+  }

  ## NCP LAUNCHER
  mkdir -p /home/www
--- a/update.sh
+++ b/update.sh
@ -25,44 +25,40 @@ fi
 CONFDIR=/usr/local/etc/ncp-config.d
 UPDATESDIR=updates

-# don't make sense in a docker container
-EXCL_DOCKER="
-nc-autoupdate-ncp
-nc-update
+# don't make sense in containers
+EXCL_CONTAINER="
 nc-automount
 nc-format-USB
-nc-datadir
-nc-database
 nc-ramlogs
 nc-swapfile
 nc-static-IP
 nc-wifi
-UFW
 nc-snapshot
 nc-snapshot-auto
 nc-snapshot-sync
 nc-restore-snapshot
-nc-audit
 nc-hdd-monitor
 nc-hdd-test
 nc-zram
-SSH
-fail2ban
 NFS
 "

-if is_docker &>/dev/null; then
-# in docker, just remove the volume for this
-EXCL_DOCKER+="
+# don't make sense in a docker container
+EXCL_DOCKER="
+$EXCL_CONTAINER
+nc-autoupdate-ncp
+nc-update
+nc-datadir
+nc-database
+UFW
+nc-audit
+SSH
+fail2ban
 nc-nextcloud
 nc-init
-"
-
-# better use a designated container
-EXCL_DOCKER+="
 samba
 "
-fi
+

 # check running apt or apt-get
 pgrep -x "apt|apt-get" &>/dev/null && { echo "apt is currently running. Try again later";  exit 1; }
@ -74,11 +70,14 @@ source /usr/local/etc/library.sh
 mkdir -p "$CONFDIR"

 # prevent installing some ncp-apps in the containerized versions
-if is_docker || is_lxc; then
-  for opt in $EXCL_DOCKER; do
-    touch $CONFDIR/$opt.cfg
-  done
-fi
+
+EXCL_APPS=""
+is_docker && EXCL_APPS="$EXCL_DOCKER"
+is_lxc && EXCL_APPS="$EXCL_CONTAINER"
+
+for opt in $EXCL_APPS; do
+  touch $CONFDIR/$opt.cfg
+done

 # copy all files in bin and etc
 cp -r bin/* /usr/local/bin/
@ -166,7 +165,7 @@ chown -R www-data:     /var/www/nextcloud/apps/nextcloudpi

 # remove unwanted ncp-apps for containerized versions
 if is_docker || is_lxc; then
-  for opt in $EXCL_DOCKER; do
+  for opt in $EXCL_APPS; do
    rm $CONFDIR/$opt.cfg
    find /usr/local/bin/ncp -name "$opt.sh" -exec rm '{}' \;
  done