External-Mirrors/nextcloudpi
2
0
Fork 0
mirror of https://github.com/nextcloud/nextcloudpi.git synced 2026-01-10 15:12:01 -03:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1905 from nextcloud/devel

Browse Source 
Support debian 12 and Nextcloud 25.0.8
This commit is contained in:
Tobias Knöppler 2024-05-01 15:48:44 +02:00 committed by GitHub
commit c4288d8e04
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
36 changed files with 1685 additions and 540 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
.github/actions/create-test-instance-bullseye/action.yml vendored Normal file
View File

@ -0,0 +1,35 @@
name: Create Test VM
description: Create NCP instance for testing in the Hetzner cloud
inputs:
version:
description: version (git rev / tag / branch) to install
required: true
uid:
description: A unique ID for labeling/naming generated resources
required: true
hcloud_token:
description: A auth token for Hetzner cloud
required: true
server_type:
description: Server type to use for hetzner servers
required: true
default: "cx11"
outputs:
server_address:
description: Adress of the test instance
snapshot_id:
description: ID of the generated postinstall snapshot
test_server_id:
description: ID of the created test server
runs:
using: docker
image: docker://thecalcaholic/ncp-test-automation:bullseye
env:
HCLOUD_TOKEN: ${{ inputs.hcloud_token }}
UID: ${{ inputs.uid }}
SERVER_TYPE: ${{ inputs.server_type }}
args:
- /ncp-test-automation/bin/actions/create-test-instance.sh
- ${{ inputs.version }}

2
.github/actions/create-test-instance/action.yml vendored
View File

@ -24,7 +24,7 @@ outputs:
description: ID of the created test server
runs:
using: docker
image: docker://thecalcaholic/ncp-test-automation
image: docker://thecalcaholic/ncp-test-automation:bookworm
env:
HCLOUD_TOKEN: ${{ inputs.hcloud_token }}

310
.github/workflows/build-lxd.yml vendored
View File

@ -34,18 +34,22 @@ jobs:
runs-on: ubuntu-latest
outputs:
runner_label: ${{ steps.script.outputs.runner_label }}
lxc_cmd: ${{ steps.script.outputs.lxc_cmd }}
steps:
- name: script
id: script
run: |
if [[ "${LXD_ARCH}" == "arm64" ]]
then
LXC_CMD="incus"
RUNNER_LABEL="ubuntu-20.04-arm64"
else
LXC_CMD="lxc"
RUNNER_LABEL="ubuntu-20.04"
fi
echo "runner_label=$RUNNER_LABEL" | tee -a $GITHUB_OUTPUT
echo "runner_label=$RUNNER_LABEL" | tee -a $GITHUB_OUTPUT
echo "lxc_cmd=$LXC_CMD" | tee -a $GITHUB_OUTPUT
build-current:
needs:
- determine-runner
@ -57,14 +61,16 @@ jobs:
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image"
LXD_ARCH: "${{ inputs.arch || 'x86' }}"
LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Cleanup lxd
run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0
run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0
- uses: whywaita/setup-lxd@v1
if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
continue-on-error: true
with:
lxd_version: latest/stable
@ -73,6 +79,8 @@ jobs:
sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT
sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
- name: Build LXD image
env:
USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
run: |
./build/build-LXD.sh
- name: Pack LXD image
@ -80,7 +88,7 @@ jobs:
run: |
. ./build/buildlib.sh
ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
lxc image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
"$LXC" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
- name: upload LXD image to artifact store
uses: actions/upload-artifact@v3
@ -90,6 +98,8 @@ jobs:
if-no-files-found: error
build-previous:
needs:
- determine-runner
runs-on: [ ubuntu-20.04 ]
if: ${{ inputs.arch == 'x86' || inputs.arch == '' }}
outputs:
@ -100,6 +110,7 @@ jobs:
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image-previous"
LXD_ARCH: "${{ inputs.arch || 'x86' }}"
LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
steps:
- name: Checkout code
uses: actions/checkout@v3
@ -134,6 +145,7 @@ jobs:
echo "VERSION=$version" >> "$GITHUB_ENV"
echo "previous_version=${version}" >> $GITHUB_OUTPUT
- uses: whywaita/setup-lxd@v1
if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
continue-on-error: true
with:
lxd_version: latest/stable
@ -143,6 +155,8 @@ jobs:
sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
- name: Build LXD image
env:
USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}"
run: |
./build/build-LXD.sh
- name: Pack LXD image
@ -150,7 +164,7 @@ jobs:
run: |
. ./build/buildlib.sh
ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}"
lxc image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
"${LXC}" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}"
echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT
- name: upload LXD image to artifact store
uses: actions/upload-artifact@v3
@ -161,6 +175,7 @@ jobs:
test-update:
needs:
- determine-runner
- build-previous
runs-on: [ubuntu-20.04]
outputs:
@ -169,8 +184,10 @@ jobs:
env:
VERSION: "${{ inputs.git_ref || github.ref }}"
ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}"
LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
steps:
- uses: whywaita/setup-lxd@v1
if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
continue-on-error: true
with:
lxd_version: latest/stable
@ -203,43 +220,185 @@ jobs:
- name: Launch ncp container
run: |
set -x
lxc delete -q -f ncp || true
lxc image import -q "./${{ needs.build-previous.outputs.artifact_file }}" --alias "ncp/update"
systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/update" ncp
lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
lxc exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
"$LXC" delete -q -f ncp || true
"$LXC" image import -q "./${{ needs.build-previous.outputs.artifact_file }}" --alias "ncp/update"
systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
"$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
sleep 30
ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Activate and Test LXD Image
working-directory: ./tests
run: |
lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
echo "Activation test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
exit 1
}
python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
echo "Nextcloud test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python system_tests.py --non-interactive --skip-update-test || {
echo "System test failed!"
exit 1
}
- name: Update ncp
run: |
set -ex
BRANCH="${VERSION/refs\/heads\//}"
BRANCH="${BRANCH/refs\/tags\//}"
if [[ "$BRANCH" =~ "refs/pull/"* ]]
then
UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION")
else
UPDATE_ARGS=("$BRANCH")
fi
echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
echo "Running update ${{ needs.build-previous.outputs.previous_version }} -> ${VERSION}"
current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
"$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
"$LXC" exec ncp -- /usr/local/bin/ncc status
if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
then
echo "Nextcloud is up to date - skipping NC update test."
else
"$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
fi
"$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
"$LXC" stop ncp
- name: Relaunch container
run: |
set -x
systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
sleep 30
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Test LXD Image
working-directory: ./tests
run: |
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
echo "Nextcloud test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python system_tests.py --non-interactive || {
echo "System test failed!"
exit 1
}
python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
test-dist-upgrade:
needs:
- determine-runner
runs-on: [ubuntu-20.04]
env:
VERSION: "${{ inputs.git_ref || github.ref }}"
LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
PREVIOUS_IMAGE_URL: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz"
steps:
- uses: whywaita/setup-lxd@v1
if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
continue-on-error: true
with:
lxd_version: latest/stable
- name: Fix LXD
run: |
sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT
sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Setup Firefox
continue-on-error: true
id: setup-firefox-browser-action
uses: browser-actions/setup-firefox@latest
- name: Setup Firefox from packages
if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }}
run: |
sudo apt-get install -y --no-install-recommends firefox
- name: Setup GeckoDriver
uses: ChlodAlejandro/setup-geckodriver@latest
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Selenium
run: pip install selenium
- name: download LXD image
run: |
wget -qO ./ncp.tar.gz "${PREVIOUS_IMAGE_URL?}"
- name: Launch ncp container
run: |
set -x
"$LXC" delete -q -f ncp || true
"$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update"
systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
"$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
sleep 30
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Activate and Test LXD Image
working-directory: ./tests
run: |
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
echo "Activation test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
exit 1
}
python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
echo "Nextcloud test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)"
lxc exec ncp -- cat "$datadir/nextcloud.log" || true
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python system_tests.py --non-interactive --skip-update-test || {
echo "System test failed!"
exit 1
}
@ -250,60 +409,103 @@ jobs:
BRANCH="${BRANCH/refs\/tags\//}"
if [[ "$BRANCH" =~ "refs/pull/"* ]]
then
UPDATE_ARGS=("${{ github.base_ref }}" "$VERSION")
UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION")
else
UPDATE_ARGS=("$BRANCH")
fi
echo "VERSION=${VERSION}" >> "$GITHUB_ENV"
echo "Running update ${{ needs.build-previous.outputs.previous_version }} -> ${VERSION}"
echo "Running update to ${VERSION}"
current_nc_version="$(lxc exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')"
latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
lxc exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
lxc exec ncp -- /usr/local/bin/ncc status
"$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
"$LXC" exec ncp -- /usr/local/bin/ncc status
if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
then
echo "Nextcloud is up to date - skipping NC update test."
else
lxc exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
"$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
fi
lxc exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
"$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
lxc stop ncp
"$LXC" stop ncp
- name: Relaunch container
run: |
set -x
systemd-run --user --scope -p "Delegate=yes" lxc start ncp
lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
sleep 30
ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Test LXD Image
working-directory: ./tests
run: |
lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python system_tests.py --non-interactive || {
python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || {
echo "Nextcloud test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python system_tests.py --non-interactive --skip-update-test || {
echo "System test failed!"
exit 1
}
- name: NCP distupgrade
id: distupgrade
run: |
set -x
"$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || {
echo "can't upgrade from Debian $("$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)"
exit 1
}
"$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade"
"$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log
"$LXC" stop ncp
- name: Relaunch container
run: |
set -x
systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
sleep 30
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Test LXD Image
working-directory: ./tests
run: |
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
echo "Nextcloud test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)"
lxc exec ncp -- cat "$datadir/nextcloud.log" || true
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
lxc stop ncp
python system_tests.py --non-interactive || {
echo "System test failed!"
exit 1
}
"$LXC" stop ncp
test-fresh-install:
needs:
@ -314,14 +516,16 @@ jobs:
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }}
ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }}
LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}"
steps:
- name: Checkout code
uses: actions/checkout@v3
with:
ref: "${{ env.VERSION }}"
- name: Cleanup lxd
run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0
run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0
- uses: whywaita/setup-lxd@v1
if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }}
continue-on-error: true
with:
lxd_version: latest/stable
@ -347,42 +551,38 @@ jobs:
- name: Remove old lxd images
if: ${{ inputs.arch == 'arm64' }}
run: |
for img in $(lxc image list -c f -f csv)
for img in $("$LXC" image list -c f -f csv)
do
lxc image delete "$img"
"$LXC" image delete "$img"
done
- name: Launch ncp container
run: |
set -x
. ./build/buildlib.sh
lxc delete -q -f ncp || true
lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true
systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp || \
sudo systemd-run --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp
lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
"$LXC" delete -q -f ncp || true
"$LXC" image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true
systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp || \
sudo systemd-run --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp
"$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
sleep 30
ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)"
ip="${ip/% *}"
echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts
- name: Test LXD Image
working-directory: ./tests
run: |
lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
"$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' &
python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
echo "Activation test failed!"
echo "Geckodriver logs:"
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)"
lxc exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python system_tests.py --non-interactive || {
echo "System test failed!"
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || {
@ -391,11 +591,15 @@ jobs:
tail -n 20 geckodriver.log >&2 || true
echo "================"
echo "ncp.log: "
lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true
"$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true
echo "================"
echo "nextcloud log: "
datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)"
lxc exec ncp -- cat "$datadir/nextcloud.log" || true
datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)"
"$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true
exit 1
}
python system_tests.py --non-interactive || {
echo "System test failed!"
exit 1
}
lxc stop ncp

549
.github/workflows/build-sd-images.yml vendored
View File

@ -186,6 +186,14 @@ jobs:
sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static
sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static
echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf
echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf
sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d
echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
sudo mkdir -p raspbian_root/etc/systemd/system/php8.1-fpm.service.d
echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf
echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf
- name: Test image
id: test
run: |
@ -308,8 +316,9 @@ jobs:
echo -e "${LOG_DIAG} ncp.log: "
"${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
echo "================"
echo "Nextcloud log: "
"${CONTAINER_CMD[@]}" -q ncp cat /opt/ncdata/data/nextcloud.log
echo "${LOG_DIAG} Nextcloud log: "
"${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
"${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
sleep 12
continue
}
@ -321,5 +330,541 @@ jobs:
[[ "$success" == "true" ]] || {
echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
echo "Apache2 logs (error):"
sudo cat ./raspbian_root/var/log/apache2/error.log || true
echo "===================="
echo "Apache2 logs (nc-error):"
sudo cat ./raspbian_root/var/log/apache2/nc-error.log || true
echo "===================="
echo "PHP FPM logs:"
sudo cat ./raspbian_root/var/log/php*-fpm.log
echo "===================="
echo "Nextcloud logs:"
sudo cat ./raspbian_root/opt/ncdata/data/nextcloud.log
echo "===================="
echo "MySQL logs:"
sudo journalctl --file ./raspbian_root/var/log/journal/"$(sudo cat ./raspbian_root/etc/machine-id)"/system.journal --no-pager -eu mariadb || true
echo "===================="
echo "Redis logs:"
sudo journalctl --file ./raspbian_root/var/log/journal/"$(sudo cat ./raspbian_root/etc/machine-id)"/system.journal --no-pager -eu redis-server || true
echo "===================="
exit 1
}
# test-distupgrade:
# runs-on: ubuntu-20.04
# env:
# VERSION: "${{ inputs.git_ref || github.ref }}"
# LOG_GUEST: "\\033[1;34mGUEST::\\033[0m"
# LOG_NCP: "\\033[1;36m~NCP::\\033[0m"
# LOG_CICD: "\\033[1;35mCICD::\\033[0m"
# LOG_TEST: "\\033[1;33mTEST::\\033[0m"
# LOG_DIAG: "\\033[1;31mDIAG::\\033[0m"
# PREVIOUS_VERSION: "v1.53.2"
# defaults:
# run:
# shell: bash
# steps:
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
# - name: Checkout code
# uses: actions/checkout@v3
# with:
# ref: "${{ env.VERSION }}"
# - name: Download previous image
# id: download-previous-image
# run: |
# set -x
# mkdir -p output
# cd output
# wgetrc=0
# wgeterr="$(wget -O ./ncp.zip "https://github.com/nextcloud/nextcloudpi/releases/download/${PREVIOUS_VERSION?}/NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.zip" 2>&1)" || wgetrc=$?
# if [[ $wgetrc -ne 0 ]]
# then
# if echo "$wgeterr" | grep '404 Not Found'
# then
# echo "Board not found in previous release - skipping."
# echo "skipped=true" >> "$GITHUB_OUTPUT"
# exit 0
# else
# echo "$wgeterr"
# exit $wgetrc
# fi
# fi
# echo "skipped=false" >> "$GITHUB_OUTPUT"
# unzip ncp.zip
# rm ncp.zip
# mv NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.img ./ncp.img
# echo "ARTIFACT_FILE=ncp.img" >> "$GITHUB_ENV"
# - name: Prepare test
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# run: |
# set -x
# mv output/${ARTIFACT_FILE?} ncp.img
# sudo apt-get install -y systemd-container
# sudo pip install selenium
# sudo rm -rf raspbian_root
# . ./build/buildlib.sh
# mount_raspbian "ncp.img"
# sudo cat raspbian_root/etc/machine-id
# sudo systemd-id128 new | sudo tee ./raspbian_root/etc/machine-id
# sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O raspbian_root/usr/bin/qemu-aarch64-static
# sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static
# sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static
# echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf
# echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf
# sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d
# echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
# echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf
# - name: Test and activate image
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# id: test
# run: |
#
# log_err() {
# rc="${1?}"
# msg="${2?}"
# echo -e "${LOG_DIAG} $msg" >&2
# return $rc
# }
#
# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
# sleep 60
#
# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
#
# success=false
# for attempt in {1..30}
# do
# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
# success=true
# break
# done
# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Could not reach container. Aborting..."
# exit 1
# }
#
# attempt=0
# success=false
# for attempt in {1..150}
# do
# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
# || log_err $? "Error retrieving redis credentials" || true
# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
# then
# echo -e "${LOG_CICD} Startup successful"
# success=true
# break
# fi
# attempt=$((attempt + 1))
# sleep 5
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Timeout reached."
# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# exit 1
# }
#
# set -x
# set +e
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) =="
# python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} Activation test failed!"
# echo -e "${LOG_DIAG} Geckodriver logs:"
# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo "================"
# echo -e "${LOG_DIAG} mysql: "
# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# sleep 12
# continue
# }
# success=true
# break
# done
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Activation test failed in all attempts!"
# exit 1
# }
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
# sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} System test failed!"
# sleep 12
# continue
# }
# success=true
# break
# done
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} System test failed in all attempts!"
# exit 1
# }
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
# python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed!"
# echo -e "{$LOG_DIAG} Geckodriver logs:"
# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo -e "${LOG_CICD} ================"
# echo -e "${LOG_DIAG} ncp.log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo "================"
# echo "${LOG_DIAG} Nextcloud log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# sleep 12
# continue
# }
# success=true
# break
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
# exit 1
# }
# - name: Update NCP
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# run: |
# set -ex
#
# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
#
# BRANCH="${VERSION/refs\/heads\//}"
# BRANCH="${BRANCH/refs\/tags\//}"
# if [[ "$BRANCH" =~ "refs/pull/"* ]]
# then
# UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION")
# else
# UPDATE_ARGS=("$BRANCH")
# fi
# current_nc_version="$("${CONTAINER_CMD[@]}" ncc status | grep "version:" | awk '{ print $3 }')"
# latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')"
#
# echo "Updating from $PREVIOUS_VERSION to $VERSION"
#
# "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}"
# "${CONTAINER_CMD[@]}" /usr/local/bin/ncc status
# # "${CONTAINER_CMD[@]}" bash -c 'curl https://download.nextcloud.com/server/releases/nextcloud-28.0.4.tar.bz2 > /var/www/nextcloud-28.0.4.tar.bz2'
# k0nKat1Nation
#
# # if [[ "$current_nc_version" =~ "$latest_nc_version".* ]]
# # then
# # echo "Nextcloud is up to date - skipping NC update test."
# # else
# # "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update-nc ${latest_nc_version?}"
# # fi
#
# sudo machinectl terminate ncp
# sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log
# - name: Test image after update
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# run: |
#
# log_err() {
# rc="${1?}"
# msg="${2?}"
# echo -e "${LOG_DIAG} $msg" >&2
# return $rc
# }
#
# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
# sleep 60
#
# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
#
# success=false
# for attempt in {1..30}
# do
# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
# success=true
# break
# done
# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Could not reach container. Aborting..."
# exit 1
# }
#
# attempt=0
# success=false
# for attempt in {1..150}
# do
# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
# || log_err $? "Error retrieving redis credentials" || true
# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
# then
# echo -e "${LOG_CICD} Startup successful"
# success=true
# break
# fi
# attempt=$((attempt + 1))
# sleep 5
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Timeout reached."
# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# exit 1
# }
#
# set -x
# set +e
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
# sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} System test failed!"
# sleep 12
# continue
# }
# success=true
# break
# done
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} System test failed in all attempts!"
# exit 1
# }
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
# python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed!"
# echo -e "{$LOG_DIAG} Geckodriver logs:"
# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo -e "${LOG_CICD} ================"
# echo -e "${LOG_DIAG} ncp.log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo "================"
# echo "${LOG_DIAG} Nextcloud log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# sleep 12
# continue
# }
# success=true
# break
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
# exit 1
# }
#
# - name: Run dist-upgrade
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# run: |
# set -ex
#
# source ./library.sh
#
# echo "Updating from $PREVIOUS_VERSION to $VERSION
#
# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
#
# sudo grep 'VERSION="11"' ./raspbian_root/etc/os-release || {
# echo "Can't dist-upgrade from debian version $(sudo grep 'Version=' ./raspbian_root/etc/os-release)"
# exit 1
# }
# "${CONTAINER_CMD[@]}" DBG=x ncp-dist-upgrade "$VERSION"
#
# sudo machinectl terminate ncp
# sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log
#
# - name: Test image after dist-upgrade
# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }}
# run: |
#
# log_err() {
# rc="${1?}"
# msg="${2?}"
# echo -e "${LOG_DIAG} $msg" >&2
# return $rc
# }
#
# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" &
# sleep 60
#
# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait)
#
# success=false
# for attempt in {1..30}
# do
# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) =="
# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')"
# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; }
# success=true
# break
# done
# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }"
# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" &
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Could not reach container. Aborting..."
# exit 1
# }
#
# attempt=0
# success=false
# for attempt in {1..150}
# do
# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..."
# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true
# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true
# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \
# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \
# || log_err $? "Error retrieving redis credentials" || true
# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \
# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \
# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; }
# then
# echo -e "${LOG_CICD} Startup successful"
# success=true
# break
# fi
# attempt=$((attempt + 1))
# sleep 5
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Timeout reached."
# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }"
# exit 1
# }
#
# set -x
# set +e
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) =="
# python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} Activation test failed!"
# echo -e "${LOG_DIAG} Geckodriver logs:"
# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo "================"
# echo -e "${LOG_DIAG} mysql: "
# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# sleep 12
# continue
# }
# success=true
# break
# done
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Activation test failed in all attempts!"
# exit 1
# }
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) =="
# sudo python tests/system_tests.py --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} System test failed!"
# sleep 12
# continue
# }
# success=true
# break
# done
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} System test failed in all attempts!"
# exit 1
# }
#
# success=false
# for attempt in {1..5}
# do
# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) =="
# python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }"
# [[ ${PIPESTATUS[0]} -eq 0 ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed!"
# echo -e "{$LOG_DIAG} Geckodriver logs:"
# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo -e "${LOG_CICD} ================"
# echo -e "${LOG_DIAG} ncp.log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# echo "================"
# echo "${LOG_DIAG} Nextcloud log: "
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true
# sleep 12
# continue
# }
# success=true
# break
# done
#
# [[ "$success" == "true" ]] || {
# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!"
# exit 1
# }

117
.github/workflows/release.yml vendored
View File

@ -16,11 +16,6 @@ on:
description: 'Build/test LXD image?'
required: false
default: true
docker:
type: boolean
description: 'Build/test docker images?'
required: false
default: true
installer:
type: boolean
description: 'Run curl-installer/upgrade tests?'
@ -28,16 +23,14 @@ on:
default: true
release:
type: boolean
description: 'Release images to GH and docker registry'
description: 'Release images to GH'
required: false
default: false
push:
tags:
- "v*"
- "docker-v*"
branches:
- master
- docker-stable
permissions:
contents: write
@ -58,30 +51,6 @@ jobs:
arch: 'arm64'
secrets: inherit
docker-x86:
if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/build-docker.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
arch: "x86"
secrets: inherit
docker-armhf:
if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/build-docker.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
arch: "armhf"
secrets: inherit
docker-arm64:
if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/build-docker.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
arch: "arm64"
secrets: inherit
test-curl-installer:
if: ${{ inputs.installer || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/vm-tests.yml
@ -89,13 +58,22 @@ jobs:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
secrets: inherit
raspberrypi:
raspberrypi-4:
if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/build-sd-images.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
board_id: raspberrypi
board_name: RaspberryPi
board_id: rpi4b
board_name: RaspberryPi4
secrets: inherit
raspberrypi-5:
if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }}
uses: ./.github/workflows/build-sd-images.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
board_id: rpi5b
board_name: RaspberryPi5
secrets: inherit
# TODO: Fix 32bit armbian images
@ -207,19 +185,31 @@ jobs:
artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}"
dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }}
raspberrypi-release:
raspberrypi-4-release:
needs:
- raspberrypi
- raspberrypi-4
- github-release
if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
uses: ./.github/workflows/publish-image.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
artifact_id: "${{ needs.raspberrypi.outputs.artifact_name }}"
artifact_file: "${{ needs.raspberrypi.outputs.artifact_file }}"
artifact_id: "${{ needs.raspberrypi-4.outputs.artifact_name }}"
artifact_file: "${{ needs.raspberrypi-4.outputs.artifact_file }}"
dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }}
secrets: inherit
raspberrypi-5-release:
needs:
- raspberrypi-5
- github-release
if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }}
uses: ./.github/workflows/publish-image.yml
with:
git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
artifact_id: "${{ needs.raspberrypi-5.outputs.artifact_name }}"
artifact_file: "${{ needs.raspberrypi-5.outputs.artifact_file }}"
dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }}
secrets: inherit
odroidxu4-release:
needs:
- odroidxu4
@ -305,55 +295,6 @@ jobs:
dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }}
secrets: inherit
docker-release:
needs:
- docker-x86
- docker-armhf
- docker-arm64
if: ${{ (inputs.release || github.event_name != 'workflow_dispatch') && github.ref_type == 'tag' && (github.ref_protected || startsWith(github.ref, 'refs/tags/docker-v')) }}
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Checkout code
uses: actions/checkout@v3
- name: Create manifest and push as tag to docker hub
run: |
. ./build/buildlib.sh
version="${version#docker-}"
for arch in x86 armhf arm64
do
docker pull "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}"
docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}"
docker tag "ownyourbits/nextcloudpi-${arch}:${version?}" "ownyourbits/nextcloudpi-${arch}:latest"
docker push "ownyourbits/nextcloudpi-${arch}:${version?}"
[[ "$version" =~ ^(docker-)?v[0-9]+'.'[0-9]+'.'[0-9]+$ ]] && docker push "ownyourbits/nextcloudpi-${arch}:latest"
done
docker manifest create ownyourbits/nextcloudpi:${version?} \
ownyourbits/nextcloudpi-armhf:${version?} \
ownyourbits/nextcloudpi-x86:${version?} \
ownyourbits/nextcloudpi-arm64:${version?}
docker manifest push ownyourbits/nextcloudpi:${version?}
- name: Create manifest and push as latest to docker hub
run: |
[[ "${{ github.ref }}" =~ ^'refs/tags/'(docker-)?'v'[0-9]+'.'[0-9]+'.'[0-9]+$ ]] || {
echo "not tagging latest - is a pre-/beta-release"
exit 0
}
docker manifest create ownyourbits/nextcloudpi:latest \
ownyourbits/nextcloudpi-armhf:latest \
ownyourbits/nextcloudpi-x86:latest \
ownyourbits/nextcloudpi-arm64:latest
docker manifest push ownyourbits/nextcloudpi:latest
github-release:
if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(github.ref_name, 'docker-') }}
needs:

509
.github/workflows/vm-tests.yml vendored
View File

@ -18,53 +18,124 @@ on:
- master
jobs:
setup-installation-test-instance:
installation-test:
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:bookworm
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-install"
outputs:
server_address: ${{ steps.create-test-instance.outputs.server_address }}
snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
version: ${{ env.VERSION }}
test_result: ${{ steps.final_test.outputs.test_result }}
ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
env:
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
HOME: /root
SSH_ARTIFACT_NAME: "${{ github.run_id }}-install-ssh"
UID: "${{ github.run_id }}-install"
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
- run: |
with:
path: /__w/nextcloudpi/nextcloudpi
- name: Generate ssh keypair
working-directory: /__w/nextcloudpi/nextcloudpi
run: |
set -e
mkdir -p ./.ssh
mkdir -p .ssh
ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
. /ncp-test-automation/bin/entrypoint.sh
- name: upload ssh private key to artifact store
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-install-ssh-privkey
path: .ssh
name: "${{ env.SSH_ARTIFACT_NAME }}"
path: /__w/nextcloudpi/nextcloudpi/.ssh
if-no-files-found: error
- id: create-test-instance
uses: ./.github/actions/create-test-instance
with:
version: ${{ env.VERSION }}
uid: "${{ github.run_id }}-install"
uid: "${{ env.UID }}"
hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
server_type: "cx11"
- name: set instance variables
run: |
echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
- name: Test postinstall VM
id: final_test
working-directory: /ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
setup-update-test-instance:
source ./library.sh
trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
setup-ssh-port-forwarding "$SERVER_ADDRESS"
echo "Run integration tests"
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
set -x
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
echo "test_result=success" >> "$GITHUB_OUTPUT";
update-test:
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:bullseye
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-update"
defaults:
run:
shell: bash
outputs:
server_address: ${{ steps.create-test-instance.outputs.server_address }}
snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
previous_version: ${{ steps.find-version.outputs.previous_version }}
version: ${{ env.VERSION }}
test_result: ${{ steps.final_test.outputs.test_result }}
ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
env:
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
HOME: /root
SSH_ARTIFACT_NAME: "${{ github.run_id }}-update-ssh"
UID: "${{ github.run_id }}-update"
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
path: /__w/nextcloudpi/nextcloudpi
- name: find reference version
working-directory: /__w/nextcloudpi/nextcloudpi
shell: bash
id: find-version
run: |
chown -R "$(id -u):$(id -g)" .
set -e
if [[ -n "${{ github.base_ref }}" ]]
then
@ -82,121 +153,46 @@ jobs:
version="${version%-*-*}"
fi
echo "Previous version is '$version'"
echo "previous_version=${version}" >> $GITHUB_OUTPUT
#TODO: Revert to dynamically found version
#echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV"
echo "PREVIOUS_VERSION=v1.53.3" >> "$GITHUB_ENV"
- name: Generate ssh key
run: |
set -x
mkdir -p ./.ssh
ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
. /ncp-test-automation/bin/entrypoint.sh
- name: upload ssh private key to artifact store
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-update-ssh-privkey
path: .ssh
name: "${{ env.SSH_ARTIFACT_NAME }}"
path: /__w/nextcloudpi/nextcloudpi/.ssh
if-no-files-found: error
- id: create-test-instance
uses: ./.github/actions/create-test-instance
uses: ./.github/actions/create-test-instance-bullseye
with:
version: "${{ steps.find-version.outputs.previous_version }}"
uid: "${{ github.run_id }}-update"
version: "${{ env.PREVIOUS_VERSION }}"
uid: "${{ env.UID }}"
hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
server_type: "cx11"
run-installation-test:
needs:
- setup-installation-test-instance
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-install"
env:
VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}"
SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}"
HOME: /root
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
repository: 'theCalcaholic/ncp-test-automation'
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
with:
name: ${{ github.run_id }}-install-ssh-privkey
path: .ssh
- name: Test postinstall VM
- name: Set instance variables
run: |
set -e
echo "Setup ssh"
chmod 0600 ./.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
cd bin
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
setup-ssh-port-forwarding "$SERVER_ADDRESS"
echo "Run integration tests"
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
run-update-test:
needs:
- setup-update-test-instance
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-update"
env:
PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }}
VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}"
SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}"
HOME: /root
defaults:
run:
shell: bash
steps:
echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
- uses: actions/checkout@v3
with:
repository: 'theCalcaholic/ncp-test-automation'
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
with:
name: ${{ github.run_id }}-update-ssh-privkey
path: .ssh
ref: "bullseye"
path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
- name: Activate and Test postinstall VM
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
chmod 0600 ./.ssh/automation_ssh_key
chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
cd bin
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
@ -205,7 +201,7 @@ jobs:
echo "Run integration tests"
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
@ -219,36 +215,40 @@ jobs:
exit 1
}
- name: perform update
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
chmod 0600 ./.ssh/automation_ssh_key
chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
. ./bin/library.sh
source ./library.sh
echo "Updating from $PREVIOUS_VERSION to $VERSION"
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
- name: Run integration tests
- name: Run integration tests after update
id: final_test
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
cd bin
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
echo "Run integration tests"
setup-ssh-port-forwarding "$SERVER_ADDRESS"
test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
NC_TEST_ARGS=()
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
set -x
test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
@ -262,11 +262,201 @@ jobs:
exit 1
}
echo "test_result=success" >> "$GITHUB_OUTPUT";
dist-upgrade-test:
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:bullseye
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-distupgrade"
defaults:
run:
shell: bash
outputs:
test_result: ${{ steps.final_test.outputs.test_result }}
ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }}
server_address: ${{ steps.create-test-instance.outputs.server_address }}
snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
previous_version: ${{ env.PREVIOUS_VERSION }}
version: ${{ env.VERSION }}
env:
PREVIOUS_VERSION: "v1.53.3"
VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
HOME: /root
SSH_ARTIFACT_NAME: "${{ github.run_id }}-distupgrade-ssh"
UID: "${{ github.run_id }}-distupgrade"
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
path: /__w/nextcloudpi/nextcloudpi
- name: Generate ssh key
run: |
set -x
chown -R "$(id -u):$(id -g)" /__w/nextcloudpi/nextcloudpi
mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh
ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key"
. /ncp-test-automation/bin/entrypoint.sh
- name: upload ssh private key to artifact store
uses: actions/upload-artifact@v3
with:
name: "${{ env.SSH_ARTIFACT_NAME }}"
path: /__w/nextcloudpi/nextcloudpi/.ssh
if-no-files-found: error
- id: create-test-instance
uses: ./.github/actions/create-test-instance-bullseye
with:
version: "${{ env.PREVIOUS_VERSION }}"
uid: "${{ env.UID }}"
hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
server_type: "cx11"
- name: Set instance variables
run: |
echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV"
echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV"
- uses: actions/checkout@v3
with:
repository: 'theCalcaholic/ncp-test-automation'
ref: "bullseye"
path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation
- name: Activate and Test postinstall VM
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
setup-ssh-port-forwarding "$SERVER_ADDRESS"
echo "Run integration tests"
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
- name: perform update
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
source ./library.sh
echo "Updating from $PREVIOUS_VERSION to $VERSION"
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
- name: Run integration tests after update
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
echo "Run integration tests"
setup-ssh-port-forwarding "$SERVER_ADDRESS"
NC_TEST_ARGS=()
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check")
set -x
test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
- name: NCP distupgrade
id: distupgrade
working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
source ./library.sh
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep 'VERSION_ID="11"' || {
echo "Can't upgrade from Debian $(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" grep 'VERSION_ID=' /etc/os-release)"
echo "skipped=yes" | tee -a $GITHUB_OUTPUT
exit 1
}
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade
echo "skipped=no" | tee -a $GITHUB_OUTPUT
- name: Run integration tests after dist-upgrade
id: final_test
working-directory: /ncp-test-automation/bin
run: |
set -e
echo "Setup ssh"
eval "$(ssh-agent)"
ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key
source ./library.sh
trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
echo "Run integration tests"
setup-ssh-port-forwarding "$SERVER_ADDRESS"
test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
echo "nextcloud.log:"
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /opt/ncdata/data/nextcloud.log;
exit 1
}
echo "test_result=success" >> "$GITHUB_OUTPUT"
install-postactivation-snapshot:
if: ${{ always() }}
needs:
- setup-installation-test-instance
- run-installation-test
- installation-test
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
@ -275,18 +465,18 @@ jobs:
env:
TEST_TYPE: install
SERVER_ADDRESS: ${{ needs.setup-installation-test-instance.outputs.server_address }}
TEST_RESULT: ${{ needs.setup-installation-test-instance.result }}
TEST_SERVER_ID: ${{ needs.setup-installation-test-instance.outputs.test_server_id }}
VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
SERVER_ADDRESS: ${{ needs.installation-test.outputs.server_address }}
TEST_RESULT: ${{ needs.installation-test.test_result }}
TEST_SERVER_ID: ${{ needs.installation-test.outputs.test_server_id }}
VERSION: ${{ needs.installation-test.outputs.version }}
SSH_ARTIFACT_NAME: "${{ needs.installation-test.outputs.ssh_artifact_name }}"
UID: ${{ github.run_id }}-install
steps:
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
if: ${{ contains('success|failure', env.TEST_RESULT) }}
with:
name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
name: ${{ env.SSH_ARTIFACT_NAME }}
path: /github/workspace/.ssh
- name: Shutdown server
if: ${{ contains('success|failure', env.TEST_RESULT) }}
@ -304,11 +494,10 @@ jobs:
- name: Create Snapshot
if: ${{ contains('success|failure', env.TEST_RESULT) }}
shell: bash
working-directory: /ncp-test-automation/bin
run: |
set -x
echo "${{ needs.setup-installation-test-instance.outputs.test_server_id }}"
echo "${TEST_SERVER_ID?}"
cd /ncp-test-automation/bin
. ./library.sh
@ -320,8 +509,7 @@ jobs:
update-postactivation-snapshot:
if: ${{ always() }}
needs:
- setup-update-test-instance
- run-update-test
- update-test
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
@ -330,17 +518,18 @@ jobs:
env:
TEST_TYPE: update
SERVER_ADDRESS: ${{ needs.setup-update-test-instance.outputs.server_address }}
TEST_RESULT: ${{ needs.setup-update-test-instance.result }}
TEST_SERVER_ID: ${{ needs.setup-update-test-instance.outputs.test_server_id }}
VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
SERVER_ADDRESS: ${{ needs.update-test.outputs.server_address }}
TEST_RESULT: ${{ needs.update-test.test_result }}
TEST_SERVER_ID: ${{ needs.update-test.outputs.test_server_id }}
VERSION: ${{ needs.update-test.outputs.version }}
UID: ${{ github.run_id }}-update
SSH_ARTIFACT_NAME: "${{ needs.update-test.outputs.ssh_artifact_name }}"
steps:
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
if: ${{ contains('success|failure', env.TEST_RESULT) }}
with:
name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
name: ${{ env.SSH_ARTIFACT_NAME }}
path: /github/workspace/.ssh
- name: Shutdown server
if: ${{ contains('success|failure', env.TEST_RESULT) }}
@ -358,11 +547,65 @@ jobs:
- name: Create Snapshot
if: ${{ contains('success|failure', env.TEST_RESULT) }}
shell: bash
working-directory: /ncp-test-automation/bin
run: |
set -x
echo "${{ needs.setup-update-test-instance.outputs.test_server_id }}"
echo "${{ needs.update-test.outputs.test_server_id }}"
echo "${TEST_SERVER_ID?}"
. ./library.sh
tf-init "$TF_SNAPSHOT"
tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"
dist-upgrade-postactivation-snapshot:
if: ${{ always() }}
needs:
- dist-upgrade-test
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
env:
TEST_TYPE: distupgrade
SERVER_ADDRESS: ${{ needs.dist-upgrade-test.outputs.server_address }}
TEST_RESULT: ${{ needs.dist-upgrade-test.outputs.test_result }}
TEST_SERVER_ID: ${{ needs.dist-upgrade-test.outputs.test_server_id }}
VERSION: ${{ needs.dist-upgrade-test.outputs.version }}
UID: ${{ github.run_id }}-distupgrade
SSH_ARTIFACT_NAME: "${{ needs.dist-upgrade-test.outputs.ssh_artifact_name }}"
steps:
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
if: ${{ contains('success|failure', env.TEST_RESULT) }}
with:
name: ${{ env.SSH_ARTIFACT_NAME }}
path: /github/workspace/.ssh
- name: Shutdown server
if: ${{ contains('success|failure', env.TEST_RESULT) }}
run: |
chmod 0600 /github/workspace/.ssh/automation_ssh_key
export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
bash /ncp-test-automation/bin/entrypoint.sh
eval "$(ssh-agent)"
ssh-add /github/workspace/.ssh/automation_ssh_key
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
systemctl stop mariadb
systemctl poweroff
EOF
- name: Create Snapshot
if: ${{ contains('success|failure', env.TEST_RESULT) }}
shell: bash
working-directory: /ncp-test-automation/bin
run: |
set -x
echo "${{ needs.dist-upgrade-test.outputs.test_server_id }}"
echo "${TEST_SERVER_ID?}"
cd /ncp-test-automation/bin
. ./library.sh
@ -376,6 +619,7 @@ jobs:
needs:
- install-postactivation-snapshot
- update-postactivation-snapshot
- dist-upgrade-postactivation-snapshot
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
@ -383,7 +627,7 @@ jobs:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
strategy:
matrix:
uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update"]
uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update", "${{ github.run_id }}-distupgrade"]
fail-fast: false
env:
HOME: '/root'
@ -410,6 +654,7 @@ jobs:
needs:
- install-postactivation-snapshot
- update-postactivation-snapshot
- dist-upgrade-postactivation-snapshot
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
@ -420,7 +665,7 @@ jobs:
steps:
- name: Delete old snapshots
run: |
for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -16)
for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -12)
do
echo "Deleting snapshot '$snapshot'..."
hcloud image delete "$snapshot"

2
bin/ncp-check-version
View File

@ -17,8 +17,6 @@ cd "$TEMPDIR" || exit 1
VER=$( git describe --always --tags | grep -oP "v\d+\.\d+\.\d+" )
canary="$(. /usr/local/etc/library.sh; find_app_param ncp-community.sh CANARY)"
# TODO: Remove temporary canary override after staged rollouts test
canary="no"
[[ "$canary" != "yes" ]] && [[ -f "/usr/local/etc/instance.cfg" ]] && {
cohorte_id="$(jq .cohorteId /usr/local/etc/instance.cfg)"

158
bin/ncp-dist-upgrade
View File

@ -1,137 +1,35 @@
#!/bin/bash
#!/usr/bin/env bash
set -eu -o pipefail
[[ -f /.dockerenv ]] && { echo "Not supported in Docker. Upgrade the container instead"; exit 0; }
new_cfg=/usr/local/etc/ncp-recommended.cfg
[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
APTINSTALL="apt-get install -y --no-install-recommends"
export DEBIAN_FRONTEND=noninteractive
echo "
>>> ATTENTION <<<
This is a dangerous process that is only guaranteed to work properly if you
have not made manual changes in the system. Backup the SD card first and
proceed at your own risk.
Note that this is not a requirement for NCP to continue working properly.
The current distribution will keep receiving updates for some time.
Do you want to continue? [y/N]"
read key
[[ "$key" == y ]] || exit 0
source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER
old_cfg="${NCPCFG}"
trap "echo 'Something went wrong. Fix it and try again'" EXIT
save_maintenance_mode
# Fix grub-pc issue in VM
if apt show grub-pc-bin &>/dev/null; then
$APTINSTALL grub
fi
apt-get update
apt-get upgrade -y
# remove old PHP version
set +e
apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \
php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \
php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json
apt-get purge -y php${PHPVER}-mysql
apt-get purge -y php${PHPVER}-redis
apt-get purge -y php${PHPVER}-exif
apt-get purge -y php${PHPVER}-bcmath
apt-get purge -y php${PHPVER}-gmp
apt-get purge -y php${PHPVER}-imagick
set -e
# update sources
sed -i 's/buster/bullseye/g' /etc/apt/sources.list
sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true
sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list
rm -f /etc/apt/sources.list.d/php.list
[[ "${EUID}" -eq 0 ]] || {
echo "ERROR: Must be run as root (try sudo ncp-dist-upgrade)"
exit 1
}
# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian
if [[ -f /usr/bin/raspi-config ]]; then
sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf
. /etc/os-release
if [[ "$VERSION_ID" -eq 10 ]]
then
UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-10.sh)
elif [[ "$VERSION_ID" -eq 11 ]]
then
UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-11.sh)
else
echo "No dist-upgrade available for OS version: Debian ${VERSION}."
exit 0
fi
# install latest distro
apt-get update
apt-get dist-upgrade -y
if { [ "$TERM" = "screen" ] && [ -n "$TMUX" ]; } || [[ "${DEBIAN_FRONTEND:-}" == "noninteractive" ]] || ! [[ -t 0 ]]
then
"${UPGRADE_CMD[@]}"
else
tmux list-sessions | grep ncp-distupgrade && {
echo "Existing distupgrade process detected. Connecting..."
sleep 5
tmux attach -t ncp-distupgrade
exit 0
}
tmux new-session -s ncp-distupgrade bash -c "${UPGRADE_CMD[*]}; bash"
fi
# install latest PHP version
release_new=$(jq -r '.release' < "${new_cfg}")
# the default repo in bullseye is bullseye-security - use bullseye if it is not available
grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security"
php_ver_new=$(jq -r '.php_version' < "${new_cfg}")
# PHP 8.1 is only supported via the
[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4
$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \
php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \
php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json
$APTINSTALL php${php_ver_new}-mysql
$APTINSTALL -t ${release_new} php${php_ver_new}-redis
$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-exif
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-gmp
#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript
# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later
{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true
apt-get autoremove -y
apt-get clean
cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <<EOF
; disable .user.ini files for performance and workaround NC update bugs
user_ini.filename =
; from Nextcloud .user.ini
upload_max_filesize=10G
post_max_size=10G
memory_limit=768M
mbstring.func_overload=0
always_populate_raw_post_data=-1
default_charset='UTF-8'
output_buffering=0
; slow transfers will be killed after this time
max_execution_time=3600
max_input_time=3600
EOF
# restart services
service php${php_ver_new}-fpm restart
a2enconf php${php_ver_new}-fpm
service apache2 restart
is_active_app unattended-upgrades && run_app unattended-upgrades || true
# mark as successful
mv "${new_cfg}" "${old_cfg}"
install_template "php/opcache.ini.sh" "/etc/php/${php_ver_new}/mods-available/opcache.ini" --defaults
clear_opcache
source /usr/local/etc/library.sh # refresh NCPCFG RELEASE PHPVER
run_app nc-limits
restore_maintenance_mode
rm -f /etc/update-motd.d/30ncp-dist-upgrade
echo "Upgrade to ${release_new} successful"
trap '' EXIT

137
bin/ncp-dist-upgrade.d/debian-10.sh Normal file
View File

@ -0,0 +1,137 @@
#!/bin/bash
set -eu -o pipefail
[[ -f /.dockerenv ]] && { echo "Not supported in Docker. Upgrade the container instead"; exit 0; }
new_cfg=/usr/local/etc/ncp-recommended.cfg
[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
APTINSTALL="apt-get install -y --no-install-recommends"
export DEBIAN_FRONTEND=noninteractive
echo "
>>> ATTENTION <<<
This is a dangerous process that is only guaranteed to work properly if you
have not made manual changes in the system. Backup the SD card first and
proceed at your own risk.
Note that this is not a requirement for NCP to continue working properly.
The current distribution will keep receiving updates for some time.
Do you want to continue? [y/N]"
read key
[[ "$key" == y ]] || exit 0
source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER
old_cfg="${NCPCFG}"
trap "echo 'Something went wrong. Fix it and try again'" EXIT
save_maintenance_mode
# Fix grub-pc issue in VM
if apt show grub-pc-bin &>/dev/null; then
$APTINSTALL grub
fi
apt-get update
apt-get upgrade -y
# remove old PHP version
set +e
apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \
php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \
php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json
apt-get purge -y php${PHPVER}-mysql
apt-get purge -y php${PHPVER}-redis
apt-get purge -y php${PHPVER}-exif
apt-get purge -y php${PHPVER}-bcmath
apt-get purge -y php${PHPVER}-gmp
apt-get purge -y php${PHPVER}-imagick
set -e
# update sources
sed -i 's/buster/bullseye/g' /etc/apt/sources.list
sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true
sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list
rm -f /etc/apt/sources.list.d/php.list
# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian
if [[ -f /usr/bin/raspi-config ]]; then
sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf
fi
# install latest distro
apt-get update
apt-get dist-upgrade -y
# install latest PHP version
release_new=$(jq -r '.release' < "${new_cfg}")
# the default repo in bullseye is bullseye-security - use bullseye if it is not available
grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security"
php_ver_new=$(jq -r '.php_version' < "${new_cfg}")
# PHP 8.1 is only supported via the
[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4
$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \
php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \
php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json
$APTINSTALL php${php_ver_new}-mysql
$APTINSTALL -t ${release_new} php${php_ver_new}-redis
$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-exif
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath
sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop
$APTINSTALL -t ${release_new} php${php_ver_new}-gmp
#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript
# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later
{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true
apt-get autoremove -y
apt-get clean
cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <<EOF
; disable .user.ini files for performance and workaround NC update bugs
user_ini.filename =
; from Nextcloud .user.ini
upload_max_filesize=10G
post_max_size=10G
memory_limit=768M
mbstring.func_overload=0
always_populate_raw_post_data=-1
default_charset='UTF-8'
output_buffering=0
; slow transfers will be killed after this time
max_execution_time=3600
max_input_time=3600
EOF
# restart services
service php${php_ver_new}-fpm restart
a2enconf php${php_ver_new}-fpm
service apache2 restart
is_active_app unattended-upgrades && run_app unattended-upgrades || true
# mark as successful
mv "${new_cfg}" "${old_cfg}"
install_template "php/opcache.ini.sh" "/etc/php/${php_ver_new}/mods-available/opcache.ini" --defaults
clear_opcache
source /usr/local/etc/library.sh # refresh NCPCFG RELEASE PHPVER
run_app nc-limits
restore_maintenance_mode
rm -f /etc/update-motd.d/30ncp-dist-upgrade
echo "Upgrade to ${release_new} successful"
trap '' EXIT

79
bin/ncp-dist-upgrade.d/debian-11.sh Normal file
View File

@ -0,0 +1,79 @@
#!/bin/bash
set -eu -o pipefail
new_cfg=/usr/local/etc/ncp-recommended.cfg
[[ -f "${new_cfg}" ]] || { echo "Already on the lastest recommended distribution. Abort." >&2; exit 1; }
echo "
>>> ATTENTION <<<
This is a dangerous process that is only guaranteed to work properly if you
have not made manual changes in the system. Backup the SD card first and
proceed at your own risk.
Note that this is not a requirement for NCP to continue working properly.
The current distribution will keep receiving updates for some time.
Do you want to continue? [y/N]"
if [[ "${DEBIAN_FRONTEND:-}" == "noninteractive" ]] || ! [[ -t 0 ]]
then
echo "Noninteractive environment detected. Automatically proceeding in 30 seconds..."
sleep 30
else
read -n1 -r key
[[ "${key,,}" == y ]] || exit 0
fi
export DEBIAN_FRONTEND=noninteractive
source /usr/local/etc/library.sh
is_more_recent_than "${PHPVER}.0" "8.0.0" || {
echo "You still have PHP version ${PHPVER} installed. Please update to the latest supported version of nextcloud (which will also update your PHP version) before proceeding with the distribution upgrade."
echo "Exiting."
exit 1
}
save_maintenance_mode
# Perform dist-upgrade
apt-get update && apt-get upgrade -y
for aptlist in /etc/apt/sources.list /etc/apt/sources.list.d/php.list /etc/apt/sources.list.d/armbian.list
do
[ -f "$aptlist" ] && sed -i -e "s/bullseye/bookworm/g" "$aptlist"
done
for aptlist in /etc/apt/sources.list.d/*.list
do
[[ "$aptlist" =~ "/etc/apt/sources.list.d/"(php|armbian)".list" ]] || continue
echo "Disabling repositories from \"$aptlist\""
sed -i -e "s/deb/#deb/g" "$aptlist"
done
apt-get update && apt-get upgrade -y --without-new-pkgs
if is_lxc
then
# Required to avoid breakage of /etc/resolv.conf
apt-get install -y --no-install-recommends systemd-resolved && systemctl enable --now systemd-resolved
fi
apt-get full-upgrade -y
sudo apt-get --purge autoremove -y
apt-get install -y --no-install-recommends exfatprogs
#mkdir -p /etc/systemd/system/php8.1-fpm.service.d
#echo '[Service]' > /etc/systemd/system/php8.1-fpm.service.d/ncp.conf
#echo 'ExecStartPre=mkdir -p /var/run/php' >> /etc/systemd/system/php8.1-fpm.service.d/ncp.conf
#[[ "$INIT_SYSTEM" != "systemd" ]] || { systemctl daemon-reload && systemctl restart php8.1-fpm; }
restore_maintenance_mode
cfg="$(jq "." "$NCPCFG")"
cfg="$(jq ".release = \"bookworm\"" <<<"$cfg")"
echo "$cfg" > "$NCPCFG"
rm -f /etc/update-motd.d/30ncp-dist-upgrade
echo "Update to Debian 12 (bookworm) successful."
is_active_app unattended-upgrades && {
echo "Setting up unattended upgrades..."
run_app unattended-upgrades || true
echo "done."
}

6
bin/ncp/CONFIG/nc-init.sh
View File

@ -111,8 +111,8 @@ EOF
# email
ncc config:system:set mail_smtpmode --value="sendmail"
ncc config:system:set mail_smtpauthtype --value="LOGIN"
ncc config:system:set mail_from_address --value="admin"
ncc config:system:set mail_domain --value="ownyourbits.com"
ncc config:system:set mail_from_address --value="noreply"
ncc config:system:set mail_domain --value="nextcloudpi.com"
# Fix NCP theme
[[ -e /usr/local/etc/logo ]] && {
@ -129,7 +129,7 @@ EOF
mysql nextcloud <<EOF
replace into oc_appconfig values ( 'theming', 'name' , "NextCloudPi" , 2, null);
replace into oc_appconfig values ( 'theming', 'slogan' , "keep your data close" , 2, null);
replace into oc_appconfig values ( 'theming', 'url' , "https://ownyourbits.com" , 2, null);
replace into oc_appconfig values ( 'theming', 'url' , "https://nextcloudpi.com" , 2, null);
replace into oc_appconfig values ( 'theming', 'logoMime' , "image/svg+xml" , 2, null);
replace into oc_appconfig values ( 'theming', 'backgroundMime', "image/png" , 2, null);
EOF

4
bin/ncp/CONFIG/nc-nextcloud.sh
View File

@ -29,8 +29,8 @@ install()
$APTINSTALL lbzip2 iputils-ping jq wget
# NOTE: php-smbclient in sury but not in Debian sources, we'll use the binary version
# https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/external_storage/smb.html
$APTINSTALL -t $RELEASE smbclient exfat-fuse exfat-utils # for external storage
$APTINSTALL -t $RELEASE exfat-fuse exfat-utils # for external storage
$APTINSTALL -t $RELEASE smbclient exfat-fuse exfatprogs # for external storage
$APTINSTALL -t $RELEASE exfat-fuse exfatprogs # for external storage
$APTINSTALL -t $RELEASE php${PHPVER}-exif # for gallery
$APTINSTALL -t $RELEASE php${PHPVER}-bcmath # for LDAP
$APTINSTALL -t $RELEASE php${PHPVER}-gmp # for bookmarks

1
bin/ncp/CONFIG/nc-passwd.sh
View File

@ -29,6 +29,7 @@ configure()
a2dissite ncp-activation
a2ensite ncp 001-nextcloud
apachectl -k graceful
sed -i -e 's/^NCP is not activated yet.*$//' /etc/issue
# Trusted Domain (local/public IP), also configures notify_push
bash /usr/local/bin/nextcloud-domain.sh

44
bin/ncp/NETWORKING/dnsmasq.sh
View File

@ -16,38 +16,30 @@ install()
apt-get install --no-install-recommends -y dnsmasq
rc=0
service dnsmasq status > /dev/null 2>&1 || rc=$?
! is_docker && [[ $rc -eq 3 ]] && ! [[ "$INIT_SYSTEM" =~ ^("chroot"|"unknown")$ ]] && {
[[ $rc -eq 3 ]] && ! [[ "$INIT_SYSTEM" =~ ^("chroot"|"unknown")$ ]] && command -v systemd-resolve > /dev/null || {
echo "Applying workaround for dnsmasq bug (compare issue #1446)"
service systemd-resolved stop || true
service dnsmasq start
service dnsmasq status
mkdir -p /etc/systemd/resolved.conf.d
if systemctl status systemd-resolved
then
cat <<EOF > /etc/systemd/resolved.conf.d/nostublistener.conf
[Resolve]
DNSStubListener=no
EOF
[[ "$INIT_SYSTEM" != "systemd" ]] || systemctl restart systemd-resolved
else
systemctl stop resolvconf
systemctl start dnsmasq
systemctl status dnsmasq
fi
# service systemd-resolved stop || true
systemctl start dnsmasq
systemctl status dnsmasq
}
service dnsmasq stop
[[ "$INIT_SYSTEM" == "systemd" ]] && service systemd-resolved start || true
[[ "$INIT_SYSTEM" != "systemd" ]] || systemctl start systemd-resolved || systemctl start resolvconf
update-rc.d dnsmasq disable || rm /etc/systemd/system/multi-user.target.wants/dnsmasq.service
[[ "$DOCKERBUILD" == 1 ]] && {
cat > /etc/services-available.d/100dnsmasq <<EOF
#!/bin/bash
source /usr/local/etc/library.sh
[[ "\$1" == "stop" ]] && {
echo "stopping dnsmasq..."
service dnsmasq stop
exit 0
}
persistent_cfg /etc/dnsmasq.conf
echo "Starting dnsmasq..."
service dnsmasq start
exit 0
EOF
chmod +x /etc/services-available.d/100dnsmasq
}
return 0
}

2
bin/ncp/SECURITY/fail2ban.sh
View File

@ -114,7 +114,7 @@ protocol = tcp
chain = INPUT
action_ = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
action_mwl = %(banaction)s[name=%(__name__)s, port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
sendmail-whois-lines[name=%(__name__)s, dest=$EMAIL, sender=ncp-fail2ban@ownyourbits.com]
sendmail-whois-lines[name=%(__name__)s, dest=$EMAIL, sender=ncp-fail2ban@nextcloudpi.com]
action = %($ACTION)s
#

6
build/armbian/armbian.sh
View File

@ -39,6 +39,12 @@ run_app_unsafe post-inst.sh
# disable SSH by default, it can be enabled through ncp-web
systemctl disable ssh
# disable armbian autologin
rm -f /etc/systemd/system/getty@.service.d/override.conf
rm -f /etc/systemd/system/serial-getty@.service.d/override.conf
rm -f /root/.not_logged_in_yet
sed -i 's|^root::|root:x:|' /etc/passwd
basename "$IMG" | tee /usr/local/etc/ncp-baseimage
cd -

36
build/build-LXD.sh
View File

@ -31,15 +31,20 @@ prepare_dirs # tmp cache output
## BUILD NCP
lxc delete -f ncp 2>/dev/null || true
LXC_CREATE=(lxc init -p default)
debian_version="$(. etc/library.sh > /dev/null 2>&1; echo "${RELEASE%%-security}")"
LXC_CMD=lxc
[[ "$USE_INCUS" == "yes" ]] && LXC_CMD=incus
$LXC_CMD delete -f ncp 2>/dev/null || true
LXC_CREATE=($LXC_CMD init -p default)
[[ -n "$LXD_EXTRA_PROFILE" ]] && LXC_CREATE+=(-p "$LXD_EXTRA_PROFILE")
if [[ -n "$LXD_ARCH" ]] && [[ "$LXD_ARCH" != "x86" ]]
then
echo "Building for architecture: $LXD_ARCH"
LXC_CREATE+=("images:debian/bullseye/$LXD_ARCH")
LXC_CREATE+=("images:debian/${debian_version}/$LXD_ARCH")
else
LXC_CREATE+=('images:debian/bullseye')
LXC_CREATE+=("images:debian/${debian_version}")
fi
LXC_CREATE+=(ncp)
"${LXC_CREATE[@]}"
@ -56,19 +61,20 @@ LXC_CREATE+=(ncp)
# fi
#fi
systemd-run --user --scope -p "Delegate=yes" lxc start ncp -q || \
sudo systemd-run --scope -p "Delegate=yes" lxc start ncp -q
lxc config device add ncp buildcode disk source="$(pwd)" path=/build
lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh'
lxc exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh'
lxc exec ncp -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage"
lxc stop ncp
lxc config device remove ncp buildcode
lxc publish -q ncp -f --alias ncp/"${version}"
set -x
systemd-run --user --scope -p "Delegate=yes" $LXC_CMD start ncp -q || \
sudo systemd-run --scope -p "Delegate=yes" $LXC_CMD start ncp -q
$LXC_CMD config device add ncp buildcode disk source="$(pwd)" path=/build
$LXC_CMD exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done'
$LXC_CMD exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh'
$LXC_CMD exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh'
$LXC_CMD exec ncp -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage"
$LXC_CMD stop ncp
$LXC_CMD config device remove ncp buildcode
$LXC_CMD publish -q ncp -f --alias ncp/"${version}"
## pack
[[ " $* " =~ .*" --pack ".* ]] && lxc image export -q ncp/"${version}" "$TAR"
[[ " $* " =~ .*" --pack ".* ]] && $LXC_CMD image export -q ncp/"${version}" "$TAR"
exit 0

2
build/build-SD-armbian.sh
View File

@ -31,7 +31,7 @@ prepare_dirs # tmp cache output
# get latest armbian
[[ -d armbian ]] || {
git clone --depth 1 --branch main https://github.com/armbian/build armbian
git clone --depth 1 --branch v24.02 https://github.com/armbian/build armbian
}
#( cd armbian && git pull --ff-only --tags && git checkout v23.02 )
#sed -i -e '/export rootfs_size=/s/du -sm/du --apparent-size -sm/' armbian/lib/functions/image/partitioning.sh

37
build/buildlib.sh
View File

@ -335,43 +335,6 @@ function generate_changelog()
sed 's|* \[tag: |\n[|' > changelog.md
}
function upload_ftp()
{
local IMGNAME="$1"
echo -e "\n\e[1m[ Upload FTP ]\e[0m"
echo "* $IMGNAME..."
[[ -f torrent/"$IMGNAME"/"$IMGNAME".tar.bz2 ]] || { echo "No image file found, abort"; return 1; }
[[ "$FTPPASS" == "" ]] && { echo "No FTPPASS variable found, skip upload"; return 0; }
cd torrent
ftp -np ftp.ownyourbits.com <<EOF
user root@ownyourbits.com $FTPPASS
mkdir testing
mkdir testing/$IMGNAME
cd testing/$IMGNAME
binary
rm $IMGNAME.torrent
put $IMGNAME.torrent
bye
EOF
cd -
cd torrent/$IMGNAME
ftp -np ftp.ownyourbits.com <<EOF
user root@ownyourbits.com $FTPPASS
cd testing/$IMGNAME
binary
rm $IMGNAME.tar.bz2
put $IMGNAME.tar.bz2
rm md5sum
put md5sum
bye
EOF
ret=$?
cd -
return $ret
}
upload_images()
{

15
changelog.md
View File

@ -1,5 +1,20 @@
# NextcloudPi Changelog
## [v1.54.0](https://github.com/nextcloud/nextcloudpi/tree/v1.54.0) (2024-05-01) Support Debian 12 and Nextcloud 28.0.5
### Changes
- Add support for Nextcloud 28.0.5
- Add ncp-dist-upgrade script to upgrade to Debian 12 (bookworm)
- Change mechanism to prevent root login by default
- Add support for Raspberry Pi 5
- Use Armbian instead of Raspbian as base for Raspberry Pi images
### Development related
- (CICD) Add integration tests for dist-upgrade
- Add support for building with incus
## [v1.53.3](https://github.com/nextcloud/nextcloudpi/tree/v1.53.3) (2024-04-24) Support Nextcloud 28.0.4
### Changes

5
etc/library.sh
View File

@ -156,8 +156,9 @@ function set-nc-domain()
if is_ncp_activated && is_app_enabled notify_push; then
ncc config:system:set trusted_proxies 11 --value="127.0.0.1"
ncc config:system:set trusted_proxies 12 --value="::1"
ncc config:system:set trusted_proxies 13 --value="${domain}"
ncc config:system:set trusted_proxies 14 --value="$(dig +short "${domain}")"
# ncc config:system:set trusted_proxies 13 --value="${domain}"
local domain_ip="$(dig +short "${domain}")"
[[ -z "$domain_ip" ]] || ncc config:system:set trusted_proxies 14 --value="$(dig +short "${domain}")"
sleep 5 # this seems to be required in the VM for some reason. We get `http2 error: protocol error` after ncp-upgrade-nc
for try in {1..5}
do

4
etc/ncp-config.d/dnsmasq.cfg
View File

@ -15,8 +15,8 @@
{
"id": "DOMAIN",
"name": "Domain",
"value": "mycloud.ownyourbits.com",
"suggest": "mycloud.ownyourbits.com"
"value": "mycloud.nextcloudpi.com",
"suggest": "mycloud.nextcloudpi.com"
},
{
"id": "DNSSERVER",

6
etc/ncp-config.d/letsencrypt.cfg
View File

@ -16,19 +16,19 @@
"id": "DOMAIN",
"name": "Domain",
"value": "",
"suggest": "mycloud.ownyourbits.com"
"suggest": "mycloud.nextcloudpi.com"
},
{
"id": "OTHER_DOMAIN",
"name": "Additional domain",
"value": "",
"suggest": "optional.cloud.ownyourbits.com"
"suggest": "optional.cloud.nextcloudpi.com"
},
{
"id": "EMAIL",
"name": "Email",
"value": "",
"suggest": "mycloud@ownyourbits.com"
"suggest": "mycloud@nextcloudpi.com"
}
]
}

2
etc/ncp-config.d/nc-nextcloud.cfg
View File

@ -9,7 +9,7 @@
{
"id": "VER",
"name": "Version",
"value": "28.0.4"
"value": "28.0.5"
},
{
"id": "MAXFILESIZE",

4
etc/ncp-config.d/nc-trusted-domains.cfg
View File

@ -10,13 +10,13 @@
"id": "DOMAIN1",
"name": "Trusted Domain #1",
"value": "",
"suggest": "example.ownyourbits.com"
"suggest": "example.nextcloudpi.com"
},
{
"id": "DOMAIN2",
"name": "Trusted Domain #2",
"value": "",
"suggest": "example.ownyourbits.com"
"suggest": "example.nextcloudpi.com"
},
{
"id": "DOMAIN3",

4
etc/ncp-config.d/no-ip.cfg
View File

@ -27,8 +27,8 @@
{
"id": "DOMAIN",
"name": "Domain",
"value": "mycloud.ownyourbits.com",
"suggest": "mycloud.ownyourbits.com"
"value": "mycloud.nextcloudpi.com",
"suggest": "mycloud.nextcloudpi.com"
},
{
"id": "TIME",

4
etc/ncp.cfg
View File

@ -1,5 +1,5 @@
{
"nextcloud_version": "28.0.4",
"nextcloud_version": "28.0.5",
"php_version": "8.1",
"release": "bullseye"
"release": "bookworm"
}

18
install.sh
View File

@ -28,9 +28,20 @@ export PATH="/usr/local/sbin:/usr/sbin:/sbin:${PATH}"
type mysqld &>/dev/null && echo ">>> WARNING: existing mysqld configuration will be changed <<<"
type mysqld &>/dev/null && mysql -e 'use nextcloud' &>/dev/null && { echo "The 'nextcloud' database already exists. Aborting"; exit 1; }
[[ "$DEBIAN_FRONTEND" == "noninteractive" ]] || {
echo "WARNING: This installer will disable SSH login for the root user and reset its password.
If you need to login with root, you should make sure, you have a root session open that you can use,
to revert these changes afterwards (set PermitRootLogin to 'yes' in /etc/ssh/sshd_config and run passwd as root)."
for i in {1..10}
do
echo "Continuing in $((30-(3*i)))s (press Ctrl+C to abort)..."
sleep 3
done
}
# get dependencies
apt-get update
apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget
DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget jq gnupg2
# get install code
if [[ "${CODE_DIR}" == "" ]]; then
@ -63,6 +74,11 @@ cp etc/ncp.cfg /usr/local/etc/
cp -r etc/ncp-templates /usr/local/etc/
install_app lamp.sh
if [[ -d "/run/systemd/system" ]] && is_lxc
then
DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y systemd-resolved
systemctl enable systemd-resolved
fi
install_app bin/ncp/CONFIG/nc-nextcloud.sh
run_app_unsafe bin/ncp/CONFIG/nc-nextcloud.sh
rm /usr/local/etc/ncp-config.d/nc-nextcloud.cfg # armbian overlay is ro

11
lamp.sh
View File

@ -37,13 +37,6 @@ install()
apache2ctl -V || true
# Create systemd users to keep uids persistent between containers
id -u systemd-resolve || {
addgroup --quiet --system systemd-journal
adduser --quiet -u 180 --system --group --no-create-home --home /run/systemd \
--gecos "systemd Network Management" systemd-network
adduser --quiet -u 181 --system --group --no-create-home --home /run/systemd \
--gecos "systemd Resolver" systemd-resolve
}
install_with_shadow_workaround --no-install-recommends systemd
$APTINSTALL -t $RELEASE php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \
php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \
@ -67,7 +60,7 @@ install()
install_template apache2/http2.conf.sh /etc/apache2/conf-available/http2.conf --defaults
# CONFIGURE PHP7
# CONFIGURE PHP
##########################################
install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" --defaults
@ -97,7 +90,7 @@ install()
# launch mariadb if not already running
if ! [[ -f /run/mysqld/mysqld.pid ]]; then
echo "Starting mariaDB"
mysqld &
sudo -u mysql mysqld &
fi
# wait for mariadb

2
ncp-app/appinfo/info.xml
View File

@ -7,7 +7,7 @@
<description><![CDATA[NextCloudPi features a preconfigured Nextcloud instance and a complete set of tools around it for easy management.]]></description>
<version>0.0.2</version>
<licence>agpl</licence>
<author mail="nachoparker@ownyourbits.com" homepage="https://ownyourbits.com">nachoparker</author>
<author homepage="https://github.com/theCalcaholic">Tobias Knöppler</author>
<namespace>NextcloudPi</namespace>
<category>tools</category>
<bugs>https://github.com/nextcloud/nextcloudpi/issues</bugs>

9
ncp.sh
View File

@ -22,7 +22,7 @@ install()
{
# NCP-CONFIG
apt-get update
$APTINSTALL git dialog whiptail jq file lsb-release
$APTINSTALL git dialog whiptail jq file lsb-release tmux
mkdir -p "$CONFDIR" "$BINDIR"
# This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager
@ -82,7 +82,8 @@ EOF
echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN"
is_docker || is_lxc || {
chsh -s /usr/sbin/nologin "$WEBADMIN"
chsh -s /usr/sbin/nologin root
passwd -l root
sed -i -e 's/^PermitRootLogin.*$/PermitRootLogin No/' /etc/ssh/sshd_config
}
## NCP LAUNCHER
@ -244,6 +245,10 @@ EOF
#!/bin/bash
/usr/local/bin/ncp-check-updates
EOF
echo '
NCP is not activated yet. Please enter https://nextcloudpi.local or this instance'"'"'s local IP address in your webbrowser to complete activation. You can find detailed instructions at https://nextcloudpi.com/activate
' >> /etc/issue
chmod a+x /etc/update-motd.d/*
## HOSTNAME AND mDNS

10
staged_rollouts/v1.54.0.txt Normal file
View File

@ -0,0 +1,10 @@
39
21
52
77
71
80
90
29
6
62

20
tests/nextcloud_tests.py
View File

@ -140,7 +140,7 @@ def close_first_run_wizard(driver: WebDriver):
time.sleep(3)
def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool):
""" Login and assert admin page checks"""
test = Test()
test.new("nextcloud page")
@ -171,7 +171,8 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
try:
wait.until(VisibilityOfElementLocatedByAnyLocator([(By.CSS_SELECTOR, "#security-warning-state-ok"),
(By.CSS_SELECTOR, "#security-warning-state-warning"),
(By.CSS_SELECTOR, "#security-warning-state-error")]))
(By.CSS_SELECTOR, "#security-warning-state-error"),
(By.CSS_SELECTOR, "#security-warning-state-failure")]))
element_ok = driver.find_element(By.ID, "security-warning-state-ok")
element_warn = driver.find_element(By.ID, "security-warning-state-warning")
@ -212,6 +213,9 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
raise ConfigTestFailure("The list of php_modules does not equal [imagick]")
elif not element_ok.is_displayed():
errors = driver.find_elements(By.CSS_SELECTOR, "#postsetupchecks > .errors > li")
for error in errors:
print(f'ERROR: {error.text}')
raise ConfigTestFailure("Neither the warnings nor the ok status is displayed "
"(so there are probably errors or the page is broken)")
@ -263,8 +267,11 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver):
expected['ncp_version'] = True
elif 'php version' in divs[0].text.lower() and divs[1].text == ncp_cfg['php_version']:
expected['php_version'] = True
elif 'debian release' in divs[0].text.lower() and divs[1].text == ncp_cfg['release']:
elif 'debian release' in divs[0].text.lower():
if divs[1].text == ncp_cfg['release'] or skip_release_check:
expected['debian_release'] = True
else:
print(f"{tc.yellow}{divs[1].text} != {ncp_cfg['release']}")
failed = list(map(lambda item: item[0], filter(lambda item: not item[1], expected.items())))
test.check(len(failed) == 0, f"checks failed for admin section: [{', '.join(failed)}]")
except Exception as e:
@ -302,11 +309,12 @@ if __name__ == "__main__":
# parse options
try:
opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui'])
opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check'])
except getopt.GetoptError:
usage()
sys.exit(2)
skip_release_check = False
options = webdriver.FirefoxOptions()
for opt, arg in opts:
if opt in ('-h', '--help'):
@ -317,6 +325,8 @@ if __name__ == "__main__":
os.unlink(test_cfg)
elif opt == '--no-gui':
options.add_argument("-headless")
elif opt == '--skip-release-check':
skip_release_check = True
else:
usage()
sys.exit(2)
@ -355,7 +365,7 @@ if __name__ == "__main__":
driver = webdriver.Firefox(options=options)
failed=False
try:
test_nextcloud(IP, nc_port, driver)
test_nextcloud(IP, nc_port, driver, skip_release_check)
except Exception as e:
print(e)
print(traceback.format_exc())

15
tests/system_tests.py
View File

@ -59,6 +59,10 @@ files_must_not_exist = [
'/.ncp-image',
]
lxc_command = 'lxc'
if 'USE_INCUS' in os.environ and os.environ['USE_INCUS'] == 'yes':
lxc_command = 'incus'
class tc:
"terminal colors"
@ -226,8 +230,13 @@ def test_autoupdates():
return False
set_cohorte_id(99)
handle_error(run(pre_cmd + ['/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE))
chk_version = handle_error(run(pre_cmd + ['/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE))
try:
result = handle_error(run(pre_cmd + ['cat', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE))
except ProcessExecutionException as e:
print("stderr:", chk_version.stderr)
print("stdout:", chk_version.stdout)
raise e
if 'v99.99.99' not in result.stdout:
print(f"{tc.red}error{tc.normal} Expected latest detected version to be v99.99.99, was {result.stdout}")
return False
@ -291,7 +300,7 @@ if __name__ == "__main__":
# detect if we are running this in a LXC instance
try:
lxc_running = run(['lxc', 'info', 'ncp'], stdout=PIPE, check = True)
lxc_running = run([lxc_command, 'info', 'ncp'], stdout=PIPE, check = True)
except:
lxc_running = False
@ -319,7 +328,7 @@ if __name__ == "__main__":
# LXC method
elif lxc_running:
print( tc.brown + "* local LXC instance detected" + tc.normal)
pre_cmd = ['lxc', 'exec', 'ncp', '--']
pre_cmd = [lxc_command, 'exec', 'ncp', '--']
elif systemd_container_running:
pre_cmd = ['systemd-run', '--wait', '-P', '--machine=ncp']

2
update.sh
View File

@ -201,8 +201,6 @@ check_distro "$NCPCFG" && check_distro etc/ncp.cfg || {
cfg="$(jq '.release = "'$release_new'"' <<<"$cfg")"
echo "$cfg" > /usr/local/etc/ncp-recommended.cfg
[[ -f /.dockerenv ]] && \
msg="Update to $release_new available. Get the latest container to upgrade" || \
msg="Update to $release_new available. Type 'sudo ncp-dist-upgrade' to upgrade"
echo "${msg}"
notify_admin "New distribution available" "${msg}"

38
updates/1.54.0.sh Normal file
View File

@ -0,0 +1,38 @@
#!/usr/bin/env bash
set -ex
echo "Update root login prevention method..."
if getent passwd "root" | grep -e '/usr/sbin/nologin'
then
chsh -s /bin/bash root
passwd -l root
sed -i -e 's/^PermitRootLogin.*$/PermitRootLogin No/' /etc/ssh/sshd_config
fi
echo "done."
echo "Fixing trusted proxies list..."
for i in {10..15}
do
proxy="$(ncc config:system:get trusted_proxies "$i" || echo 'NONE')"
[[ "$proxy" == 'NONE' ]] || python3 -c "import ipaddress; ipaddress.ip_address('${proxy}')" > /dev/null 2>&1 || ncc config:system:delete trusted_proxies "$i"
done
echo "done."
echo "Updating PHP package signing key..."
apt-get update
apt-get install --no-install-recommends -y gnupg2
apt-key adv --fetch-keys https://packages.sury.org/php/apt.gpg
echo "done."
echo "Installing dependencies..."
apt-get install --no-install-recommends -y tmux
echo "done."
echo "Updating obsolete theming URL"
if [[ "$(ncc config:app:get theming url)" == "https://ownyourbits.com" ]]
then
ncc config:app:set theming url --value="https://nextcloudpi.com"
fi
echo "done."