From 26edf1f7b0594c4923828e1979cb53f50e5c23a0 Mon Sep 17 00:00:00 2001 From: nachoparker Date: Wed, 29 Sep 2021 16:19:40 -0600 Subject: [PATCH] build: add LXC/LXD support Signed-off-by: nachoparker --- .gitignore | 22 +++--- README.md | 20 ++++- bin/ncp-provisioning.sh | 4 +- bin/ncp/BACKUPS/nc-restore.sh | 2 +- bin/ncp/CONFIG/nc-nextcloud.sh | 14 +++- bin/ncp/NETWORKING/letsencrypt.sh | 4 +- bin/ncp/SYSTEM/nc-swapfile.sh | 3 +- Vagrantfile => build/Vagrantfile | 2 +- armbian.sh => build/armbian/armbian.sh | 17 +---- .../armbian/config-odroidc2.conf | 0 .../armbian/config-odroidhc4.conf | 0 .../armbian/config-odroidxu4.conf | 0 .../armbian/config-orangepizeroplus2-h5.conf | 0 .../armbian/config-rock64.conf | 0 batch.sh => build/batch.sh | 36 ++++----- build/build-LXC.sh | 73 +++++++++++++++++++ build/build-LXD.sh | 70 ++++++++++++++++++ .../build-SD-armbian.sh | 7 +- .../build-SD-berryboot.sh | 6 +- build-SD-rpi.sh => build/build-SD-rpi.sh | 5 +- build-VM.sh => build/build-VM.sh | 3 +- build-docker.sh => build/build-docker.sh | 11 ++- buildlib.sh => build/buildlib.sh | 3 + .../docker}/debian-ncp/Dockerfile | 2 +- .../docker}/debian-ncp/run-parts.sh | 0 .../docker/docker-compose-ncpdev.yml | 0 .../docker/docker-compose.yml | 0 {docker => build/docker}/lamp/010lamp | 0 {docker => build/docker}/lamp/Dockerfile | 2 +- .../docker}/nextcloud/020nextcloud | 0 {docker => build/docker}/nextcloud/Dockerfile | 2 +- {docker => build/docker}/nextcloudpi/000ncp | 0 .../docker}/nextcloudpi/Dockerfile | 4 +- build/lxc_config | 31 ++++++++ changelog.md | 4 +- docker-compose-armhf.yml | 16 ---- docker-compose-nc-armhf.yml | 15 ---- docker-compose-nc.yml | 15 ---- etc/library.sh | 10 ++- etc/ncp-templates/nextcloud.conf.sh | 8 +- install.sh | 15 ++-- ncp.sh | 3 +- tag_and_push.sh | 2 +- tests/libvirt_forwarding.sh | 40 ++++++++++ tests/lxd_forwarding.sh | 33 +++++++++ update.sh | 30 +++++--- 46 files changed, 391 insertions(+), 143 deletions(-) rename Vagrantfile => build/Vagrantfile (96%) rename armbian.sh => build/armbian/armbian.sh (80%) rename config-odroidc2.conf => build/armbian/config-odroidc2.conf (100%) rename config-odroidhc4.conf => build/armbian/config-odroidhc4.conf (100%) rename config-odroidxu4.conf => build/armbian/config-odroidxu4.conf (100%) rename config-orangepizeroplus2-h5.conf => build/armbian/config-orangepizeroplus2-h5.conf (100%) rename config-rock64.conf => build/armbian/config-rock64.conf (100%) rename batch.sh => build/batch.sh (86%) create mode 100755 build/build-LXC.sh create mode 100755 build/build-LXD.sh rename build-SD-armbian.sh => build/build-SD-armbian.sh (95%) rename build-SD-berryboot.sh => build/build-SD-berryboot.sh (98%) rename build-SD-rpi.sh => build/build-SD-rpi.sh (97%) rename build-VM.sh => build/build-VM.sh (98%) rename build-docker.sh => build/build-docker.sh (70%) rename buildlib.sh => build/buildlib.sh (99%) rename {docker => build/docker}/debian-ncp/Dockerfile (91%) rename {docker => build/docker}/debian-ncp/run-parts.sh (100%) rename docker-compose-ncpdev.yml => build/docker/docker-compose-ncpdev.yml (100%) rename docker-compose.yml => build/docker/docker-compose.yml (100%) rename {docker => build/docker}/lamp/010lamp (100%) rename {docker => build/docker}/lamp/Dockerfile (95%) rename {docker => build/docker}/nextcloud/020nextcloud (100%) rename {docker => build/docker}/nextcloud/Dockerfile (95%) rename {docker => build/docker}/nextcloudpi/000ncp (100%) rename {docker => build/docker}/nextcloudpi/Dockerfile (94%) create mode 100644 build/lxc_config delete mode 100644 docker-compose-armhf.yml delete mode 100644 docker-compose-nc-armhf.yml delete mode 100644 docker-compose-nc.yml create mode 100755 tests/libvirt_forwarding.sh create mode 100755 tests/lxd_forwarding.sh diff --git a/.gitignore b/.gitignore index 5124aef2..419998b2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,13 @@ .*.swp -qemu-raspbian-network/ -archive/ -output/ -cache/ -torrent/ -armbian/ -raspbian_root -raspbian_boot -ncp-web/wizard.cfg -ncp-web/ncp-web.cfg -docker-armhf/qemu-arm-static +/qemu-raspbian-network/ +/archive/ +/output/ +/cache/ +/torrent/ +/armbian/ +/raspbian_root +/raspbian_boot +/ncp-web/wizard.cfg +/ncp-web/ncp-web.cfg +/docker-armhf/qemu-arm-static .vagrant/ diff --git a/README.md b/README.md index f9b3b3b1..15ce8dda 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,12 @@ sudo ncp-config docker run -d -p 4443:4443 -p 443:443 -p 80:80 -v ncdata:/data --name nextcloudpi ownyourbits/nextcloudpi $DOMAIN ``` +## Run in LXD + +``` +lxc import NextCloudPi_LXD_09-29-21.tar.bz +lxc start ncp +``` ## How to build @@ -96,7 +102,7 @@ Install git, docker, qemu-user-static, chroot and all the usual building tools. ``` git clone https://github.com/nextcloud/nextcloudpi.git cd nextcloudpi -./build-SD-rpi.sh +build/build-SD-rpi.sh ``` , or for an Armbian based board @@ -110,9 +116,15 @@ In order to generate the Docker images ``` git clone https://github.com/nextcloud/nextcloudpi.git cd nextcloudpi -./build-docker.sh x86 -./build-docker.sh armhf -./build-docker.sh arm64 +build/build-docker.sh x86 +build/build-docker.sh armhf +build/build-docker.sh arm64 +``` + +, for LXD + +``` +build/build-LXD.sh ``` NextCloudPi can be installed in any architecture running the latest Debian diff --git a/bin/ncp-provisioning.sh b/bin/ncp-provisioning.sh index 8e51556d..cda1c70c 100644 --- a/bin/ncp-provisioning.sh +++ b/bin/ncp-provisioning.sh @@ -2,6 +2,8 @@ # this script runs at startup to provide an unique random passwords for each instance +source /usr/local/etc/library.sh + ## redis provisioning CFG=/var/www/nextcloud/config/config.php @@ -14,7 +16,7 @@ REDISPASS="$( grep "^requirepass" /etc/redis/redis.conf | cut -f2 -d' ' )" echo Provisioning Redis password sed -i -E "s|^requirepass .*|requirepass $REDISPASS|" /etc/redis/redis.conf chown redis:redis /etc/redis/redis.conf - [[ "$DOCKERBUILD" != 1 ]] && systemctl restart redis + is_docker || systemctl restart redis } ### If there exists already a configuration adjust the password diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 0ada9950..26301346 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -158,7 +158,7 @@ sed -i "s|^;\?sys_temp_dir =.*$|sys_temp_dir = $DATADIR/tmp|" /etc/php/${PHP ncc config:system:set logfile --value="$DATADIR/nextcloud.log" # update fail2ban logpath -[[ ! -f /.docker-image ]] && { +[[ -f /etc/fail2ban/jail.conf ]] && { sed -i "s|logpath =.*|logpath = $DATADIR/nextcloud.log|" /etc/fail2ban/jail.conf pgrep fail2ban &>/dev/null && service fail2ban restart } diff --git a/bin/ncp/CONFIG/nc-nextcloud.sh b/bin/ncp/CONFIG/nc-nextcloud.sh index 4b9cf9de..17333bd6 100644 --- a/bin/ncp/CONFIG/nc-nextcloud.sh +++ b/bin/ncp/CONFIG/nc-nextcloud.sh @@ -19,11 +19,9 @@ install() # During build, this step is run before ncp.sh. Avoid executing twice [[ -f /usr/lib/systemd/system/nc-provisioning.service ]] && return 0 - source /usr/local/etc/library.sh # sets PHPVER RELEASE - # Optional packets for Nextcloud and Apps apt-get update - $APTINSTALL lbzip2 iputils-ping jq + $APTINSTALL lbzip2 iputils-ping jq wget $APTINSTALL -t $RELEASE php-smbclient exfat-fuse exfat-utils # for external storage $APTINSTALL -t $RELEASE php${PHPVER}-exif # for gallery $APTINSTALL -t $RELEASE php${PHPVER}-gmp # for bookmarks @@ -56,6 +54,16 @@ install() echo "maxmemory $REDIS_MEM" >> $REDIS_CONF echo 'vm.overcommit_memory = 1' >> /etc/sysctl.conf + if is_lxc; then + # Otherwise it fails to start in Buster LXC container + mkdir -p /etc/systemd/system/redis-server.service.d + cat > /etc/systemd/system/redis-server.service.d/lxc_fix.conf <<'EOF' +[Service] +ReadOnlyDirectories= +EOF + systemctl daemon-reload + fi + chown redis: "$REDIS_CONF" usermod -a -G redis www-data diff --git a/bin/ncp/NETWORKING/letsencrypt.sh b/bin/ncp/NETWORKING/letsencrypt.sh index f2e48ae2..70e10285 100644 --- a/bin/ncp/NETWORKING/letsencrypt.sh +++ b/bin/ncp/NETWORKING/letsencrypt.sh @@ -35,7 +35,7 @@ install() rm -f /etc/cron.d/certbot mkdir -p /etc/letsencrypt/live - [[ "$DOCKERBUILD" == 1 ]] && { + is_docker && { # execute before lamp stack cat > /etc/services-available.d/009letsencrypt < +# GPL licensed (see end of file) * Use at your own risk! +# +# Usage: +# + +set -e +source build/buildlib.sh + +#CLEAN=0 # Pass this envvar to skip cleaning download cache +IMG="NextCloudPi_LXC_$( date "+%m-%d-%y" ).img" +IMG=tmp/"$IMG" + +TAR=output/"$( basename "$IMG" .img ).tar.bz2" + +test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } + +############################################################################## + +## preparations + +test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } +set -e +prepare_dirs # tmp cache output + +## BUILD NCP + +echo -e "\e[1m\n[ Build NCP ]\e[0m" + +# TODO sudo +sudo lxc-destroy ncp -f +sudo lxc-create -n ncp -t download -B btrfs -- --dist debian --release buster --arch amd64 # TODO vars for distro and stuff +sudo cp lxc_config /var/lib/lxc/ncp/config +sudo lxc-start -n ncp +sudo lxc-attach -n ncp --clear-env -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' +sudo lxc-attach -n ncp --clear-env -- bash /build/install.sh +sudo lxc-attach -n ncp --clear-env -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +sudo lxc-attach -n ncp --clear-env -- poweroff + +exit 0 # TODO + +## pack +pack_image "$IMG" "$TAR" + +## test +#set_static_IP "$IMG" "$IP" +#test_image "$IMG" "$IP" # TODO fix tests + +# upload +create_torrent "$TAR" +upload_ftp "$( basename "$TAR" .tar.bz2 )" + + +# License +# +# This script is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this script; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, +# Boston, MA 02111-1307 USA diff --git a/build/build-LXD.sh b/build/build-LXD.sh new file mode 100755 index 00000000..b1cb2cf5 --- /dev/null +++ b/build/build-LXD.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Batch creation of NextCloudPi LXD image +# +# Copyleft 2021 by Ignacio Nunez Hernanz +# GPL licensed (see end of file) * Use at your own risk! +# +# Usage: +# + +set -e +source build/buildlib.sh + +#CLEAN=0 # Pass this envvar to skip cleaning download cache +IMG="NextCloudPi_LXD_$( date "+%m-%d-%y" ).img" +IMG=tmp/"$IMG" + +TAR=output/"$( basename "$IMG" .img ).tar.bz2" + +test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } + +############################################################################## + +## preparations + +test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } +set -e +prepare_dirs # tmp cache output + +## BUILD NCP + +echo -e "\e[1m\n[ Build NCP ]\e[0m" + +lxc delete -f ncp 2>/dev/null || true +systemd-run --user --scope -p "Delegate=yes" lxc launch images:debian/buster ncp +lxc config device add ncp buildcode disk source="$(pwd)" path=/build +lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' +lxc exec ncp -- bash -c 'CODE_DIR=/build bash /build/install.sh' +lxc exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +lxc config device remove ncp buildcode +lxc publish ncp -f --alias ncp/"${version}" + +## pack +lxc export ncp "$TAR" + +## test +#set_static_IP "$IMG" "$IP" +#test_image "$IMG" "$IP" + +# upload +create_torrent "$TAR" +upload_ftp "$( basename "$TAR" .tar.bz2 )" + + +# License +# +# This script is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this script; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, +# Boston, MA 02111-1307 USA diff --git a/build-SD-armbian.sh b/build/build-SD-armbian.sh similarity index 95% rename from build-SD-armbian.sh rename to build/build-SD-armbian.sh index da1a1945..93069e53 100755 --- a/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -17,10 +17,11 @@ IMG="NextCloudPi_${BNAME}_$( date "+%m-%d-%y" ).img" IMG=tmp/"$IMG" TAR=output/"$( basename "$IMG" .img ).tar.bz2" +set -e +source build/buildlib.sh + test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } -set -e -source buildlib.sh source etc/library.sh # sets RELEASE prepare_dirs # tmp cache output @@ -32,7 +33,7 @@ prepare_dirs # tmp cache output # add NCP modifications mkdir -p armbian/userpatches armbian/userpatches/overlay rm -f ncp-web/{wizard.cfg,ncp-web.cfg} -cp armbian.sh armbian/userpatches/customize-image.sh +cp build/armbian/armbian.sh armbian/userpatches/customize-image.sh rsync -Aax --delete --exclude-from .gitignore --exclude *.img --exclude *.bz2 . armbian/userpatches/overlay/ # GENERATE IMAGE diff --git a/build-SD-berryboot.sh b/build/build-SD-berryboot.sh similarity index 98% rename from build-SD-berryboot.sh rename to build/build-SD-berryboot.sh index 4857902d..115ed994 100755 --- a/build-SD-berryboot.sh +++ b/build/build-SD-berryboot.sh @@ -8,13 +8,13 @@ # Usage: ./build-SD-berryboot.sh # +set -e +source build/buildlib.sh + SRC="$1" IMG="NextCloudPi_RPi_Berryboot_$( date "+%m-%d-%y" ).img" TAR=output/"$( basename "$IMG" .img ).tar.bz2" -set -e -source buildlib.sh - test -f "$TAR" && { echo "$TAR already exists. Skipping... "; exit 0; } [[ -f "$SRC" ]] || { echo "$SRC not found"; exit 1; } diff --git a/build-SD-rpi.sh b/build/build-SD-rpi.sh similarity index 97% rename from build-SD-rpi.sh rename to build/build-SD-rpi.sh index 91e22e0a..c2ecb50f 100755 --- a/build-SD-rpi.sh +++ b/build/build-SD-rpi.sh @@ -9,7 +9,7 @@ # set -e -source buildlib.sh +source build/buildlib.sh URL="https://downloads.raspberrypi.org/raspios_lite_arm64/images/raspios_lite_arm64-2020-08-24/2020-08-20-raspios-buster-arm64-lite.zip" SIZE=3G # Raspbian image size @@ -50,6 +50,9 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ # mark the image as an image build touch /.ncp-image + # allow oldstable + apt-get update --allow-releaseinfo-change + # As of 10-2018 this upgrades raspi-kernel and messes up wifi and BTRFS #apt-get upgrade -y #apt-get dist-upgrade -y diff --git a/build-VM.sh b/build/build-VM.sh similarity index 98% rename from build-VM.sh rename to build/build-VM.sh index a5c85980..72cfa186 100755 --- a/build-VM.sh +++ b/build/build-VM.sh @@ -9,7 +9,7 @@ # set -e -source buildlib.sh +source build/buildlib.sh IP=${1:-192.168.0.145} # For QEMU automated testing (optional) SIZE=3G # Raspbian image size @@ -34,6 +34,7 @@ prepare_dirs # tmp cache output echo -e "\e[1m\n[ Build NCP ]\e[0m" export DEB_RELEASE=$(jq -r .release < etc/ncp.cfg) +cd build/ vagrant destroy -f vagrant box update vagrant up --provider=libvirt --provision diff --git a/build-docker.sh b/build/build-docker.sh similarity index 70% rename from build-docker.sh rename to build/build-docker.sh index cd72dc06..019026cf 100755 --- a/build-docker.sh +++ b/build/build-docker.sh @@ -6,8 +6,7 @@ set -e -version=$(git describe --tags --always) -version=${version%-*-*} +source build/buildlib.sh release=$(jq -r .release < etc/ncp.cfg) function docker_build() { DOCKER_BUILDKIT=1 docker build --progress=plain . "$@"; } @@ -19,10 +18,10 @@ function build_arch() local arch_qemu="${3}" local ncp_tag="${4:-$arch}" - docker_build -f docker/debian-ncp/Dockerfile -t ownyourbits/debian-ncp-${ncp_tag}:latest --pull --build-arg release=${release} --build-arg arch=${arch} --build-arg arch_qemu=${arch_qemu} - docker_build -f docker/lamp/Dockerfile -t ownyourbits/lamp-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} - docker_build -f docker/nextcloud/Dockerfile -t ownyourbits/nextcloud-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} - docker_build -f docker/nextcloudpi/Dockerfile -t ownyourbits/nextcloudpi-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} --build-arg ncp_ver=${version} + docker_build -f build/docker/debian-ncp/Dockerfile -t ownyourbits/debian-ncp-${ncp_tag}:latest --pull --build-arg release=${release} --build-arg arch=${arch} --build-arg arch_qemu=${arch_qemu} + docker_build -f build/docker/lamp/Dockerfile -t ownyourbits/lamp-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} + docker_build -f build/docker/nextcloud/Dockerfile -t ownyourbits/nextcloud-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} + docker_build -f build/docker/nextcloudpi/Dockerfile -t ownyourbits/nextcloudpi-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} --build-arg ncp_ver=${version} docker tag ownyourbits/debian-ncp-${ncp_tag}:latest ownyourbits/debian-ncp-${ncp_tag}:"${version}" docker tag ownyourbits/lamp-${ncp_tag}:latest ownyourbits/lamp-${ncp_tag}:"${version}" diff --git a/buildlib.sh b/build/buildlib.sh similarity index 99% rename from buildlib.sh rename to build/buildlib.sh index d3507916..d589111f 100644 --- a/buildlib.sh +++ b/build/buildlib.sh @@ -10,6 +10,9 @@ DBG=x +version=$(git describe --tags --always) +version=${version%-*-*} + # $IMG is the source image # $IP is the IP of the QEMU images # $IMGOUT will contain the name of the generated image diff --git a/docker/debian-ncp/Dockerfile b/build/docker/debian-ncp/Dockerfile similarity index 91% rename from docker/debian-ncp/Dockerfile rename to build/docker/debian-ncp/Dockerfile index ff5717d0..c2196d94 100644 --- a/docker/debian-ncp/Dockerfile +++ b/build/docker/debian-ncp/Dockerfile @@ -19,4 +19,4 @@ COPY --from=qemu /usr/bin/qemu-${arch_qemu}-static /usr/bin/ RUN mkdir -p /etc/services-available.d /etc/services-enabled.d -COPY docker/debian-ncp/run-parts.sh / +COPY build/docker/debian-ncp/run-parts.sh / diff --git a/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh similarity index 100% rename from docker/debian-ncp/run-parts.sh rename to build/docker/debian-ncp/run-parts.sh diff --git a/docker-compose-ncpdev.yml b/build/docker/docker-compose-ncpdev.yml similarity index 100% rename from docker-compose-ncpdev.yml rename to build/docker/docker-compose-ncpdev.yml diff --git a/docker-compose.yml b/build/docker/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to build/docker/docker-compose.yml diff --git a/docker/lamp/010lamp b/build/docker/lamp/010lamp similarity index 100% rename from docker/lamp/010lamp rename to build/docker/lamp/010lamp diff --git a/docker/lamp/Dockerfile b/build/docker/lamp/Dockerfile similarity index 95% rename from docker/lamp/Dockerfile rename to build/docker/lamp/Dockerfile index 41c3b213..61c5eb4b 100644 --- a/docker/lamp/Dockerfile +++ b/build/docker/lamp/Dockerfile @@ -44,7 +44,7 @@ rm -f /var/log/alternatives.log /var/log/apt/*; \ rm /data-ro/database/ib_logfile*; \ rm /usr/local/etc/lamp.sh -COPY docker/lamp/010lamp /etc/services-enabled.d/ +COPY build/docker/lamp/010lamp /etc/services-enabled.d/ ENTRYPOINT ["/run-parts.sh"] diff --git a/docker/nextcloud/020nextcloud b/build/docker/nextcloud/020nextcloud similarity index 100% rename from docker/nextcloud/020nextcloud rename to build/docker/nextcloud/020nextcloud diff --git a/docker/nextcloud/Dockerfile b/build/docker/nextcloud/Dockerfile similarity index 95% rename from docker/nextcloud/Dockerfile rename to build/docker/nextcloud/Dockerfile index bc802187..a20465e2 100644 --- a/docker/nextcloud/Dockerfile +++ b/build/docker/nextcloud/Dockerfile @@ -50,5 +50,5 @@ apt-get purge -y wget ca-certificates; \ rm /nc-nextcloud.sh /usr/local/etc/ncp-config.d/nc-nextcloud.cfg; \ rm /.ncp-image; -COPY docker/nextcloud/020nextcloud /etc/services-enabled.d/ +COPY build/docker/nextcloud/020nextcloud /etc/services-enabled.d/ COPY bin/ncp-provisioning.sh /usr/local/bin/ diff --git a/docker/nextcloudpi/000ncp b/build/docker/nextcloudpi/000ncp similarity index 100% rename from docker/nextcloudpi/000ncp rename to build/docker/nextcloudpi/000ncp diff --git a/docker/nextcloudpi/Dockerfile b/build/docker/nextcloudpi/Dockerfile similarity index 94% rename from docker/nextcloudpi/Dockerfile rename to build/docker/nextcloudpi/Dockerfile index 7d7d6864..be63ba09 100644 --- a/docker/nextcloudpi/Dockerfile +++ b/build/docker/nextcloudpi/Dockerfile @@ -19,7 +19,7 @@ COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/ COPY ncp-web /tmp/ncp-build/ncp-web/ COPY ncp-app /tmp/ncp-build/ncp-app/ COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/ -COPY docker /tmp/ncp-build/docker/ +COPY build/docker /tmp/ncp-build/docker/ COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ @@ -66,4 +66,4 @@ rm /var/cache/debconf/*-old; \ # set version echo "${ncp_ver}" > /usr/local/etc/ncp-version -COPY docker/nextcloudpi/000ncp /etc/services-enabled.d/ +COPY build/docker/nextcloudpi/000ncp /etc/services-enabled.d/ diff --git a/build/lxc_config b/build/lxc_config new file mode 100644 index 00000000..59640994 --- /dev/null +++ b/build/lxc_config @@ -0,0 +1,31 @@ +# Template used to create this container: /usr/share/lxc/templates/lxc-download +# Parameters passed to the template: --dist debian --release buster --arch amd64 +# Template script checksum (SHA-1): 1ad14dde73ae69ed485f71e26793af87573947d4 +# For additional config options, please look at lxc.container.conf(5) + +# Uncomment the following line to support nesting containers: +#lxc.include = /usr/share/lxc/config/nesting.conf +# (Be aware this has security implications) + + +# Distribution configuration +lxc.include = /usr/share/lxc/config/common.conf +lxc.arch = linux64 + +# Container specific configuration +lxc.rootfs.path = btrfs:/var/lib/lxc/ncp/rootfs +lxc.uts.name = ncp + +# Network configuration +#lxc.net.0.type = empty +lxc.net.0.type = veth +lxc.net.0.link = lxcbr0 +lxc.net.0.flags = up +lxc.net.0.hwaddr = 00:16:3e:2d:5d:80 + +# Mounts +lxc.mount.entry = /media/DATA/src/nextcloudpi build none bind,create=dir 0 0 + +# Moar privileges (TODO review) +#lxc.apparmor.profile = unconfined +#lxc.cgroup2.devices.allow = a diff --git a/changelog.md b/changelog.md index e85f0333..abc29447 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,7 @@ -[v1.39.18](https://github.com/nextcloud/nextcloudpi/commit/edaf4f1) (2021-09-30) nc-httpsonly: always use overwriteprotocol https in all cases +[v1.39.19](https://github.com/nextcloud/nextcloudpi/commit/b177bd1) (2021-09-30) ncp-web: fix upload from local file path + +[v1.39.18](https://github.com/nextcloud/nextcloudpi/commit/f1c90f5) (2021-09-30) nc-httpsonly: always use overwriteprotocol https in all cases [v1.39.17](https://github.com/nextcloud/nextcloudpi/commit/c037c11) (2021-09-29) add bash completion to ncc diff --git a/docker-compose-armhf.yml b/docker-compose-armhf.yml deleted file mode 100644 index 27e033cb..00000000 --- a/docker-compose-armhf.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '3' -services: - nextcloudpi: - image: ownyourbits/nextcloudpi-armhf - command: "${IP}" - ports: - - "80:80" - - "443:443" - - "4443:4443" - volumes: - - ncdata:/data - - /etc/localtime:/etc/localtime:ro - container_name: nextcloudpi - -volumes: - ncdata: diff --git a/docker-compose-nc-armhf.yml b/docker-compose-nc-armhf.yml deleted file mode 100644 index 3ed6c7e1..00000000 --- a/docker-compose-nc-armhf.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - nextcloud: - image: ownyourbits/nextcloud-armhf - command: "${IP}" - ports: - - "80:80" - - "443:443" - volumes: - - ncdata:/data - - /etc/localtime:/etc/localtime:ro - container_name: nextcloudpi - -volumes: - ncdata: diff --git a/docker-compose-nc.yml b/docker-compose-nc.yml deleted file mode 100644 index e45f7bed..00000000 --- a/docker-compose-nc.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: '3' -services: - nextcloud: - image: ownyourbits/nextcloud-x86 - command: "${IP}" - ports: - - "80:80" - - "443:443" - volumes: - - ncdata:/data - - /etc/localtime:/etc/localtime:ro - container_name: nextcloudpi - -volumes: - ncdata: diff --git a/etc/library.sh b/etc/library.sh index 2e147645..06405aa8 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -417,7 +417,15 @@ function clear_password_fields() function apt_install() { apt-get update --allow-releaseinfo-change - apt-get install -y --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::="--force-confold" "$@" + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends -o Dpkg::Options::=--force-confdef -o Dpkg::Options::="--force-confold" "$@" +} + +function is_docker() { + [[ -f /.dockerenv ]] || [[ "$DOCKERBUILD" == 1 ]] +} + +function is_lxc() { + grep -q container=lxc /proc/1/environ &>/dev/null } function notify_admin() diff --git a/etc/ncp-templates/nextcloud.conf.sh b/etc/ncp-templates/nextcloud.conf.sh index 728217be..caf987f3 100644 --- a/etc/ncp-templates/nextcloud.conf.sh +++ b/etc/ncp-templates/nextcloud.conf.sh @@ -13,7 +13,7 @@ if [[ "$1" != "--defaults" ]]; then )" fi -if ! [[ -f /.docker-image ]] && [[ "$1" != "--defaults" ]]; then +if ! [[ -f /.ncp-image ]] && [[ "$1" != "--defaults" ]]; then METRICS_IS_ENABLED="$( source "${BINDIR}/SYSTEM/metrics.sh" tmpl_metrics_enabled && echo yes || echo no @@ -103,5 +103,7 @@ cat < EOF -echo "Apache self check:" >> /var/log/ncp.log -apache2ctl -t >> /var/log/ncp.log 2>&1 +if ! [[ -f /.ncp-image ]]; then + echo "Apache self check:" >> /var/log/ncp.log + apache2ctl -t >> /var/log/ncp.log 2>&1 +fi diff --git a/install.sh b/install.sh index b5c2b9ee..4149617e 100644 --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ # more details at https://ownyourbits.com BRANCH="${BRANCH:-master}" -DBG=x +#DBG=x set -e$DBG @@ -27,13 +27,17 @@ export PATH="/usr/local/sbin:/usr/sbin:/sbin:${PATH}" # check installed software type mysqld &>/dev/null && echo ">>> WARNING: existing mysqld configuration will be changed <<<" -# get install code -echo "Getting build code..." +# get dependencies apt-get update apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release -git clone -b "${BRANCH}" https://github.com/nextcloud/nextcloudpi.git "${TMPDIR}"/nextcloudpi -cd "${TMPDIR}"/nextcloudpi +# get install code +if [[ "${CODE_DIR}" == "" ]]; then + echo "Getting build code..." + CODE_DIR="${TMPDIR}"/nextcloudpi + git clone -b "${BRANCH}" https://github.com/nextcloud/nextcloudpi.git "${CODE_DIR}" +fi +cd "${CODE_DIR}" # install NCP echo -e "\nInstalling NextCloudPi..." @@ -58,6 +62,7 @@ cp -r etc/ncp-templates /usr/local/etc/ install_app lamp.sh install_app bin/ncp/CONFIG/nc-nextcloud.sh run_app_unsafe bin/ncp/CONFIG/nc-nextcloud.sh +rm /usr/local/etc/ncp-config.d/nc-nextcloud.cfg # armbian overlay is ro systemctl restart mysqld # TODO this shouldn't be necessary, but somehow it's needed in Debian 9.6. Fixme install_app ncp.sh run_app_unsafe bin/ncp/CONFIG/nc-init.sh diff --git a/ncp.sh b/ncp.sh index 26bc0fe4..2f3ac751 100644 --- a/ncp.sh +++ b/ncp.sh @@ -248,7 +248,8 @@ EOF echo nextcloudpi > /etc/hostname ## tag image - [[ -f /.docker-image ]] && local DOCKER_TAG="_docker" + is_docker && local DOCKER_TAG="_docker" + is_lxc && local DOCKER_TAG="_lxc" echo "NextCloudPi${DOCKER_TAG}_$( date "+%m-%d-%y" )" > /usr/local/etc/ncp-baseimage ## SSH hardening diff --git a/tag_and_push.sh b/tag_and_push.sh index a3c39d73..d6e3d835 100755 --- a/tag_and_push.sh +++ b/tag_and_push.sh @@ -18,7 +18,7 @@ set -e TAG="$@" -source buildlib.sh +source build/buildlib.sh git tag "$TAG" generate_changelog git add changelog.md diff --git a/tests/libvirt_forwarding.sh b/tests/libvirt_forwarding.sh new file mode 100755 index 00000000..0c3d715d --- /dev/null +++ b/tests/libvirt_forwarding.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# copy to /etc/libvirt/hooks/qemu and restart libvirtd + +function manage_ports() +{ + local GUEST_IP=$1 + local GUEST_PORT=$2 + local HOST_PORT=$3 + local OP=$4 + + if [ "${OP}" = "stopped" ] || [ "${OP}" = "reconnect" ]; then + /sbin/iptables -D FORWARD -o virbr1 -p tcp -d $GUEST_IP --dport $GUEST_PORT -j ACCEPT + /sbin/iptables -t nat -D PREROUTING -p tcp --dport $HOST_PORT -j DNAT --to $GUEST_IP:$GUEST_PORT + fi + if [ "${OP}" = "start" ] || [ "${OP}" = "reconnect" ]; then + /sbin/iptables -I FORWARD -o virbr1 -p tcp -d $GUEST_IP --dport $GUEST_PORT -j ACCEPT + /sbin/iptables -t nat -I PREROUTING -p tcp --dport $HOST_PORT -j DNAT --to $GUEST_IP:$GUEST_PORT + fi +} + +VM_NAME="${1}" +OP="${2}" +GUEST_IP=192.168.121.243 + +# IMPORTANT: Change the "VM NAME" string to match your actual VM Name. +# In order to create rules to other VMs, just duplicate the below block and configure +# it accordingly. +[ "${VM_NAME}" = "nextcloudpi_default" ] || exit 0 + +manage_ports "${GUEST_IP}" 80 80 "${OP}" +manage_ports "${GUEST_IP}" 443 443 "${OP}" +manage_ports "${GUEST_IP}" 4443 4443 "${OP}" + +# these are for SMB +manage_ports "${GUEST_IP}" 137 137 "${OP}" +manage_ports "${GUEST_IP}" 138 138 "${OP}" +manage_ports "${GUEST_IP}" 139 139 "${OP}" +manage_ports "${GUEST_IP}" 445 445 "${OP}" +manage_ports "${GUEST_IP}" 900 900 "${OP}" diff --git a/tests/lxd_forwarding.sh b/tests/lxd_forwarding.sh new file mode 100755 index 00000000..6642713e --- /dev/null +++ b/tests/lxd_forwarding.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +GUEST_IP="${1}" +OP="${2}" +IFACE=lxdbr0 + +function manage_ports() +{ + local GUEST_IP=$1 + local GUEST_PORT=$2 + local HOST_PORT=$3 + local OP=$4 + + if [ "${OP}" = "stopped" ] || [ "${OP}" = "reconnect" ]; then + /sbin/iptables -D FORWARD -o "${IFACE}" -p tcp -d $GUEST_IP --dport $GUEST_PORT -j ACCEPT + /sbin/iptables -t nat -D PREROUTING -p tcp --dport $HOST_PORT -j DNAT --to $GUEST_IP:$GUEST_PORT + fi + if [ "${OP}" = "start" ] || [ "${OP}" = "reconnect" ]; then + /sbin/iptables -I FORWARD -o "${IFACE}" -p tcp -d $GUEST_IP --dport $GUEST_PORT -j ACCEPT + /sbin/iptables -t nat -I PREROUTING -p tcp --dport $HOST_PORT -j DNAT --to $GUEST_IP:$GUEST_PORT + fi +} + +manage_ports "${GUEST_IP}" 80 80 "${OP}" +manage_ports "${GUEST_IP}" 443 443 "${OP}" +manage_ports "${GUEST_IP}" 4443 4443 "${OP}" + +# these are for SMB +manage_ports "${GUEST_IP}" 137 137 "${OP}" +manage_ports "${GUEST_IP}" 138 138 "${OP}" +manage_ports "${GUEST_IP}" 139 139 "${OP}" +manage_ports "${GUEST_IP}" 445 445 "${OP}" +manage_ports "${GUEST_IP}" 900 900 "${OP}" diff --git a/update.sh b/update.sh index 4fa4e70e..9d0ebecd 100755 --- a/update.sh +++ b/update.sh @@ -8,6 +8,8 @@ # More at https://ownyourbits.com/ # +source /usr/local/etc/library.sh + set -e CONFDIR=/usr/local/etc/ncp-config.d/ @@ -23,8 +25,6 @@ nc-ramlogs nc-swapfile nc-static-IP nc-wifi -nc-nextcloud -nc-init UFW nc-snapshot nc-snapshot-auto @@ -39,10 +39,18 @@ NFS metrics " +if is_docker &>/dev/null; then +# in docker, just remove the volume for this +EXCL_DOCKER+=" +nc-nextcloud +nc-init +" + # better use a designated container EXCL_DOCKER+=" samba " +fi # check running apt pgrep apt &>/dev/null && { echo "apt is currently running. Try again later"; exit 1; } @@ -53,12 +61,12 @@ source /usr/local/etc/library.sh mkdir -p "$CONFDIR" -# prevent installing some ncp-apps in the docker version -[[ -f /.docker-image ]] && { +# prevent installing some ncp-apps in the containerized versions +if is_docker || is_lxc; then for opt in $EXCL_DOCKER; do touch $CONFDIR/$opt.cfg done -} +fi # copy all files in bin and etc cp -r bin/* /usr/local/bin/ @@ -144,16 +152,18 @@ rm -rf /var/www/nextcloud/apps/nextcloudpi cp -r /var/www/ncp-app /var/www/nextcloud/apps/nextcloudpi chown -R www-data: /var/www/nextcloud/apps/nextcloudpi -[[ -f /.docker-image ]] && { - # remove unwanted ncp-apps for the docker version +# remove unwanted ncp-apps for containerized versions +if is_docker || is_lxc; then for opt in $EXCL_DOCKER; do rm $CONFDIR/$opt.cfg find /usr/local/bin/ncp -name "$opt.sh" -exec rm '{}' \; done +fi - # update services - cp docker/{lamp/010lamp,nextcloud/020nextcloud,nextcloudpi/000ncp} /etc/services-enabled.d -} +# update services for docker +if is_docker; then + cp build/docker/{lamp/010lamp,nextcloud/020nextcloud,nextcloudpi/000ncp} /etc/services-enabled.d +fi # only live updates from here [[ -f /.ncp-image ]] && exit 0