From b675d61e61e11f14581eb82faeb5407cf61e0137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 15 Sep 2022 17:21:58 +0200 Subject: [PATCH] Upgrade to PHP 8.1 when installing NC >= 24 (#1554) * Update nextcloud to 24.0.4 * ncp-update-nc: Upgrade php to version 8.1 when installing NC >= 24 * ncp-update-nc: Use /etc/shadow workaround for installing systemd * ncp-update-nc: Run nc-limits after php upgrade * ncp-update-nc: Rollback after failed php upgrade * ncp-update-nc: Add success message * ncp-update-nc: Prevent installation of NC >= 24 on debian 10/PHP <= 7.3 * lamp.sh: Install php8.1 from sury.org * lamp.sh: Use /etc/shadow workaround for installing systemd * Dockerfile: Install wget, ca-certificates, lsb-release and procps before installing lamp.sh * Dockerfile: Make sure, ncp-templates are available when installing lamp.sh * Migrate all scripts to use template for writing opcache.ini and get_nc_config_value for retrieving datadir * nc-nextcloud.sh Fix crash if nc-datadir has not been installed yet * opcache.ini.sh: Don't try to get tmpl values from nc-datadir in containers --- .github/workflows/build-docker.yml | 8 --- bin/ncp-diag | 3 +- bin/ncp-dist-upgrade | 16 +---- bin/ncp-report | 5 +- bin/ncp-update-nc | 88 ++++++++++++++++++++++++- bin/ncp/BACKUPS/nc-backup.sh | 2 +- bin/ncp/BACKUPS/nc-restore-snapshot.sh | 2 +- bin/ncp/BACKUPS/nc-restore.sh | 6 +- bin/ncp/BACKUPS/nc-rsync-auto.sh | 4 +- bin/ncp/BACKUPS/nc-rsync.sh | 2 +- bin/ncp/BACKUPS/nc-snapshot-auto.sh | 2 +- bin/ncp/BACKUPS/nc-snapshot.sh | 2 +- bin/ncp/CONFIG/nc-database.sh | 16 +++-- bin/ncp/CONFIG/nc-datadir.sh | 39 +++++++++-- bin/ncp/CONFIG/nc-limits.sh | 62 ++++++++++------- bin/ncp/CONFIG/nc-nextcloud.sh | 21 ++++-- bin/ncp/NETWORKING/samba.sh | 14 ++-- bin/ncp/SYSTEM/metrics.sh | 27 +------- bin/ncp/TOOLS/nc-fix-permissions.sh | 4 +- bin/ncp/TOOLS/nc-format-USB.sh | 2 +- bin/ncp/TOOLS/nc-previews.sh | 2 +- build/build-LXC.sh | 3 +- build/build-LXD.sh | 1 + build/build-SD-rpi.sh | 2 + build/build-VM.sh | 2 +- build/docker/Dockerfile | 8 +-- build/docker/lamp/010lamp | 2 +- build/docker/nextcloud/020nextcloud | 4 +- etc/library.sh | 29 +++++++- etc/ncp-config.d/nc-nextcloud.cfg | 2 +- etc/ncp-templates/apache2/http2.conf.sh | 29 ++++++++ etc/ncp-templates/mysql/90-ncp.cnf.sh | 28 ++++++++ etc/ncp-templates/mysql/91-ncp.cnf.sh | 38 +++++++++++ etc/ncp-templates/ncp-metrics.cfg.sh | 4 +- etc/ncp-templates/nextcloud.conf.sh | 6 +- etc/ncp-templates/php/90-ncp.ini.sh | 33 ++++++++++ etc/ncp-templates/php/opcache.ini.sh | 28 ++++++++ etc/ncp.cfg | 4 +- install.sh | 2 +- lamp.sh | 76 +++------------------ updates/1.20.0.sh | 2 +- 41 files changed, 430 insertions(+), 200 deletions(-) create mode 100644 etc/ncp-templates/apache2/http2.conf.sh create mode 100644 etc/ncp-templates/mysql/90-ncp.cnf.sh create mode 100644 etc/ncp-templates/mysql/91-ncp.cnf.sh create mode 100644 etc/ncp-templates/php/90-ncp.ini.sh create mode 100644 etc/ncp-templates/php/opcache.ini.sh diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 4ad19b11..965e1fd5 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -35,14 +35,6 @@ jobs: steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 - - name: debugging - run: | - mount - echo 'DOCKER_OPTS="--storage-driver=devicemapper"' | sudo tee -a /etc/default/docker - sudo apt-get update && sudo apt-get -y upgrade - sudo systemctl restart docker - sudo apt-get install -y qemu-user-static - docker run --rm -v /usr/bin/qemu-arm-static:/usr/bin/qemu-arm-static arm32v7/debian:bullseye-slim bash -c 'apt-get update && apt-get install -y apache2 && apache2ctl -V' || true - name: Setup Docker Buildx uses: docker/setup-buildx-action@v1 diff --git a/bin/ncp-diag b/bin/ncp-diag index 0d64a30f..679b9f60 100644 --- a/bin/ncp-diag +++ b/bin/ncp-diag @@ -19,8 +19,7 @@ echo "NextCloudPi version|$( cat /usr/local/etc/ncp-version )" echo "OS|$(cat /etc/issue | sed 's| \\n \\l||'). $(uname -r) ($(uname -m))" # Data -DATADIR="$( grep datadirectory /var/www/nextcloud/config/config.php | - awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )" +DATADIR="$( get_nc_config_value datadirectory )" test -d "$DATADIR" || DIRINFO=" (doesn't exist)" USBDEVS="$( lsblk -S -o NAME,TRAN | awk '{ if ( $2 == "usb" ) print $1; }' | tr '\n' ' ' )" [[ "$USBDEVS" == "" ]] && USBDEVS="none" diff --git a/bin/ncp-dist-upgrade b/bin/ncp-dist-upgrade index a9e37d5e..f5fd13ba 100755 --- a/bin/ncp-dist-upgrade +++ b/bin/ncp-dist-upgrade @@ -95,20 +95,6 @@ $APTINSTALL -t ${release_new} php${php_ver_new}-gmp apt-get autoremove -y apt-get clean -# configure latest PHP version -cat > /etc/php/${php_ver_new}/mods-available/opcache.ini < /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <"; exit 1; } [[ -f /.docker-image ]] && BASEDIR=/data || BASEDIR=/var/www cd "$BASEDIR" -DATADIR="$( grep datadirectory nextcloud/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1 )" +DATADIR="$( get_nc_config_value datadirectory )" ncc status &>/dev/null || { echo "Nextcloud is currently down"; exit 1; } [[ -d /var/www/nextcloud-old ]] && { echo "Nextcloud backup directory found. Interrupted or already running installation?"; exit 1; } [[ -d /var/www/nextcloud ]] || { echo "Nextcloud directory not found" ; exit 1; } @@ -48,6 +48,12 @@ if [[ $((MAJOR_NEW - MAJOR_CUR)) -gt 1 ]]; then exit 1 fi +if [[ "$MAJOR_NEW" -ge 24 ]] && [[ "$(lsb_release -r)" =~ .*10 ]] +then + echo -e "Nextcloud version greater than 23 are not supported with Debian 10 (Buster). Please run ncp-dist-upgrade." + exit 1 +fi + grep -qP "\d+\.\d+\.\d+" <<<"$CURRENT" || { echo "Malformed version $CURRENT"; exit 1; } grep -qP "\d+\.\d+\.\d+" <<<"$VER" || { echo "Malformed version $VER" ; exit 1; } @@ -55,6 +61,12 @@ echo "Current Nextcloud version $CURRENT" echo "Available Nextcloud version $VER" is_more_recent_than "${VER}" "${CURRENT}" || { echo "Nothing to update"; exit 1; } # we want `exit 1` so the autoupdate doesn't notify success in this case +if ! is_more_recent_than "24.0.0" "${VER}" && is_more_recent_than "7.4.0" "${PHPVER}.0" +then + echo -e "Upgrading to Nextcloud versions > 23 requires the latest debian and PHP versions. Please run \`ncp-dist-upgrade\` and then run the update again." + exit 1 +fi + # make sure that cron.php is not running and there are no pending jobs # https://github.com/nextcloud/server/issues/10949 pgrep -cf cron.php &>/dev/null && { pkill -f cron.php; sleep 3; } @@ -179,7 +191,6 @@ $ncc | grep -q db:add-missing-columns && $ncc db:add-missing-columns -n $ncc | grep -q db:add-missing-primary-keys && $ncc db:add-missing-primary-keys -n $ncc | grep -q db:convert-filecache-bigint && $ncc db:convert-filecache-bigint -n - # use the correct version for custom apps NCVER="$(ncc status | grep "version:" | awk '{ print $3 }')" if is_more_recent_than "21.0.0" "${NCVER}"; then @@ -189,7 +200,7 @@ else if ! is_app_enabled notify_push; then ncc app:install notify_push ncc app:enable notify_push - bash /usr/local/etc/ncp-templates/nextcloud.conf.sh > /etc/apache2/sites-available/nextcloud.conf + install_template nextcloud.conf.sh /etc/apache2/sites-available/nextcloud.conf a2enmod proxy proxy_http proxy_wstunnel apachectl -k graceful ## make sure the notify_push daemon is runnnig @@ -224,9 +235,80 @@ fi rm -rf /var/www/nextcloud/apps/previewgenerator ln -snf "${NCPREV}" /var/www/nextcloud/apps/previewgenerator + +if ! is_more_recent_than "24.0.0" "${NCVER}" && is_more_recent_than "8.1.0" "${PHPVER}.0" +then + ( + echo "Upgrading PHP..." + export DEBIAN_FRONTEND=noninteractive + PHPVER_OLD="$PHPVER" + PHPVER_NEW="8.1" + PHP_PACKAGES_OLD=(php-{common,igbinary,redis} "php${PHPVER_OLD}" \ + "php${PHPVER_OLD}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,json,common,readline,mysql,bcmath,gmp}) + PHP_PACKAGES_NEW=("php${PHPVER_NEW}" php-json \ + "php${PHPVER_NEW}"-{curl,gd,fpm,cli,opcache,mbstring,xml,zip,fileinfo,ldap,intl,bz2,mysql,bcmath,gmp,redis,common}) + + php_restore() { + trap "" INT TERM HUP ERR + echo "Something went wrong while upgrading PHP. Rolling back to version ${PHPVER_OLD}..." + set +e + service "php${PHPVER_NEW}-fpm" stop + a2disconf php${PHPVER_NEW}-fpm + rm /etc/apt/sources.list.d/php.list + apt-get update + apt-get remove --purge -y "${PHP_PACKAGES_NEW[@]}" systemd + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_OLD[@]}" + set_ncpcfg "php_version" "${PHPVER_OLD}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + run_app nc-limits + a2enconf "php${PHPVER_OLD}-fpm" + service "php${PHPVER_OLD}-fpm" start + service apache2 restart + echo "PHP upgrade has been successfully reverted" + set -e + } + + trap php_restore INT TERM HUP ERR + + # Setup apt repository for php 8 + wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg + echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list + apt-get update + + echo "Stopping apache and php-fpm..." + service "php${PHPVER_OLD}-fpm" stop + service apache2 stop + + echo "Remove old PHP (${PHPVER_OLD})..." + a2disconf "php${PHPVER_OLD}-fpm" + + apt-get remove --purge -y "${PHP_PACKAGES_OLD[@]}" + + echo "Install PHP ${PHPVER_NEW}..." + install_with_shadow_workaround --no-install-recommends systemd + apt-get install -y --no-install-recommends -t "$RELEASE" "${PHP_PACKAGES_NEW[@]}" + + set_ncpcfg "php_version" "${PHPVER_NEW}" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER_NEW}/mods-available/opcache.ini" + ( export PHPVER="${PHPVER_NEW}"; run_app nc-limits ) + a2enconf php${PHPVER_NEW}-fpm + + echo "Starting apache and php-fpm..." + service "php${PHPVER_NEW}-fpm" start + service apache2 start + ncc status + ) + + # Reload library.sh to reset PHPVER + source /usr/local/etc/library.sh + +fi + + # refresh completions ncc _completion -g --shell-type bash -p ncc | sed 's|/var/www/nextcloud/occ|ncc|g' > /usr/share/bash-completion/completions/ncp +echo "Update completed successfully." # done #################### mkdir -p "$DATADIR"/ncp-update-backups diff --git a/bin/ncp/BACKUPS/nc-backup.sh b/bin/ncp/BACKUPS/nc-backup.sh index c00298e0..240ab409 100644 --- a/bin/ncp/BACKUPS/nc-backup.sh +++ b/bin/ncp/BACKUPS/nc-backup.sh @@ -43,7 +43,7 @@ occ="sudo -u www-data php /var/www/nextcloud/occ" [[ "$compress" == "yes" ]] && destfile="$destfile".gz -datadir=$( $occ config:system:get datadirectory ) || { +datadir=$( get_nc_config_value datadirectory ) || { echo "Error reading data directory. Is NextCloud running and configured?"; exit 1; } diff --git a/bin/ncp/BACKUPS/nc-restore-snapshot.sh b/bin/ncp/BACKUPS/nc-restore-snapshot.sh index e10bc148..eef88b85 100644 --- a/bin/ncp/BACKUPS/nc-restore-snapshot.sh +++ b/bin/ncp/BACKUPS/nc-restore-snapshot.sh @@ -16,7 +16,7 @@ configure() [[ -d "$SNAPSHOT" ]] || { echo "$SNAPSHOT doesn't exist"; return 1; } local datadir mountpoint - datadir=$( ncc config:system:get datadirectory ) || { + datadir=$( get_nc_config_value datadirectory ) || { echo -e "Error reading data directory. Is NextCloud running?"; return 1; } diff --git a/bin/ncp/BACKUPS/nc-restore.sh b/bin/ncp/BACKUPS/nc-restore.sh index 19deef5f..fc4de1b9 100644 --- a/bin/ncp/BACKUPS/nc-restore.sh +++ b/bin/ncp/BACKUPS/nc-restore.sh @@ -96,7 +96,7 @@ mysql -u root nextcloud < "$TMPDIR"/nextcloud-sqlbkp_*.bak || { echo "Error res if is_docker; then DATADIR=/data/nextcloud/data else - DATADIR="$(grep datadirectory "$NCDIR"/config/config.php | awk '{ print $3 }' | grep -oP "[^']*[^']" | head -1)" + DATADIR="$(get_nc_config_value datadirectory)" fi [[ "$DATADIR" == "" ]] && { echo "Error reading data directory"; exit 1; } @@ -107,7 +107,7 @@ cd "$NCDIR" NUMFILES=2 if [[ $( ls "$TMPDIR" | wc -l ) -eq $NUMFILES ]]; then - [[ -e "$DATADIR" ]] && { + [[ -e "$DATADIR" ]] && { echo "backing up existing $DATADIR to $DATADIR-$( date "+%m-%d-%y" )..." mv "$DATADIR" "$DATADIR-$( date "+%m-%d-%y" )" || exit 1 } @@ -149,7 +149,7 @@ fi sed -i "s|'datadirectory' =>.*|'datadirectory' => '${DATADIR}',|" "${NCDIR}"/config/config.php # Just in case we moved the opcache dir -sed -i "s|^opcache.file_cache=.*|opcache.file_cache=$DATADIR/.opcache|" /etc/php/${PHPVER}/mods-available/opcache.ini +install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" # tmp upload dir mkdir -p "$DATADIR/tmp" diff --git a/bin/ncp/BACKUPS/nc-rsync-auto.sh b/bin/ncp/BACKUPS/nc-rsync-auto.sh index ed2510a1..c9d5ae5f 100644 --- a/bin/ncp/BACKUPS/nc-rsync-auto.sh +++ b/bin/ncp/BACKUPS/nc-rsync-auto.sh @@ -16,14 +16,14 @@ install() configure() { - [[ $ACTIVE != "yes" ]] && { + [[ $ACTIVE != "yes" ]] && { rm -f /etc/cron.d/ncp-rsync-auto echo "automatic rsync disabled" return 0 } local DATADIR - DATADIR=$( ncc config:system:get datadirectory ) || { + DATADIR=$( get_nc_config_value datadirectory ) || { echo -e "Error reading data directory. Is NextCloud running and configured?"; return 1; } diff --git a/bin/ncp/BACKUPS/nc-rsync.sh b/bin/ncp/BACKUPS/nc-rsync.sh index b10e297a..91bc5399 100644 --- a/bin/ncp/BACKUPS/nc-rsync.sh +++ b/bin/ncp/BACKUPS/nc-rsync.sh @@ -19,7 +19,7 @@ configure() save_maintenance_mode local DATADIR - DATADIR=$( sudo -u www-data php /var/www/nextcloud/occ config:system:get datadirectory ) || { + DATADIR=$( get_nc_config_value datadirectory ) || { echo -e "Error reading data directory. Is NextCloud running and configured?"; return 1; } diff --git a/bin/ncp/BACKUPS/nc-snapshot-auto.sh b/bin/ncp/BACKUPS/nc-snapshot-auto.sh index 4d9d5b3a..51e26ce6 100644 --- a/bin/ncp/BACKUPS/nc-snapshot-auto.sh +++ b/bin/ncp/BACKUPS/nc-snapshot-auto.sh @@ -26,7 +26,7 @@ configure() cat > /etc/cron.hourly/btrfs-snp < /dev/null || TOTAL_MEM="$(( 1024 * 1024 * 1024 * 4 ))" - AUTOMEM=$(( TOTAL_MEM * 75 / 100 )) + local AUTOMEM=$(( TOTAL_MEM * 75 / 100 )) # MAX FILESIZE - local CONF=/etc/php/${PHPVER}/fpm/conf.d/90-ncp.ini - local CURRENT_FILE_SIZE="$( grep "^upload_max_filesize" "$CONF" | sed 's|.*=||' )" - [[ "$MAXFILESIZE" == "0" ]] && MAXFILESIZE=10G # MAX PHP MEMORY + local require_fpm_restart=false local CONF=/etc/php/${PHPVER}/fpm/conf.d/90-ncp.ini - local CURRENT_PHP_MEM="$( grep "^memory_limit" "$CONF" | sed 's|.*=||' )" - [[ "$MEMORYLIMIT" == "0" ]] && MEMORYLIMIT=$AUTOMEM && echo "Using ${AUTOMEM}B for PHP" - sed -i "s/^post_max_size=.*/post_max_size=$MAXFILESIZE/" "$CONF" - sed -i "s/^upload_max_filesize=.*/upload_max_filesize=$MAXFILESIZE/" "$CONF" - sed -i "s/^memory_limit=.*/memory_limit=$MEMORYLIMIT/" "$CONF" + local CONF_VALUE="$(cat "$CONF" || true)" + echo "Using $(tmpl_php_max_memory) for PHP max memory" + install_template "php/90-ncp.ini.sh" "$CONF" + [[ "$CONF_VALUE" == "$(cat "$CONF")" ]] || require_fpm_restart=true # MAX PHP THREADS local CONF=/etc/php/${PHPVER}/fpm/pool.d/www.conf @@ -37,25 +59,15 @@ configure() echo "Using $PHPTHREADS PHP threads" sed -i "s|^pm =.*|pm = static|" "$CONF" sed -i "s|^pm.max_children =.*|pm.max_children = $PHPTHREADS|" "$CONF" + [[ "$PHPTHREADS" == "$CURRENT_THREADS" ]] || require_fpm_restart=true - # DATABASE MEMORY (25%) - AUTOMEM=$(( TOTAL_MEM * 25 / 100 )) - # Maximum MySQL Memory Usage = innodb_buffer_pool_size + key_buffer_size + (read_buffer_size + sort_buffer_size) X max_connections - # leave 16MiB for key_buffer_size and a bit more - AUTOMEM=$(( AUTOMEM - (16 + 32) * 1024 * 1024 )) local CONF=/etc/mysql/mariadb.conf.d/91-ncp.cnf - local CURRENT_DB_MEM=$(grep "^innodb_buffer_pool_size" "$CONF" | awk '{ print $3 }') - echo "Using $AUTOMEM memory for the database" - [[ "$CURRENT_DB_MEM" != "$AUTOMEM" ]] && { - sed -i "s|^innodb_buffer_pool_size =.*|innodb_buffer_pool_size = $AUTOMEM|" "$CONF" - service mariadb restart - } + CONF_VALUE="$(cat "$CONF" || true)" + install_template "mysql/91-ncp.cnf.sh" "$CONF" + [[ "$CONF_VALUE" == "$(cat "$CONF")" ]] || service mariadb restart # RESTART PHP - [[ "$PHPTHREADS" != "$CURRENT_THREADS" ]] || \ - [[ "$MEMORYLIMIT" != "$CURRENT_PHP_MEM" ]] || \ - [[ "$MAXFILESIZE" != "$CURRENT_FILE_SIZE" ]] && \ - bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & + [[ "$require_fpm_restart" == "true" ]] && bash -c "sleep 3; service php${PHPVER}-fpm restart" &>/dev/null & # redis max memory local CONF=/etc/redis/redis.conf diff --git a/bin/ncp/CONFIG/nc-nextcloud.sh b/bin/ncp/CONFIG/nc-nextcloud.sh index 381aeb07..6a0c988a 100644 --- a/bin/ncp/CONFIG/nc-nextcloud.sh +++ b/bin/ncp/CONFIG/nc-nextcloud.sh @@ -14,6 +14,11 @@ REDIS_MEM=3gb APTINSTALL="apt-get install -y --no-install-recommends" export DEBIAN_FRONTEND=noninteractive +tmpl_max_transfer_time() +{ + find_app_param nc-nextcloud MAXTRANSFERTIME +} + install() { # During build, this step is run before ncp.sh. Avoid executing twice @@ -145,10 +150,18 @@ configure() fi # create and configure opcache dir - local OPCACHEDIR=/var/www/nextcloud/data/.opcache - sed -i "s|^opcache.file_cache=.*|opcache.file_cache=$OPCACHEDIR|" /etc/php/${PHPVER}/mods-available/opcache.ini - mkdir -p $OPCACHEDIR - chown -R www-data:www-data $OPCACHEDIR + local OPCACHEDIR="$( + # shellcheck disable=SC2015 + [ -f "${BINDIR}/CONFIG/nc-datadir.sh" ] && { source "${BINDIR}/CONFIG/nc-datadir.sh"; tmpl_opcache_dir; } || true + )" + if [[ -z "${OPCACHEDIR}" ]] + then + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" --defaults + else + mkdir -p "$OPCACHEDIR" + chown -R www-data:www-data "$OPCACHEDIR" + install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" + fi ## RE-CREATE DATABASE TABLE # launch mariadb if not already running (for docker build) diff --git a/bin/ncp/NETWORKING/samba.sh b/bin/ncp/NETWORKING/samba.sh index 6d33882c..ca58cb9d 100644 --- a/bin/ncp/NETWORKING/samba.sh +++ b/bin/ncp/NETWORKING/samba.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SAMBA server for Raspbian +# SAMBA server for Raspbian # # Copyleft 2017 by Ignacio Nunez Hernanz # GPL licensed (see end of file) * Use at your own risk! @@ -33,26 +33,26 @@ EOF configure() { - [[ $ACTIVE != "yes" ]] && { + [[ $ACTIVE != "yes" ]] && { service smbd stop update-rc.d smbd disable update-rc.d nmbd disable echo "SMB disabled" return - } + } # CHECKS ################################ local DATADIR - DATADIR=$( sudo -u www-data php /var/www/nextcloud/occ config:system:get datadirectory ) || { - echo -e "Error reading data directory. Is NextCloud running and configured?"; + DATADIR=$( get_nc_config_value datadirectory ) || { + echo -e "Error reading data directory. Is NextCloud running and configured?"; return 1; } [ -d "$DATADIR" ] || { echo -e "data directory $DATADIR not found" ; return 1; } # CONFIG ################################ - + # remove files from this line to the end sed -i '/# NextCloudPi automatically/,/\$/d' /etc/samba/smb.conf @@ -63,7 +63,7 @@ EOF # create a share per Nextcloud user local USERS=() - while read -r path; do + while read -r path; do USERS+=( "$( basename "$(dirname "$path")" )" ) done < <( ls -d "$DATADIR"/*/files ) diff --git a/bin/ncp/SYSTEM/metrics.sh b/bin/ncp/SYSTEM/metrics.sh index 9acccecb..82e58220 100644 --- a/bin/ncp/SYSTEM/metrics.sh +++ b/bin/ncp/SYSTEM/metrics.sh @@ -1,10 +1,5 @@ #!/bin/bash -apt_install_with_recommends() { - apt-get update --allow-releaseinfo-change - DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::="--force-confold" "$@" -} - is_supported() { [[ "${DOCKERBUILD:-0}" == 1 ]] && [[ "$(lsb_release -r)" =~ .*10 ]] && return 1 return 0 @@ -111,26 +106,8 @@ EOF # shellcheck disable=SC2016 sed -i 's|status_of_proc "$DAEMON" "$NAME" ${PIDFILE:="-p ${PIDFILE}"}|status_of_proc ${PIDFILE:+-p "$PIDFILE"} "$DAEMON" "$NAME"|' /lib/init/init-d-script - if is_docker - then - # during installation of prometheus-node-exporter `useradd` is used to create a user. - # However, `useradd` doesn't the symlink in /etc/shadow, so we need to temporarily move it back - restore_shadow=true - [[ -L /etc/shadow ]] || restore_shadow=false - [[ "$restore_shadow" == "false" ]] || { - trap "mv /etc/shadow /data/etc/shadow; ln -s /data/etc/shadow /etc/shadow" EXIT - rm /etc/shadow - cp /data/etc/shadow /etc/shadow - } - apt_install_with_recommends prometheus-node-exporter - [[ "$restore_shadow" == "false" ]] || { - mv /etc/shadow /data/etc/shadow - ln -s /data/etc/shadow /etc/shadow - } - trap - EXIT - else - apt_install_with_recommends prometheus-node-exporter - fi + apt-get update --allow-releaseinfo-change + install_with_shadow_workaround -o Dpkg::Options::=--force-confdef -o Dpkg::Options::="--force-confold" prometheus-node-exporter if is_docker then diff --git a/bin/ncp/TOOLS/nc-fix-permissions.sh b/bin/ncp/TOOLS/nc-fix-permissions.sh index de08731e..68a2438e 100644 --- a/bin/ncp/TOOLS/nc-fix-permissions.sh +++ b/bin/ncp/TOOLS/nc-fix-permissions.sh @@ -9,10 +9,10 @@ # -configure() +configure() { local DATADIR - DATADIR=$( cd /var/www/nextcloud; sudo -u www-data php occ config:system:get datadirectory ) || { + DATADIR=$( get_nc_config_value datadirectory ) || { echo "data directory not found"; return 1; } diff --git a/bin/ncp/TOOLS/nc-format-USB.sh b/bin/ncp/TOOLS/nc-format-USB.sh index 14d3970b..f3483429 100644 --- a/bin/ncp/TOOLS/nc-format-USB.sh +++ b/bin/ncp/TOOLS/nc-format-USB.sh @@ -50,7 +50,7 @@ configure() return 1; } - DATADIR="$(ncc config:system:get datadirectory || true)" + DATADIR="$(get_nc_config_value datadirectory || true)" if [[ $( stat -fc%d / ) != $( stat -fc%d "$DATADIR" ) ]] || [[ -z "$DATADIR" ]] && [[ "$ALLOW_DATA_DIR_REMOVAL" != "yes" ]] then echo "ERROR: Data directory is on USB drive (or can't be determined) and removal of data directory was not explicitly allowed." \ diff --git a/bin/ncp/TOOLS/nc-previews.sh b/bin/ncp/TOOLS/nc-previews.sh index 13251455..dc18661d 100644 --- a/bin/ncp/TOOLS/nc-previews.sh +++ b/bin/ncp/TOOLS/nc-previews.sh @@ -18,7 +18,7 @@ configure() [[ "$CLEAN" == "yes" ]] && { local datadir - datadir=$( ncc config:system:get datadirectory ) || { + datadir=$( get_nc_config_value datadirectory ) || { echo "data directory not found"; return 1; } diff --git a/build/build-LXC.sh b/build/build-LXC.sh index dc0456b5..76996ff3 100755 --- a/build/build-LXC.sh +++ b/build/build-LXC.sh @@ -33,12 +33,13 @@ prepare_dirs # tmp cache output # TODO sudo sudo lxc-destroy ncp -f -sudo lxc-create -n ncp -t download -B btrfs -- --dist debian --release buster --arch amd64 # TODO vars for distro and stuff +sudo lxc-create -n ncp -t download -B btrfs -- --dist debian --release buster --arch amd64 # TODO vars for distro and stuff sudo cp lxc_config /var/lib/lxc/ncp/config sudo lxc-start -n ncp sudo lxc-attach -n ncp --clear-env -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sudo lxc-attach -n ncp --clear-env -- CODE_DIR="$(pwd)" bash /build/install.sh sudo lxc-attach -n ncp --clear-env -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +sudo lxc-attach -n ncp --clear-env -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" sudo lxc-attach -n ncp --clear-env -- poweroff exit 0 # TODO diff --git a/build/build-LXD.sh b/build/build-LXD.sh index bf639347..c695e769 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -37,6 +37,7 @@ lxc config device add ncp buildcode disk source="$(pwd)" path=/build lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' lxc exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +lxc exec ncp -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" lxc stop ncp lxc config device remove ncp buildcode lxc publish -q ncp -f --alias ncp/"${version}" diff --git a/build/build-SD-rpi.sh b/build/build-SD-rpi.sh index e93fff51..9da95db0 100755 --- a/build/build-SD-rpi.sh +++ b/build/build-SD-rpi.sh @@ -92,6 +92,8 @@ PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ rm -rf /tmp/ncp-build EOFCHROOT +basename "$IMG" > raspbian_root/usr/local/etc/ncp-baseimage + trap '' EXIT clean_chroot_raspbian diff --git a/build/build-VM.sh b/build/build-VM.sh index c00f5539..87e4e654 100755 --- a/build/build-VM.sh +++ b/build/build-VM.sh @@ -16,7 +16,7 @@ echo -e "\e[1m\n[ Build NCP VM ]\e[0m" IP=${1:-192.168.0.145} # For QEMU automated testing (optional) SIZE=3G # Raspbian image size #CLEAN=0 # Pass this envvar to skip cleaning download cache -IMG="NextCloudPi_VM_$( date "+%m-%d-%y" ).img" +IMG="${IMG:-NextCloudPi_VM_$( date "+%m-%d-%y" ).img}" IMG=tmp/"$IMG" VM="/var/lib/libvirt/images/ncp-vm.img" diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 6ca995c5..e995d9b5 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -30,13 +30,14 @@ SHELL ["/bin/bash", "-c"] ENV DOCKERBUILD 1 COPY etc/ncp.cfg etc/library.sh lamp.sh /usr/local/etc/ +COPY etc/ncp-templates /usr/local/etc/ncp-templates RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ # installation apt-get update; \ -apt-get install --no-install-recommends -y jq; \ +apt-get install --no-install-recommends -y jq wget procps ca-certificates lsb-release; \ source /usr/local/etc/library.sh; \ set +x; \ @@ -48,7 +49,7 @@ mysqladmin -u root shutdown; \ # mariaDB fixups (move database to /data-ro, which will be in a persistent volume) mkdir -p /data-ro /data; \ mv /var/lib/mysql /data-ro/database; \ -sed -i "s|^datadir.*|datadir = /data-ro/database|" /etc/mysql/mariadb.conf.d/90-ncp.cnf; \ +install_template "mysql/90-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/90-ncp.cnf" \ # package cleanup apt-get autoremove -y; \ @@ -83,7 +84,6 @@ ENV DOCKERBUILD 1 COPY etc/library.sh /usr/local/etc/ COPY bin/ncp/CONFIG/nc-nextcloud.sh / COPY etc/ncp-config.d/nc-nextcloud.cfg /usr/local/etc/ncp-config.d/ -COPY etc/ncp-templates /usr/local/etc/ncp-templates RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ @@ -96,7 +96,7 @@ touch /.docker-image; \ # installation ( /var/www/nextcloud -> /data/app which will be in a volume ) apt-get update; \ -apt-get install --no-install-recommends -y wget ca-certificates sudo jq; \ +apt-get install --no-install-recommends -y sudo jq; \ source /usr/local/etc/library.sh; \ install_app /nc-nextcloud.sh; \ run_app_unsafe /nc-nextcloud.sh; \ diff --git a/build/docker/lamp/010lamp b/build/docker/lamp/010lamp index e7ca59c1..5139c01c 100755 --- a/build/docker/lamp/010lamp +++ b/build/docker/lamp/010lamp @@ -31,7 +31,7 @@ echo "Starting Apache" /usr/sbin/apache2ctl start # adjust the dbdir to the persistent storage -sed -i "s|^datadir.*|datadir = /data/database|" /etc/mysql/mariadb.conf.d/90-ncp.cnf +install_template "mysql/90-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/90-ncp.cnf" # start echo "Starting mariaDB" diff --git a/build/docker/nextcloud/020nextcloud b/build/docker/nextcloud/020nextcloud index e2ff13c6..edfe4b1d 100755 --- a/build/docker/nextcloud/020nextcloud +++ b/build/docker/nextcloud/020nextcloud @@ -71,8 +71,10 @@ echo "Configuring Domain" bash /usr/local/bin/nextcloud-domain.sh # Trusted Domain (as an argument) -[[ "$2" != "" ]] && \ +[[ -z "$2" ]] || { ncc config:system:set trusted_domains 6 --value="$2" + ncc notify_push:self-test > /dev/null 2>&1 || ncc notify_push:setup "https://$2/push" +} echo "Nextcloud version $(nc_version). NextCloudPi version $(cat /usr/local/etc/ncp-version)" diff --git a/etc/library.sh b/etc/library.sh index b9cca167..b7fc0354 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -217,7 +217,7 @@ install_template() { { bash "/usr/local/etc/ncp-templates/$template" --defaults > "$target"; } 2>&1 else { bash "/usr/local/etc/ncp-templates/$template" > "$target"; } 2>&1 || \ - if [[ "${3:}" == "--allow-fallback" ]]; then + if [[ "${3:-}" == "--allow-fallback" ]]; then { bash "/usr/local/etc/ncp-templates/$template" --defaults > "$target"; } 2>&1 fi fi @@ -385,6 +385,26 @@ function persistent_cfg() ln -s "$DST" "$SRC" } +function install_with_shadow_workaround() +{ + # Subshell to trap trap :P + ( + restore_shadow=true + [[ -L /etc/shadow ]] || restore_shadow=false + [[ "$restore_shadow" == "false" ]] || { + trap "mv /etc/shadow /data/etc/shadow; ln -s /data/etc/shadow /etc/shadow" EXIT + rm /etc/shadow + cp /data/etc/shadow /etc/shadow + } + DEBIAN_FRONTEND=noninteractive apt-get install -y "$@" + [[ "$restore_shadow" == "false" ]] || { + mv /etc/shadow /data/etc/shadow + ln -s /data/etc/shadow /etc/shadow + } + trap - EXIT + ) +} + function is_more_recent_than() { local version_A="$1" @@ -470,7 +490,7 @@ function apt_install() } function is_docker() { - [[ -f /.dockerenv ]] || [[ "$DOCKERBUILD" == 1 ]] + [[ -f /.dockerenv ]] || [[ -f /.docker-image ]] || [[ "$DOCKERBUILD" == 1 ]] } function is_lxc() { @@ -525,6 +545,11 @@ function get_ncpcfg() jq -r ".${name}" < "${NCPCFG}" } +function get_nc_config_value() { + sudo -u www-data php -r "include(\"/var/www/nextcloud/config/config.php\"); echo(\$CONFIG[\"${1?Missing required argument: config key}\"]);" + #ncc config:system:get "${1?Missing required argument: config key}" +} + # License # # This script is free software; you can redistribute it and/or modify it diff --git a/etc/ncp-config.d/nc-nextcloud.cfg b/etc/ncp-config.d/nc-nextcloud.cfg index e3dff293..c240b908 100644 --- a/etc/ncp-config.d/nc-nextcloud.cfg +++ b/etc/ncp-config.d/nc-nextcloud.cfg @@ -9,7 +9,7 @@ { "id": "VER", "name": "Version", - "value": "24.0.3" + "value": "24.0.4" }, { "id": "MAXFILESIZE", diff --git a/etc/ncp-templates/apache2/http2.conf.sh b/etc/ncp-templates/apache2/http2.conf.sh new file mode 100644 index 00000000..17dea501 --- /dev/null +++ b/etc/ncp-templates/apache2/http2.conf.sh @@ -0,0 +1,29 @@ +#! /bin/bash + +set -e +source /usr/local/etc/library.sh + +cat > /etc/apache2/conf-available/http2.conf < /etc/mysql/mariadb.conf.d/91-ncp.cnf <&2 - apache2ctl -t 2>&1 | tee /var/log/ncp.log >&2 + echo -e "Apache self check:" + apache2ctl -t 1>&2 fi diff --git a/etc/ncp-templates/php/90-ncp.ini.sh b/etc/ncp-templates/php/90-ncp.ini.sh new file mode 100644 index 00000000..bc686b19 --- /dev/null +++ b/etc/ncp-templates/php/90-ncp.ini.sh @@ -0,0 +1,33 @@ +#! /bin/bash + +set -e +source /usr/local/etc/library.sh + +MAXTRANSFERTIME="3600" +if [[ "$1" == "--defaults" ]] +then + MAXFILESIZE="10G" + MEMORYLIMIT="768M" +else + MAXFILESIZE="$(source "${BINDIR}/CONFIG/nc-limits.sh" && tmpl_php_max_filesize)" + MEMORYLIMIT="$(source "${BINDIR}/CONFIG/nc-limits.sh" && tmpl_php_max_memory)" + [[ -f "${BINDIR}/CONFIG/nc-nextcloud.sh" ]] && MAXTRANSFERTIME="$(source "${BINDIR}/CONFIG/nc-nextcloud.sh" && tmpl_max_transfer_time)" +fi + +cat </dev/null && mysql -e 'use nextcloud' &>/dev/null && { echo "The ' # get dependencies apt-get update -apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release +apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget # get install code if [[ "${CODE_DIR}" == "" ]]; then diff --git a/lamp.sh b/lamp.sh index 11fbc237..d535fa9b 100644 --- a/lamp.sh +++ b/lamp.sh @@ -25,6 +25,9 @@ export DEBIAN_FRONTEND=noninteractive install() { set -x + # Setup apt repository for php 8 + wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg + echo "deb https://packages.sury.org/php/ ${RELEASE%-security} main" > /etc/apt/sources.list.d/php.list apt-get update $APTINSTALL apt-utils cron curl ls -l /var/lock || true @@ -33,9 +36,10 @@ install() mkdir -p /run/lock apache2ctl -V || true + install_with_shadow_workaround --no-install-recommends systemd $APTINSTALL -t $RELEASE php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ - php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json + php${PHPVER}-intl php${PHPVER}-bz2 php-json mkdir -p /run/php @@ -53,46 +57,12 @@ install() # CONFIGURE APACHE ########################################## - cat > /etc/apache2/conf-available/http2.conf < /etc/php/${PHPVER}/mods-available/opcache.ini < /etc/mysql/mariadb.conf.d/90-ncp.cnf < /etc/mysql/mariadb.conf.d/91-ncp.cnf < " '{ print $2 }' | sed "s|[,']||g") logo_dir="${datadir}/appdata_${id}/theming/images" [[ -f "${logo_dir}"/logo ]] && {