External-Mirrors/nextcloudpi
2
0
Fork 0
mirror of https://github.com/nextcloud/nextcloudpi.git synced 2026-01-09 14:42:01 -03:30
Code Issues Packages Projects Releases Wiki Activity

build-docker.yml: Add integration tests for docker images and only push

Browse Source 
on success

- build/docker/, build/build-docker.sh: Migrate to multistage docker images
- tests/: Fix tests for GH workflow scenarios
This commit is contained in:
Tobias K 2022-04-28 21:41:15 +02:00
commit 98be0220c2
No known key found for this signature in database
GPG Key ID: 3510056072886A8F
17 changed files with 825 additions and 247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
.github/actions/create-test-instance/action.yml vendored Normal file
View File

@ -0,0 +1,35 @@
name: Create Test VM
description: Create NCP instance for testing in the Hetzner cloud
inputs:
version:
description: version (git rev / tag / branch) to install
required: true
uid:
description: A unique ID for labeling/naming generated resources
required: true
hcloud_token:
description: A auth token for Hetzner cloud
required: true
server_type:
description: Server type to use for hetzner servers
required: true
default: "cx11"
outputs:
server_address:
description: Adress of the test instance
snapshot_id:
description: ID of the generated postinstall snapshot
test_server_id:
description: ID of the created test server
runs:
using: docker
image: docker://thecalcaholic/ncp-test-automation
env:
HCLOUD_TOKEN: ${{ inputs.hcloud_token }}
UID: ${{ inputs.uid }}
SERVER_TYPE: ${{ inputs.server_type }}
args:
- /ncp-test-automation/bin/actions/create-test-instance.sh
- ${{ inputs.version }}

147
.github/workflows/build-docker.yml vendored Normal file
View File

@ -0,0 +1,147 @@
# https://www.docker.com/blog/docker-v2-github-action-is-now-ga/
# https://www.docker.com/blog/multi-arch-build-and-images-the-simple-way/
# https://docs.github.com/en/actions/guides/publishing-docker-images
name: 'Docker Integration Tests and Release'
on:
push:
branches:
- "*"
tags:
- 'v*'
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
arch:
- x86
- armhf
- arm64
fail-fast: false
steps:
- name: Setup Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Checkout code
uses: actions/checkout@v3
- name: Login to docker
run: |
echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin
- name: Build images
id: build-container
run: |
./build/build-docker.sh "${{ matrix.arch }}"
docker tag "ownyourbits/nextcloudpi-${{ matrix.arch }}:latest" "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}"
docker push "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}"
test:
needs:
- build
runs-on: ubuntu-latest
strategy:
matrix:
arch:
- armhf
- x86
- arm64
fail-fast: false
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Login to docker
run: |
echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin
- name: Start ncp container
run: |
docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}
- name: Checkout code
uses: actions/checkout@v3
- name: Setup Firefox
uses: browser-actions/setup-firefox@latest
- name: Setup GeckoDriver
uses: browser-actions/setup-geckodriver@latest
- name: Setup Selenium
run: pip install selenium
- name: Integration Tests
working-directory: ./tests
run: |
python activation_tests.py --no-gui localhost 8443 4443 || {
tail -n 20 geckodriver.log >&2 || true
echo "======================="
echo "Activation test failed!"
echo "Container logs:"
echo "==========================================="
docker logs nextcloudpi
echo "Last lines of ncp.log:"
echo "==========================================="
docker exec nextcloudpi tail /var/log/ncp.log;
exit 1
}
echo "Activation test successful"
python system_tests.py --no-ping --non-interactive || {
echo "System test failed!"
exit 1
}
echo "System test successful"
python nextcloud_tests.py --no-gui localhost 8443 4443 || {
tail -n 20 geckodriver.log >&2 || true
echo "======================="
echo "Nextcloud test failed!"
echo "Container logs:"
echo "==========================================="
docker logs nextcloudpi
echo "Last lines of ncp.log:"
echo "==========================================="
docker exec nextcloudpi tail /var/log/ncp.log;
exit 1
}
echo "Nextcloud test successful"
release:
needs:
- test
if: ${{ github.event_name == 'push' && github.github.ref_type == 'tag' }}
runs-on: ubuntu-latest
steps:
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Create manifest and push as tag to docker hub
run: |
. ./build/buildlib.sh
for arch in x86 armhf arm64
do
docker pull "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}"
docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}"
done
docker manifest create ownyourbits/nextcloudpi:${version?} \
ownyourbits/nextcloudpi-armhf:${version?} \
ownyourbits/nextcloudpi-x86:${version?} \
ownyourbits/nextcloudpi-arm64:${version?}
docker manifest push ownyourbits/nextcloudpi:${version?}
- name: Create manifest and push as latest to docker hub
run: |
docker manifest create ownyourbits/nextcloudpi:latest \
ownyourbits/nextcloudpi-armhf:latest \
ownyourbits/nextcloudpi-x86:latest \
ownyourbits/nextcloudpi-arm64:latest
docker manifest push ownyourbits/nextcloudpi:latest

350
.github/workflows/vm-tests.yml vendored Normal file
View File

@ -0,0 +1,350 @@
name: 'VM Integration Tests'
on:
workflow_dispatch:
inputs:
version:
description: git ref, branch or tag to test against
required: false
type: string
push:
branches:
- master
- devel
tags:
- v*
pull_request:
jobs:
setup-installation-test-instance:
runs-on: ubuntu-latest
outputs:
server_address: ${{ steps.create-test-instance.outputs.server_address }}
snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
version: ${{ env.VERSION }}
env:
VERSION: "${{ github.event.inputs.version || github.head_ref || github.ref_name }}"
steps:
- uses: actions/checkout@v3
- run: |
set -e
mkdir -p ./.ssh
ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
- name: upload ssh private key to artifact store
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-install-ssh-privkey
path: .ssh
if-no-files-found: error
- id: create-test-instance
uses: ./.github/actions/create-test-instance
with:
version: ${{ env.VERSION }}
uid: "${{ github.run_id }}-install"
hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
server_type: "cx11"
setup-update-test-instance:
runs-on: ubuntu-latest
outputs:
server_address: ${{ steps.create-test-instance.outputs.server_address }}
snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
previous_version: ${{ steps.find-version.outputs.previous_version }}
version: ${{ env.VERSION }}
env:
VERSION: "${{ github.event.inputs.version || github.head_ref || github.ref_name }}"
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- shell: bash
id: find-version
run: |
set -e
if [[ -n "${{ github.base_ref }}" ]]
then
version="${{ github.base_ref }}"
elif [[ "${{ github.ref }}" == "refs/heads/devel" ]]
then
version="master"
else
git fetch -fu --tags origin ${{ github.ref }}:${{ github.ref }}
version="$(git describe --tags)"
[[ "$version" =~ .*-.*-.* ]] || {
git checkout HEAD~1
version="$(git describe --tags)"
}
version="${version%-*-*}"
fi
echo "Previous version is '$version'"
echo "::set-output name=previous_version::${version}"
- run: |
set -x
mkdir -p ./.ssh
ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
- name: upload ssh private key to artifact store
uses: actions/upload-artifact@v3
with:
name: ${{ github.run_id }}-update-ssh-privkey
path: .ssh
if-no-files-found: error
- id: create-test-instance
uses: ./.github/actions/create-test-instance
with:
version: "${{ steps.find-version.outputs.previous_version }}"
uid: "${{ github.run_id }}-update"
hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
server_type: "cx11"
run-installation-test:
needs:
- setup-installation-test-instance
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-install"
env:
VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}"
SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}"
HOME: /root
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
repository: 'theCalcaholic/ncp-test-automation'
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
with:
name: ${{ github.run_id }}-install-ssh-privkey
path: .ssh
- name: Test postinstall VM
run: |
set -e
echo "Setup ssh"
chmod 0600 ./.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
cd bin
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
setup-ssh-port-forwarding "$SERVER_ADDRESS"
echo "Run integration tests"
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
run-update-test:
needs:
- setup-update-test-instance
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
UID: "${{ github.run_id }}-update"
env:
PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }}
VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}"
SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}"
HOME: /root
defaults:
run:
shell: bash
steps:
- uses: actions/checkout@v3
with:
repository: 'theCalcaholic/ncp-test-automation'
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
with:
name: ${{ github.run_id }}-update-ssh-privkey
path: .ssh
- name: perform update
run: |
set -e
echo "Setup ssh"
chmod 0600 ./.ssh/automation_ssh_key
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
. ./bin/library.sh
echo "Updating from $PREVIOUS_VERSION to $VERSION"
ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
- name: Run integration tests
run: |
set -e
echo "Setup ssh"
eval "$(ssh-agent)"
ssh-add ./.ssh/automation_ssh_key
cd bin
source ./library.sh
trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
echo "Run integration tests"
setup-ssh-port-forwarding "$SERVER_ADDRESS"
test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
echo "Integration tests failed"
echo "Here are the last lines of ncp-install.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
echo "==========================================="
echo "and ncp.log:"
echo "==========================================="
ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
echo "==========================================="
exit 1
}
create-postactivation-snapshots:
if: ${{ always() }}
needs:
- setup-installation-test-instance
- setup-update-test-instance
- run-installation-test
- run-update-test
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
strategy:
matrix:
test_type: [install, update]
include:
- test_type: install
server_address: ${{ needs.setup-installation-test-instance.outputs.server_address }}
test_result: ${{ needs.setup-installation-test-instance.result }}
test_server_id: ${{ needs.setup-installation-test-instance.outputs.test_server_id }}
version: ${{ needs.setup-installation-test-instance.outputs.version }}
- test_type: update
server_address: ${{ needs.setup-update-test-instance.outputs.server_address }}
test_result: ${{ needs.setup-update-test-instance.result }}
test_server_id: ${{ needs.setup-update-test-instance.outputs.test_server_id }}
version: ${{ needs.setup-update-test-instance.outputs.version }}
fail-fast: false
env:
UID: ${{ github.run_id }}-${{ matrix.test_type }}
VERSION: ${{ matrix.version }}
steps:
- name: download ssh private key from artifact store
uses: actions/download-artifact@v3
if: ${{ contains('success|failure', matrix.test_result) }}
with:
name: ${{ github.run_id }}-${{ matrix.test_type }}-ssh-privkey
path: /github/workspace/.ssh
- name: Shutdown server
if: ${{ contains('success|failure', matrix.test_result) }}
run: |
chmod 0600 /github/workspace/.ssh/automation_ssh_key
export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
bash /ncp-test-automation/bin/entrypoint.sh
eval "$(ssh-agent)"
ssh-add /github/workspace/.ssh/automation_ssh_key
ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${{ matrix.server_address }}" <<EOF
systemctl stop mariadb
systemctl poweroff
EOF
- name: Create Snapshot
if: ${{ contains('success|failure', matrix.test_result) }}
shell: bash
run: |
set -x
echo "${{ needs.setup-installation-test-instance.outputs.test_server_id }}"
echo "${{ needs.setup-update-test-instance.outputs.test_server_id }}"
echo "${{ matrix.test_server_id }}"
cd /ncp-test-automation/bin
. ./library.sh
tf-init "$TF_SNAPSHOT"
tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${{ matrix.version }}" -var="snapshot_provider_id=${{ matrix.test_server_id }}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
hcloud image add-label -o "$snapshot_id" "test-result=${{ matrix.test_result }}"
cleanup:
if: ${{ always() }}
needs:
- create-postactivation-snapshots
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
strategy:
matrix:
uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update"]
fail-fast: false
env:
HOME: '/root'
UID: ${{ matrix.uid }}
defaults:
run:
shell: bash
working-directory: /ncp-test-automation/bin
steps:
- name: Teardown VMs
run: |
for server in $(hcloud server list -o noheader -o columns=id -l "ci=${UID}")
do
echo "Deleting server '$server'..."
hcloud server delete "$server"
echo "done."
done
- name: Delete ssh key
run: |
source ./library.sh
hcloud-clear-root-key
cleanup-snapshots:
if: ${{ always() }}
needs:
- cleanup
runs-on: ubuntu-latest
container:
image: thecalcaholic/ncp-test-automation:latest
env:
HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
env:
HOME: '/root'
steps:
- name: Delete old snapshots
run: |
for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -20)
do
echo "Deleting snapshot '$snapshot'..."
hcloud image delete "$snapshot"
echo "done."
done

2
README.md
View File

@ -13,6 +13,8 @@ This code also generates the NextCloudPi [docker image](https://hub.docker.com/r
Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com)
[![VM Integration Tests](https://github.com/nextcloud/nextcloudpi/workflows/VM%20Integration%20Tests/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/vm-tests.yml)
## Features
* Debian/Raspbian 11 Bullseye

2
bin/ncp/NETWORKING/dnsmasq.sh
View File

@ -15,7 +15,7 @@ install()
apt-get install --no-install-recommends -y dnsmasq
rc=0
service dnsmasq status > /dev/null 2>&1 || rc=$?
[[ $rc -eq 3 ]] && {
! is_docker && [[ $rc -eq 3 ]] && {
echo "Applying workaround for dnsmasq bug (compare issue #1446)"
service systemd-resolved stop
service dnsmasq start

33
bin/ncp/SYSTEM/metrics.sh
View File

@ -62,10 +62,14 @@ metrics_services() {
exit 1
fi
if [[ "$DOCKERBUILD" == 1 ]]
if is_docker
then
update-rc.d ncp-metrics "$cmd"
return $?
rc1=0
rc2=0
update-rc.d ncp-metrics-exporter "$cmd" || rc1=$?
update-rc.d prometheus-node-exporter "$cmd" || rc2=$?
[[ $rc1 > $rc2 ]] && return $rc1
return $rc2
else
systemctl "$cmd" prometheus-node-exporter ncp-metrics-exporter
return $?
@ -92,7 +96,7 @@ EOF
[[ "${arch}" =~ ^"arm" ]] && arch="armv7"
mkdir -p /usr/local/lib/ncp-metrics
wget -O "/usr/local/lib/ncp-metrics/ncp-metrics-exporter" \
wget -qO "/usr/local/lib/ncp-metrics/ncp-metrics-exporter" \
"https://github.com/theCalcaholic/ncp-metrics-exporter/releases/download/v1.1.0/ncp-metrics-exporter-${arch}"
chmod +x /usr/local/lib/ncp-metrics/ncp-metrics-exporter
@ -101,22 +105,28 @@ EOF
# shellcheck disable=SC2016
sed -i 's|status_of_proc "$DAEMON" "$NAME" ${PIDFILE:="-p ${PIDFILE}"}|status_of_proc ${PIDFILE:+-p "$PIDFILE"} "$DAEMON" "$NAME"|' /lib/init/init-d-script
if [[ "$DOCKERBUILD" == 1 ]]
if is_docker
then
# during installation of prometheus-node-exporter `useradd` is used to create a user.
# However, `useradd` doesn't the symlink in /etc/shadow, so we need to temporarily move it back
trap "mv /etc/shadow /data/etc/shadow; ln -s /data/etc/shadow /etc/shadow" EXIT
rm /etc/shadow
cp /data/etc/shadow /etc/shadow
restore_shadow=true
[[ -L /etc/shadow ]] || restore_shadow=false
[[ "$restore_shadow" == "false" ]] || {
trap "mv /etc/shadow /data/etc/shadow; ln -s /data/etc/shadow /etc/shadow" EXIT
rm /etc/shadow
cp /data/etc/shadow /etc/shadow
}
apt_install_with_recommends prometheus-node-exporter
mv /etc/shadow /data/etc/shadow
ln -s /data/etc/shadow /etc/shadow
[[ "$restore_shadow" == "false" ]] || {
mv /etc/shadow /data/etc/shadow
ln -s /data/etc/shadow /etc/shadow
}
trap - EXIT
else
apt_install_with_recommends prometheus-node-exporter
fi
if [[ "$DOCKERBUILD" == 1 ]]
if is_docker
then
cat > /etc/init.d/ncp-metrics-exporter <<'EOF'
#!/bin/sh
@ -144,6 +154,7 @@ LOGFILE=/var/log/ncp-metrics.log
START_ARGS="--background --make-pidfile"
EOF
chmod +x /etc/init.d/ncp-metrics-exporter
update-rc.d ncp-metrics-exporter defaults
cat > /etc/services-available.d/101ncp-metrics <<EOF
#!/bin/bash

61
build/build-docker.sh
View File

@ -11,34 +11,53 @@ release=$(jq -r .release < etc/ncp.cfg)
function docker_build() { DOCKER_BUILDKIT=1 docker build --progress=plain . "$@"; }
function build_arch()
{
local release="${1}"
local arch="${2}"
local arch_qemu="${3}"
local ncp_tag="${4:-$arch}"
build_arch() {
local target="${1?}"
local release="${2?}"
local arch="${3?}"
local arch_qemu="${4?}"
local suffix="${5:-$arch}"
echo -e "\e[1m\n[ Build NCP Docker ${arch} ]\e[0m"
DOCKER_BUILDKIT=1 docker build --pull --progress=plain . -f build/docker/Dockerfile \
--target "$target" -t "ownyourbits/$target-${suffix}:latest" \
--cache-from "ownyourbits/nextcloudpi-${suffix}" --build-arg "release=$release" --build-arg "arch=${arch}" \
--build-arg "arch_qemu=$arch_qemu" --build-arg "ncp_ver=${version?}"
docker_build -f build/docker/debian-ncp/Dockerfile -t ownyourbits/debian-ncp-${ncp_tag}:latest --pull --build-arg release=${release} --build-arg arch=${arch} --build-arg arch_qemu=${arch_qemu}
docker_build -f build/docker/lamp/Dockerfile -t ownyourbits/lamp-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} --build-arg arch_qemu=${arch_qemu}
docker_build -f build/docker/nextcloud/Dockerfile -t ownyourbits/nextcloud-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} --build-arg arch_qemu=${arch_qemu}
docker_build -f build/docker/nextcloudpi/Dockerfile -t ownyourbits/nextcloudpi-${ncp_tag}:latest --build-arg release=${release} --build-arg arch=${ncp_tag} --build-arg arch_qemu=${arch_qemu} --build-arg ncp_ver=${version}
docker tag ownyourbits/debian-ncp-${ncp_tag}:latest ownyourbits/debian-ncp-${ncp_tag}:"${version}"
docker tag ownyourbits/lamp-${ncp_tag}:latest ownyourbits/lamp-${ncp_tag}:"${version}"
docker tag ownyourbits/nextcloud-${ncp_tag}:latest ownyourbits/nextcloud-${ncp_tag}:"${version}"
docker tag ownyourbits/nextcloudpi-${ncp_tag}:latest ownyourbits/nextcloudpi-${ncp_tag}:"${version}"
docker tag "ownyourbits/${target}-${suffix}:latest" "ownyourbits/${target}-${suffix}:${version}"
}
# make sure we don't accidentally include this
rm -f ncp-web/wizard.cfg
get_arch_args() {
[[ "$@" =~ "x86" ]] && build_arch "${release}" amd64 x86_64 x86
[[ "$@" =~ "armhf" ]] && build_arch "${release}" arm32v7 arm armhf
[[ "$@" =~ "arm64" ]] && build_arch "${release}" arm64v8 aarch64 arm64
[[ "${1?}" =~ "x86" ]] && { echo "amd64 x86_64 x86"; return 0; }
[[ "$1" =~ "armhf" ]] && { echo "arm32v7 arm armhf"; return 0; }
[[ "$1" =~ "arm64" ]] && { echo "arm64v8 aarch64 arm64"; return 0; }
exit 0
echo -e "Unsupported architecture: '${arch}'!"
return 1
}
clean_workspace() {
# make sure we don't accidentally include this
rm -f ncp-web/wizard.cfg
}
# Only execute script if not sourced
[[ "${BASH_SOURCE[0]}" == "$0" ]] && {
arch="${1?Missing argument: target architecture}"
shopt -s lastpipe
get_arch_args "$arch" | read -r -a arch_args
# Pull latest image for caching
docker pull ownyourbits/nextcloudpi
for target in qemu nextcloudpi debian-ncp lamp nextcloud
do
build_arch "$target" "${release}" "${arch_args[@]}"
done
exit 0
}
# License
#

188
build/docker/Dockerfile Normal file
View File

@ -0,0 +1,188 @@
ARG arch=armhf
ARG arch_qemu=arm
ARG release=bullseye
FROM debian:${release}-slim AS qemu
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends qemu-user-static
FROM ${arch}/debian:${release}-slim as debian-ncp
ARG arch_qemu
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
CMD /bin/bash
COPY --from=qemu /usr/bin/qemu-${arch_qemu}-static /usr/bin/
RUN mkdir -p /etc/services-available.d /etc/services-enabled.d
COPY build/docker/debian-ncp/run-parts.sh /
# syntax=docker/dockerfile:experimental
FROM --platform=linux/${arch_qemu} debian-ncp as lamp
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
COPY etc/ncp.cfg etc/library.sh lamp.sh /usr/local/etc/
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# installation
apt-get update; \
apt-get install --no-install-recommends -y jq; \
source /usr/local/etc/library.sh; \
set +x; \
install_app /usr/local/etc/lamp.sh; \
# stop mysqld
mysqladmin -u root shutdown; \
# mariaDB fixups (move database to /data-ro, which will be in a persistent volume)
mkdir -p /data-ro /data; \
mv /var/lib/mysql /data-ro/database; \
sed -i "s|^datadir.*|datadir = /data-ro/database|" /etc/mysql/mariadb.conf.d/90-ncp.cnf; \
# package cleanup
apt-get autoremove -y; \
apt-get clean; \
find /var/lib/apt/lists -type f | xargs rm; \
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm /var/cache/debconf/*-old; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
# specific cleanup
rm /data-ro/database/ib_logfile*; \
rm /usr/local/etc/lamp.sh
COPY build/docker/lamp/010lamp /etc/services-enabled.d/
ENTRYPOINT ["/run-parts.sh"]
EXPOSE 80 443
FROM --platform=linux/${arch_qemu} lamp as nextcloud
# syntax=docker/dockerfile:experimental
ARG arch_qemu
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
COPY etc/library.sh /usr/local/etc/
COPY bin/ncp/CONFIG/nc-nextcloud.sh /
COPY etc/ncp-config.d/nc-nextcloud.cfg /usr/local/etc/ncp-config.d/
COPY etc/ncp-templates /usr/local/etc/ncp-templates
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# mark as image build
touch /.ncp-image; \
# mark as docker image
touch /.docker-image; \
# installation ( /var/www/nextcloud -> /data/app which will be in a volume )
apt-get update; \
apt-get install --no-install-recommends -y wget ca-certificates sudo jq; \
source /usr/local/etc/library.sh; \
install_app /nc-nextcloud.sh; \
run_app_unsafe /nc-nextcloud.sh; \
mv /var/www/nextcloud /data-ro/nextcloud; \
ln -s /data-ro/nextcloud /var/www/nextcloud; \
# package cleanup
apt-get autoremove -y; \
apt-get clean; \
find /var/lib/apt/lists -type f | xargs rm; \
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm /var/cache/debconf/*-old; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
# specific cleanup
apt-get purge -y wget ca-certificates; \
rm /nc-nextcloud.sh /usr/local/etc/ncp-config.d/nc-nextcloud.cfg; \
rm /.ncp-image;
COPY build/docker/nextcloud/020nextcloud /etc/services-enabled.d/
COPY bin/ncp-provisioning.sh /usr/local/bin/
# syntax=docker/dockerfile:experimental
FROM --platform=linux/${arch_qemu} nextcloud as nextcloudpi
ARG ncp_ver=v0.0.0
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
RUN mkdir -p /tmp/ncp-build
COPY bin/ /tmp/ncp-build/bin/
COPY etc /tmp/ncp-build/etc/
COPY ncp-web /tmp/ncp-build/ncp-web/
COPY ncp-app /tmp/ncp-build/ncp-app/
COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/
COPY build/docker /tmp/ncp-build/build/docker/
COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/
COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# make sure we don't accidentally disable first run wizard
rm -f ncp-web/{wizard.cfg,ncp-web.cfg}; \
# mark as image build
touch /.ncp-image; \
# mark as docker image
touch /.docker-image; \
apt-get update; \
apt-get install --no-install-recommends -y wget ca-certificates; \
# install nextcloudpi
source /usr/local/etc/library.sh; \
cd /tmp/ncp-build/; \
install_app ncp.sh; \
mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \
run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \
sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \
# fix default paths
sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup.cfg; \
sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup-auto.cfg; \
# cleanup all NCP extras
run_app_unsafe post-inst.sh; \
# specific cleanup
cd /; rm -r /tmp/ncp-build; \
rm /usr/local/etc/ncp-config.d/nc-init.cfg; \
# package installation clean up
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
rm /var/cache/debconf/*-old; \
# set version
echo "${ncp_ver}" > /usr/local/etc/ncp-version
COPY build/docker/nextcloudpi/000ncp /etc/services-enabled.d/

20
build/docker/debian-ncp/Dockerfile
View File

@ -1,20 +0,0 @@
ARG arch=armhf
ARG arch_qemu=arm
ARG release=bullseye
FROM debian:${release}-slim AS qemu
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends qemu-user-static
FROM ${arch}/debian:${release}-slim
ARG arch_qemu
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
CMD /bin/bash
COPY --from=qemu /usr/bin/qemu-${arch_qemu}-static /usr/bin/
RUN mkdir -p /etc/services-available.d /etc/services-enabled.d
COPY build/docker/debian-ncp/run-parts.sh /

52
build/docker/lamp/Dockerfile
View File

@ -1,52 +0,0 @@
# syntax=docker/dockerfile:experimental
ARG arch=armhf
ARG arch_qemu=arm
FROM --platform=linux/${arch_qemu} ownyourbits/debian-ncp-${arch}
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
COPY etc/ncp.cfg etc/library.sh lamp.sh /usr/local/etc/
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# installation
apt-get update; \
apt-get install --no-install-recommends -y jq; \
source /usr/local/etc/library.sh; \
set +x; \
install_app /usr/local/etc/lamp.sh; \
# stop mysqld
mysqladmin -u root shutdown; \
# mariaDB fixups (move database to /data-ro, which will be in a persistent volume)
mkdir -p /data-ro /data; \
mv /var/lib/mysql /data-ro/database; \
sed -i "s|^datadir.*|datadir = /data-ro/database|" /etc/mysql/mariadb.conf.d/90-ncp.cnf; \
# package cleanup
apt-get autoremove -y; \
apt-get clean; \
find /var/lib/apt/lists -type f | xargs rm; \
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm /var/cache/debconf/*-old; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
# specific cleanup
rm /data-ro/database/ib_logfile*; \
rm /usr/local/etc/lamp.sh
COPY build/docker/lamp/010lamp /etc/services-enabled.d/
ENTRYPOINT ["/run-parts.sh"]
EXPOSE 80 443

52
build/docker/nextcloud/Dockerfile
View File

@ -1,52 +0,0 @@
# syntax=docker/dockerfile:experimental
ARG arch=armhf
ARG arch_qemu=arm
FROM --platform=linux/${arch_qemu} ownyourbits/lamp-${arch}
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
COPY etc/library.sh /usr/local/etc/
COPY bin/ncp/CONFIG/nc-nextcloud.sh /
COPY etc/ncp-config.d/nc-nextcloud.cfg /usr/local/etc/ncp-config.d/
COPY etc/ncp-templates /usr/local/etc/ncp-templates
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# mark as image build
touch /.ncp-image; \
# mark as docker image
touch /.docker-image; \
# installation ( /var/www/nextcloud -> /data/app which will be in a volume )
apt-get update; \
apt-get install --no-install-recommends -y wget ca-certificates sudo jq; \
source /usr/local/etc/library.sh; \
install_app /nc-nextcloud.sh; \
run_app_unsafe /nc-nextcloud.sh; \
mv /var/www/nextcloud /data-ro/nextcloud; \
ln -s /data-ro/nextcloud /var/www/nextcloud; \
# package cleanup
apt-get autoremove -y; \
apt-get clean; \
find /var/lib/apt/lists -type f | xargs rm; \
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm /var/cache/debconf/*-old; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
# specific cleanup
apt-get purge -y wget ca-certificates; \
rm /nc-nextcloud.sh /usr/local/etc/ncp-config.d/nc-nextcloud.cfg; \
rm /.ncp-image;
COPY build/docker/nextcloud/020nextcloud /etc/services-enabled.d/
COPY bin/ncp-provisioning.sh /usr/local/bin/

70
build/docker/nextcloudpi/Dockerfile
View File

@ -1,70 +0,0 @@
# syntax=docker/dockerfile:experimental
ARG arch=armhf
ARG arch_qemu=arm
FROM --platform=linux/${arch_qemu} ownyourbits/nextcloud-${arch}
ARG ncp_ver=v0.0.0
LABEL maintainer="Ignacio Núñez Hernanz <nacho@ownyourbits.com>"
SHELL ["/bin/bash", "-c"]
ENV DOCKERBUILD 1
RUN mkdir -p /tmp/ncp-build
COPY bin/ /tmp/ncp-build/bin/
COPY etc /tmp/ncp-build/etc/
COPY ncp-web /tmp/ncp-build/ncp-web/
COPY ncp-app /tmp/ncp-build/ncp-app/
COPY ncp-previewgenerator /tmp/ncp-build/ncp-previewgenerator/
COPY build/docker /tmp/ncp-build/build/docker/
COPY ncp.sh update.sh post-inst.sh /tmp/ncp-build/
COPY etc/ncp-config.d/nc-init.cfg /usr/local/etc/ncp-config.d/nc-init-copy.cfg
RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
set -e; \
# make sure we don't accidentally disable first run wizard
rm -f ncp-web/{wizard.cfg,ncp-web.cfg}; \
# mark as image build
touch /.ncp-image; \
# mark as docker image
touch /.docker-image; \
apt-get update; \
apt-get install --no-install-recommends -y wget ca-certificates; \
# install nextcloudpi
source /usr/local/etc/library.sh; \
cd /tmp/ncp-build/; \
install_app ncp.sh; \
mv /usr/local/etc/ncp-config.d/nc-init-copy.cfg /usr/local/etc/ncp-config.d/nc-init.cfg; \
run_app_unsafe bin/ncp/CONFIG/nc-init.sh; \
sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \
# fix default paths
sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup.cfg; \
sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup-auto.cfg; \
# cleanup all NCP extras
run_app_unsafe post-inst.sh; \
# specific cleanup
cd /; rm -r /tmp/ncp-build; \
rm /usr/local/etc/ncp-config.d/nc-init.cfg; \
# package installation clean up
rm -rf /usr/share/man/*; \
rm -rf /usr/share/doc/*; \
rm -f /var/log/alternatives.log /var/log/apt/*; \
rm /var/cache/debconf/*-old; \
# set version
echo "${ncp_ver}" > /usr/local/etc/ncp-version
COPY build/docker/nextcloudpi/000ncp /etc/services-enabled.d/

2
etc/ncp-templates/ncp-metrics.cfg.sh
View File

@ -50,7 +50,7 @@ EOF
EOF
}
[[ "$DOCKERBUILD" == 1 ]] || {
is_docker || {
DATADIR=$( ncc config:system:get datadirectory ) || {
echo -e "ERROR: Could not get data directory. Is NextCloud running?";

2
etc/ncp-templates/nextcloud.conf.sh
View File

@ -4,7 +4,7 @@ set -e
source /usr/local/etc/library.sh
[[ "$1" != "--defaults" ]] || echo "INFO: Restoring template to default settings" >&2
[[ ! -f /.docker-image ]] || echo "INFO: Docker installation detected" >&2
is_docker && echo "INFO: Docker installation detected" >&2
if [[ "$1" != "--defaults" ]]; then
LETSENCRYPT_DOMAIN="$(

18
tests/activation_tests.py
View File

@ -24,6 +24,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import UnexpectedAlertPresentException
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import TimeoutException
@ -31,6 +32,7 @@ from selenium.common.exceptions import TimeoutException
suite_name = "activation tests"
test_cfg = 'test_cfg.txt'
test_log = 'test_log.txt'
wait_timeout = 120
class tc:
@ -88,12 +90,12 @@ def signal_handler(sig, frame):
sys.exit(0)
def test_activation(IP, nc_port, admin_port):
def test_activation(IP, nc_port, admin_port, options):
""" Activation process checks"""
# activation page
test = Test()
driver = webdriver.Firefox(service_log_path='/dev/null')
driver = webdriver.Firefox(options=options)
driver.implicitly_wait(5)
test.new("activation opens")
driver.get(f"https://{IP}:{nc_port}")
@ -120,7 +122,7 @@ def test_activation(IP, nc_port, admin_port):
test.new("activation ends")
try:
wait = WebDriverWait(driver, 60)
wait = WebDriverWait(driver, wait_timeout)
wait.until(EC.text_to_be_present_in_element((By.ID,'error-box'), "ACTIVATION SUCCESSFUL"))
test.check(True)
except TimeoutException:
@ -134,7 +136,7 @@ def test_activation(IP, nc_port, admin_port):
# ncp-web
test.new("ncp-web")
driver = webdriver.Firefox(service_log_path='/dev/null')
driver = webdriver.Firefox(options=options)
try:
driver.get(f"https://ncp:{urllib.parse.quote_plus(ncp_pass)}@{IP}:{admin_port}")
except UnexpectedAlertPresentException:
@ -150,15 +152,18 @@ if __name__ == "__main__":
# parse options
try:
opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-gui'])
except getopt.GetoptError:
usage()
sys.exit(2)
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
sys.exit(2)
elif opt == '--no-gui':
options.headless = True
else:
usage()
sys.exit(2)
@ -170,7 +175,8 @@ if __name__ == "__main__":
admin_port = args[2] if len(args) > 2 else "4443"
print("Activation tests " + tc.yellow + IP + tc.normal)
print("---------------------------")
test_activation(IP, nc_port, admin_port)
test_activation(IP, nc_port, admin_port, options)
# License
#

8
tests/nextcloud_tests.py
View File

@ -25,6 +25,7 @@ from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.firefox.options import Options
from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException
from typing import List, Tuple
import traceback
@ -175,11 +176,12 @@ if __name__ == "__main__":
# parse options
try:
opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help'])
opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui'])
except getopt.GetoptError:
usage()
sys.exit(2)
options = Options()
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
@ -187,6 +189,8 @@ if __name__ == "__main__":
elif opt in ('-n', '--new'):
if os.path.exists(test_cfg):
os.unlink(test_cfg)
elif opt == '--no-gui':
options.headless = True
else:
usage()
sys.exit(2)
@ -222,7 +226,7 @@ if __name__ == "__main__":
print("Nextcloud tests " + tc.yellow + IP + tc.normal)
print("---------------------------")
driver = webdriver.Firefox(service_log_path='/dev/null')
driver = webdriver.Firefox(service_log_path='/dev/null', options=options)
try:
test_nextcloud(IP, nc_port, driver)
finally:

30
tests/system_tests.py
View File

@ -181,15 +181,21 @@ if __name__ == "__main__":
# parse options
try:
opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-ping', 'non-interactive'])
except getopt.GetoptError:
usage()
sys.exit(2)
skip_ping = False
interactive = True
for opt, arg in opts:
if opt in ('-h', '--help'):
usage()
sys.exit(2)
elif opt == '--no-ping':
skip_ping = True
elif opt == '--non-interactive':
interactive = False
else:
usage()
sys.exit(2)
@ -205,7 +211,7 @@ if __name__ == "__main__":
# detect if we are running this in a NCP instance
try:
dockers_running = run(['docker', 'ps', '--format', '{{.Image}}'], stdout=PIPE).stdout.decode('utf-8')
dockers_running = run(['docker', 'ps', '--format', '{{.Names}}'], stdout=PIPE).stdout.decode('utf-8')
except:
dockers_running = ''
@ -223,9 +229,12 @@ if __name__ == "__main__":
pre_cmd = []
# docker method
elif 'ownyourbits/nextcloudpi-' in dockers_running:
elif 'nextcloudpi' in dockers_running:
print( tc.brown + "* local NCP docker instance detected" + tc.normal)
pre_cmd = ['docker', 'exec', '-ti', 'nextcloudpi']
pre_cmd = ['docker', 'exec']
if interactive:
pre_cmd.append('-ti')
pre_cmd.append('nextcloudpi')
# LXC method
elif lxc_running:
@ -241,12 +250,13 @@ if __name__ == "__main__":
pre_cmd = ['ssh', '-o UserKnownHostsFile=/dev/null' , '-o PasswordAuthentication=no',
'-o StrictHostKeyChecking=no', '-o ConnectTimeout=1', ssh_cmd[4:]]
at_char = ssh_cmd.index('@')
ip = ssh_cmd[at_char+1:]
ping_cmd = run(['ping', '-c1', '-w1', ip], stdout=PIPE, stderr=PIPE)
if ping_cmd.returncode != 0:
print(tc.red + "No connectivity to " + tc.yellow + ip + tc.normal)
sys.exit(1)
if not skip_ping:
at_char = ssh_cmd.index('@')
ip = ssh_cmd[at_char+1:]
ping_cmd = run(['ping', '-c1', '-w10', ip], stdout=PIPE, stderr=PIPE)
if ping_cmd.returncode != 0:
print(tc.red + "No connectivity to " + tc.yellow + ip + tc.normal)
#sys.exit(1)
ssh_test = run(pre_cmd + [':'], stdout=PIPE, stderr=PIPE)
if ssh_test.returncode != 0: