From 6fc815937b553516e8be8da335305e73ec983fbc Mon Sep 17 00:00:00 2001 From: Shane McDonald Date: Sat, 4 Apr 2020 15:08:09 -0400 Subject: [PATCH] Tidy up the dev environment a bit --- .../roles/image_build/files/Dockerfile.sdist | 17 +- .../image_build/files/google-cloud-sdk.repo | 3 +- .../roles/image_build/templates/Dockerfile.j2 | 176 ++++++++-------- tools/docker-compose/Dockerfile | 197 ++++++++---------- tools/docker-compose/google-cloud-sdk.repo | 3 +- 5 files changed, 186 insertions(+), 210 deletions(-) diff --git a/installer/roles/image_build/files/Dockerfile.sdist b/installer/roles/image_build/files/Dockerfile.sdist index 8b35e7d11b..960df7db76 100644 --- a/installer/roles/image_build/files/Dockerfile.sdist +++ b/installer/roles/image_build/files/Dockerfile.sdist @@ -1,20 +1,17 @@ -FROM centos:7 +FROM centos:8 -RUN yum -y update && yum -y install epel-release && yum -y install https://centos7.iuscommunity.org/ius-release.rpm - -RUN yum install -y bzip2 \ +RUN dnf -y update && dnf -y install epel-release && \ + dnf install -y bzip2 \ gcc-c++ \ gettext \ - git2u \ - git2u-core \ + git \ make \ nodejs \ - python36-setuptools + python3 \ + python3-setuptools # Use the distro provided npm to bootstrap our required version of node -RUN npm install -g n -RUN n 10.15.0 -RUN yum remove -y nodejs +RUN npm install -g n && n 10.15.0 && dnf remove -y nodejs ENV PATH=/usr/local/n/versions/node/10.15.0/bin:$PATH diff --git a/installer/roles/image_build/files/google-cloud-sdk.repo b/installer/roles/image_build/files/google-cloud-sdk.repo index bff8d01ccd..1aa0e1d7e0 100644 --- a/installer/roles/image_build/files/google-cloud-sdk.repo +++ b/installer/roles/image_build/files/google-cloud-sdk.repo @@ -3,6 +3,5 @@ name=Google Cloud SDK baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64 enabled=1 gpgcheck=1 -repo_gpgcheck=1 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg \ No newline at end of file + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg diff --git a/installer/roles/image_build/templates/Dockerfile.j2 b/installer/roles/image_build/templates/Dockerfile.j2 index bf78f88a9b..fa1e3cc1d7 100644 --- a/installer/roles/image_build/templates/Dockerfile.j2 +++ b/installer/roles/image_build/templates/Dockerfile.j2 @@ -1,64 +1,69 @@ FROM centos:8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + USER root +ADD google-cloud-sdk.repo /etc/yum.repos.d/ + # sync with installer/roles/image_build/templates/Dockerfile.j2 RUN dnf -y update && \ - dnf -y install epel-release 'dnf-command(config-manager)' && \ - dnf module -y enable 'postgresql:10' && \ - dnf config-manager --set-enabled PowerTools && \ - dnf -y install acl \ - ansible \ - bubblewrap \ - curl \ - diffutils \ - dnf-utils \ - gcc \ - gcc-c++ \ - gettext \ - git-core \ - glibc-langpack-en \ - krb5-workstation \ - libcurl-devel \ - libffi-devel \ - libstdc++.so.6 \ - libtool-ltdl-devel \ - libcgroup-tools \ - make \ - mercurial \ - nginx \ - nodejs \ - nss \ - openldap-devel \ - openssh-server \ - patch \ - @postgresql:10 \ - postgresql-devel \ - python3-devel \ - python3-libselinux \ - python3-pip \ - python3-psycopg2 \ - python3-setuptools \ - python3-pycurl \ - rsync \ - subversion \ - sudo \ - swig \ - tmux \ - unzip \ - vim-minimal \ - which \ - xmlsec1 \ - xmlsec1-devel \ - xmlsec1-openssl \ - xmlsec1-openssl-devel + dnf -y install https://github.com/krallin/tini/releases/download/v0.18.0/tini_0.18.0.rpm && \ + dnf -y install epel-release 'dnf-command(config-manager)' && \ + dnf module -y enable 'postgresql:10' && \ + dnf config-manager --set-enabled PowerTools && \ + dnf -y install acl \ + ansible \ + bubblewrap \ + curl \ + diffutils \ + dnf-utils \ + gcc \ + gcc-c++ \ + gettext \ + git-core \ + glibc-langpack-en \ + krb5-workstation \ + kubectl \ + libcurl-devel \ + libffi-devel \ + libstdc++.so.6 \ + libtool-ltdl-devel \ + libcgroup-tools \ + make \ + mercurial \ + nginx \ + nodejs \ + nss \ + openldap-devel \ + openssh-server \ + patch \ + @postgresql:10 \ + postgresql-devel \ + python3-devel \ + python3-libselinux \ + python3-pip \ + python3-psycopg2 \ + python3-setuptools \ + python3-pycurl \ + rsync \ + subversion \ + sudo \ + swig \ + tmux \ + unzip \ + vim-minimal \ + which \ + xmlsec1 \ + xmlsec1-devel \ + xmlsec1-openssl \ + xmlsec1-openssl-devel -ADD https://github.com/krallin/tini/releases/download/v0.14.0/tini /tini -RUN chmod +x /tini - -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" -RUN pip3 install supervisor +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" supervisor +# Install AWX + Requirements ADD Makefile /tmp/Makefile RUN mkdir /tmp/requirements ADD requirements/requirements_ansible.txt \ @@ -70,27 +75,15 @@ ADD requirements/requirements_ansible.txt \ /tmp/requirements/ RUN cd /tmp && VENV_BASE="/var/lib/awx/venv" make requirements_awx requirements_ansible_py3 -RUN dnf -y remove *-devel \ - gcc \ - gcc-c++ \ - nodejs - -# makes postgres available for data migrations when upgrading -RUN export PATH=/usr/pgsql-10/bin:$PATH - -RUN dnf -y clean all -RUN rm -rf /root/.cache - -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - -RUN ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage -RUN rm -rf /tmp/* - -RUN echo "{{ awx_version }}" > /var/lib/awx/.tower_version COPY {{ awx_sdist_file }} /tmp/{{ awx_sdist_file }} -RUN OFFICIAL=yes /var/lib/awx/venv/awx/bin/pip install /tmp/{{ awx_sdist_file }} +RUN echo "{{ awx_version }}" > /var/lib/awx/.tower_version && \ + OFFICIAL=yes /var/lib/awx/venv/awx/bin/pip install /tmp/{{ awx_sdist_file }} && \ + ln -s /var/lib/awx/venv/awx/bin/awx-manage /usr/bin/awx-manage + +RUN dnf -y remove *-devel \ + gcc \ + gcc-c++ \ + nodejs ADD settings.py /etc/tower/settings.py ADD supervisor.conf /supervisor.conf @@ -98,39 +91,40 @@ ADD supervisor_task.conf /supervisor_task.conf ADD launch_awx.sh /usr/bin/launch_awx.sh ADD launch_awx_task.sh /usr/bin/launch_awx_task.sh ADD config-watcher /usr/bin/config-watcher -RUN chmod +rx /usr/bin/launch_awx.sh && \ - chmod +rx /usr/bin/launch_awx_task.sh && \ - chmod +rx /usr/bin/config-watcher # Install OpenShift CLI RUN cd /usr/local/bin && \ curl -L https://github.com/openshift/origin/releases/download/v3.11.0/openshift-origin-client-tools-v3.11.0-0cbc58b-linux-64bit.tar.gz | \ tar -xz --strip-components=1 --wildcards --no-anchored 'oc' -ADD google-cloud-sdk.repo /etc/yum.repos.d/ -RUN dnf install -y kubectl - -RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root -RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chmod g+w - # Pre-create things that we need to write to RUN for dir in /home/awx /var/log/tower /var/log/nginx /var/lib/nginx; \ - do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done + do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ + \ + for file in /etc/passwd /var/run/nginx.pid; \ + do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done -RUN for file in /etc/passwd /var/run/nginx.pid; \ - do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done +# Fix up permissions +RUN find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chgrp root && \ + find /var/lib/awx -not -path '/var/lib/awx/venv*' | xargs chmod g+w && \ + chmod +rx /usr/bin/launch_awx.sh && \ + chmod +rx /usr/bin/launch_awx_task.sh && \ + chmod +rx /usr/bin/config-watcher && \ + chmod u+s /usr/bin/bwrap # https://github.com/ansible/awx/issues/5224 -# https://github.com/ansible/awx/issues/5224 -RUN chmod u+s /usr/bin/bwrap +RUN ln -sf /dev/stdout /var/log/nginx/access.log && \ + ln -sf /dev/stderr /var/log/nginx/error.log -VOLUME /var/lib/nginx -RUN ln -sf /dev/stdout /var/log/nginx/access.log \ - && ln -sf /dev/stderr /var/log/nginx/error.log +RUN dnf -y clean all && rm -rf /root/.cache rm -rf /tmp/* ENV HOME=/home/awx +ENV PATH="/usr/pgsql-10/bin:${PATH}" + WORKDIR ${HOME} USER 1000 EXPOSE 8052 -ENTRYPOINT ["/tini", "--"] +ENTRYPOINT ["tini", "--"] CMD /usr/bin/launch_awx.sh + +VOLUME /var/lib/nginx diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 1bca54f43f..0ec22e499d 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -2,147 +2,134 @@ FROM centos:8 ARG UID=0 -# Add ansible-devel so that we get collections! +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + ADD tools/docker-compose/ansible_nightly.repo /etc/yum.repos.d/ansible_nightly.repo +ADD tools/docker-compose/google-cloud-sdk.repo /etc/yum.repos.d/ # sync with installer/roles/image_build/templates/Dockerfile.j2 RUN dnf -y update && \ - dnf -y install epel-release 'dnf-command(config-manager)' && \ - dnf module -y enable 'postgresql:10' && \ - dnf config-manager --set-enabled PowerTools && \ - dnf -y install acl \ - ansible \ - bubblewrap \ - curl \ - diffutils \ - dnf-utils \ - gcc \ - gcc-c++ \ - gettext \ - git-core \ - glibc-langpack-en \ - krb5-workstation \ - libcurl-devel \ - libffi-devel \ - libstdc++.so.6 \ - libtool-ltdl-devel \ - libcgroup-tools \ - make \ - mercurial \ - nginx \ - nmap-ncat \ - nodejs \ - nss \ - openldap-devel \ - openssh-server \ - patch \ - @postgresql:10 \ - postgresql-devel \ - python3-devel \ - python3-libselinux \ - python3-pip \ - python3-psycopg2 \ - python3-setuptools \ - python3-pycurl \ - rsync \ - subversion \ - strace \ - sudo \ - swig \ - tcpdump \ - tmux \ - unzip \ - vim \ - which \ - xmlsec1 \ - xmlsec1-devel \ - xmlsec1-openssl \ - xmlsec1-openssl-devel + dnf -y install https://github.com/krallin/tini/releases/download/v0.18.0/tini_0.18.0.rpm && \ + dnf -y install epel-release 'dnf-command(config-manager)' && \ + dnf module -y enable 'postgresql:10' && \ + dnf config-manager --set-enabled PowerTools && \ + dnf -y install acl \ + ansible \ + bubblewrap \ + curl \ + diffutils \ + dnf-utils \ + gcc \ + gcc-c++ \ + gettext \ + git-core \ + glibc-langpack-en \ + krb5-workstation \ + kubectl \ + libcurl-devel \ + libffi-devel \ + libstdc++.so.6 \ + libtool-ltdl-devel \ + libcgroup-tools \ + make \ + mercurial \ + nginx \ + nodejs \ + nss \ + openldap-devel \ + openssh-server \ + patch \ + @postgresql:10 \ + postgresql-devel \ + python3-devel \ + python3-libselinux \ + python3-pip \ + python3-psycopg2 \ + python3-setuptools \ + python3-pycurl \ + rsync \ + subversion \ + sudo \ + swig \ + tmux \ + unzip \ + vim-minimal \ + which \ + xmlsec1 \ + xmlsec1-devel \ + xmlsec1-openssl \ + xmlsec1-openssl-devel -# UI tests only, do not put in installer/roles/image_build/templates/Dockerfile.j2 +# Dev dependencies / UI tests only, do not put in installer/roles/image_build/templates/Dockerfile.j2 RUN dnf -y install \ - gtk3 \ - alsa-lib \ - libX11-xcb \ - libXScrnSaver + gtk3 \ + alsa-lib \ + libX11-xcb \ + libXScrnSaver \ + strace \ + vim \ + nmap-ncat -ENV PATH="/usr/pgsql-10/bin:${PATH}" +RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa && \ + openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr \ + -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" && \ + openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt && \ + chmod 640 /etc/nginx/nginx.{csr,key,crt} -RUN dnf install -y https://github.com/krallin/tini/releases/download/v0.18.0/tini_0.18.0.rpm - -RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa -RUN mkdir -p /data/db - -ADD tools/docker-compose/awx.egg-link /tmp/awx.egg-link -ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage -ADD tools/docker-compose/awx.egg-info /tmp/awx.egg-info - -RUN openssl req -nodes -newkey rsa:2048 -keyout /etc/nginx/nginx.key -out /etc/nginx/nginx.csr -subj "/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost" -RUN openssl x509 -req -days 365 -in /etc/nginx/nginx.csr -signkey /etc/nginx/nginx.key -out /etc/nginx/nginx.crt -RUN chmod 640 /etc/nginx/nginx.{csr,key,crt} - -RUN python3 -m ensurepip && pip3 install "virtualenv < 20" flake8 -RUN pip3 install supervisor +RUN python3 -m ensurepip && pip3 install "virtualenv < 20" flake8 supervisor +# Install Requirements ADD Makefile /tmp/Makefile RUN mkdir /tmp/requirements ADD requirements/requirements.txt \ - requirements/requirements_git.txt \ - requirements/requirements_ansible.txt \ - requirements/requirements_ansible_git.txt \ - requirements/requirements_dev.txt \ - requirements/requirements_ansible_uninstall.txt \ - requirements/requirements_tower_uninstall.txt \ - /tmp/requirements/ + requirements/requirements_git.txt \ + requirements/requirements_ansible.txt \ + requirements/requirements_ansible_git.txt \ + requirements/requirements_dev.txt \ + requirements/requirements_ansible_uninstall.txt \ + requirements/requirements_tower_uninstall.txt \ + /tmp/requirements/ RUN mkdir -p /venv && chmod g+w /venv RUN cd /tmp && VENV_BASE="/venv" make requirements_dev # Use the distro provided npm to bootstrap our required version of node -RUN npm install -g n -RUN n 10.15.0 +RUN npm install -g n && n 10.15.0 && dnf remove -y nodejs # Install OpenShift CLI RUN cd /usr/local/bin && \ curl -L https://github.com/openshift/origin/releases/download/v3.9.0/openshift-origin-client-tools-v3.9.0-191fece-linux-64bit.tar.gz | \ tar -xz --strip-components=1 --wildcards --no-anchored 'oc' -ADD tools/docker-compose/google-cloud-sdk.repo /etc/yum.repos.d/ -RUN dnf install -y kubectl - -RUN dnf -y remove nodejs - -RUN dnf -y clean all - -RUN rm -rf /root/.cache +RUN dnf -y clean all && rm -rf /root/.cache # https://github.com/ansible/awx/issues/5224 RUN chmod u+s /usr/bin/bwrap -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 - +ADD tools/docker-compose/awx.egg-link /tmp/awx.egg-link +ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage +ADD tools/docker-compose/awx.egg-info /tmp/awx.egg-info ADD tools/docker-compose/nginx.conf /etc/nginx/nginx.conf ADD tools/docker-compose/nginx.vh.default.conf /etc/nginx/conf.d/nginx.vh.default.conf ADD tools/docker-compose/launch_awx.sh /usr/bin/launch_awx.sh ADD tools/docker-compose/start_tests.sh /start_tests.sh ADD tools/docker-compose/bootstrap_development.sh /usr/bin/bootstrap_development.sh - -EXPOSE 8043 8013 8080 22 - ADD tools/docker-compose/entrypoint.sh / ADD tools/scripts/awx-python /usr/bin/awx-python -ENTRYPOINT ["/entrypoint.sh"] -CMD ["/bin/bash"] # Pre-create things that we need to write to RUN for dir in /var/lib/awx/ /var/log/tower/ /var/lib/awx/projects /.ansible /var/log/nginx /var/lib/nginx /.local; \ - do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done - -RUN for file in /etc/passwd /etc/supervisord.conf \ - /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \ - do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done + do mkdir -p $dir; chmod -R g+rwx $dir; chgrp -R root $dir; done && \ + \ + for file in /etc/passwd /etc/supervisord.conf /venv/awx/lib/python3.6/site-packages/awx.egg-link /var/run/nginx.pid; \ + do touch $file; chmod -R g+rwx $file; chgrp -R root $file; done ENV HOME /var/lib/awx +ENV PATH="/usr/local/n/versions/node/10.15.0/bin:${PATH}" +ENV PATH="/usr/pgsql-10/bin:${PATH}" -ENV PATH=/usr/local/n/versions/node/10.15.0/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +EXPOSE 8043 8013 8080 22 + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/bin/bash"] diff --git a/tools/docker-compose/google-cloud-sdk.repo b/tools/docker-compose/google-cloud-sdk.repo index bff8d01ccd..1aa0e1d7e0 100644 --- a/tools/docker-compose/google-cloud-sdk.repo +++ b/tools/docker-compose/google-cloud-sdk.repo @@ -3,6 +3,5 @@ name=Google Cloud SDK baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64 enabled=1 gpgcheck=1 -repo_gpgcheck=1 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg \ No newline at end of file + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg