Initial commit

roles/docker/files/docker-gc

@@ -0,0 +1,211 @@
+#!/bin/bash
+
+# Copyright (c) 2014 Spotify AB.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This script attempts to garbage collect docker containers and images.
+# Containers that exited more than an hour ago are removed.
+# Images that have existed more than an hour and are not in use by any
+# containers are removed.
+
+# Note: Although docker normally prevents removal of images that are in use by
+#       containers, we take extra care to not remove any image tags (e.g.
+#       ubuntu:14.04, busybox, etc) that are used by containers. A naive
+#       "docker rmi `docker images -q`" will leave images stripped of all tags,
+#       forcing users to re-pull the repositories even though the images
+#       themselves are still on disk.
+
+# Note: State is stored in $STATE_DIR, defaulting to /var/lib/docker-gc
+
+set -o nounset
+set -o errexit
+
+GRACE_PERIOD_SECONDS=${GRACE_PERIOD_SECONDS:=3600}
+STATE_DIR=${STATE_DIR:=/var/lib/docker-gc}
+DOCKER=${DOCKER:=docker}
+PID_DIR=${PID_DIR:=/var/run}
+
+for pid in $(pidof -s docker-gc); do
+    if [[ $pid != $$ ]]; then
+        echo "[$(date)] : docker-gc : Process is already running with PID $pid"
+        exit 1
+    fi
+done
+
+trap "rm -f -- '$PID_DIR/dockergc'" EXIT
+
+echo $$ > $PID_DIR/dockergc
+
+
+EXCLUDE_FROM_GC=${EXCLUDE_FROM_GC:=/etc/docker-gc-exclude}
+if [ ! -f "$EXCLUDE_FROM_GC" ]
+then
+  EXCLUDE_FROM_GC=/dev/null
+fi
+
+EXCLUDE_CONTAINERS_FROM_GC=${EXCLUDE_CONTAINERS_FROM_GC:=/etc/docker-gc-exclude-containers}
+if [ ! -f "$EXCLUDE_CONTAINERS_FROM_GC" ]
+then
+  EXCLUDE_CONTAINERS_FROM_GC=/dev/null  
+fi
+
+EXCLUDE_IDS_FILE="exclude_ids"
+EXCLUDE_CONTAINER_IDS_FILE="exclude_container_ids"
+
+function date_parse {
+  if date --utc >/dev/null 2>&1; then
+    # GNU/date
+    echo $(date -u --date "${1}" "+%s")
+  else
+    # BSD/date
+    echo $(date -j -u -f "%F %T" "${1}" "+%s")
+  fi
+}
+
+# Elapsed time since a docker timestamp, in seconds
+function elapsed_time() {
+    # Docker 1.5.0 datetime format is 2015-07-03T02:39:00.390284991
+    # Docker 1.7.0 datetime format is 2015-07-03 02:39:00.390284991 +0000 UTC
+    utcnow=$(date -u "+%s")
+    replace_q="${1#\"}"
+    without_ms="${replace_q:0:19}"
+    replace_t="${without_ms/T/ }"
+    epoch=$(date_parse "${replace_t}")
+    echo $(($utcnow - $epoch))
+}
+
+function compute_exclude_ids() {
+    # Find images that match patterns in the EXCLUDE_FROM_GC file and put their
+    # id prefixes into $EXCLUDE_IDS_FILE, prefixed with ^
+
+    PROCESSED_EXCLUDES="processed_excludes.tmp"
+    # Take each line and put a space at the beginning and end, so when we
+    # grep for them below, it will effectively be: "match either repo:tag
+    # or imageid".  Also delete blank lines or lines that only contain
+    # whitespace
+    sed 's/^\(.*\)$/ \1 /' $EXCLUDE_FROM_GC | sed '/^ *$/d' > $PROCESSED_EXCLUDES
+    # The following looks a bit of a mess, but here's what it does:
+    # 1. Get images
+    # 2. Skip header line
+    # 3. Turn columnar display of 'REPO TAG IMAGEID ....' to 'REPO:TAG IMAGEID'
+    # 4. find lines that contain things mentioned in PROCESSED_EXCLUDES
+    # 5. Grab the image id from the line
+    # 6. Prepend ^ to the beginning of each line
+
+    # What this does is make grep patterns to match image ids mentioned by
+    # either repo:tag or image id for later greppage
+    $DOCKER images \
+        | tail -n+2 \
+        | sed 's/^\([^ ]*\) *\([^ ]*\) *\([^ ]*\).*/ \1:\2 \3 /' \
+        | grep -f $PROCESSED_EXCLUDES 2>/dev/null \
+        | cut -d' ' -f3 \
+        | sed 's/^/^/' > $EXCLUDE_IDS_FILE
+}
+
+function compute_exclude_container_ids() {
+    # Find containers matching to patterns listed in EXCLUDE_CONTAINERS_FROM_GC file
+    # Implode their values with a \| separator on a single line
+    PROCESSED_EXCLUDES=`cat $EXCLUDE_CONTAINERS_FROM_GC \
+        | xargs \
+        | sed -e 's/ /\|/g'`
+    # The empty string would match everything
+    if [ "$PROCESSED_EXCLUDES" = "" ]; then
+        touch $EXCLUDE_CONTAINER_IDS_FILE
+        return
+    fi
+    # Find all docker images
+    # Filter out with matching names 
+    # and put them to $EXCLUDE_CONTAINER_IDS_FILE
+    $DOCKER ps -a \
+        | grep -E "$PROCESSED_EXCLUDES" \
+        | awk '{ print $1 }' \
+        | tr -s " " "\012" \
+        | sort -u > $EXCLUDE_CONTAINER_IDS_FILE
+}
+
+# Change into the state directory (and create it if it doesn't exist)
+if [ ! -d "$STATE_DIR" ]
+then
+  mkdir -p $STATE_DIR
+fi
+cd "$STATE_DIR"
+
+# Verify that docker is reachable
+$DOCKER version 1>/dev/null
+
+# List all currently existing containers
+$DOCKER ps -a -q --no-trunc | sort | uniq > containers.all
+
+# List running containers
+$DOCKER ps -q --no-trunc | sort | uniq > containers.running
+
+# compute ids of container images to exclude from GC
+compute_exclude_ids
+
+# compute ids of containers to exclude from GC
+compute_exclude_container_ids
+
+# List containers that are not running
+comm -23 containers.all containers.running > containers.exited
+
+# Find exited containers that finished at least GRACE_PERIOD_SECONDS ago
+echo -n "" > containers.reap.tmp
+cat containers.exited | while read line
+do
+    EXITED=$(${DOCKER} inspect -f "{{json .State.FinishedAt}}" ${line})
+    ELAPSED=$(elapsed_time $EXITED)
+    if [[ $ELAPSED -gt $GRACE_PERIOD_SECONDS ]]; then
+        echo $line >> containers.reap.tmp
+    fi
+done
+
+# List containers that we will remove and exclude ids.
+cat containers.reap.tmp | sort | uniq | grep -v -f $EXCLUDE_CONTAINER_IDS_FILE > containers.reap || true
+
+# List containers that we will keep.
+comm -23 containers.all containers.reap > containers.keep
+
+# List images used by containers that we keep.
+# This may be both image id's and repo/name:tag, so normalize to image id's only
+cat containers.keep |
+xargs -n 1 $DOCKER inspect -f '{{.Config.Image}}' 2>/dev/null |
+sort | uniq |
+xargs -n 1 $DOCKER inspect -f '{{.Id}}' 2>/dev/null |
+sort | uniq > images.used
+
+# List images to reap; images that existed last run and are not in use.
+$DOCKER images -q --no-trunc | sort | uniq > images.all
+
+# Find images that are created at least GRACE_PERIOD_SECONDS ago
+echo -n "" > images.reap.tmp
+cat images.all | while read line
+do
+    CREATED=$(${DOCKER} inspect -f "{{.Created}}" ${line})
+    ELAPSED=$(elapsed_time $CREATED)
+    if [[ $ELAPSED -gt $GRACE_PERIOD_SECONDS ]]; then
+        echo $line >> images.reap.tmp
+    fi
+done
+comm -23 images.reap.tmp images.used | grep -v -f $EXCLUDE_IDS_FILE > images.reap || true
+
+# Reap containers.
+xargs -n 1 $DOCKER rm --volumes=true < containers.reap &>/dev/null || true
+
+# Reap images.
+xargs -n 1 $DOCKER rmi < images.reap &>/dev/null || true

roles/docker/files/systemd-docker.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+After=network.target docker.socket
+Requires=docker.socket
+
+[Service]
+EnvironmentFile=-/etc/default/docker
+Type=notify
+ExecStart=/usr/bin/docker daemon -H fd:// $DOCKER_OPTS
+MountFlags=slave
+LimitNOFILE=1048576
+LimitNPROC=1048576
+LimitCORE=infinity
+
+[Install]
+WantedBy=multi-user.target

roles/docker/handlers/main.yml

@@ -0,0 +1,12 @@
+---
+- name: restart docker
+  command: /bin/true
+  notify:
+    - reload systemd
+    - restart docker service
+
+- name: reload systemd
+  shell: systemctl daemon-reload
+
+- name: restart docker service
+  service: name=docker state=restarted

roles/docker/tasks/configure.yml

@@ -0,0 +1,41 @@
+---
+- name: Write script for calico/docker bridge configuration
+  template: src=create_cbr.j2 dest=/etc/network/if-up.d/create_cbr mode=u+x
+  when: overlay_network_plugin is defined and overlay_network_plugin == "calico"
+
+- name: Configure calico/docker bridge
+  shell: /etc/network/if-up.d/create_cbr
+  when: overlay_network_plugin is defined and overlay_network_plugin == "calico"
+
+- name: Configure docker to use cbr0 bridge
+  lineinfile:
+    dest=/etc/default/docker
+    regexp='.*DOCKER_OPTS=.*'
+    line='DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"'
+  notify:
+    - restart docker
+  when: overlay_network_plugin is defined and overlay_network_plugin == "calico"
+
+- name: enable docker
+  service:
+    name: docker
+    enabled: yes
+    state: started
+  tags:
+    - docker
+
+- meta: flush_handlers
+
+#- name: login to arkena's docker registry
+#  shell : >
+#    docker login --username={{ dockerhub_user }}
+#    --password={{ dockerhub_pass }}
+#    --email={{ dockerhub_email }}
+
+#- pause: prompt='WARNING The next task will remove all exited containers, enter to continue'
+#
+#- name: Purge all exited containers
+#  shell: >
+#    if [ ! -z "$(docker ps -aq -f status=exited)" ]; then
+#    docker rm $(docker ps -aq -f status=exited);
+#    fi

roles/docker/tasks/install.yml

@@ -0,0 +1,33 @@
+---
+- name: Configure debian distribution apt repository
+  template: src=debian.list.j2 dest=/etc/apt/sources.list.d/{{ ansible_distribution_release }}.list
+
+- name: Install prerequisites for https transport
+  apt: pkg={{ item }} state=present update_cache=yes
+  with_items:
+    - apt-transport-https
+    - ca-certificates
+
+- name: Configure docker apt repository
+  template: src=docker.list.j2 dest=/etc/apt/sources.list.d/docker.list
+
+- name: Install docker-engine
+  apt: pkg={{ item }} state=present force=yes update_cache=yes
+  with_items:
+    - aufs-tools
+    - cgroupfs-mount
+    - docker-engine=1.8.2-0~{{ ansible_distribution_release }}
+
+- name: Copy default docker configuration
+  template: src=default-docker.j2 dest=/etc/default/docker
+  notify: restart docker
+
+- name: Copy Docker systemd unit file
+  copy: src=systemd-docker.service dest=/lib/systemd/system/docker.service
+  notify: restart docker
+
+- name: Copy Docker garbage collection script
+  copy: src=docker-gc dest={{ bin_dir }}/docker-gc mode=700
+
+- name: Copy Cron for garbage collection script 
+  template: src=cron_docker-gc.j2 dest=/etc/cron.hourly/cron_docker-gc

roles/docker/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+- include: install.yml
+- include: configure.yml

roles/docker/templates/create_cbr.j2

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Create calico bridge cbr0 if it doesn't exist
+ifaces=$(ifconfig -a | sed 's/[ \t].*//;/^\(lo\|\)$/d' |tr '\n' ' ')
+if ! [[ "${ifaces}" =~ "cbr0" ]];then
+   brctl addbr cbr0
+   ip link set cbr0 up
+fi
+
+# Configure calico bridge ip
+br_ips=$(ip addr list cbr0 |grep "inet " |cut -d' ' -f6)
+if ! [[ "${br_ips}" =~ "{{ br_addr }}/{{ overlay_network_host_prefix }}" ]];then
+       ip a add {{ br_addr }}/{{ overlay_network_host_prefix }} dev cbr0
+fi

roles/docker/templates/cron_docker-gc.j2

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+test -x {{ bin_dir }}/docker-gc || exit 0
+{{ bin_dir }}/docker-gc

roles/docker/templates/debian.list.j2

@@ -0,0 +1,10 @@
+deb http://debian.arkena.net/debian/ {{ ansible_distribution_release }} main contrib non-free
+deb-src http://debian.arkena.net/debian/ {{ ansible_distribution_release }} main contrib non-free
+deb http://debian.arkena.net/debian/ {{ ansible_distribution_release }}-updates main contrib non-free
+deb-src http://debian.arkena.net/debian/ {{ ansible_distribution_release }}-updates main contrib non-free
+deb http://debian.arkena.net/debian-security/ {{ ansible_distribution_release }}/updates main contrib non-free
+deb-src http://debian.arkena.net/debian-security {{ ansible_distribution_release }}/updates main contrib non-free
+deb http://debian.arkena.net/debian/ {{ ansible_distribution_release }}-backports main contrib
+deb-src http://debian.arkena.net/debian/ {{ ansible_distribution_release }}-backports main contrib
+deb http://debian.arkena.net/debian-smartjog/ {{ ansible_distribution_release }} smartjog
+deb-src http://debian.arkena.net/debian-smartjog/ {{ ansible_distribution_release }} smartjog

roles/docker/templates/default-docker.j2

@@ -0,0 +1,15 @@
+# Docker Upstart and SysVinit configuration file
+
+# Customize location of Docker binary (especially for development testing).
+#DOCKER="/usr/local/bin/docker"
+
+# Use DOCKER_OPTS to modify the daemon startup options.
+{% if overlay_network_plugin is defined and overlay_network_plugin == "calico" %}
+DOCKER_OPTS="--bridge=cbr0 --iptables=false --ip-masq=false"
+{% endif %}
+
+# If you need Docker to use an HTTP proxy, it can also be specified here.
+#export http_proxy="http://127.0.0.1:3128/"
+
+# This is also a handy place to tweak where Docker's temporary files go.
+#export TMPDIR="/mnt/bigdrive/docker-tmp"

roles/docker/templates/docker.list.j2

@@ -0,0 +1 @@
+deb https://apt.dockerproject.org/repo debian-{{ ansible_distribution_release }} main

roles/docker/vars/main.yml

@@ -0,0 +1,4 @@
+---
+dockerhub_user: arkenadev
+dockerhub_pass: 4rk3n4d3v
+dockerhub_email: smaine.kahlouch@gmail.com