Handling docker clean up during docker upgrade and docker config changes. (#3321)

* Added changes to clean up orphan containers and reload docker & kubelet directories.

* Added new files for cleaning up orphans and docker & kubelet directories

* Added new lines at the end of these files

* removed the trailing whitespaces from main.yml and clean-up.yml

* Updated as per the review comments

* Updated as per the review comments

* Removed service_facts and package_facts because they are not supported in ansible 2.4.0

* Corrected yaml syntax errors

* Removed the use of json_query filter and utilized selectattr

* Removed trailing spaces

* Changed the default value of docker_clean_up to false

* Added Changes to only include cleanup-docker-orphans.sh

* Reverted back changes done inside handler.

* Removed trailing spaces and made default value of docker_orphan_clean_up as true

* Reverted the default value of docker_orphan_clean_up as false

* Made the docker clean up as drop in

* Made the docker clean up as drop in

* Reverted the value of boolean docker_orphan_clean_up to false

roles/container-engine/docker/files/cleanup-docker-orphans.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+list_descendants ()
+{
+  local children=$(ps -o pid= --ppid "$1")
+  for pid in $children
+  do
+    list_descendants "$pid"
+  done
+  [[ -n "$children" ]] && echo "$children"
+}
+
+count_shim_processes=$(pgrep -f ^docker-containerd-shim | wc -l)
+live_restore=$(docker info --format {{.LiveRestoreEnabled}} 2>/dev/null)
+
+if [ ${count_shim_processes} -gt 0 ] && [ -n "${live_restore}" -a "${live_restore}" == "true" ]; then
+        # Find all container pids from shims
+        orphans=$(pgrep -P $(pgrep -d ',' -f ^docker-containerd-shim) |\
+        # Filter out valid docker pids, leaving the orphans
+        egrep -v $(docker ps -q | xargs docker inspect --format '{{.State.Pid}}' | awk '{printf "%s%s",sep,$1; sep="|"}'))
+
+        if [[ -n "$orphans" ]]
+        then
+                # Get shim pids of orphans
+                orphan_shim_pids=$(ps -o pid= $(ps -o ppid= $orphans))
+
+                # Find all orphaned container PIDs
+                orphan_container_pids=$(for pid in $orphan_shim_pids; do list_descendants $pid; done)
+
+                # Recursively kill all child PIDs of orphan shims
+                echo -e "Killing orphan container PIDs and descendants: \n$(ps -O ppid= $orphan_container_pids)"
+                #kill -9 $orphan_container_pids || true
+
+        else
+                echo "No orphaned containers found"
+        fi
+else
+        echo "Either live-restore is turned off or the node doesn't have any shim processes."
+fi