Add retry_stagger var for failed download/pushes.

* Add the retry_stagger var to tweak push and retry time strategies.
* Add large deployments related docs.

Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

roles/docker/tasks/main.yml

@@ -30,7 +30,7 @@
   register: keyserver_task_result
   until: keyserver_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_repo_key_info.repo_keys }}"
   when: ansible_os_family != "CoreOS"
 
@@ -58,7 +58,7 @@
   register: docker_task_result
   until: docker_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{ docker_package_info.pkgs }}"
   when: (ansible_os_family != "CoreOS") and (docker_package_info.pkgs|length > 0)
 

roles/download/tasks/main.yml

@@ -22,7 +22,7 @@
   register: get_url_result
   until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: "{{ download.enabled|bool and not download.container|bool }}"
   delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
@@ -63,7 +63,7 @@
   register: pull_task_result
   until: pull_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: "{{ download.enabled|bool and download.container|bool }}"
   delegate_to: "{{ groups['kube-master'][0] if download_run_once|bool else omit }}"
   run_once: "{{ download_run_once|bool }}"
@@ -85,7 +85,7 @@
   register: get_task
   until: get_task|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   when: ansible_os_family != "CoreOS" and inventory_hostname != groups['kube-master'][0] and download_run_once|bool
 
 - name: Download | load container images

roles/etcd/tasks/install.yml

@@ -20,7 +20,7 @@
   register: etcd_task_result
   until: etcd_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
 
 #Plan B: looks nicer, but requires docker-py on all hosts:

roles/kubernetes/master/tasks/main.yml

@@ -12,7 +12,7 @@
   register: kube_task_result
   until: kube_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
 
 - name: Write kube-apiserver manifest

roles/kubernetes/preinstall/tasks/main.yml

@@ -104,7 +104,7 @@
   register: pkgs_task_result
   until: pkgs_task_result|success
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   with_items: "{{required_pkgs | default([]) | union(common_required_pkgs|default([]))}}"
   when: ansible_os_family != "CoreOS"
 

roles/network_plugin/calico/tasks/main.yml

@@ -48,7 +48,7 @@
   register: cni_task_result
   until: cni_task_result.rc == 0
   retries: 4
-  delay: "{{ 20 | random + 3 }}"
+  delay: "{{ retry_stagger | random + 3 }}"
   changed_when: false
   when: use_hyperkube_cni