Update OpenStack Terraform: Modules, Bastions, and New Floating IP config (#1958)

* Adding bastion and private network provisioning for openstack terraform

* Remove usage of floating-ip property

* Combine openstack instances + floating ips

* Fix relating floating IPs to hosts for openstack builds

* Tighten up security groups

Allow ssh into all instances with floating IP

* Add the gluster hosts to the no-floating group

* Break terraform into modules

* Update README and var descriptions to match current config

* Remove volume property in gluster compute def

* Include cluster name in internal network and router names

* Make dns_nameservers a variable

contrib/terraform/openstack/modules/compute/main.tf

@@ -0,0 +1,280 @@
+
+
+variable user_data  {
+  type    = "string"
+  default = <<EOF
+#cloud-config
+manage_etc_hosts: localhost
+package_update: true
+package_upgrade: true
+EOF
+}
+resource "openstack_compute_keypair_v2" "k8s" {
+    name = "kubernetes-${var.cluster_name}"
+    public_key = "${chomp(file(var.public_key_path))}"
+}
+
+resource "openstack_compute_secgroup_v2" "k8s_master" {
+    name = "${var.cluster_name}-k8s-master"
+    description = "${var.cluster_name} - Kubernetes Master"
+    rule {
+        ip_protocol = "tcp"
+        from_port = "6443"
+        to_port = "6443"
+        cidr = "0.0.0.0/0"
+    }
+}
+
+resource "openstack_compute_secgroup_v2" "bastion" {
+    name = "${var.cluster_name}-bastion"
+    description = "${var.cluster_name} - Bastion Server"
+    rule {
+        ip_protocol = "tcp"
+        from_port = "22"
+        to_port = "22"
+        cidr = "0.0.0.0/0"
+    }
+}
+
+resource "openstack_compute_secgroup_v2" "k8s" {
+    name = "${var.cluster_name}-k8s"
+    description = "${var.cluster_name} - Kubernetes"
+    rule {
+        ip_protocol = "icmp"
+        from_port = "-1"
+        to_port = "-1"
+        cidr = "0.0.0.0/0"
+    }
+    rule {
+        ip_protocol = "tcp"
+        from_port = "1"
+        to_port = "65535"
+        self = true
+    }
+    rule {
+        ip_protocol = "udp"
+        from_port = "1"
+        to_port = "65535"
+        self = true
+    }
+    rule {
+        ip_protocol = "icmp"
+        from_port = "-1"
+        to_port = "-1"
+        self = true
+    }
+}
+
+resource "openstack_compute_instance_v2" "bastion" {
+    name = "${var.cluster_name}-bastion-${count.index+1}"
+    count = "${var.number_of_bastions}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_bastion}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}",
+                        "${openstack_compute_secgroup_v2.bastion.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "bastion"
+        depends_on = "${var.network_id}"
+    }
+
+    provisioner "local-exec" {
+	     command = "sed s/USER/${var.ssh_user}/ contrib/terraform/openstack/ansible_bastion_template.txt | sed s/BASTION_ADDRESS/${var.bastion_fips[0]}/ > contrib/terraform/openstack/group_vars/no-floating.yml"
+    }
+
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_instance_v2" "k8s_master" {
+    name = "${var.cluster_name}-k8s-master-${count.index+1}"
+    count = "${var.number_of_k8s_masters}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.bastion.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_instance_v2" "k8s_master_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_instance_v2" "etcd" {
+    name = "${var.cluster_name}-etcd-${count.index+1}"
+    count = "${var.number_of_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_etcd}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "etcd,vault,no-floating"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+
+resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip" {
+    name = "${var.cluster_name}-k8s-master-nf-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_floating_ip}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "etcd,kube-master,kube-node,k8s-cluster,vault,no-floating"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_instance_v2" "k8s_master_no_floating_ip_no_etcd" {
+    name = "${var.cluster_name}-k8s-master-ne-nf-${count.index+1}"
+    count = "${var.number_of_k8s_masters_no_floating_ip_no_etcd}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_master}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s_master.name}",
+                        "${openstack_compute_secgroup_v2.k8s.name}" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-master,kube-node,k8s-cluster,vault,no-floating"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+
+resource "openstack_compute_instance_v2" "k8s_node" {
+    name = "${var.cluster_name}-k8s-node-${count.index+1}"
+    count = "${var.number_of_k8s_nodes}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_node}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}",
+                        "${openstack_compute_secgroup_v2.bastion.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-node,k8s-cluster"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_instance_v2" "k8s_node_no_floating_ip" {
+    name = "${var.cluster_name}-k8s-node-nf-${count.index+1}"
+    count = "${var.number_of_k8s_nodes_no_floating_ip}"
+    image_name = "${var.image}"
+    flavor_id = "${var.flavor_k8s_node}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = [ "${openstack_compute_secgroup_v2.k8s.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user}"
+        kubespray_groups = "kube-node,k8s-cluster,no-floating"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "${var.user_data}"
+}
+
+resource "openstack_compute_floatingip_associate_v2" "bastion" {
+    count = "${var.number_of_bastions}"
+    floating_ip = "${var.bastion_fips[count.index]}"
+    instance_id = "${element(openstack_compute_instance_v2.bastion.*.id, count.index)}"
+}
+
+resource "openstack_compute_floatingip_associate_v2" "k8s_master" {
+    count = "${var.number_of_k8s_masters}"
+    instance_id = "${element(openstack_compute_instance_v2.k8s_master.*.id, count.index)}"
+    floating_ip = "${var.k8s_master_fips[count.index]}"
+}
+
+resource "openstack_compute_floatingip_associate_v2" "k8s_node" {
+    count = "${var.number_of_k8s_nodes}"
+    floating_ip = "${var.k8s_node_fips[count.index]}"
+    instance_id = "${element(openstack_compute_instance_v2.k8s_node.*.id, count.index)}"
+}
+
+
+resource "openstack_blockstorage_volume_v2" "glusterfs_volume" {
+  name = "${var.cluster_name}-glusterfs_volume-${count.index+1}"
+  count = "${var.number_of_gfs_nodes_no_floating_ip}"
+  description = "Non-ephemeral volume for GlusterFS"
+  size = "${var.gfs_volume_size_in_gb}"
+}
+
+resource "openstack_compute_instance_v2" "glusterfs_node_no_floating_ip" {
+    name = "${var.cluster_name}-gfs-node-nf-${count.index+1}"
+    count = "${var.number_of_gfs_nodes_no_floating_ip}"
+    image_name = "${var.image_gfs}"
+    flavor_id = "${var.flavor_gfs_node}"
+    key_pair = "${openstack_compute_keypair_v2.k8s.name}"
+    network {
+        name = "${var.network_name}"
+    }
+    security_groups = ["${openstack_compute_secgroup_v2.k8s.name}",
+                        "default" ]
+    metadata = {
+        ssh_user = "${var.ssh_user_gfs}"
+        kubespray_groups = "gfs-cluster,network-storage,no-floating"
+        depends_on = "${var.network_id}"
+    }
+    user_data = "#cloud-config\nmanage_etc_hosts: localhost\npackage_update: true\npackage_upgrade: true"
+}
+
+resource "openstack_compute_volume_attach_v2" "glusterfs_volume" {
+  count = "${var.number_of_gfs_nodes_no_floating_ip}"
+  instance_id = "${element(openstack_compute_instance_v2.glusterfs_node_no_floating_ip.*.id, count.index)}"
+  volume_id   = "${element(openstack_blockstorage_volume_v2.glusterfs_volume.*.id, count.index)}"
+}

contrib/terraform/openstack/modules/compute/variables.tf

@@ -0,0 +1,83 @@
+variable "cluster_name" {
+}
+
+variable "number_of_k8s_masters" {
+}
+
+variable "number_of_k8s_masters_no_etcd" {
+}
+
+variable "number_of_etcd" {
+}
+
+variable "number_of_k8s_masters_no_floating_ip" {
+}
+
+variable "number_of_k8s_masters_no_floating_ip_no_etcd" {
+}
+
+variable "number_of_k8s_nodes" {
+}
+
+variable "number_of_k8s_nodes_no_floating_ip" {
+}
+
+variable "number_of_bastions" {
+}
+
+variable "number_of_gfs_nodes_no_floating_ip" {
+}
+
+variable "gfs_volume_size_in_gb" {
+}
+
+variable "public_key_path" {
+}
+
+variable "image" {
+}
+
+variable "image_gfs" {
+}
+
+variable "ssh_user" {
+}
+
+variable "ssh_user_gfs" {
+}
+
+variable "flavor_k8s_master" {
+}
+
+variable "flavor_k8s_node" {
+}
+
+variable "flavor_etcd" {
+}
+
+variable "flavor_gfs_node" {
+}
+
+variable "network_name" {
+}
+
+variable "flavor_bastion" {
+}
+
+
+variable "network_id"{
+
+}
+
+
+variable "k8s_master_fips" {
+  type = "list"
+}
+
+variable "k8s_node_fips" {
+  type = "list"
+}
+
+variable "bastion_fips" {
+  type = "list"
+}

contrib/terraform/openstack/modules/ips/main.tf

@@ -0,0 +1,24 @@
+
+resource "null_resource" "dummy_dependency" {
+  triggers {
+    dependency_id = "${var.router_id}"
+  }
+}
+
+resource "openstack_networking_floatingip_v2" "k8s_master" {
+    count = "${var.number_of_k8s_masters}"
+    pool = "${var.floatingip_pool}"
+    depends_on = ["null_resource.dummy_dependency"]
+}
+
+resource "openstack_networking_floatingip_v2" "k8s_node" {
+    count = "${var.number_of_k8s_nodes}"
+    pool = "${var.floatingip_pool}"
+    depends_on = ["null_resource.dummy_dependency"]
+}
+
+resource "openstack_networking_floatingip_v2" "bastion" {
+    count = "${var.number_of_bastions}"
+    pool = "${var.floatingip_pool}"
+    depends_on = ["null_resource.dummy_dependency"]
+}

contrib/terraform/openstack/modules/ips/outputs.tf

@@ -0,0 +1,11 @@
+output "k8s_master_fips" {
+    value = ["${openstack_networking_floatingip_v2.k8s_master.*.address}"]
+}
+
+output "k8s_node_fips" {
+    value = ["${openstack_networking_floatingip_v2.k8s_node.*.address}"]
+}
+
+output "bastion_fips" {
+    value = ["${openstack_networking_floatingip_v2.bastion.*.address}"]
+}

contrib/terraform/openstack/modules/ips/variables.tf

@@ -0,0 +1,26 @@
+variable "number_of_k8s_masters" {
+}
+
+variable "number_of_k8s_masters_no_etcd" {
+}
+
+variable "number_of_k8s_nodes" {
+}
+
+variable "floatingip_pool" {
+}
+
+variable "number_of_bastions" {
+
+ }
+
+variable "external_net" {
+
+}
+
+variable "network_name" {
+}
+
+variable "router_id"{
+
+}

contrib/terraform/openstack/modules/network/main.tf

@@ -0,0 +1,24 @@
+
+resource "openstack_networking_router_v2" "k8s" {
+  name             = "${var.cluster_name}-router"
+  admin_state_up   = "true"
+  external_gateway = "${var.external_net}"
+}
+
+resource "openstack_networking_network_v2" "k8s" {
+  name           = "${var.network_name}"
+  admin_state_up = "true"
+}
+
+resource "openstack_networking_subnet_v2" "k8s" {
+  name            = "${var.cluster_name}-internal-network"
+  network_id      = "${openstack_networking_network_v2.k8s.id}"
+  cidr            = "10.0.0.0/24"
+  ip_version      = 4
+  dns_nameservers = "${var.dns_nameservers}"
+}
+
+resource "openstack_networking_router_interface_v2" "k8s" {
+  router_id = "${openstack_networking_router_v2.k8s.id}"
+  subnet_id = "${openstack_networking_subnet_v2.k8s.id}"
+}

contrib/terraform/openstack/modules/network/outputs.tf

@@ -0,0 +1,7 @@
+output "router_id" {
+    value = "${openstack_networking_router_interface_v2.k8s.id}"
+}
+
+output "network_id" {
+    value = "${openstack_networking_subnet_v2.k8s.id}"
+}

contrib/terraform/openstack/modules/network/variables.tf

@@ -0,0 +1,13 @@
+variable "external_net" {
+
+}
+
+variable "network_name" {
+}
+
+variable "cluster_name" {
+}
+
+variable "dns_nameservers"{
+  type = "list"
+}