From 5ca00279b8fb5313bb9b5f3a35690a8ced214643 Mon Sep 17 00:00:00 2001 From: Razvan Petrescu <7141388+razvan01@users.noreply.github.com> Date: Tue, 8 Sep 2026 12:16:29 +0200 Subject: [PATCH] Add contrib/terraform/opennebula: provision Kubespray VMs on OpenNebula (#13390) --- .gitlab-ci/terraform.yml | 1 + contrib/terraform/opennebula/README.md | 187 ++++++++++++++++++ contrib/terraform/opennebula/default.tfvars | 50 +++++ contrib/terraform/opennebula/main.tf | 96 +++++++++ .../modules/kubernetes-cluster/main.tf | 173 ++++++++++++++++ .../modules/kubernetes-cluster/output.tf | 13 ++ .../modules/kubernetes-cluster/variables.tf | 75 +++++++ .../modules/kubernetes-cluster/versions.tf | 9 + contrib/terraform/opennebula/output.tf | 15 ++ .../sample-inventory/cluster.tfvars | 50 +++++ .../opennebula/sample-inventory/group_vars | 1 + .../opennebula/templates/inventory.tpl | 17 ++ contrib/terraform/opennebula/variables.tf | 187 ++++++++++++++++++ contrib/terraform/opennebula/versions.tf | 13 ++ 14 files changed, 887 insertions(+) create mode 100644 contrib/terraform/opennebula/README.md create mode 100644 contrib/terraform/opennebula/default.tfvars create mode 100644 contrib/terraform/opennebula/main.tf create mode 100644 contrib/terraform/opennebula/modules/kubernetes-cluster/main.tf create mode 100644 contrib/terraform/opennebula/modules/kubernetes-cluster/output.tf create mode 100644 contrib/terraform/opennebula/modules/kubernetes-cluster/variables.tf create mode 100644 contrib/terraform/opennebula/modules/kubernetes-cluster/versions.tf create mode 100644 contrib/terraform/opennebula/output.tf create mode 100644 contrib/terraform/opennebula/sample-inventory/cluster.tfvars create mode 120000 contrib/terraform/opennebula/sample-inventory/group_vars create mode 100644 contrib/terraform/opennebula/templates/inventory.tpl create mode 100644 contrib/terraform/opennebula/variables.tf create mode 100644 contrib/terraform/opennebula/versions.tf diff --git a/.gitlab-ci/terraform.yml b/.gitlab-ci/terraform.yml index de17ce198..be371854c 100644 --- a/.gitlab-ci/terraform.yml +++ b/.gitlab-ci/terraform.yml @@ -36,6 +36,7 @@ terraform_validate: - exoscale - hetzner - vsphere + - opennebula - upcloud .terraform_apply: diff --git a/contrib/terraform/opennebula/README.md b/contrib/terraform/opennebula/README.md new file mode 100644 index 000000000..27781d5c3 --- /dev/null +++ b/contrib/terraform/opennebula/README.md @@ -0,0 +1,187 @@ +# Kubernetes on OpenNebula with Terraform + +Provision a Kubespray-ready cluster on OpenNebula 7.2+: Terraform creates the +VMs from an existing contextualized template and generates the Ansible +inventory, then the standard Kubespray playbook deploys Kubernetes. + +## Requirements + +- OpenNebula >= 7.2 with the XML-RPC endpoint reachable from this machine +- An existing VM template with the [one-context](https://github.com/OpenNebula/one-apps) + package installed (all OpenNebula marketplace OS appliances qualify) (for + the contextualization requirement — see the disk-options note below for a + separate template constraint). The template's network context must be + enabled (`NETWORK = "YES"` is set by this module). +- An existing Virtual Network whose addresses are reachable from the machine + running Terraform and Ansible +- Terraform >= 1.3 or OpenTofu (provider `OpenNebula/opennebula ~> 1.5` — the + release line adapted to the OpenNebula 7.x API) +- The VM template should NOT define its own NIC — the module attaches the + network explicitly, and a template NIC would leave nodes dual-homed with + the default route on the wrong interface. +- To use any disk option (`master_disk_size`/`worker_disk_size`/`additional_disk_size`) + the template's first disk must reference its image by numeric `IMAGE_ID` — + templates created via Sunstone wizards or the CLI often reference the image + by name (`IMAGE=`), which the provider cannot read; the module fails at + plan time with a clear error in that case. + +## Quickstart + +```ShellSession +$ cd kubespray +$ cp -LRp contrib/terraform/opennebula/sample-inventory inventory/mycluster +$ cd inventory/mycluster +$ $EDITOR cluster.tfvars # template_name, network_name, master/worker_count, ssh_public_keys +$ export OPENNEBULA_ENDPOINT="https://one.example.com:2634/RPC2" +$ export OPENNEBULA_USERNAME="oneadmin" +$ export OPENNEBULA_PASSWORD="opennebula" +$ terraform -chdir=../../contrib/terraform/opennebula init +$ terraform -chdir=../../contrib/terraform/opennebula apply \ + -var-file=$PWD/cluster.tfvars \ + -var inventory_file=$PWD/inventory.ini +``` + +**Pass `-var inventory_file=$PWD/inventory.ini` on every subsequent `apply` +as well** — without it the variable falls back to a path relative to the +module directory and the previously rendered inventory file is removed. + +After `apply`, the VMs are RUNNING in OpenNebula and `inventory.ini` contains +all Kubespray groups. Kubespray pins its Ansible version — install it in a +virtualenv from the repo root before deploying: + +```ShellSession +$ python3 -m venv ~/.venvs/kubespray && source ~/.venvs/kubespray/bin/activate && \ + pip install -r requirements.txt +``` + +Deploy Kubernetes: + +```ShellSession +$ cd ../.. && \ + ansible-playbook -i inventory/mycluster/inventory.ini cluster.yml -b +``` + +Smoke-test checklist: VMs visible and RUNNING in Sunstone; `inventory.ini` +written next to your `cluster.tfvars`; `ansible -i inventory/mycluster/inventory.ini -m ping all` +succeeds (one-context may need a few seconds after RUNNING before SSH is up). + +To tear the cluster down: `terraform -chdir=contrib/terraform/opennebula destroy +-var-file=$PWD/inventory/mycluster/cluster.tfvars` (run from the repo root; +this also deletes the rendered `inventory.ini`). + +The Terraform state lives in `contrib/terraform/opennebula/` — manage one +cluster per checkout, or use `terraform workspace` for more. + +## Variables + +### Required + +- `template_name`: name of the existing contextualized OpenNebula VM template +- `network_name`: name of the existing Virtual Network for the nodes +- `master_count` / `worker_count`: how many control-plane and worker nodes to + create (named `master-0..`, `worker-0..`; at least one master required) — + the simple way to define the cluster +- `machines`: advanced alternative to the counts (takes precedence when set); + map of machines, key = machine name (VM/hostname becomes `-`) + - `node_type`: `master` or `worker` (at least one master required) + - `ip`: optional static IP inside an address range of the network; empty + string lets OpenNebula assign a lease. When `network_reservation_size > 0` + the static IP must fall inside the **reservation's** address range, not + just the parent network's +- `ssh_public_keys`: list of SSH public keys injected into the VMs via + contextualization + +### Connection + +Set `OPENNEBULA_ENDPOINT`, `OPENNEBULA_USERNAME`, `OPENNEBULA_PASSWORD` +(and optionally `OPENNEBULA_INSECURE=true`) in the environment, or the +`one_endpoint` / `one_username` / `one_password` / `one_insecure` variables +(e.g. `one_endpoint = "https://one.example.com:2634/RPC2"`). +OpenNebula XML-RPC sends the username:password with every request — use a +TLS endpoint in production (`http://…:2633` is the plaintext out-of-the-box +default). + +### Optional + +- `prefix` (default `k8s`): VM name prefix — must be unique per OpenNebula + user when the extras are enabled (the extra-disk image, VM group and + reservation are named `-...` and OpenNebula rejects duplicate + names) +- `ansible_user` (default `root`): SSH user written into the inventory +- `inventory_file` (default `inventory.ini`): where the inventory is + rendered — relative paths resolve against `contrib/terraform/opennebula/` + because of `-chdir`; prefer an absolute path (e.g. set it in your + `cluster.tfvars`) +- `master_cpu`/`master_vcpu`/`master_memory` (1 / 2 / 4096 MB) +- `worker_cpu`/`worker_vcpu`/`worker_memory` (1 / 4 / 8192 MB) +- `master_disk_size`/`worker_disk_size` (MB, default 0 = keep the template's + disk; grow-only and create-time — raising it later does NOT grow an + existing cluster's root disks). Any disk option drops the template's + secondary disks (only the first disk is re-declared). +- `vm_create_timeout` (default `20m`) +- `additional_disk_size` (MB, default 0): extra non-persistent DATABLOCK disk + cloned per node (requires `image_datastore_name`) — strictly create-time in + BOTH directions: enabling it on an existing cluster silently hot-attaches + the re-declared OS disk as a duplicate OS-image clone on every node (apply + succeeds, wrong result), resizing it later fails against the in-use image, + and disabling it can attempt to detach the boot disk. Decide disk options + at cluster creation; changing them means recreating the VMs. The module + automatically re-declares the template's first disk alongside the extra + disk (preserving the template's `SIZE=` override; `TARGET`/`DRIVER` fall + back to image defaults); templates with more than one disk lose their + secondary disks whenever any disk option is used. +- `masters_anti_affinity` (default `false`): spread masters across hosts with + an `ANTI_AFFINED` VM group; requires at least as many hosts as masters — on + fewer hosts the VM stays pending until `vm_create_timeout` — changing this + on an existing cluster replaces every master VM. +- `network_reservation_size` (default 0): carve an address reservation of + this size out of `network_name` and attach the nodes to it — create-time + only (the provider cannot grow a reservation in place); use + `network_reservation_first_ip`/`network_reservation_ar_id` to pin the + reserved range so static `machines` IPs can target it. Toggling the + reservation on/off (0↔N) on an existing cluster re-attaches every node's + NIC on the other network and re-leases all IPs — it will break a deployed + cluster; treat it as a create-time choice. The reservation must be at + least as large as the number of nodes (enforced at plan time). Leaving + `network_reservation_first_ip`/`network_reservation_ar_id` unpinned causes + a harmless perpetual in-place diff on `plan` (the provider reads back the + allocated values) — pin both to avoid it. +- `network_reservation_first_ip` (default `""`): first IP of the reservation + carved from the parent network (empty = let OpenNebula choose) +- `network_reservation_ar_id` (default `null`): address-range ID of the + parent network to reserve from (null = provider default) + +## Troubleshooting + +- **CoreDNS / nodelocaldns in CrashLoopBackOff after deploy** (validated on + Ubuntu 24.04 marketplace images): the nodes' `/etc/resolv.conf` points at + the systemd-resolved stub (`127.0.0.53`), which CoreDNS detects as a + forwarding loop — or the OpenNebula VNet defines no `DNS` attribute at all. + Fix: set `upstream_dns_servers` (e.g. `[8.8.8.8, 1.1.1.1]` or your internal + resolver) in `inventory/$CLUSTER/group_vars/all/all.yml` and re-run + `cluster.yml`. Infrastructure-level alternative: set the `DNS` attribute on + the OpenNebula Virtual Network so contextualized guests get a real resolver. +- **A failed first run can leave the cluster half-installed**: if + `cluster.yml` dies before the network plugin is deployed, a re-run may wait + forever on "control plane nodes to be Ready" (the nodes stay NotReady + without a CNI). Recreate the VMs (`terraform destroy` + `apply`) and deploy + again on clean machines. +- **Transient `Service is in unknown state` handler failures** during heavy + image-pull phases are usually a one-off systemd/dbus hiccup — re-running + `cluster.yml` is safe and idempotent. + +## Known issues + +- Deleting VMs/images/networks manually in OpenNebula breaks `terraform + refresh`/`plan` with `NO_EXISTS` ([provider #623](https://github.com/OpenNebula/terraform-provider-opennebula/issues/623)); + workaround: `terraform state rm `. +- Overriding the template disk (any disk option: `*_disk_size > 0` or + `additional_disk_size > 0`) can produce perpetual + in-place updates in `plan` with some template layouts + ([provider #598](https://github.com/OpenNebula/terraform-provider-opennebula/issues/598)). +- A network reservation supports a single address range + ([provider #625](https://github.com/OpenNebula/terraform-provider-opennebula/issues/625)). +- IPv4 only: IPv6 read-back from templates is broken in provider 1.5.0 + ([provider #615](https://github.com/OpenNebula/terraform-provider-opennebula/issues/615)). +- Provider CI upstream covers OpenNebula 6.10/7.0; 7.2 keeps XML-RPC + compatibility and has no known provider issues. diff --git a/contrib/terraform/opennebula/default.tfvars b/contrib/terraform/opennebula/default.tfvars new file mode 100644 index 000000000..cc8824310 --- /dev/null +++ b/contrib/terraform/opennebula/default.tfvars @@ -0,0 +1,50 @@ +prefix = "k8s" + +# Prefer an absolute path (relative paths resolve against the module directory): +# inventory_file = "/path/to/inventory/mycluster/inventory.ini" + +# Existing OpenNebula objects +template_name = "i-did-not-read-the-docs" # e.g. ubuntu2404-context +network_name = "i-did-not-read-the-docs" # e.g. private + +# Node counts (nodes are named master-0.., worker-0..) +master_count = 1 +worker_count = 2 + +# Advanced alternative: named nodes and/or static IPs. Takes precedence over +# master_count/worker_count when set. +# machines = { +# "master-0" : { +# "node_type" : "master", +# "ip" : "" # optional static IP from an address range of the network, e.g. 192.168.0.10 +# }, +# "worker-0" : { +# "node_type" : "worker", +# "ip" : "" +# } +# } + +ssh_public_keys = [ + # Put your public SSH key here + "ssh-rsa I-did-not-read-the-docs", +] + +# SSH user for the generated inventory (one-context installs keys for root +# on standard OpenNebula marketplace images) +# ansible_user = "root" + +## Sizing +# master_vcpu = 2 +# master_memory = 4096 +# worker_vcpu = 4 +# worker_memory = 8192 +# master_disk_size = 0 # MB; 0 keeps the template disk size +# worker_disk_size = 0 # MB; 0 keeps the template disk size + +## OpenNebula extras +# additional_disk_size = 0 # MB; extra DATABLOCK disk on every node +# image_datastore_name = "" # required when additional_disk_size > 0 +# masters_anti_affinity = false +# network_reservation_size = 0 # carve an address reservation out of network_name +# network_reservation_first_ip = "" # pin the reserved range's first IP +# network_reservation_ar_id = null # parent address-range ID to reserve from (null = provider default) diff --git a/contrib/terraform/opennebula/main.tf b/contrib/terraform/opennebula/main.tf new file mode 100644 index 000000000..3d36fe23b --- /dev/null +++ b/contrib/terraform/opennebula/main.tf @@ -0,0 +1,96 @@ +provider "opennebula" { + # Credentials/endpoint fall back to the OPENNEBULA_* environment variables + # when the corresponding variable is left empty. + endpoint = var.one_endpoint != "" ? var.one_endpoint : null + username = var.one_username != "" ? var.one_username : null + password = var.one_password != "" ? var.one_password : null + insecure = var.one_insecure +} + +data "opennebula_template" "node" { + name = var.template_name + + lifecycle { + precondition { + condition = length([ + for machine in local.machines : machine if machine.node_type == "master" + ]) > 0 + error_message = "Define at least one control-plane node: set master_count >= 1 or add a machines entry with node_type \"master\"." + } + } +} + +data "opennebula_virtual_network" "network" { + name = var.network_name +} + +locals { + # Effective node set: the explicit machines map wins; otherwise it is + # generated from master_count/worker_count (master-0.., worker-0..). + machines = length(var.machines) > 0 ? var.machines : merge( + { for i in range(var.master_count) : "master-${i}" => { node_type = "master", ip = "" } }, + { for i in range(var.worker_count) : "worker-${i}" => { node_type = "worker", ip = "" } }, + ) + # The provider's data source flattens template disks that reference their + # image by name (IMAGE=) as image_id = -1; treat that sentinel as unusable. + template_disk_image_id = try(data.opennebula_template.node.disk[0].image_id, -1) >= 0 ? data.opennebula_template.node.disk[0].image_id : null + # Preserve the template's DISK-level SIZE override when the OS disk has to + # be re-declared (0 = no override, use the image's size) + template_disk_size = try(data.opennebula_template.node.disk[0].size, 0) +} + +module "kubernetes" { + source = "./modules/kubernetes-cluster" + + prefix = var.prefix + machines = local.machines + + template_id = data.opennebula_template.node.id + template_disk_image_id = local.template_disk_image_id + template_disk_size = local.template_disk_size + network_id = data.opennebula_virtual_network.network.id + + ## Master ## + master_cpu = var.master_cpu + master_vcpu = var.master_vcpu + master_memory = var.master_memory + master_disk_size = var.master_disk_size + + ## Worker ## + worker_cpu = var.worker_cpu + worker_vcpu = var.worker_vcpu + worker_memory = var.worker_memory + worker_disk_size = var.worker_disk_size + + ## OpenNebula extras ## + additional_disk_size = var.additional_disk_size + image_datastore_name = var.image_datastore_name + masters_anti_affinity = var.masters_anti_affinity + network_reservation_size = var.network_reservation_size + network_reservation_first_ip = var.network_reservation_first_ip + network_reservation_ar_id = var.network_reservation_ar_id + + ssh_public_keys = var.ssh_public_keys + vm_create_timeout = var.vm_create_timeout +} + +# +# Generate ansible inventory +# + +resource "local_file" "inventory" { + content = templatefile("${path.module}/templates/inventory.tpl", { + connection_strings_master = join("\n", formatlist("%s ansible_user=${var.ansible_user} ansible_host=%s ip=%s etcd_member_name=etcd%d", + keys(module.kubernetes.master_ip), + values(module.kubernetes.master_ip), + values(module.kubernetes.master_ip), + range(1, length(module.kubernetes.master_ip) + 1))), + connection_strings_worker = join("\n", formatlist("%s ansible_user=${var.ansible_user} ansible_host=%s ip=%s", + keys(module.kubernetes.worker_ip), + values(module.kubernetes.worker_ip), + values(module.kubernetes.worker_ip))), + list_master = join("\n", keys(module.kubernetes.master_ip)), + list_worker = join("\n", keys(module.kubernetes.worker_ip)) + }) + filename = var.inventory_file +} diff --git a/contrib/terraform/opennebula/modules/kubernetes-cluster/main.tf b/contrib/terraform/opennebula/modules/kubernetes-cluster/main.tf new file mode 100644 index 000000000..11cb7d5af --- /dev/null +++ b/contrib/terraform/opennebula/modules/kubernetes-cluster/main.tf @@ -0,0 +1,173 @@ +locals { + masters = { + for name, machine in var.machines : + name => machine + if machine.node_type == "master" + } + workers = { + for name, machine in var.machines : + name => machine + if machine.node_type == "worker" + } + context = { + NETWORK = "YES" + SET_HOSTNAME = "$NAME" + SSH_PUBLIC_KEY = join("\n", var.ssh_public_keys) + } + node_network_id = var.network_reservation_size > 0 ? opennebula_virtual_network.reservation[0].id : var.network_id +} + +# Optional reservation carved out of the parent network (one AR only — +# provider issue #625) +resource "opennebula_virtual_network" "reservation" { + count = var.network_reservation_size > 0 ? 1 : 0 + name = "${var.prefix}-reservation" + reservation_vnet = var.network_id + reservation_size = var.network_reservation_size + reservation_first_ip = var.network_reservation_first_ip != "" ? var.network_reservation_first_ip : null + reservation_ar_id = var.network_reservation_ar_id + + lifecycle { + precondition { + condition = var.network_reservation_size >= length(var.machines) + error_message = "network_reservation_size must be at least the number of machines (each node leases one address from the reservation)." + } + } +} + +# Optional shared non-persistent DATABLOCK image; every VM attaching it gets +# its own clone, deleted together with the VM +data "opennebula_datastore" "image" { + count = var.additional_disk_size > 0 ? 1 : 0 + name = var.image_datastore_name + + lifecycle { + precondition { + condition = var.image_datastore_name != "" + error_message = "image_datastore_name must be set when additional_disk_size > 0." + } + } +} + +resource "opennebula_image" "extra_disk" { + count = var.additional_disk_size > 0 ? 1 : 0 + name = "${var.prefix}-extra-disk" + datastore_id = data.opennebula_datastore.image[0].id + type = "DATABLOCK" + size = var.additional_disk_size + persistent = false + permissions = "600" +} + +# Anti-affinity: spread control-plane VMs across hosts +resource "opennebula_virtual_machine_group" "cluster" { + count = var.masters_anti_affinity ? 1 : 0 + name = "${var.prefix}-vm-group" + + role { + name = "master" + policy = "ANTI_AFFINED" + } +} + +resource "opennebula_virtual_machine" "master" { + for_each = local.masters + + name = "${var.prefix}-${each.key}" + template_id = var.template_id + cpu = var.master_cpu + vcpu = var.master_vcpu + memory = var.master_memory + + context = local.context + + nic { + model = "virtio" + network_id = local.node_network_id + ip = each.value.ip != "" ? each.value.ip : null + } + + # Any disk block replaces ALL of the template's disks (provider merge + # semantics), so the OS disk must be re-declared whenever another disk + # block is emitted. + dynamic "disk" { + for_each = var.master_disk_size > 0 || var.additional_disk_size > 0 ? [1] : [] + content { + image_id = var.template_disk_image_id + size = var.master_disk_size > 0 ? var.master_disk_size : (var.template_disk_size > 0 ? var.template_disk_size : null) + } + } + + dynamic "disk" { + for_each = var.additional_disk_size > 0 ? [1] : [] + content { + image_id = opennebula_image.extra_disk[0].id + } + } + + dynamic "vmgroup" { + for_each = var.masters_anti_affinity ? [1] : [] + content { + vmgroup_id = opennebula_virtual_machine_group.cluster[0].id + role = "master" + } + } + + timeouts { + create = var.vm_create_timeout + } + + lifecycle { + precondition { + condition = (var.master_disk_size <= 0 && var.additional_disk_size <= 0) || var.template_disk_image_id != null + error_message = "master_disk_size/additional_disk_size require the template's first disk to reference its image by numeric IMAGE_ID (templates referencing the image by name are not readable through the provider data source)." + } + } +} + +resource "opennebula_virtual_machine" "worker" { + for_each = local.workers + + name = "${var.prefix}-${each.key}" + template_id = var.template_id + cpu = var.worker_cpu + vcpu = var.worker_vcpu + memory = var.worker_memory + + context = local.context + + nic { + model = "virtio" + network_id = local.node_network_id + ip = each.value.ip != "" ? each.value.ip : null + } + + # Any disk block replaces ALL of the template's disks (provider merge + # semantics), so the OS disk must be re-declared whenever another disk + # block is emitted. + dynamic "disk" { + for_each = var.worker_disk_size > 0 || var.additional_disk_size > 0 ? [1] : [] + content { + image_id = var.template_disk_image_id + size = var.worker_disk_size > 0 ? var.worker_disk_size : (var.template_disk_size > 0 ? var.template_disk_size : null) + } + } + + dynamic "disk" { + for_each = var.additional_disk_size > 0 ? [1] : [] + content { + image_id = opennebula_image.extra_disk[0].id + } + } + + timeouts { + create = var.vm_create_timeout + } + + lifecycle { + precondition { + condition = (var.worker_disk_size <= 0 && var.additional_disk_size <= 0) || var.template_disk_image_id != null + error_message = "worker_disk_size/additional_disk_size require the template's first disk to reference its image by numeric IMAGE_ID (templates referencing the image by name are not readable through the provider data source)." + } + } +} diff --git a/contrib/terraform/opennebula/modules/kubernetes-cluster/output.tf b/contrib/terraform/opennebula/modules/kubernetes-cluster/output.tf new file mode 100644 index 000000000..60e050462 --- /dev/null +++ b/contrib/terraform/opennebula/modules/kubernetes-cluster/output.tf @@ -0,0 +1,13 @@ +output "master_ip" { + value = { + for name, vm in opennebula_virtual_machine.master : + vm.name => vm.nic[0].computed_ip + } +} + +output "worker_ip" { + value = { + for name, vm in opennebula_virtual_machine.worker : + vm.name => vm.nic[0].computed_ip + } +} diff --git a/contrib/terraform/opennebula/modules/kubernetes-cluster/variables.tf b/contrib/terraform/opennebula/modules/kubernetes-cluster/variables.tf new file mode 100644 index 000000000..da4263584 --- /dev/null +++ b/contrib/terraform/opennebula/modules/kubernetes-cluster/variables.tf @@ -0,0 +1,75 @@ +variable "prefix" {} + +variable "machines" { + description = "Cluster machines" + type = map(object({ + node_type = string + ip = optional(string, "") + })) +} + +variable "template_id" { + description = "ID of the existing contextualized OpenNebula VM template" + type = number +} + +variable "template_disk_image_id" { + description = "Image ID of the template's first disk (null when the template defines no image disk); required only for disk resizing" + type = number + default = null +} + +variable "template_disk_size" { + description = "SIZE (MB) declared on the template's first disk (0 = image default); used when re-declaring the OS disk" + type = number + default = 0 +} + +variable "network_id" { + description = "ID of the OpenNebula Virtual Network the nodes attach to" + type = number +} + +variable "master_cpu" {} +variable "master_vcpu" {} +variable "master_memory" {} +variable "master_disk_size" {} + +variable "worker_cpu" {} +variable "worker_vcpu" {} +variable "worker_memory" {} +variable "worker_disk_size" {} + +variable "ssh_public_keys" { + type = list(string) +} + +variable "vm_create_timeout" {} + +variable "additional_disk_size" { + default = 0 +} + +variable "image_datastore_name" { + default = "" +} + +variable "masters_anti_affinity" { + type = bool + default = false +} + +variable "network_reservation_size" { + default = 0 +} + +variable "network_reservation_first_ip" { + description = "First IP of the reservation carved from the parent network (empty = let OpenNebula choose)" + default = "" +} + +variable "network_reservation_ar_id" { + description = "Address-range ID of the parent network to reserve from (null = provider default)" + type = number + default = null +} diff --git a/contrib/terraform/opennebula/modules/kubernetes-cluster/versions.tf b/contrib/terraform/opennebula/modules/kubernetes-cluster/versions.tf new file mode 100644 index 000000000..2c0385399 --- /dev/null +++ b/contrib/terraform/opennebula/modules/kubernetes-cluster/versions.tf @@ -0,0 +1,9 @@ +terraform { + required_providers { + opennebula = { + source = "OpenNebula/opennebula" + version = "~> 1.5" + } + } + required_version = ">= 1.3.0" +} diff --git a/contrib/terraform/opennebula/output.tf b/contrib/terraform/opennebula/output.tf new file mode 100644 index 000000000..ad27091df --- /dev/null +++ b/contrib/terraform/opennebula/output.tf @@ -0,0 +1,15 @@ +output "master_ip_addresses" { + value = module.kubernetes.master_ip +} + +output "worker_ip_addresses" { + value = module.kubernetes.worker_ip +} + +output "one_template" { + value = var.template_name +} + +output "one_network" { + value = var.network_name +} diff --git a/contrib/terraform/opennebula/sample-inventory/cluster.tfvars b/contrib/terraform/opennebula/sample-inventory/cluster.tfvars new file mode 100644 index 000000000..cc8824310 --- /dev/null +++ b/contrib/terraform/opennebula/sample-inventory/cluster.tfvars @@ -0,0 +1,50 @@ +prefix = "k8s" + +# Prefer an absolute path (relative paths resolve against the module directory): +# inventory_file = "/path/to/inventory/mycluster/inventory.ini" + +# Existing OpenNebula objects +template_name = "i-did-not-read-the-docs" # e.g. ubuntu2404-context +network_name = "i-did-not-read-the-docs" # e.g. private + +# Node counts (nodes are named master-0.., worker-0..) +master_count = 1 +worker_count = 2 + +# Advanced alternative: named nodes and/or static IPs. Takes precedence over +# master_count/worker_count when set. +# machines = { +# "master-0" : { +# "node_type" : "master", +# "ip" : "" # optional static IP from an address range of the network, e.g. 192.168.0.10 +# }, +# "worker-0" : { +# "node_type" : "worker", +# "ip" : "" +# } +# } + +ssh_public_keys = [ + # Put your public SSH key here + "ssh-rsa I-did-not-read-the-docs", +] + +# SSH user for the generated inventory (one-context installs keys for root +# on standard OpenNebula marketplace images) +# ansible_user = "root" + +## Sizing +# master_vcpu = 2 +# master_memory = 4096 +# worker_vcpu = 4 +# worker_memory = 8192 +# master_disk_size = 0 # MB; 0 keeps the template disk size +# worker_disk_size = 0 # MB; 0 keeps the template disk size + +## OpenNebula extras +# additional_disk_size = 0 # MB; extra DATABLOCK disk on every node +# image_datastore_name = "" # required when additional_disk_size > 0 +# masters_anti_affinity = false +# network_reservation_size = 0 # carve an address reservation out of network_name +# network_reservation_first_ip = "" # pin the reserved range's first IP +# network_reservation_ar_id = null # parent address-range ID to reserve from (null = provider default) diff --git a/contrib/terraform/opennebula/sample-inventory/group_vars b/contrib/terraform/opennebula/sample-inventory/group_vars new file mode 120000 index 000000000..373595823 --- /dev/null +++ b/contrib/terraform/opennebula/sample-inventory/group_vars @@ -0,0 +1 @@ +../../../../inventory/sample/group_vars \ No newline at end of file diff --git a/contrib/terraform/opennebula/templates/inventory.tpl b/contrib/terraform/opennebula/templates/inventory.tpl new file mode 100644 index 000000000..28ff28ac2 --- /dev/null +++ b/contrib/terraform/opennebula/templates/inventory.tpl @@ -0,0 +1,17 @@ + +[all] +${connection_strings_master} +${connection_strings_worker} + +[kube_control_plane] +${list_master} + +[etcd] +${list_master} + +[kube_node] +${list_worker} + +[k8s_cluster:children] +kube_control_plane +kube_node diff --git a/contrib/terraform/opennebula/variables.tf b/contrib/terraform/opennebula/variables.tf new file mode 100644 index 000000000..8b753af9d --- /dev/null +++ b/contrib/terraform/opennebula/variables.tf @@ -0,0 +1,187 @@ +## OpenNebula connection. Leave empty to use the OPENNEBULA_ENDPOINT, +## OPENNEBULA_USERNAME, OPENNEBULA_PASSWORD and OPENNEBULA_INSECURE +## environment variables. + +variable "one_endpoint" { + description = "OpenNebula XML-RPC API endpoint, e.g. https://one.example.com:2634/RPC2" + default = "" +} + +variable "one_username" { + default = "" +} + +variable "one_password" { + default = "" + sensitive = true +} + +variable "one_insecure" { + description = "Skip TLS certificate verification of the OpenNebula endpoint (null = use OPENNEBULA_INSECURE if set)" + type = bool + default = null +} + +## Cluster + +variable "prefix" { + default = "k8s" +} + +variable "inventory_file" { + default = "inventory.ini" +} + +variable "template_name" { + description = "Name of the existing contextualized OpenNebula VM template" +} + +variable "network_name" { + description = "Name of the existing OpenNebula Virtual Network the nodes attach to" +} + +variable "master_count" { + description = "Number of control-plane nodes to create (named master-0..N-1); ignored when machines is set" + type = number + default = 0 + + validation { + condition = var.master_count >= 0 + error_message = "master_count must be >= 0." + } +} + +variable "worker_count" { + description = "Number of worker nodes to create (named worker-0..N-1); ignored when machines is set" + type = number + default = 0 + + validation { + condition = var.worker_count >= 0 + error_message = "worker_count must be >= 0." + } +} + +variable "machines" { + description = "Advanced node definition (named nodes, optional static IPs); takes precedence over master_count/worker_count when non-empty" + type = map(object({ + node_type = string + ip = optional(string, "") + })) + default = {} + + validation { + condition = alltrue([ + for machine in var.machines : contains(["master", "worker"], machine.node_type) + ]) + error_message = "machines[*].node_type must be \"master\" or \"worker\"." + } +} + +variable "ssh_public_keys" { + description = "List of public SSH keys which are injected into the VMs." + type = list(string) +} + +variable "ansible_user" { + description = "SSH user written into the generated inventory (the user one-context injects keys for; root on standard OpenNebula images)" + default = "root" +} + +## Master + +variable "master_cpu" { + default = 1 +} + +variable "master_vcpu" { + default = 2 +} + +variable "master_memory" { + description = "Memory in MB" + default = 4096 +} + +variable "master_disk_size" { + description = "OS disk size in MB; 0 keeps the template's disk unchanged" + default = 0 + + validation { + condition = var.master_disk_size >= 0 + error_message = "master_disk_size must be >= 0 (MB)." + } +} + +## Worker + +variable "worker_cpu" { + default = 1 +} + +variable "worker_vcpu" { + default = 4 +} + +variable "worker_memory" { + description = "Memory in MB" + default = 8192 +} + +variable "worker_disk_size" { + description = "OS disk size in MB; 0 keeps the template's disk unchanged" + default = 0 + + validation { + condition = var.worker_disk_size >= 0 + error_message = "worker_disk_size must be >= 0 (MB)." + } +} + +variable "vm_create_timeout" { + default = "20m" +} + +## OpenNebula extras + +variable "additional_disk_size" { + description = "Size (MB) of an extra DATABLOCK disk attached to every node; 0 disables it" + default = 0 + + validation { + condition = var.additional_disk_size >= 0 + error_message = "additional_disk_size must be >= 0 (MB)." + } +} + +variable "image_datastore_name" { + description = "IMAGE datastore for the additional DATABLOCK disk (required when additional_disk_size > 0)" + default = "" +} + +variable "masters_anti_affinity" { + description = "Spread master VMs across hosts using an ANTI_AFFINED VM group" + type = bool + default = false +} + +variable "network_reservation_size" { + description = "If > 0, create a reservation of this many addresses from network_name and attach the nodes to it" + default = 0 + + validation { + condition = var.network_reservation_size >= 0 + error_message = "network_reservation_size must be >= 0." + } +} + +variable "network_reservation_first_ip" { + description = "First IP of the reservation carved from the parent network (empty = let OpenNebula choose)" + default = "" +} + +variable "network_reservation_ar_id" { + description = "Address-range ID of the parent network to reserve from (null = provider default)" + type = number + default = null +} diff --git a/contrib/terraform/opennebula/versions.tf b/contrib/terraform/opennebula/versions.tf new file mode 100644 index 000000000..8edcc5cde --- /dev/null +++ b/contrib/terraform/opennebula/versions.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + opennebula = { + source = "OpenNebula/opennebula" + version = "~> 1.5" + } + local = { + source = "hashicorp/local" + version = ">= 2.0" + } + } + required_version = ">= 1.3.0" +}