mirror of
https://github.com/PyratLabs/ansible-role-k3s
synced 2024-11-05 13:23:30 +01:00
Merge pull request #6 from PyratLabs/multi_master_support
Auto-deploy templates, HA support now possible.
This commit is contained in:
commit
df253b504a
27 changed files with 508 additions and 68 deletions
30
.travis.yml
30
.travis.yml
|
@ -8,6 +8,9 @@ services: docker
|
|||
env:
|
||||
global:
|
||||
- ROLE_NAME: k3s
|
||||
- ROLE_OWNER: xanmanning
|
||||
- REPO_OWNER: PyratLabs
|
||||
- MOLECULE_SCENARIO: default
|
||||
matrix:
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-centos7-ansible:latest
|
||||
|
@ -34,6 +37,27 @@ env:
|
|||
- MOLECULE_DISTRO: geerlingguy/docker-fedora31-ansible:latest
|
||||
MOLECULE_PLAYBOOK: playbook-no-deploy.yml
|
||||
|
||||
# Test auto deploying manifests
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-ubuntu1804-ansible:latest
|
||||
MOLECULE_PLAYBOOK: playbook-auto-deploying-manifests.yml
|
||||
|
||||
# Test multiple masters in control plane with PostgreSQL
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
|
||||
MOLECULE_SCENARIO_DIRECTORY: /home/travis/build/${REPO_OWNER}/${ROLE_OWNER}.${ROLE_NAME}/molecule/highavailability
|
||||
MOLECULE_SCENARIO: highavailability
|
||||
|
||||
# Test multiple masters with auto deploying manifests
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
|
||||
MOLECULE_SCENARIO_DIRECTORY: /home/travis/build/${REPO_OWNER}/${ROLE_OWNER}.${ROLE_NAME}/molecule/highavailability
|
||||
MOLECULE_SCENARIO: highavailability
|
||||
MOLECULE_PLAYBOOK: playbook-auto-deploying-manifests.yml
|
||||
|
||||
# Test multiple masters in control plane with DQLite
|
||||
- MOLECULE_DISTRO: geerlingguy/docker-centos8-ansible:latest
|
||||
MOLECULE_SCENARIO_DIRECTORY: /home/travis/build/${REPO_OWNER}/${ROLE_OWNER}.${ROLE_NAME}/molecule/highavailability
|
||||
MOLECULE_SCENARIO: highavailability
|
||||
MOLECULE_PLAYBOOK: playbook-dqlite.yml
|
||||
|
||||
install:
|
||||
# Install test dependencies.
|
||||
- pip install molecule docker jmespath
|
||||
|
@ -41,12 +65,12 @@ install:
|
|||
before_script:
|
||||
# Use actual Ansible Galaxy role name for the project directory.
|
||||
- cd ../
|
||||
- mv ansible-role-$ROLE_NAME xanmanning.$ROLE_NAME
|
||||
- cd xanmanning.$ROLE_NAME
|
||||
- mv ansible-role-${ROLE_NAME} ${ROLE_OWNER}.${ROLE_NAME}
|
||||
- cd ${ROLE_OWNER}.${ROLE_NAME}
|
||||
|
||||
script:
|
||||
# Run tests.
|
||||
- molecule test
|
||||
- molecule test --scenario-name "${MOLECULE_SCENARIO}"
|
||||
|
||||
notifications:
|
||||
webhooks: https://galaxy.ansible.com/api/v1/notifications/
|
||||
|
|
136
README.md
136
README.md
|
@ -43,35 +43,42 @@ my spare time so I cannot promise a speedy fix delivery.
|
|||
Below are variables that are set against all of the play hosts for environment
|
||||
consistency.
|
||||
|
||||
| Variable | Description | Default Value |
|
||||
|--------------------------------|--------------------------------------------------------------------------|--------------------------------|
|
||||
| `k3s_cluster_state` | State of cluster: installed, started, stopped, restarted, downloaded. | installed |
|
||||
| `k3s_release_version` | Use a specific version of k3s, eg. `v0.2.0`. Specify `false` for latest. | `false` |
|
||||
| `k3s_github_url` | Set the GitHub URL to install k3s from. | https://github.com/rancher/k3s |
|
||||
| `k3s_install_dir` | Installation directory for k3s. | `/usr/local/bin` |
|
||||
| `k3s_use_experimental` | Allow the use of experimental features in k3s. | `false` |
|
||||
| `k3s_non_root` | Install k3s as non-root user. See notes below. | `false` |
|
||||
| `k3s_control_workers` | Are control hosts also workers? | `true` |
|
||||
| `k3s_cluster_cidr` | Network CIDR to use for pod IPs | 10.42.0.0/16 |
|
||||
| `k3s_service_cidr` | Network CIDR to use for service IPs | 10.43.0.0/16 |
|
||||
| `k3s_control_node_address` | Use a specific control node address. IP or FQDN. | _NULL_ |
|
||||
| `k3s_control_token` | Use a specific control token, please read notes below. | _NULL_ |
|
||||
| `k3s_https_port` | HTTPS port listening port. | 6443 |
|
||||
| `k3s_use_docker` | Use Docker rather than Containerd? | `false` |
|
||||
| `k3s_no_flannel` | Do not use Flannel | `false` |
|
||||
| `k3s_flannel_backend` | Flannel backend ('none', 'vxlan', 'ipsec', or 'wireguard') | vxlan |
|
||||
| `k3s_no_coredns` | Do not use CoreDNS | `false` |
|
||||
| `k3s_cluster_dns` | Cluster IP for CoreDNS service. Should be in your service-cidr range. | _NULL_ |
|
||||
| `k3s_cluster_domain` | Cluster Domain. | cluster.local |
|
||||
| `k3s_resolv_conf` | Kubelet resolv.conf file | _NULL_ |
|
||||
| `k3s_no_traefik` | Do not use Traefik | `false` |
|
||||
| `k3s_no_servicelb` | Do not use ServiceLB, necessary for using something like MetalLB. | `false` |
|
||||
| `k3s_no_local_storage` | Do not use Local Storage | `false` |
|
||||
| `k3s_no_metrics_server` | Do not deploy metrics server | `false` |
|
||||
| `k3s_disable_scheduler` | Disable Kubernetes default scheduler | `false` |
|
||||
| `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager. | `false` |
|
||||
| `k3s_disable_network_policy` | Disable k3s default network policy controller. | `false` |
|
||||
| `k3s_write_kubeconfig_mode` | Define the file mode from the generated KubeConfig, eg. `644` | _NULL_ |
|
||||
| Variable | Description | Default Value |
|
||||
|----------------------------------|--------------------------------------------------------------------------|-----------------------------------------|
|
||||
| `k3s_cluster_state` | State of cluster: installed, started, stopped, restarted, downloaded. | installed |
|
||||
| `k3s_release_version` | Use a specific version of k3s, eg. `v0.2.0`. Specify `false` for latest. | `false` |
|
||||
| `k3s_github_url` | Set the GitHub URL to install k3s from. | https://github.com/rancher/k3s |
|
||||
| `k3s_install_dir` | Installation directory for k3s. | `/usr/local/bin` |
|
||||
| `k3s_server_manifests_dir` | Path for place the `k3s_server_manifests_templates`. | `/var/lib/rancher/k3s/server/manifests` |
|
||||
| `k3s_server_manifests_templates` | A list of Auto-Deploying Manifests Templates. | [] |
|
||||
| `k3s_use_experimental` | Allow the use of experimental features in k3s. | `false` |
|
||||
| `k3s_non_root` | Install k3s as non-root user. See notes below. | `false` |
|
||||
| `k3s_control_workers` | Are control hosts also workers? | `true` |
|
||||
| `k3s_cluster_cidr` | Network CIDR to use for pod IPs | 10.42.0.0/16 |
|
||||
| `k3s_service_cidr` | Network CIDR to use for service IPs | 10.43.0.0/16 |
|
||||
| `k3s_control_node_address` | Use a specific control node address. IP or FQDN. | _NULL_ |
|
||||
| `k3s_control_token` | Use a specific control token, please read notes below. | _NULL_ |
|
||||
| `k3s_https_port` | HTTPS port listening port. | 6443 |
|
||||
| `k3s_use_docker` | Use Docker rather than Containerd? | `false` |
|
||||
| `k3s_no_flannel` | Do not use Flannel | `false` |
|
||||
| `k3s_flannel_backend` | Flannel backend ('none', 'vxlan', 'ipsec', or 'wireguard') | vxlan |
|
||||
| `k3s_no_coredns` | Do not use CoreDNS | `false` |
|
||||
| `k3s_cluster_dns` | Cluster IP for CoreDNS service. Should be in your service-cidr range. | _NULL_ |
|
||||
| `k3s_cluster_domain` | Cluster Domain. | cluster.local |
|
||||
| `k3s_resolv_conf` | Kubelet resolv.conf file | _NULL_ |
|
||||
| `k3s_no_traefik` | Do not use Traefik | `false` |
|
||||
| `k3s_no_servicelb` | Do not use ServiceLB, necessary for using something like MetalLB. | `false` |
|
||||
| `k3s_no_local_storage` | Do not use Local Storage | `false` |
|
||||
| `k3s_no_metrics_server` | Do not deploy metrics server | `false` |
|
||||
| `k3s_disable_scheduler` | Disable Kubernetes default scheduler | `false` |
|
||||
| `k3s_disable_cloud_controller` | Disable k3s default cloud controller manager. | `false` |
|
||||
| `k3s_disable_network_policy` | Disable k3s default network policy controller. | `false` |
|
||||
| `k3s_write_kubeconfig_mode` | Define the file mode from the generated KubeConfig, eg. `644` | _NULL_ |
|
||||
| `k3s_datastore_endpoint` | Define the database or etcd cluster endpoint for HA. | _NULL_ |
|
||||
| `k3s_datastore_cafile` | Define the database TLS CA file. | _NULL_ |
|
||||
| `k3s_datastore_certfile` | Define the database TLS Cert file. | _NULL_ |
|
||||
| `k3s_datastore_keyfile` | Define the database TLS Key file. | _NULL_ |
|
||||
| `k3s_dqlite_datastore` | Use DQLite as the database backend for HA. (EXPERIMENTAL) | `false` |
|
||||
|
||||
#### Important note about `k3s_release_version`
|
||||
|
||||
|
@ -107,27 +114,39 @@ Please note that this may potentially break setting up agents.
|
|||
|
||||
Below are variables that are set against specific hosts in your inventory.
|
||||
|
||||
| Variable | Description | Default Value |
|
||||
|-----------------------------|--------------------------------------------------------------------------|------------------------|
|
||||
| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` |
|
||||
| `k3s_node_name` | Define the name of this node. | `$(hostname)` |
|
||||
| `k3s_node_id` | Define the ID of this node. | _NULL_ |
|
||||
| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | _NULL_ |
|
||||
| `k3s_bind_address` | Define the bind address for this node. | localhost |
|
||||
| `k3s_node_ip_address` | IP Address to advertise for this node. | _NULL_ |
|
||||
| `k3s_node_external_address` | External IP Address to advertise for this node. | _NULL_ |
|
||||
| `k3s_node_labels` | List of node labels. | _NULL_ |
|
||||
| `k3s_node_taints` | List of node taints. | _NULL_ |
|
||||
| `k3s_node_data_dir` | Folder to hold state. | `/var/lib/rancher/k3s` |
|
||||
| `k3s_tls_san` | Add additional hosname or IP as Subject Alternate Name in the TLS cert. | _NULL_ |
|
||||
| Variable | Description | Default Value |
|
||||
|----------------------------------|--------------------------------------------------------------------------|------------------------|
|
||||
| `k3s_control_node` | Define the host as a control plane node, (True/False). | `false` |
|
||||
| `k3s_node_name` | Define the name of this node. | `$(hostname)` |
|
||||
| `k3s_node_id` | Define the ID of this node. | _NULL_ |
|
||||
| `k3s_flannel_interface` | Define the flannel proxy interface for this node. | _NULL_ |
|
||||
| `k3s_bind_address` | Define the bind address for this node. | localhost |
|
||||
| `k3s_node_ip_address` | IP Address to advertise for this node. | _NULL_ |
|
||||
| `k3s_node_external_address` | External IP Address to advertise for this node. | _NULL_ |
|
||||
| `k3s_node_labels` | List of node labels. | _NULL_ |
|
||||
| `k3s_node_taints` | List of node taints. | _NULL_ |
|
||||
| `k3s_node_data_dir` | Folder to hold state. | `/var/lib/rancher/k3s` |
|
||||
| `k3s_tls_san` | Add additional hosname or IP as Subject Alternate Name in the TLS cert. | _NULL_ |
|
||||
|
||||
#### Important note about `k3s_control_node`
|
||||
#### Important note about `k3s_control_node` and High Availability (HA)
|
||||
|
||||
Currently only one host can be defined as a control node, if multiple hosts are
|
||||
set to true the play will fail.
|
||||
By default only one host will be defined as a control node by Ansible, If you
|
||||
do not set a host as a control node, the role will automatically delegate
|
||||
the first play host as a control node (master). This is not suitable for use in
|
||||
a Production workload.
|
||||
|
||||
If you do not set a host as a control node, the role will automatically delegate
|
||||
the first play host as a control node.
|
||||
If multiple hosts have `k3s_control_node` set to true, you must also set
|
||||
`k3s_datastore_endpoint` as the connection string to a MySQL or PostgreSQL
|
||||
database, or etcd cluster else the play will fail.
|
||||
|
||||
If using TLS, the CA, Certificate and Key need to already be available on
|
||||
the play hosts.
|
||||
|
||||
See: [High Availability with an External DB](https://rancher.com/docs/k3s/latest/en/installation/ha/)
|
||||
|
||||
Since K3s v1.0.0 it is possible to use DQLite as the backend database, and this
|
||||
is done by setting `k3s_dqlite_datastore` to true. As this is an experimental
|
||||
feature you will also need to set `k3s_use_experimental` to true.
|
||||
|
||||
#### Important note about `k3s_flannel_interface`
|
||||
|
||||
|
@ -163,16 +182,35 @@ k3s_node_taints:
|
|||
|
||||
No dependencies on other roles.
|
||||
|
||||
## Example Playbook
|
||||
## Example Playbooks
|
||||
|
||||
Example playbook:
|
||||
Example playbook, single master node running v0.10.2:
|
||||
|
||||
```yaml
|
||||
- hosts: k3s_nodes
|
||||
become: true
|
||||
roles:
|
||||
- { role: xanmanning.k3s, k3s_release_version: v0.10.2 }
|
||||
```
|
||||
|
||||
Example playbook, Highly Available running the latest release:
|
||||
|
||||
```yaml
|
||||
- hosts: k3s_nodes
|
||||
become: true
|
||||
vars:
|
||||
molecule_is_test: true
|
||||
k3s_control_node_address: loadbalancer
|
||||
k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
|
||||
pre_tasks:
|
||||
- name: Set each node to be a control node
|
||||
set_fact:
|
||||
k3s_control_node: true
|
||||
when: inventory_hostname in ['node2', 'node3']
|
||||
roles:
|
||||
- role: xanmanning.k3s
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
BSD
|
||||
|
|
|
@ -14,9 +14,37 @@ k3s_github_url: https://github.com/rancher/k3s
|
|||
# Installation directory for k3s
|
||||
k3s_install_dir: /usr/local/bin
|
||||
|
||||
# Path for additional Kubernetes Manifests
|
||||
# https://rancher.com/docs/k3s/latest/en/advanced/#auto-deploying-manifests
|
||||
k3s_server_manifests_dir: /var/lib/rancher/k3s/server/manifests
|
||||
|
||||
# A list of templates used for preconfigure the cluster.
|
||||
k3s_server_manifests_templates: []
|
||||
|
||||
# Use experimental features in k3s?
|
||||
k3s_use_experimental: false
|
||||
|
||||
# Use a database or etcd cluster to enable HA. Examples below:
|
||||
# MySQL:
|
||||
# k3s_datastore_endpoint "mysql://username:password@tcp(hostname:3306)/database-name"
|
||||
# PostgreSQL:
|
||||
# k3s_datastore_endpoint: "postgres://username:password@hostname:port/database-name"
|
||||
# Etcd:
|
||||
# k3s_datastore_endpoint: "https://etcd-host-1:2379,https://etcd-host-2:2379,https://etcd-host-3:2379"
|
||||
k3s_datastore_endpoint: false
|
||||
|
||||
# If using a database endpoint for HA, you can optionally set the CA file,
|
||||
# Cert file and Key file for connecting to the database using TLS.
|
||||
#
|
||||
# These need to already be present on the play hosts.
|
||||
#
|
||||
# k3s_datastore_cafile: /path/to/ca.crt
|
||||
# k3s_datastore_certfile: /path/to/cert.crt
|
||||
# k3s_datastore_keyfile: /path/to/key.pem
|
||||
|
||||
# Use DQLite for HA Datastore? (EXPERIMENTAL)
|
||||
k3s_dqlite_datastore: false
|
||||
|
||||
# Are control hosts also worker nodes?
|
||||
k3s_control_workers: true
|
||||
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
galaxy_info:
|
||||
role_name: k3s
|
||||
author: Xan Manning
|
||||
description: Ansible role for installing k3s as either a standalone server or cluster
|
||||
description: Ansible role for installing k3s as either a standalone server or HA cluster
|
||||
company: Pyrat Ltd.
|
||||
|
||||
# If the issue tracker for your role is not on github, uncomment the
|
||||
|
|
10
molecule/default/playbook-auto-deploying-manifests.yml
Normal file
10
molecule/default/playbook-auto-deploying-manifests.yml
Normal file
|
@ -0,0 +1,10 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: all
|
||||
become: true
|
||||
vars:
|
||||
molecule_is_test: true
|
||||
k3s_server_manifests_templates:
|
||||
- "molecule/default/templates/00-ns-monitoring.yml.j2"
|
||||
roles:
|
||||
- role: xanmanning.k3s
|
|
@ -1,6 +1,6 @@
|
|||
---
|
||||
- name: Prepare
|
||||
hosts: all
|
||||
hosts: node1
|
||||
become: true
|
||||
tasks:
|
||||
- name: Ensure a user group exists
|
||||
|
|
4
molecule/default/templates/00-ns-monitoring.yml.j2
Normal file
4
molecule/default/templates/00-ns-monitoring.yml.j2
Normal file
|
@ -0,0 +1,4 @@
|
|||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: monitoring
|
14
molecule/highavailability/Dockerfile.j2
Normal file
14
molecule/highavailability/Dockerfile.j2
Normal file
|
@ -0,0 +1,14 @@
|
|||
# Molecule managed
|
||||
|
||||
{% if item.registry is defined %}
|
||||
FROM {{ item.registry.url }}/{{ item.image }}
|
||||
{% else %}
|
||||
FROM {{ item.image }}
|
||||
{% endif %}
|
||||
|
||||
RUN if [ $(command -v apt-get) ]; then apt-get update && apt-get install -y python sudo bash ca-certificates && apt-get clean; \
|
||||
elif [ $(command -v dnf) ]; then dnf makecache && dnf --assumeyes install python sudo python-devel python*-dnf bash && dnf clean all; \
|
||||
elif [ $(command -v yum) ]; then yum makecache fast && yum install -y python sudo yum-plugin-ovl bash && sed -i 's/plugins=0/plugins=1/g' /etc/yum.conf && yum clean all; \
|
||||
elif [ $(command -v zypper) ]; then zypper refresh && zypper install -y python sudo bash python-xml && zypper clean -a; \
|
||||
elif [ $(command -v apk) ]; then apk update && apk add --no-cache python sudo bash ca-certificates; \
|
||||
elif [ $(command -v xbps-install) ]; then xbps-install -Syu && xbps-install -y python sudo bash ca-certificates && xbps-remove -O; fi
|
22
molecule/highavailability/INSTALL.rst
Normal file
22
molecule/highavailability/INSTALL.rst
Normal file
|
@ -0,0 +1,22 @@
|
|||
*******
|
||||
Docker driver installation guide
|
||||
*******
|
||||
|
||||
Requirements
|
||||
============
|
||||
|
||||
* Docker Engine
|
||||
|
||||
Install
|
||||
=======
|
||||
|
||||
Please refer to the `Virtual environment`_ documentation for installation best
|
||||
practices. If not using a virtual environment, please consider passing the
|
||||
widely recommended `'--user' flag`_ when invoking ``pip``.
|
||||
|
||||
.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
|
||||
.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ pip install 'molecule[docker]'
|
67
molecule/highavailability/molecule.yml
Normal file
67
molecule/highavailability/molecule.yml
Normal file
|
@ -0,0 +1,67 @@
|
|||
---
|
||||
|
||||
dependency:
|
||||
name: galaxy
|
||||
driver:
|
||||
name: docker
|
||||
lint:
|
||||
name: yamllint
|
||||
platforms:
|
||||
- name: node1
|
||||
image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
|
||||
command: ${MOLECULE_DOCKER_COMMAND:-""}
|
||||
volumes:
|
||||
- /sys/fs/cgroup:/sys/fs/cgroup:ro
|
||||
privileged: true
|
||||
pre_build_image: ${MOLECULE_PREBUILT:-true}
|
||||
networks:
|
||||
- name: k3snet
|
||||
- name: node2
|
||||
image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
|
||||
command: ${MOLECULE_DOCKER_COMMAND:-""}
|
||||
volumes:
|
||||
- /sys/fs/cgroup:/sys/fs/cgroup:ro
|
||||
privileged: true
|
||||
pre_build_image: ${MOLECULE_PREBUILT:-true}
|
||||
networks:
|
||||
- name: k3snet
|
||||
- name: node3
|
||||
image: "${MOLECULE_DISTRO:-geerlingguy/docker-centos8-ansible:latest}"
|
||||
command: ${MOLECULE_DOCKER_COMMAND:-""}
|
||||
volumes:
|
||||
- /sys/fs/cgroup:/sys/fs/cgroup:ro
|
||||
privileged: true
|
||||
pre_build_image: ${MOLECULE_PREBUILT:-true}
|
||||
networks:
|
||||
- name: k3snet
|
||||
- name: database
|
||||
image: postgres:11-alpine
|
||||
pre_build_image: true
|
||||
command: "postgres"
|
||||
env:
|
||||
POSTGRES_PASSWORD: "verybadpass"
|
||||
networks:
|
||||
- name: k3snet
|
||||
- name: loadbalancer
|
||||
image: nginx:1.17-alpine
|
||||
pre_build_image: true
|
||||
ports:
|
||||
- "6443:6443"
|
||||
volumes:
|
||||
- ${MOLECULE_SCENARIO_DIRECTORY}/nginx-loadbalancer.conf:/etc/nginx/nginx.conf:ro
|
||||
command: "nginx -g 'daemon off;'"
|
||||
networks:
|
||||
- name: k3snet
|
||||
provisioner:
|
||||
name: ansible
|
||||
options:
|
||||
verbose: true
|
||||
lint:
|
||||
name: ansible-lint
|
||||
playbooks:
|
||||
prepare: ${MOLECULE_PREPARE_PLAYBOOK:-prepare.yml}
|
||||
converge: ${MOLECULE_PLAYBOOK:-playbook.yml}
|
||||
verifier:
|
||||
name: testinfra
|
||||
lint:
|
||||
name: flake8
|
34
molecule/highavailability/nginx-loadbalancer.conf
Normal file
34
molecule/highavailability/nginx-loadbalancer.conf
Normal file
|
@ -0,0 +1,34 @@
|
|||
worker_processes 4;
|
||||
worker_rlimit_nofile 40000;
|
||||
|
||||
events {
|
||||
worker_connections 8192;
|
||||
}
|
||||
|
||||
stream {
|
||||
# Set up our upstream of control (master) nodes. The default load balancing
|
||||
# algorithm for nginx is to round-robin. Perfect!
|
||||
upstream control_plane {
|
||||
server node2:6443 max_fails=3 fail_timeout=5s;
|
||||
server node3:6443 max_fails=3 fail_timeout=5s;
|
||||
}
|
||||
|
||||
upstream control_plane_443 {
|
||||
server node2:443 max_fails=3 fail_timeout=5s;
|
||||
server node3:443 max_fails=3 fail_timeout=5s;
|
||||
}
|
||||
|
||||
# Listen on port 6443, this is our default control plane port, then pass
|
||||
# all traffic to one of the control (master) nodes.
|
||||
server {
|
||||
listen 6443;
|
||||
proxy_pass control_plane;
|
||||
}
|
||||
|
||||
# Listen on port 443, this is our default ssl port, then pass
|
||||
# all traffic to one of the control (master) nodes.
|
||||
server {
|
||||
listen 443;
|
||||
proxy_pass control_plane_443;
|
||||
}
|
||||
}
|
|
@ -0,0 +1,17 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: node*
|
||||
become: true
|
||||
vars:
|
||||
molecule_is_test: true
|
||||
k3s_control_node_address: loadbalancer
|
||||
k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
|
||||
k3s_server_manifests_templates:
|
||||
- "molecule/highavailability/templates/00-ns-monitoring.yml.j2"
|
||||
pre_tasks:
|
||||
- name: Set each node to be a control node
|
||||
set_fact:
|
||||
k3s_control_node: true
|
||||
when: inventory_hostname in ['node2', 'node3']
|
||||
roles:
|
||||
- role: xanmanning.k3s
|
15
molecule/highavailability/playbook-dqlite.yml
Normal file
15
molecule/highavailability/playbook-dqlite.yml
Normal file
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: node*
|
||||
become: true
|
||||
vars:
|
||||
molecule_is_test: true
|
||||
k3s_dqlite_datastore: true
|
||||
k3s_use_experimental: true
|
||||
pre_tasks:
|
||||
- name: Set each node to be a control node
|
||||
set_fact:
|
||||
k3s_control_node: true
|
||||
when: inventory_hostname in ['node2', 'node3']
|
||||
roles:
|
||||
- role: xanmanning.k3s
|
15
molecule/highavailability/playbook.yml
Normal file
15
molecule/highavailability/playbook.yml
Normal file
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
- name: Converge
|
||||
hosts: node*
|
||||
become: true
|
||||
vars:
|
||||
molecule_is_test: true
|
||||
k3s_control_node_address: loadbalancer
|
||||
k3s_datastore_endpoint: "postgres://postgres:verybadpass@database:5432/postgres?sslmode=disable"
|
||||
pre_tasks:
|
||||
- name: Set each node to be a control node
|
||||
set_fact:
|
||||
k3s_control_node: true
|
||||
when: inventory_hostname in ['node2', 'node3']
|
||||
roles:
|
||||
- role: xanmanning.k3s
|
7
molecule/highavailability/prepare.yml
Normal file
7
molecule/highavailability/prepare.yml
Normal file
|
@ -0,0 +1,7 @@
|
|||
---
|
||||
- name: Prepare
|
||||
hosts: node*
|
||||
tasks:
|
||||
- name: Debug Message
|
||||
debug:
|
||||
msg: No prepare steps required
|
|
@ -0,0 +1,4 @@
|
|||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: monitoring
|
14
molecule/highavailability/tests/test_default.py
Normal file
14
molecule/highavailability/tests/test_default.py
Normal file
|
@ -0,0 +1,14 @@
|
|||
import os
|
||||
|
||||
import testinfra.utils.ansible_runner
|
||||
|
||||
testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
|
||||
os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
|
||||
|
||||
|
||||
def test_hosts_file(host):
|
||||
f = host.file('/etc/hosts')
|
||||
|
||||
assert f.exists
|
||||
assert f.user == 'root'
|
||||
assert f.group == 'root'
|
BIN
molecule/highavailability/tests/test_default.pyc
Normal file
BIN
molecule/highavailability/tests/test_default.pyc
Normal file
Binary file not shown.
|
@ -8,12 +8,14 @@
|
|||
@@@
|
||||
{{ hostvars[item].ansible_host | default(hostvars[item].ansible_fqdn) }}
|
||||
@@@
|
||||
{{ hostvars[item].k3s_control_node }}
|
||||
C_{{ hostvars[item].k3s_control_node }}
|
||||
@@@
|
||||
P_{{ hostvars[item].k3s_primary_control_node | default(False) }}
|
||||
create: true
|
||||
loop: "{{ play_hosts }}"
|
||||
|
||||
- name: Lookup control node from file
|
||||
command: "grep 'True' /tmp/inventory.txt"
|
||||
command: "grep '{{ 'P_True' if (k3s_controller_count | length > 1) else 'C_True' }}' /tmp/inventory.txt"
|
||||
changed_when: false
|
||||
register: k3s_control_delegate_raw
|
||||
|
||||
|
@ -46,7 +48,12 @@
|
|||
- reload systemd
|
||||
- restart k3s
|
||||
|
||||
- meta: flush_handlers
|
||||
- name: Ensure secondary masters are started
|
||||
service:
|
||||
name: k3s
|
||||
state: started
|
||||
enabled: true
|
||||
when: k3s_control_node and not k3s_primary_control_node
|
||||
|
||||
- name: Wait for control plane to be ready to accept connections
|
||||
wait_for:
|
||||
|
@ -56,6 +63,8 @@
|
|||
timeout: 300
|
||||
when: k3s_control_node
|
||||
|
||||
- meta: flush_handlers
|
||||
|
||||
- name: Wait for all nodes to be ready
|
||||
command: "{{ k3s_install_dir }}/kubectl get nodes"
|
||||
changed_when: false
|
||||
|
|
|
@ -5,13 +5,15 @@
|
|||
path: "{{ k3s_systemd_unit_directory }}"
|
||||
state: directory
|
||||
recurse: true
|
||||
when: k3s_control_node
|
||||
when: (k3s_control_node and k3s_controller_count | length == 1)
|
||||
or (k3s_primary_control_node and k3s_controller_count | length > 1)
|
||||
|
||||
- name: Ensure k3s service unit file is present on control plane
|
||||
template:
|
||||
src: k3s.service.j2
|
||||
dest: "{{ k3s_systemd_unit_directory }}/k3s.service"
|
||||
when: k3s_control_node
|
||||
when: (k3s_control_node and k3s_controller_count | length == 1)
|
||||
or (k3s_primary_control_node and k3s_controller_count | length > 1)
|
||||
notify:
|
||||
- reload systemd
|
||||
|
||||
|
@ -36,4 +38,5 @@
|
|||
state: started
|
||||
enabled: true
|
||||
scope: "{{ k3s_systemd_context }}"
|
||||
when: k3s_control_node
|
||||
when: (k3s_control_node and k3s_controller_count | length == 1)
|
||||
or (k3s_primary_control_node and k3s_controller_count | length > 1)
|
||||
|
|
14
tasks/build/preconfigure-k3s-auto-deploying-manifests.yml
Normal file
14
tasks/build/preconfigure-k3s-auto-deploying-manifests.yml
Normal file
|
@ -0,0 +1,14 @@
|
|||
---
|
||||
|
||||
- name: Ensure that the manifests directory exists
|
||||
file:
|
||||
state: directory
|
||||
path: "{{ k3s_server_manifests_dir }}"
|
||||
when: k3s_server_manifests_templates | length > 0
|
||||
|
||||
# https://rancher.com/docs/k3s/latest/en/advanced/#auto-deploying-manifests
|
||||
- name: Ensure Auto-Deploying Manifests are copied to controllers
|
||||
template:
|
||||
src: "{{ item }}"
|
||||
dest: "{{ k3s_server_manifests_dir }}/{{ item | basename | replace('.j2','') }}"
|
||||
loop: "{{ k3s_server_manifests_templates }}"
|
|
@ -5,6 +5,11 @@
|
|||
k3s_control_node: false
|
||||
when: k3s_control_node is not defined
|
||||
|
||||
- name: Ensure k3s master control node fact is set
|
||||
set_fact:
|
||||
k3s_primary_control_node: false
|
||||
when: k3s_primary_control_node is not defined
|
||||
|
||||
- name: Ensure a k3s control node is defined if none are found in play_hosts
|
||||
block:
|
||||
- name: Set control host
|
||||
|
@ -16,5 +21,11 @@
|
|||
- name: Ensure a count of control masters is generated
|
||||
set_fact:
|
||||
k3s_controller_count: "{{ k3s_controller_count + [ item ] }}"
|
||||
when: item
|
||||
loop: "{{ hostvars | json_query('*.k3s_control_node') }}"
|
||||
when: hostvars[item].k3s_control_node
|
||||
loop: "{{ play_hosts }}"
|
||||
|
||||
- name: Ensure a primary k3s control node is defined if multiple are found in play_hosts
|
||||
set_fact:
|
||||
k3s_primary_control_node: true
|
||||
when: k3s_controller_count | length > 1
|
||||
and inventory_hostname == k3s_controller_count[0]
|
||||
|
|
|
@ -28,6 +28,10 @@
|
|||
|
||||
- import_tasks: build/download-k3s.yml
|
||||
|
||||
- import_tasks: build/preconfigure-k3s-auto-deploying-manifests.yml
|
||||
when: k3s_control_node
|
||||
and k3s_server_manifests_templates | length > 0
|
||||
|
||||
- import_tasks: build/install-k3s.yml
|
||||
|
||||
- import_tasks: build/configure-k3s-cluster.yml
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
---
|
||||
|
||||
- name: Check k3s_non_root is enabled with k3s_use_experimental
|
||||
- name: Check if any experimental variables are configure and if they are enabled with k3s_use_experimental
|
||||
assert:
|
||||
that:
|
||||
- k3s_use_experimental is defined and k3s_use_experimental
|
||||
success_msg: "Experimental variables are defined and enabled."
|
||||
fail_msg: "Experimental variables have been configured. If you want to use them ensure you set k3s_use_experimental"
|
||||
when: k3s_non_root is defined and k3s_non_root
|
||||
when: (k3s_non_root is defined and k3s_non_root)
|
||||
or (k3s_dqlite_datastore is defined and k3s_dqlite_datastore)
|
||||
|
|
|
@ -1,8 +1,21 @@
|
|||
---
|
||||
|
||||
- name: Check if multiple controllers defined
|
||||
- name: Check the conditions when a single controller is defined
|
||||
assert:
|
||||
that:
|
||||
- k3s_controller_count | length == 1
|
||||
success_msg: "Only one controller has been defined."
|
||||
fail_msg: "Multiple controllers have been defined. This is not yet supported."
|
||||
- (k3s_controller_count | length == 1)
|
||||
and (k3s_datastore_endpoint is not defined or not k3s_datastore_endpoint)
|
||||
and (k3s_dqlite_datastore is not defined or not k3s_dqlite_datastore)
|
||||
success_msg: "Control plane configuration is valid."
|
||||
fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
|
||||
when: k3s_controller_count | length == 1
|
||||
|
||||
- name: Check the conditions when multiple controllers are defined
|
||||
assert:
|
||||
that:
|
||||
- (k3s_controller_count | length >= 2)
|
||||
and ((k3s_datastore_endpoint is defined and k3s_datastore_endpoint)
|
||||
or k3s_dqlite_datastore is defined and k3s_dqlite_datastore)
|
||||
success_msg: "Control plane configuration is valid."
|
||||
fail_msg: "Control plane configuration is invalid. Please see notes about k3s_control_node and HA in README.md."
|
||||
when: k3s_controller_count | length >= 2
|
||||
|
|
|
@ -90,6 +90,18 @@
|
|||
fail_msg: "--bind-address is not supported in {{ k3s_release_version }}"
|
||||
when: k3s_bind_address is defined and k3s_bind_address
|
||||
|
||||
- name: Check k3s_bind_address against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('0.5.0', '>=')
|
||||
success_msg: "Auto deploy manifests is supported in {{ k3s_release_version }}"
|
||||
fail_msg: |
|
||||
Auto deploy manifests supported is limited in {{ k3s_release_version }}.
|
||||
To disable this message ensure k3s_use_experimental is set to true.
|
||||
when: k3s_server_manifests_templates is defined
|
||||
and k3s_server_manifests_templates | length > 0
|
||||
and (k3s_use_experimental is not defined or not k3s_use_experimental)
|
||||
|
||||
- name: Check k3s_node_labels against k3s version
|
||||
assert:
|
||||
that:
|
||||
|
@ -137,3 +149,43 @@
|
|||
success_msg: "--disable-scheduler supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--disable-scheduler not supported in {{ k3s_release_version }}"
|
||||
when: k3s_disable_scheduler is defined and k3s_disable_scheduler
|
||||
|
||||
- name: Check k3s_datastore_endpoint against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
|
||||
success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
|
||||
when: k3s_datastore_endpoint is defined and k3s_datastore_endpoint
|
||||
|
||||
- name: Check k3s_dqlite_datastore against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
|
||||
success_msg: "--cluster-init supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--cluster-init not supported in {{ k3s_release_version }}"
|
||||
when: k3s_dqlite_datastore is defined and k3s_dqlite_datastore
|
||||
|
||||
- name: Check k3s_datastore_cafile against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
|
||||
success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
|
||||
when: k3s_datastore_cafile is defined and k3s_datastore_cafile
|
||||
|
||||
- name: Check k3s_datastore_certfile against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
|
||||
success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
|
||||
when: k3s_datastore_certfile is defined and k3s_datastore_certfile
|
||||
|
||||
- name: Check k3s_datastore_keyfile against k3s version
|
||||
assert:
|
||||
that:
|
||||
- (k3s_release_version | replace('v', '')) is version_compare('1.0.0', '>=')
|
||||
success_msg: "--datastore-endpoint supported in {{ k3s_release_version }}"
|
||||
fail_msg: "--datastore-endpoint not supported in {{ k3s_release_version }}"
|
||||
when: k3s_datastore_keyfile is defined and k3s_datastore_keyfile
|
||||
|
|
|
@ -54,6 +54,26 @@ ExecStart={{ k3s_install_dir }}/k3s
|
|||
{% if k3s_cluster_domain is defined and k3s_cluster_domain != "cluster.local" %}
|
||||
--cluster-domain {{ k3s_cluster_domain }}
|
||||
{% endif %}
|
||||
{% if k3s_datastore_endpoint is defined and k3s_datastore_endpoint %}
|
||||
--datastore-endpoint "{{ k3s_datastore_endpoint }}"
|
||||
{% if k3s_datastore_cafile is defined and k3s_datastore_cafile %}
|
||||
--datastore-cafile {{ k3s_datastore_cafile }}
|
||||
{% endif %}
|
||||
{% if k3s_datastore_certfile is defined and k3s_datastore_certfile %}
|
||||
--datastore-certfile {{ k3s_datastore_certfile }}
|
||||
{% endif %}
|
||||
{% if k3s_datastore_keyfile is defined and k3s_datastore_keyfile %}
|
||||
--datastore-keyfile {{ k3s_datastore_keyfile }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% if k3s_dqlite_datastore is defined and k3s_dqlite_datastore %}
|
||||
{% if k3s_primary_control_node is defined and k3s_primary_control_node %}
|
||||
--cluster-init
|
||||
{% else %}
|
||||
--server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}
|
||||
--token {{ k3s_control_token }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% else %}
|
||||
agent
|
||||
--server https://{{ k3s_control_node_address }}:{{ k3s_https_port }}
|
||||
|
|
Loading…
Reference in a new issue