diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..1e66a0ca --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +cluster/ansible/venv \ No newline at end of file diff --git a/cluster/ansible/activate.sh b/cluster/ansible/activate.sh new file mode 100755 index 00000000..1f4c3ac3 --- /dev/null +++ b/cluster/ansible/activate.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# Check if `init.sh` has not run +if [ ! -d venv ]; then + echo "Please run init.sh first." + exit 1 +fi + +# Allows venv/bin/activate has permission to run +chmod +x venv/bin/activate + +# Activate virtual env +source venv/bin/activate \ No newline at end of file diff --git a/cluster/ansible/init.sh b/cluster/ansible/init.sh new file mode 100755 index 00000000..814ccd3c --- /dev/null +++ b/cluster/ansible/init.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +# Check if there has no Python 3 installed +if ! command -v python3 &> /dev/null; then + echo "Python3 is not installed. Please install python3 first." + exit 1 +fi + +# Check if there has no virtualenv installed +if ! command -v virtualenv &> /dev/null; then + echo "Virtualenv is not installed. Please install virtualenv first." + exit 1 +fi + +# Create virtual env in current workspace +virtualenv venv --python=python3 --prompt="(freeleaps-cluster-maintain)" --clear + +# Install requirements with ../../3rd/kubespary/requirements.txt +source venv/bin/activate +pip install -r ../../3rd/kubespray/requirements.txt +deactivate diff --git a/cluster/group_vars/all/all.yml b/cluster/ansible/manifests/group_vars/all/all.yml similarity index 98% rename from cluster/group_vars/all/all.yml rename to cluster/ansible/manifests/group_vars/all/all.yml index efabd051..d0227974 100644 --- a/cluster/group_vars/all/all.yml +++ b/cluster/ansible/manifests/group_vars/all/all.yml @@ -90,7 +90,7 @@ cert_management: script # kube_read_only_port: 10255 ## Set true to download and cache container -# download_container: true +download_container: true ## Deploy container engine # Set false if you want to deploy container engine manually. @@ -134,4 +134,4 @@ ntp_servers: unsafe_show_logs: false ## If enabled it will allow kubespray to attempt setup even if the distribution is not supported. For unsupported distributions this can lead to unexpected failures in some cases. -allow_unsupported_distribution_setup: false +allow_unsupported_distribution_setup: false \ No newline at end of file diff --git a/cluster/ansible/manifests/group_vars/all/docker.yml b/cluster/ansible/manifests/group_vars/all/docker.yml new file mode 100644 index 00000000..4e968c30 --- /dev/null +++ b/cluster/ansible/manifests/group_vars/all/docker.yml @@ -0,0 +1,59 @@ +--- +## Uncomment this if you want to force overlay/overlay2 as docker storage driver +## Please note that overlay2 is only supported on newer kernels +# docker_storage_options: -s overlay2 + +## Enable docker_container_storage_setup, it will configure devicemapper driver on Centos7 or RedHat7. +docker_container_storage_setup: false + +## It must be define a disk path for docker_container_storage_setup_devs. +## Otherwise docker-storage-setup will be executed incorrectly. +# docker_container_storage_setup_devs: /dev/vdb + +## Uncomment this if you want to change the Docker Cgroup driver (native.cgroupdriver) +## Valid options are systemd or cgroupfs, default is systemd +# docker_cgroup_driver: systemd + +## Only set this if you have more than 3 nameservers: +## If true Kubespray will only use the first 3, otherwise it will fail +docker_dns_servers_strict: false + +# Path used to store Docker data +docker_daemon_graph: "/var/lib/docker" + +## Used to set docker daemon iptables options to true +docker_iptables_enabled: "false" + +# Docker log options +# Rotate container stderr/stdout logs at 50m and keep last 5 +docker_log_opts: "--log-opt max-size=50m --log-opt max-file=5" + +# define docker bin_dir +docker_bin_dir: "/usr/bin" + +# keep docker packages after installation; speeds up repeated ansible provisioning runs when '1' +# kubespray deletes the docker package on each run, so caching the package makes sense +docker_rpm_keepcache: 1 + +## An obvious use case is allowing insecure-registry access to self hosted registries. +## Can be ipaddress and domain_name. +## example define 172.19.16.11 or mirror.registry.io +# docker_insecure_registries: +# - mirror.registry.io +# - 172.19.16.11 + +## Add other registry,example China registry mirror. +# docker_registry_mirrors: +# - https://registry.docker-cn.com +# - https://mirror.aliyuncs.com + +## If non-empty will override default system MountFlags value. +## This option takes a mount propagation flag: shared, slave +## or private, which control whether mounts in the file system +## namespace set up for docker will receive or propagate mounts +## and unmounts. Leave empty for system default +# docker_mount_flags: + +## A string of extra options to pass to the docker daemon. +## This string should be exactly as you wish it to appear. +# docker_options: "" diff --git a/cluster/group_vars/all/etcd.yml b/cluster/ansible/manifests/group_vars/all/etcd.yml similarity index 95% rename from cluster/group_vars/all/etcd.yml rename to cluster/ansible/manifests/group_vars/all/etcd.yml index 39600c35..eaf5ff90 100644 --- a/cluster/group_vars/all/etcd.yml +++ b/cluster/ansible/manifests/group_vars/all/etcd.yml @@ -9,7 +9,7 @@ etcd_data_dir: /var/lib/etcd ## If this is not set, container manager will be inherited from the Kubespray defaults ## and not from k8s_cluster/k8s-cluster.yml, which might not be what you want. ## Also this makes possible to use different container manager for etcd nodes. -# container_manager: containerd +container_manager: docker ## Settings for etcd deployment type # Set this to docker if you are using container_manager: docker diff --git a/cluster/group_vars/etcd.yml b/cluster/ansible/manifests/group_vars/etcd.yml similarity index 100% rename from cluster/group_vars/etcd.yml rename to cluster/ansible/manifests/group_vars/etcd.yml diff --git a/cluster/group_vars/k8s_cluster/addons.yml b/cluster/ansible/manifests/group_vars/k8s_cluster/addons.yml similarity index 100% rename from cluster/group_vars/k8s_cluster/addons.yml rename to cluster/ansible/manifests/group_vars/k8s_cluster/addons.yml diff --git a/cluster/group_vars/k8s_cluster/k8s-cluster.yml b/cluster/ansible/manifests/group_vars/k8s_cluster/k8s-cluster.yml similarity index 99% rename from cluster/group_vars/k8s_cluster/k8s-cluster.yml rename to cluster/ansible/manifests/group_vars/k8s_cluster/k8s-cluster.yml index 96652983..0b3ac911 100644 --- a/cluster/group_vars/k8s_cluster/k8s-cluster.yml +++ b/cluster/ansible/manifests/group_vars/k8s_cluster/k8s-cluster.yml @@ -222,7 +222,7 @@ dns_domain: "{{ cluster_name }}" ## Container runtime ## docker for docker, crio for cri-o and containerd for containerd. ## Default: containerd -container_manager: containerd +container_manager: docker # Additional container runtimes kata_containers_enabled: false @@ -240,11 +240,11 @@ kubernetes_audit: true default_kubelet_config_dir: "{{ kube_config_dir }}/dynamic_kubelet_dir" # Make a copy of kubeconfig on the host that runs Ansible in {{ inventory_dir }}/artifacts -# kubeconfig_localhost: false +kubeconfig_localhost: true # Use ansible_host as external api ip when copying over kubeconfig. # kubeconfig_localhost_ansible_host: false # Download kubectl onto the host that runs Ansible in {{ bin_dir }} -# kubectl_localhost: false +kubectl_localhost: true # A comma separated list of levels of node allocatable enforcement to be enforced by kubelet. # Acceptable options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "". diff --git a/cluster/group_vars/k8s_cluster/k8s-net-calico.yml b/cluster/ansible/manifests/group_vars/k8s_cluster/k8s-net-calico.yml similarity index 100% rename from cluster/group_vars/k8s_cluster/k8s-net-calico.yml rename to cluster/ansible/manifests/group_vars/k8s_cluster/k8s-net-calico.yml diff --git a/cluster/ansible/manifests/inventory.ini b/cluster/ansible/manifests/inventory.ini new file mode 100644 index 00000000..68b9de3e --- /dev/null +++ b/cluster/ansible/manifests/inventory.ini @@ -0,0 +1,19 @@ +[bastion] +bastion ansible_host=jumper.mathmast.com ansible_user=zhenyus@mathmast.com + +[kube_control_plane] +prod-usw2-k8s-freeleaps-master-01 ansible_host=prod-usw2-k8s-freeleaps-master-01.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-01 +prod-usw2-k8s-freeleaps-master-02 ansible_host=prod-usw2-k8s-freeleaps-master-02.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-02 +prod-usw2-k8s-freeleaps-master-03 ansible_host=prod-usw2-k8s-freeleaps-master-03.mathmast.com ansible_user=zhenyus@mathmast.com etcd_member_name=freeleaps-etcd-03 + +[etcd:children] +kube_control_plane + +[kube_node] +prod-usw2-k8s-freeleaps-worker-01 ansible_host=prod-usw2-k8s-freeleaps-worker-01.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-worker-02 ansible_host=prod-usw2-k8s-freeleaps-worker-02.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-worker-03 ansible_host=prod-usw2-k8s-freeleaps-worker-03.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-worker-04 ansible_host=prod-usw2-k8s-freeleaps-worker-04.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-ingress-worker-01 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-01.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-ingress-worker-02 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-02.mathmast.com ansible_user=zhenyus@mathmast.com +prod-usw2-k8s-freeleaps-ingress-worker-03 ansible_host=prod-usw2-k8s-freeleaps-ingress-worker-03.mathmast.com ansible_user=zhenyus@mathmast.com diff --git a/cluster/group_vars/all/containerd.yml b/cluster/group_vars/all/containerd.yml deleted file mode 100644 index 906437df..00000000 --- a/cluster/group_vars/all/containerd.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -# Please see roles/container-engine/containerd/defaults/main.yml for more configuration options - -# containerd_storage_dir: "/var/lib/containerd" -# containerd_state_dir: "/run/containerd" -# containerd_oom_score: 0 - -# containerd_default_runtime: "runc" -# containerd_snapshotter: "native" - -# containerd_runc_runtime: -# name: runc -# type: "io.containerd.runc.v2" -# engine: "" -# root: "" - -# containerd_additional_runtimes: -# Example for Kata Containers as additional runtime: -# - name: kata -# type: "io.containerd.kata.v2" -# engine: "" -# root: "" - -# containerd_grpc_max_recv_message_size: 16777216 -# containerd_grpc_max_send_message_size: 16777216 - -# Containerd debug socket location: unix or tcp format -# containerd_debug_address: "" - -# Containerd log level -# containerd_debug_level: "info" - -# Containerd logs format, supported values: text, json -# containerd_debug_format: "" - -# Containerd debug socket UID -# containerd_debug_uid: 0 - -# Containerd debug socket GID -# containerd_debug_gid: 0 - -# containerd_metrics_address: "" - -# containerd_metrics_grpc_histogram: false - -# Registries defined within containerd. -# containerd_registries_mirrors: -# - prefix: docker.io -# mirrors: -# - host: https://registry-1.docker.io -# capabilities: ["pull", "resolve"] -# skip_verify: false - -# containerd_max_container_log_line_size: 16384 - -# containerd_registry_auth: -# - registry: 10.0.0.2:5000 -# username: user -# password: pass diff --git a/cluster/inventory.ini b/cluster/inventory.ini deleted file mode 100644 index 344de3d9..00000000 --- a/cluster/inventory.ini +++ /dev/null @@ -1,8 +0,0 @@ -# Configure 'ip' variable to bind kubernetes services on a different ip than the default iface -# We should set etcd_member_name for etcd cluster. The node that are not etcd members do not need to set the value, -# or can set the empty string value. -[kube_control_plane] - -[etcd:children] - -[kube_node] \ No newline at end of file