-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathbootstrap-k3s.sh
executable file
·193 lines (167 loc) · 7.73 KB
/
bootstrap-k3s.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/bin/bash
KEY="$HOME/.ssh/simplyblock-ohio.pem"
print_help() {
echo "Usage: $0 [options]"
echo "Options:"
echo " --k8s-snode <value> Set Storage node to run on k8s (default: false)"
echo " --help Print this help message"
exit 0
}
K8S_SNODE="false"
while [[ $# -gt 0 ]]; do
arg="$1"
case $arg in
--k8s-snode)
K8S_SNODE="true"
;;
--help)
print_help
;;
*)
echo "Unknown option: $1"
print_help
;;
esac
shift
done
SECRET_VALUE=$(terraform output -raw secret_value)
KEY_NAME=$(terraform output -raw key_name)
ssh_dir="$HOME/.ssh"
if [ ! -d "$ssh_dir" ]; then
mkdir -p "$ssh_dir"
echo "Directory $ssh_dir created."
else
echo "Directory $ssh_dir already exists."
fi
if [[ -n "$SECRET_VALUE" ]]; then
KEY="$HOME/.ssh/$KEY_NAME"
if [ -f "$HOME/.ssh/$KEY_NAME" ]; then
echo "the ssh key: ${KEY} already exits on local"
else
echo "$SECRET_VALUE" >"$KEY"
chmod 400 "$KEY"
fi
else
echo "Failed to retrieve secret value. Falling back to default key."
fi
BASTION_IP=$(terraform output -raw bastion_public_ip)
mnodes=($(terraform output -raw extra_nodes_public_ips))
mnodes_private_ips=$(terraform output -raw extra_nodes_private_ips)
IFS=' ' read -ra mnodes_private_ips <<<"$mnodes_private_ips"
storage_private_ips=$(terraform output -raw storage_private_ips)
sec_storage_private_ips=$(terraform output -raw sec_storage_private_ips)
echo "::set-output name=KEY::$KEY"
echo "::set-output name=extra_node_ip::${mnodes[0]}"
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "
sudo yum install -y fio nvme-cli;
sudo modprobe nvme-tcp
sudo modprobe nbd
total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}')
total_memory_mb=\$((total_memory_kb / 1024))
hugepages=\$((total_memory_mb / 4 ))
sudo sysctl -w vm.nr_hugepages=\$hugepages
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC='--advertise-address=${mnodes[0]}' bash
sudo /usr/local/bin/k3s kubectl taint nodes --all node-role.kubernetes.io/master-
sudo /usr/local/bin/k3s kubectl get node
sudo yum install -y pciutils
lspci
sudo chown ec2-user:ec2-user /etc/rancher/k3s/k3s.yaml
sudo yum install -y make golang
echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf
echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf
echo \"vm.nr_hugepages=\$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf
sudo sysctl --system
"
MASTER_NODE_NAME=$(ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl get nodes -o wide | grep -w ${mnodes_private_ips[0]} | awk '{print \$1}'")
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl label nodes $MASTER_NODE_NAME type=simplyblock-cache --overwrite"
TOKEN=$(ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "sudo cat /var/lib/rancher/k3s/server/node-token")
for ((i=1; i<${#mnodes[@]}; i++)); do
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[${i}]} "
sudo yum install -y fio nvme-cli;
sudo modprobe nvme-tcp
sudo modprobe nbd
total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}')
total_memory_mb=\$((total_memory_kb / 1024))
hugepages=\$((total_memory_mb / 4 / 2))
sudo sysctl -w vm.nr_hugepages=\$hugepages
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
curl -sfL https://get.k3s.io | K3S_URL=https://${mnodes[0]}:6443 K3S_TOKEN=$TOKEN bash
sudo /usr/local/bin/k3s kubectl get node
sudo yum install -y pciutils
lspci
sudo yum install -y make golang
echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf
echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf
echo \"vm.nr_hugepages=\$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf
sudo sysctl --system
"
NODE_NAME=$(ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl get nodes -o wide | grep -w ${mnodes_private_ips[${i}]} | awk '{print \$1}'")
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl label nodes $NODE_NAME type=simplyblock-cache --overwrite"
done
if [ "$K8S_SNODE" == "true" ]; then
for node in ${storage_private_ips[@]}; do
echo ""
echo "Adding primary storage node ${node}.."
echo ""
ssh -i "$KEY" -o StrictHostKeyChecking=no \
-o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \
ec2-user@${node} "
sudo yum install -y fio nvme-cli;
sudo modprobe nvme-tcp
sudo modprobe nbd
total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}')
total_memory_mb=\$((total_memory_kb / 1024))
hugepages=\$((total_memory_mb / 4 / 2))
sudo sysctl -w vm.nr_hugepages=\$hugepages
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
curl -sfL https://get.k3s.io | K3S_URL=https://${mnodes[0]}:6443 K3S_TOKEN=$TOKEN bash
sudo /usr/local/bin/k3s kubectl get node
sudo yum install -y pciutils
lspci
sudo yum install -y make golang
echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf
echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf
echo \"vm.nr_hugepages=\$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf
sudo sysctl --system
"
NODE_NAME=$(ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl get nodes -o wide | grep -w ${node} | awk '{print \$1}'")
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl label nodes $NODE_NAME type=simplyblock-storage-plane --overwrite"
done
for node in ${sec_storage_private_ips[@]}; do
echo ""
echo "Adding secondary storage node ${node}.."
echo ""
ssh -i "$KEY" -o StrictHostKeyChecking=no \
-o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \
ec2-user@${node} "
sudo yum install -y fio nvme-cli;
sudo modprobe nvme-tcp
sudo modprobe nbd
total_memory_kb=\$(grep MemTotal /proc/meminfo | awk '{print \$2}')
total_memory_mb=\$((total_memory_kb / 1024))
hugepages=\$((total_memory_mb / 4 / 2))
sudo sysctl -w vm.nr_hugepages=\$hugepages
sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1
sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1
sudo systemctl disable nm-cloud-setup.service nm-cloud-setup.timer
curl -sfL https://get.k3s.io | K3S_URL=https://${mnodes[0]}:6443 K3S_TOKEN=$TOKEN bash
sudo /usr/local/bin/k3s kubectl get node
sudo yum install -y pciutils
lspci
sudo yum install -y make golang
echo 'nvme-tcp' | sudo tee /etc/modules-load.d/nvme-tcp.conf
echo 'nbd' | sudo tee /etc/modules-load.d/nbd.conf
echo \"vm.nr_hugepages=\$hugepages\" | sudo tee /etc/sysctl.d/hugepages.conf
sudo sysctl --system
"
NODE_NAME=$(ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl get nodes -o wide | grep -w ${node} | awk '{print \$1}'")
ssh -i $KEY -o StrictHostKeyChecking=no ec2-user@${mnodes[0]} "kubectl label nodes $NODE_NAME type=simplyblock-storage-plane-reserve --overwrite"
done
fi