Karmada多集群管理高可用部署

1
2
3
4
5
6

wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssl_1.6.4_linux_amd64
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.4/cfssljson_1.6.4_linux_amd64

chmod +x cfssl_1.6.4_linux_amd64 cfssljson_1.6.4_linux_amd64
mv cfssl_1.6.4_linux_amd64 /usr/local/bin/cfssl
mv cfssljson_1.6.4_linux_amd64 /usr/local/bin/cfssljson

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74

set -ex
VIP=10.90.209.105
expiry_houry=87600h
clusterDomain="cluster.local"
basePath="/opt/certs"
mkdir -p ${basePath}/{etcd,karmada}

cat << EOF >${basePath}/etcd/ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}"
    },
    "profiles": {
      "etcd": {
         "expiry": "${expiry_houry}",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

cat << EOF >${basePath}/etcd/ca-csr.json
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Guangzhou",
            "ST": "Guangzhou"
        }
    ]
}
EOF

cfssl gencert -initca ${basePath}/etcd/ca-csr.json | cfssljson -bare ${basePath}/etcd/ca

cat << EOF >${basePath}/etcd/server-csr.json
{
    "CN": "etcd",
    "hosts": [
      "kubernetes.default.svc",
      "*.etcd.karmada-system.svc.${clusterDomain}",
      "*.karmada-system.svc.${clusterDomain}",
      "*.karmada-system.svc",
      "localhost",
      "127.0.0.1",
      "${VIP}"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Guangzhou",
            "ST": "Guangzhou"
        }
    ]
}
EOF

cfssl gencert -ca=${basePath}/etcd/ca.pem -ca-key=${basePath}/etcd/ca-key.pem -config=${basePath}/etcd/ca-config.json -profile=etcd ${basePath}/etcd/server-csr.json | cfssljson  -f - -bare ${basePath}/etcd/server

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

docker run -d --name etcd1 -p 2379:2379 \
    -v ${basePath}/etcd:/opt/etcd/ssl/ \
    registry.cn-hangzhou.aliyuncs.com/seam/etcd:3.5.9-0 \
    etcd --name etcd1 --initial-advertise-peer-urls https://${VIP}:2380 \
     --listen-peer-urls https://0.0.0.0:2380 \
     --listen-client-urls https://0.0.0.0:2379 \
     --advertise-client-urls https://${VIP}:2379 \
     --initial-cluster-token etcd-cluster-1 \
     --initial-cluster-state new \
     --client-cert-auth --trusted-ca-file=/opt/etcd/ssl/ca.pem \
     --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem \
     --peer-client-cert-auth --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem \
     --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem

docker exec -ti etcd1 sh

ETCDCTL_API=3 etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem  endpoint health
#  127.0.0.1:2379 is healthy: successfully committed proposal: took = 6.695865ms

ETCDCTL_API=3 etcdctl --cacert=/opt/etcd/ssl/ca.pem --cert=/opt/etcd/ssl/server.pem --key=/opt/etcd/ssl/server-key.pem  get --keys-only --prefix=true "/"

exit

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94

openssl req -x509 -sha256 -new -nodes -days 3650 -newkey rsa:2048 -keyout "${basePath}/karmada/server-ca.key" -out "${basePath}/karmada/server-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
openssl req -x509 -sha256 -new -nodes -days 3650 -newkey rsa:2048 -keyout "${basePath}/karmada/front-proxy-ca.key" -out "${basePath}/karmada/front-proxy-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"

cat << EOF >${basePath}/karmada/server-ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}",
      "usages": [
        "signing",
        "key encipherment",
        "client auth",
        "server auth"
      ]
    }
  }
}
EOF

cat << EOF  | cfssl gencert -ca=${basePath}/karmada/server-ca.crt -ca-key=${basePath}/karmada/server-ca.key -config=${basePath}/karmada/server-ca-config.json - | cfssljson -bare ${basePath}/karmada/karmada
{
  "CN": "system:admin",
  "hosts": [
    "kubernetes.default.svc",
    "*.etcd.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc",
    "localhost",
    "127.0.0.1",
    "${VIP}"
  ],
  "names": [
    {
      "O": "system:masters"
    }
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  }
}
EOF

cat << EOF > ${basePath}/karmada/front-proxy-ca-config.json
{
  "signing": {
    "default": {
      "expiry": "${expiry_houry}",
      "usages": [
        "signing",
        "key encipherment",
        "client auth",
        "server auth"
      ]
    }
  }
}
EOF

cat << EOF | cfssl gencert -ca=${basePath}/karmada/front-proxy-ca.crt -ca-key=${basePath}/karmada/front-proxy-ca.key -config=${basePath}/karmada/front-proxy-ca-config.json - | cfssljson -bare ${basePath}/karmada/front-proxy-client
{
  "CN": "front-proxy-client",
  "hosts": [
    "kubernetes.default.svc",
    "*.etcd.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc.${clusterDomain}",
    "*.karmada-system.svc",
    "localhost",
    "127.0.0.1",
    "${VIP}"
  ],
  "names": [
    {
      "O": "system:masters"
    }
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  }
}
EOF

karmada_ca=$(cat ${basePath}/karmada/server-ca.crt | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_ca_key=$(cat ${basePath}/karmada/server-ca.key | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_crt=$(cat ${basePath}/karmada/karmada.pem | sed ":tag;N;s/\n/\\\n/;b tag")
karmada_key=$(cat ${basePath}/karmada/karmada-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_ca=$(cat ${basePath}/karmada/front-proxy-ca.crt | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_client_crt=$(cat ${basePath}/karmada/front-proxy-client.pem | sed ":tag;N;s/\n/\\\n/;b tag")
front_proxy_client_key=$(cat ${basePath}/karmada/front-proxy-client-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")

etcdcaCrt=$(cat ${basePath}/etcd/ca.pem | sed ":tag;N;s/\n/\\\n/;b tag")
etcdcrt=$(cat ${basePath}/etcd/server.pem | sed ":tag;N;s/\n/\\\n/;b tag")
etcdkey=$(cat ${basePath}/etcd/server-key.pem | sed ":tag;N;s/\n/\\\n/;b tag")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

cat << EOF > primary.yaml
kind: Cluster
apiVersion: "kind.x-k8s.io/v1alpha4"
kubeadmConfigPatches:
- |
  apiVersion: kubeadm.k8s.io/v1beta3
  kind: ClusterConfiguration
  metadata:
    name: primary
  imageRepository: registry.aliyuncs.com/google_containers
  networking:
    dnsDomain: "${clusterDomain}"
networking:
  apiServerAddress: "${VIP}"
  podSubnet: "10.8.0.0/16"
  serviceSubnet: "10.9.0.0/16"
  dnsSearch: ["${clusterDomain}"]
nodes:
  - role: control-plane
    image: registry.cn-hangzhou.aliyuncs.com/seam/node:v1.29.0
    extraPortMappings:
      - containerPort: 5443
        hostPort: 5443
        protocol: TCP
EOF


kind create cluster --name=primary --kubeconfig=primary --config=primary.yaml --retain


cat << EOF > secondary.yaml
kind: Cluster
apiVersion: "kind.x-k8s.io/v1alpha4"
kubeadmConfigPatches:
- |
  apiVersion: kubeadm.k8s.io/v1beta3
  kind: ClusterConfiguration
  metadata:
    name: secondary
  imageRepository: registry.aliyuncs.com/google_containers
  networking:
    dnsDomain: "${clusterDomain}"
networking:
  apiServerAddress: "${VIP}"
  podSubnet: "10.10.0.0/16"
  serviceSubnet: "10.11.0.0/16"
  dnsSearch: ["${clusterDomain}"]
nodes:
  - role: control-plane
    image: registry.cn-hangzhou.aliyuncs.com/seam/node:v1.29.0
    extraPortMappings:
      - containerPort: 5443
        hostPort: 7443
        protocol: TCP
EOF

kind create cluster --name=secondary --kubeconfig=secondary --config=secondary.yaml --retain

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130

cat << EOF > values.json
{
  "installMode": "host",
  "clusterDomain": "${clusterDomain}",
  "systemNamespace": "karmada-system",
  "components": [],
  "cfssl": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/cfssl",
      "tag": "latest",
      "pullPolicy": "IfNotPresent"
    }
  },
  "kubectl": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/kubectl",
      "tag": "latest",
      "pullPolicy": "IfNotPresent"
    }
  },
  "certs": {
    "mode": "custom",
    "auto": {
      "expiry": "${expiry_houry}",
      "hosts": [
        "kubernetes.default.svc",
        "*.etcd.karmada-system.svc.${clusterDomain}",
        "*.karmada-system.svc.${clusterDomain}",
        "*.karmada-system.svc",
        "localhost",
        "127.0.0.1",
        "${VIP}"
      ]
    },
    "custom": {
      "caCrt": "${karmada_ca}",
      "caKey": "${karmada_ca_key}",
      "crt": "${karmada_crt}",
      "key": "${karmada_key}",
      "frontProxyCaCrt": "${front_proxy_ca}",
      "frontProxyCrt": "${front_proxy_client_crt}",
      "frontProxyKey": "${front_proxy_client_key}"
    }
  },
  "scheduler": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-scheduler"
    }
  },
  "webhook": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-webhook"
    }
  },
  "controllerManager": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-controller-manager"
    }
  },
  "apiServer": {
    "hostNetwork": true,
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/kube-apiserver",
      "tag": "v1.25.4",
      "pullPolicy": "IfNotPresent"
    }
  },
  "aggregatedApiServer": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-aggregated-apiserver"
    }
  },
  "metricsAdapter": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-metrics-adapter"
    }
  },
  "kubeControllerManager": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/kube-controller-manager",
      "tag": "v1.25.4"
    }
  },
  "etcd": {
    "mode": "external",
    "external": {
      "servers": "https://${VIP}:2379",
      "registryPrefix": "/registry/karmada",
      "certs": {
        "caCrt": "${etcdcaCrt}",
        "crt": "${etcdcrt}",
        "key": "${etcdkey}"
      }
    }
  },
  "agent": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-agent"
    }
  },
  "schedulerEstimator": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-scheduler-estimator"
    }
  },
  "descheduler": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-descheduler"
    }
  },
  "search": {
    "image": {
      "registry": "registry.cn-hangzhou.aliyuncs.com",
      "repository": "seam/karmada-search"
    }
  }
}
EOF

1
2
3
4

helm --namespace karmada-system upgrade --install   karmada karmada-charts/karmada -f values.json --kubeconfig primary --create-namespace


helm --namespace karmada-system upgrade --install   karmada karmada-charts/karmada -f values.json --kubeconfig secondary --create-namespace

1
2
3

# kubectl  get secret -n karmada-system --kubeconfig primary karmada-kubeconfig -o jsonpath="{.data.kubeconfig}" | base64 -d | sed "s/karmada-apiserver.karmada-system.svc.${clusterDomain}:5443/${VIP}:5443/g" > karmada-primary

# kubectl get secret -n karmada-system --kubeconfig secondary karmada-kubeconfig -o jsonpath="{.data.kubeconfig}" | base64 -d | sed "s/karmada-apiserver.karmada-system.svc.${clusterDomain}:5443/${VIP}:7443/g" > karmada-secondary

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# kubectl karmada join primary --kubeconfig=karmada-primary --cluster-kubeconfig=primary
cluster(primary) is joined successfully

# kubectl karmada join secondary --kubeconfig=karmada-primary --cluster-kubeconfig=secondary
cluster(secondary) is joined successfully

# kubectl get cluster --kubeconfig karmada-primary
NAME        VERSION   MODE   READY   AGE
primary     v1.29.0   Push   True    20s
secondary   v1.29.0   Push   True    15s
# kubectl get cluster --kubeconfig karmada-secondary
NAME        VERSION   MODE   READY   AGE
primary     v1.29.0   Push   True    12s
secondary   v1.29.0   Push   True    10s

1
2

# karmadactl join primary --kubeconfig=karmada-primary --cluster-kubeconfig=primary --cluster-context kind-primary
# karmadactl join secondary --kubeconfig=karmada-primary --cluster-kubeconfig=secondary --cluster-context kind-secondary

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

cat << EOF >propagationpolicy.yaml
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: nginx-propagation
spec:
  resourceSelectors:
    - apiVersion: apps/v1
      kind: Deployment
      name: nginx
  placement:
    clusterAffinity:
      clusterNames:
        - primary
        - secondary
    replicaScheduling:
      replicaDivisionPreference: Weighted
      replicaSchedulingType: Divided
EOF

cat << EOF >deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
  labels:
    app: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - image: nginx
        name: nginx
EOF

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

#  kubectl apply -f deployment.yaml --kubeconfig karmada-primary
deployment.apps/nginx created
#  kubectl apply -f propagationpolicy.yaml --kubeconfig karmada-primary
propagationpolicy.policy.karmada.io/nginx-propagation created

#  kubectl get pod  --kubeconfig secondary
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7854ff8877-6m42g   1/1     Running   0          40m
#  kubectl get pod  --kubeconfig primary
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7854ff8877-zzj45   1/1     Running   0          40m
# 

 1
 2
 3
 4
 5
 6
 7
 8
 9
10

kubectl exec -ti --kubeconfig secondary -n karmada-system etcd-0  \
  -- sh -c 'ETCDCTL_CACERT=/etc/kubernetes/pki/etcd/server-ca.crt \
            ETCDCTL_CERT=/etc/kubernetes/pki/etcd/karmada.crt \
            ETCDCTL_KEY=/etc/kubernetes/pki/etcd/karmada.key \
            ETCDCTL_API=3  \
            etcdctl \
              get \
              --keys-only \
              --prefix=true \
              "/registry/karmada/" '