|
| 1 | +package helpers |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + toxiproxyapi "github.com/Shopify/toxiproxy/v2/client" |
| 7 | + . "github.com/onsi/gomega" |
| 8 | + corev1 "k8s.io/api/core/v1" |
| 9 | + "net" |
| 10 | + "os" |
| 11 | + "path" |
| 12 | + "regexp" |
| 13 | + "sigs.k8s.io/cluster-api/test/framework" |
| 14 | + "sigs.k8s.io/cluster-api/test/framework/clusterctl" |
| 15 | + "sigs.k8s.io/cluster-api/test/framework/exec" |
| 16 | + "sigs.k8s.io/controller-runtime/pkg/client" |
| 17 | + "strconv" |
| 18 | + "strings" |
| 19 | +) |
| 20 | + |
| 21 | +func ToxiProxyServerExec(ctx context.Context) error { |
| 22 | + execArgs := []string{"run", "-d", "--name=capc-e2e-toxiproxy", "--net=host", "--rm", "ghcr.io/shopify/toxiproxy"} |
| 23 | + runCmd := exec.NewCommand( |
| 24 | + exec.WithCommand("docker"), |
| 25 | + exec.WithArgs(execArgs...), |
| 26 | + ) |
| 27 | + _, stderr, err := runCmd.Run(ctx) |
| 28 | + if err != nil { |
| 29 | + fmt.Println(string(stderr)) |
| 30 | + } |
| 31 | + return err |
| 32 | +} |
| 33 | + |
| 34 | +func ToxiProxyServerKill(ctx context.Context) error { |
| 35 | + execArgs := []string{"stop", "capc-e2e-toxiproxy"} |
| 36 | + runCmd := exec.NewCommand( |
| 37 | + exec.WithCommand("docker"), |
| 38 | + exec.WithArgs(execArgs...), |
| 39 | + ) |
| 40 | + _, _, err := runCmd.Run(ctx) |
| 41 | + return err |
| 42 | +} |
| 43 | + |
| 44 | +type ToxiProxyContext struct { |
| 45 | + KubeconfigPath string |
| 46 | + Secret corev1.Secret |
| 47 | + ClusterProxy framework.ClusterProxy |
| 48 | + ToxiProxy *toxiproxyapi.Proxy |
| 49 | + ConfigPath string |
| 50 | +} |
| 51 | + |
| 52 | +func SetupForToxiProxyTestingBootstrapCluster(bootstrapClusterProxy framework.ClusterProxy, clusterName string) *ToxiProxyContext { |
| 53 | + // Read/parse the actual kubeconfig for the cluster |
| 54 | + kubeConfig := NewKubeconfig() |
| 55 | + unproxiedKubeconfigPath := bootstrapClusterProxy.GetKubeconfigPath() |
| 56 | + err := kubeConfig.Load(unproxiedKubeconfigPath) |
| 57 | + Expect(err).To(BeNil()) |
| 58 | + |
| 59 | + // Get the cluster's server url from the kubeconfig |
| 60 | + server, err := kubeConfig.GetCurrentServer() |
| 61 | + Expect(err).To(BeNil()) |
| 62 | + |
| 63 | + // Decompose server url into protocol, address and port |
| 64 | + protocol, address, port, _ := parseUrl(server) |
| 65 | + |
| 66 | + // Format into the needed addresses/URL form |
| 67 | + actualBootstrapClusterAddress := fmt.Sprintf("%v:%v", address, port) |
| 68 | + |
| 69 | + // Create the toxiProxy for this test |
| 70 | + toxiProxyClient := toxiproxyapi.NewClient("127.0.0.1:8474") |
| 71 | + toxiProxyName := fmt.Sprintf("deploy_app_toxi_test_%v_bootstrap", clusterName) |
| 72 | + proxy, err := toxiProxyClient.CreateProxy(toxiProxyName, "127.0.0.1:0", actualBootstrapClusterAddress) |
| 73 | + Expect(err).To(BeNil()) |
| 74 | + |
| 75 | + // Get the actual listen address (having the toxiproxy-assigned port #). |
| 76 | + toxiProxyServerUrl := fmt.Sprintf("%v://%v", protocol, proxy.Listen) |
| 77 | + |
| 78 | + // Modify the kubeconfig to use the toxiproxy's server url |
| 79 | + err = kubeConfig.SetCurrentServer(toxiProxyServerUrl) |
| 80 | + Expect(err).To(BeNil()) |
| 81 | + |
| 82 | + // Write the modified kubeconfig using a new name. |
| 83 | + extension := path.Ext(unproxiedKubeconfigPath) |
| 84 | + baseWithoutExtension := strings.TrimSuffix(path.Base(unproxiedKubeconfigPath), extension) |
| 85 | + toxiProxyKubeconfigFileName := fmt.Sprintf("toxiProxy_%v_%v%v", baseWithoutExtension, clusterName, extension) |
| 86 | + toxiProxyKubeconfigPath := path.Join("/tmp", toxiProxyKubeconfigFileName) |
| 87 | + err = kubeConfig.Save(toxiProxyKubeconfigPath) |
| 88 | + Expect(err).To(BeNil()) |
| 89 | + |
| 90 | + // Create a new ClusterProxy using the new kubeconfig |
| 91 | + toxiproxyBootstrapClusterProxy := framework.NewClusterProxy( |
| 92 | + "toxiproxy-bootstrap", |
| 93 | + toxiProxyKubeconfigPath, |
| 94 | + bootstrapClusterProxy.GetScheme(), |
| 95 | + framework.WithMachineLogCollector(framework.DockerLogCollector{}), |
| 96 | + ) |
| 97 | + |
| 98 | + return &ToxiProxyContext{ |
| 99 | + KubeconfigPath: toxiProxyKubeconfigPath, |
| 100 | + ClusterProxy: toxiproxyBootstrapClusterProxy, |
| 101 | + ToxiProxy: proxy, |
| 102 | + } |
| 103 | +} |
| 104 | + |
| 105 | +func TearDownToxiProxyBootstrap(toxiProxyContext *ToxiProxyContext) { |
| 106 | + // Tear down the proxy |
| 107 | + err := toxiProxyContext.ToxiProxy.Delete() |
| 108 | + Expect(err).To(BeNil()) |
| 109 | + |
| 110 | + // Delete the kubeconfig pointing to the proxy |
| 111 | + err = os.Remove(toxiProxyContext.KubeconfigPath) |
| 112 | + Expect(err).To(BeNil()) |
| 113 | +} |
| 114 | + |
| 115 | +func (tp *ToxiProxyContext) RemoveToxic(toxicName string) { |
| 116 | + err := tp.ToxiProxy.RemoveToxic(toxicName) |
| 117 | + Expect(err).To(BeNil()) |
| 118 | +} |
| 119 | + |
| 120 | +func (tp *ToxiProxyContext) AddLatencyToxic(latencyMs int, jitterMs int, toxicity float32, upstream bool) string { |
| 121 | + stream := "downstream" |
| 122 | + if upstream == true { |
| 123 | + stream = "upstream" |
| 124 | + } |
| 125 | + toxicName := fmt.Sprintf("latency_%v", stream) |
| 126 | + |
| 127 | + _, err := tp.ToxiProxy.AddToxic(toxicName, "latency", stream, toxicity, toxiproxyapi.Attributes{ |
| 128 | + "latency": latencyMs, |
| 129 | + "jitter": jitterMs, |
| 130 | + }) |
| 131 | + Expect(err).To(BeNil()) |
| 132 | + |
| 133 | + return toxicName |
| 134 | +} |
| 135 | + |
| 136 | +func SetupForToxiProxyTestingACS(ctx context.Context, clusterName string, clusterProxy framework.ClusterProxy, e2eConfig *clusterctl.E2EConfig, configPath string) *ToxiProxyContext { |
| 137 | + // Get the cloud-config secret that CAPC will use to access CloudStack |
| 138 | + fdEndpointSecretObjectKey := client.ObjectKey{ |
| 139 | + Namespace: e2eConfig.GetVariable("CLOUDSTACK_FD1_SECRET_NAMESPACE"), |
| 140 | + Name: e2eConfig.GetVariable("CLOUDSTACK_FD1_SECRET_NAME"), |
| 141 | + } |
| 142 | + fdEndpointSecret := corev1.Secret{} |
| 143 | + err := clusterProxy.GetClient().Get(ctx, fdEndpointSecretObjectKey, &fdEndpointSecret) |
| 144 | + Expect(err).To(BeNil()) |
| 145 | + |
| 146 | + // Extract and parse the URL for CloudStack from the secret |
| 147 | + cloudstackUrl := string(fdEndpointSecret.Data["api-url"]) |
| 148 | + protocol, address, port, path := parseUrl(cloudstackUrl) |
| 149 | + upstreamAddress := fmt.Sprintf("%v:%v", address, port) |
| 150 | + |
| 151 | + // Create the CloudStack toxiProxy for this test |
| 152 | + toxiProxyClient := toxiproxyapi.NewClient("127.0.0.1:8474") |
| 153 | + toxiProxyName := fmt.Sprintf("%v_cloudstack", clusterName) |
| 154 | + |
| 155 | + // Formulate the proxy listen address. |
| 156 | + // CAPC can't route to the actual host's localhost. We have to use a real host IP address for the proxy listen address. |
| 157 | + hostIP := getOutboundIP() |
| 158 | + proxyAddress := fmt.Sprintf("%v:0", hostIP) |
| 159 | + proxy, err := toxiProxyClient.CreateProxy(toxiProxyName, proxyAddress, upstreamAddress) |
| 160 | + Expect(err).To(BeNil()) |
| 161 | + |
| 162 | + // Retrieve the actual listen address (having the toxiproxy-assigned port #). |
| 163 | + toxiProxyUrl := fmt.Sprintf("%v://%v%v", protocol, proxy.Listen, path) |
| 164 | + |
| 165 | + // Create a new cloud-config secret using the proxy listen address |
| 166 | + toxiProxyFdEndpointSecret := corev1.Secret{} |
| 167 | + toxiProxyFdEndpointSecret.Type = fdEndpointSecret.Type |
| 168 | + toxiProxyFdEndpointSecret.Namespace = fdEndpointSecret.Namespace |
| 169 | + toxiProxyFdEndpointSecret.Name = fdEndpointSecret.Name + "-toxiproxy" |
| 170 | + toxiProxyFdEndpointSecret.Data = make(map[string][]byte) |
| 171 | + toxiProxyFdEndpointSecret.Data["api-key"] = fdEndpointSecret.Data["api-key"] |
| 172 | + toxiProxyFdEndpointSecret.Data["secret-key"] = fdEndpointSecret.Data["secret-key"] |
| 173 | + toxiProxyFdEndpointSecret.Data["verify-ssl"] = fdEndpointSecret.Data["verify-ssl"] |
| 174 | + toxiProxyFdEndpointSecret.Data["api-url"] = []byte(toxiProxyUrl) |
| 175 | + |
| 176 | + err = clusterProxy.GetClient().Create(ctx, &toxiProxyFdEndpointSecret) |
| 177 | + Expect(err).To(BeNil()) |
| 178 | + |
| 179 | + // Override the test config to use this alternate cloud-config secret |
| 180 | + e2eConfig.Variables["CLOUDSTACK_FD1_SECRET_NAME"] = toxiProxyFdEndpointSecret.Name |
| 181 | + |
| 182 | + // Overriding e2e config file into a new temp copy, so as not to inadvertently override the other e2e tests. |
| 183 | + newConfigFilePath := fmt.Sprintf("/tmp/%v.yaml", toxiProxyName) |
| 184 | + editConfigFile(newConfigFilePath, configPath, "CLOUDSTACK_FD1_SECRET_NAME", toxiProxyFdEndpointSecret.Name) |
| 185 | + |
| 186 | + // Return a context |
| 187 | + return &ToxiProxyContext{ |
| 188 | + Secret: toxiProxyFdEndpointSecret, |
| 189 | + ToxiProxy: proxy, |
| 190 | + ConfigPath: newConfigFilePath, |
| 191 | + } |
| 192 | +} |
| 193 | + |
| 194 | +func TearDownToxiProxyACS(ctx context.Context, clusterProxy framework.ClusterProxy, toxiProxyContext *ToxiProxyContext) { |
| 195 | + // Tear down the proxy |
| 196 | + err := toxiProxyContext.ToxiProxy.Delete() |
| 197 | + Expect(err).To(BeNil()) |
| 198 | + |
| 199 | + // Delete the secret |
| 200 | + err = clusterProxy.GetClient().Delete(ctx, &toxiProxyContext.Secret) |
| 201 | + Expect(err).To(BeNil()) |
| 202 | + |
| 203 | + // Delete the overridden e2e config |
| 204 | + err = os.Remove(toxiProxyContext.ConfigPath) |
| 205 | + Expect(err).To(BeNil()) |
| 206 | + |
| 207 | +} |
| 208 | + |
| 209 | +func parseUrl(url string) (string, string, int, string) { |
| 210 | + serverRegex := regexp.MustCompilePOSIX("(https?)://([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+):([0-9]+)?(.*)") |
| 211 | + |
| 212 | + urlComponents := serverRegex.FindStringSubmatch(url) |
| 213 | + Expect(len(urlComponents)).To(BeNumerically(">=", 4)) |
| 214 | + protocol := urlComponents[1] |
| 215 | + address := urlComponents[2] |
| 216 | + port, err := strconv.Atoi(urlComponents[3]) |
| 217 | + Expect(err).To(BeNil()) |
| 218 | + path := urlComponents[4] |
| 219 | + return protocol, address, port, path |
| 220 | +} |
| 221 | + |
| 222 | +func getOutboundIP() net.IP { |
| 223 | + conn, err := net.Dial("udp", "8.8.8.8:80") // 8.8.8.8:80 is arbitrary. Any IP will do, reachable or not. |
| 224 | + Expect(err).To(BeNil()) |
| 225 | + |
| 226 | + defer conn.Close() |
| 227 | + |
| 228 | + localAddr := conn.LocalAddr().(*net.UDPAddr) |
| 229 | + |
| 230 | + return localAddr.IP |
| 231 | +} |
| 232 | + |
| 233 | +func editConfigFile(destFilename string, sourceFilename string, key string, newValue string) { |
| 234 | + // For config files with key: value on each line. |
| 235 | + |
| 236 | + dat, err := os.ReadFile(sourceFilename) |
| 237 | + Expect(err).To(BeNil()) |
| 238 | + |
| 239 | + lines := strings.Split(string(dat), "\n") |
| 240 | + |
| 241 | + keyFound := false |
| 242 | + for index, line := range lines { |
| 243 | + if strings.HasPrefix(line, "CLOUDSTACK_FD1_SECRET_NAME:") { |
| 244 | + keyFound = true |
| 245 | + lines[index] = fmt.Sprintf("%v: %v", key, newValue) |
| 246 | + break |
| 247 | + } |
| 248 | + } |
| 249 | + Expect(keyFound).To(BeTrue()) |
| 250 | + |
| 251 | + dat = []byte(strings.Join(lines[:], "\n")) |
| 252 | + err = os.WriteFile(destFilename, dat, 0600) |
| 253 | + Expect(err).To(BeNil()) |
| 254 | +} |
0 commit comments