339 lines
14 KiB
Go
339 lines
14 KiB
Go
//go:build e2e
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"net/netip"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestE2E(t *testing.T) {
|
|
t.Log("Running E2E tests")
|
|
|
|
imageRef := os.Getenv("IMG_REF")
|
|
require.NotEmpty(t, imageRef)
|
|
proxyMode := os.Getenv("E2E_PROXY_MODE")
|
|
require.NotEmpty(t, proxyMode)
|
|
ipFamily := os.Getenv("E2E_IP_FAMILY")
|
|
require.NotEmpty(t, ipFamily)
|
|
kindName := "spegel-e2e"
|
|
|
|
// Create kind cluster.
|
|
kcPath := createKindCluster(t.Context(), t, kindName, proxyMode, ipFamily)
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
|
|
if t.Failed() {
|
|
t.Log("Test failure debug output")
|
|
|
|
out, _ := commandWithError(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods", kcPath))
|
|
t.Logf("Spegel Pods:\n\n%s\n\n", out)
|
|
out, _ = commandWithError(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel logs -l app.kubernetes.io/name=spegel --max-log-requests 50", kcPath))
|
|
t.Logf("Spegel Logs:\n\n%s\n\n", out)
|
|
|
|
out, _ = commandWithError(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s --namespace conformance get pods", kcPath))
|
|
t.Logf("Conformance Pods:\n\n%s\n\n", out)
|
|
out, _ = commandWithError(ctx, t, fmt.Sprintf("kubectl --kubeconfig %s --namespace conformance logs -l job-name=conformance", kcPath))
|
|
t.Logf("Conformance Logs:\n\n%s\n\n", out)
|
|
}
|
|
|
|
t.Log("Deleting Kind cluster")
|
|
command(ctx, t, fmt.Sprintf("kind delete cluster --name %s", kindName))
|
|
})
|
|
|
|
// Pull test images.
|
|
g, gCtx := errgroup.WithContext(t.Context())
|
|
images := []string{
|
|
"ghcr.io/spegel-org/conformance:583e014",
|
|
"docker.io/library/nginx:1.23.0",
|
|
"docker.io/library/nginx@sha256:b3a676a9145dc005062d5e79b92d90574fb3bf2396f4913dc1732f9065f55c4b",
|
|
"mcr.microsoft.com/containernetworking/azure-cns@sha256:7944413c630746a35d5596f56093706e8d6a3db0569bec0c8e58323f965f7416",
|
|
}
|
|
for _, image := range images {
|
|
g.Go(func() error {
|
|
t.Logf("Pulling image %s", image)
|
|
_, err := commandWithError(gCtx, t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io image pull %s", kindName, image))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
err := g.Wait()
|
|
require.NoError(t, err)
|
|
|
|
// Write existing configuration to test backup.
|
|
hostsToml := `server = https://docker.io
|
|
|
|
[host.https://registry-1.docker.io]
|
|
capabilities = [push]`
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 bash -c \"mkdir -p /etc/containerd/certs.d/docker.io; echo -e '%s' > /etc/containerd/certs.d/docker.io/hosts.toml\"", kindName, hostsToml))
|
|
|
|
// Deploy Spegel.
|
|
deploySpegel(t.Context(), t, kindName, imageRef, kcPath)
|
|
podOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers", kcPath))
|
|
require.Len(t, strings.Split(podOutput, "\n"), 5)
|
|
|
|
// Verify that configuration has been backed up.
|
|
backupHostToml := command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 cat /etc/containerd/certs.d/_backup/docker.io/hosts.toml", kindName))
|
|
require.Equal(t, hostsToml, backupHostToml)
|
|
|
|
// Cleanup backup for uninstall tests
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 rm -rf /etc/containerd/certs.d/_backup", kindName))
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker2 mkdir /etc/containerd/certs.d/_backup", kindName))
|
|
|
|
// Run conformance tests.
|
|
t.Log("Running conformance tests")
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s create namespace conformance --dry-run=client -o yaml | kubectl --kubeconfig %s apply -f -", kcPath, kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s apply --namespace conformance -f ./testdata/conformance-job.yaml", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace conformance wait --timeout 60s --for=condition=complete job/conformance", kcPath))
|
|
|
|
// Remove Spegel from the last node to test that the mirror fallback is working.
|
|
workerPod := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers -o name --field-selector spec.nodeName=%s-worker4", kcPath, kindName))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s label nodes %s-worker4 spegel-", kcPath, kindName))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel wait --for=delete %s --timeout=60s", kcPath, workerPod))
|
|
|
|
// Pull image from registry after Spegel has started.
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io image pull docker.io/library/nginx:1.21.0@sha256:2f1cd90e00fe2c991e18272bb35d6a8258eeb27785d121aa4cc1ae4235167cfd", kindName))
|
|
|
|
// Verify that both local and external ports are working.
|
|
tests := []struct {
|
|
node string
|
|
port string
|
|
expected string
|
|
}{
|
|
{
|
|
node: "worker",
|
|
port: "30020",
|
|
expected: "200",
|
|
},
|
|
{
|
|
node: "worker",
|
|
port: "30021",
|
|
expected: "200",
|
|
},
|
|
{
|
|
node: "worker4",
|
|
port: "30020",
|
|
expected: "000",
|
|
},
|
|
{
|
|
node: "worker4",
|
|
port: "30021",
|
|
expected: "200",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
hostIP := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get nodes %s-%s -o jsonpath='{.status.addresses[?(@.type==\"InternalIP\")].address}'", kcPath, kindName, tt.node))
|
|
addr, err := netip.ParseAddr(hostIP)
|
|
require.NoError(t, err)
|
|
if addr.Is6() {
|
|
hostIP = fmt.Sprintf("[%s]", hostIP)
|
|
}
|
|
httpCode := command(t.Context(), t, fmt.Sprintf("docker exec %s-worker curl -s -o /dev/null -w \"%%{http_code}\" http://%s:%s/healthz || true", kindName, hostIP, tt.port))
|
|
require.Equal(t, tt.expected, httpCode)
|
|
}
|
|
|
|
// Block internet access by only allowing RFC1918 CIDR.
|
|
nodes := getNodes(t.Context(), t, kindName)
|
|
for _, node := range nodes {
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 10.0.0.0/8 -j ACCEPT", kindName, node))
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 172.16.0.0/12 -j ACCEPT", kindName, node))
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -d 192.168.0.0/16 -j ACCEPT", kindName, node))
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-%s iptables -A OUTPUT -o eth0 -j REJECT", kindName, node))
|
|
}
|
|
|
|
// Pull test image that does not contain any media types.
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker3 crictl pull mcr.microsoft.com/containernetworking/azure-cns@sha256:7944413c630746a35d5596f56093706e8d6a3db0569bec0c8e58323f965f7416", kindName))
|
|
|
|
// Deploy test Nginx pods and verify deployment status.
|
|
t.Log("Deploy test Nginx pods")
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s apply -f ./testdata/test-nginx.yaml", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-tag --for condition=available", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-digest --for condition=available", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s deployment/nginx-tag-and-digest --for condition=available", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace nginx wait --timeout=30s -l app=nginx-not-present --for jsonpath='{.status.containerStatuses[*].state.waiting.reason}'=ImagePullBackOff pod", kcPath))
|
|
|
|
// Verify that Spegel has never restarted.
|
|
restartOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath))
|
|
require.Equal(t, "0 0 0 0", restartOutput)
|
|
|
|
// Remove all Spegel Pods and only restart one to verify that running a single instance works.
|
|
t.Log("Scale down Spegel to single instance")
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s label nodes %s-control-plane %s-worker %s-worker2 spegel-", kcPath, kindName, kindName, kindName))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel delete pods --all", kcPath))
|
|
command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel rollout status daemonset spegel --timeout 60s", kcPath))
|
|
podOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods --no-headers", kcPath))
|
|
require.Len(t, strings.Split(podOutput, "\n"), 1)
|
|
|
|
// Verify that Spegel has never restarted
|
|
restartOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath))
|
|
require.Equal(t, "0", restartOutput)
|
|
|
|
// Restart Containerd and verify that Spegel restarts
|
|
t.Log("Restarting Containerd")
|
|
command(t.Context(), t, fmt.Sprintf("docker exec %s-worker3 systemctl restart containerd", kindName))
|
|
require.Eventually(t, func() bool {
|
|
restartOutput = command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s --namespace spegel get pods -o=jsonpath='{.items[*].status.containerStatuses[0].restartCount}'", kcPath))
|
|
return restartOutput == "1"
|
|
}, 5*time.Second, 1*time.Second)
|
|
|
|
// Uninstall Spegel and make sure cleanup is run
|
|
t.Log("Uninstalling Spegel")
|
|
command(t.Context(), t, fmt.Sprintf("helm --kubeconfig %s uninstall --timeout 60s --namespace spegel spegel", kcPath))
|
|
require.Eventually(t, func() bool {
|
|
allOutput := command(t.Context(), t, fmt.Sprintf("kubectl --kubeconfig %s get all --namespace spegel", kcPath))
|
|
return allOutput == ""
|
|
}, 10*time.Second, 1*time.Second)
|
|
nodes = getNodes(t.Context(), t, kindName)
|
|
for _, node := range nodes {
|
|
lsOutput := command(t.Context(), t, fmt.Sprintf("docker exec %s-%s ls /etc/containerd/certs.d", kindName, node))
|
|
require.Empty(t, lsOutput)
|
|
}
|
|
}
|
|
|
|
func TestDevDeploy(t *testing.T) {
|
|
t.Log("Running Dev Deploy")
|
|
|
|
imageRef := os.Getenv("IMG_REF")
|
|
require.NotEmpty(t, imageRef)
|
|
kindName := "spegel-dev"
|
|
|
|
clusterOutput := command(t.Context(), t, "kind get clusters")
|
|
exists := false
|
|
for _, cluster := range strings.Split(clusterOutput, "\n") {
|
|
if cluster != kindName {
|
|
continue
|
|
}
|
|
exists = true
|
|
break
|
|
}
|
|
kcPath := ""
|
|
if exists {
|
|
kcPath = filepath.Join(t.TempDir(), "kind.kubeconfig")
|
|
kcOutput := command(t.Context(), t, fmt.Sprintf("kind get kubeconfig --name %s", kindName))
|
|
err := os.WriteFile(kcPath, []byte(kcOutput), 0o644)
|
|
require.NoError(t, err)
|
|
} else {
|
|
kcPath = createKindCluster(t.Context(), t, kindName, "iptables", "ipv4")
|
|
}
|
|
deploySpegel(t.Context(), t, kindName, imageRef, kcPath)
|
|
}
|
|
|
|
func createKindCluster(ctx context.Context, t *testing.T, kindName, proxyMode, ipFamily string) string {
|
|
t.Helper()
|
|
|
|
kindConfig := fmt.Sprintf(`apiVersion: kind.x-k8s.io/v1alpha4
|
|
kind: Cluster
|
|
networking:
|
|
kubeProxyMode: %s
|
|
ipFamily: %s
|
|
containerdConfigPatches:
|
|
- |-
|
|
[plugins."io.containerd.grpc.v1.cri".registry]
|
|
config_path = "/etc/containerd/certs.d"
|
|
# Discarding unpacked layers causes them to be removed, which defeats the purpose of a local cache.
|
|
# Aditioanlly nodes will report having layers which no long exist.
|
|
# This is by default false in containerd.
|
|
[plugins."io.containerd.grpc.v1.cri".containerd]
|
|
discard_unpacked_layers = false
|
|
# This is just to make sure that images are not shared between namespaces.
|
|
[plugins."io.containerd.metadata.v1.bolt"]
|
|
content_sharing_policy = "isolated"
|
|
nodes:
|
|
- role: control-plane
|
|
labels:
|
|
spegel: schedule
|
|
- role: worker
|
|
labels:
|
|
spegel: schedule
|
|
- role: worker
|
|
labels:
|
|
spegel: schedule
|
|
test: true
|
|
- role: worker
|
|
labels:
|
|
spegel: schedule
|
|
test: true
|
|
- role: worker
|
|
labels:
|
|
spegel: schedule
|
|
test: true`, proxyMode, ipFamily)
|
|
path := filepath.Join(t.TempDir(), "kind-config.yaml")
|
|
err := os.WriteFile(path, []byte(kindConfig), 0o644)
|
|
require.NoError(t, err)
|
|
|
|
t.Log("Creating Kind cluster", "proxy mode", proxyMode, "ip family", ipFamily)
|
|
kcPath := filepath.Join(t.TempDir(), "kind.kubeconfig")
|
|
command(ctx, t, fmt.Sprintf("kind create cluster --kubeconfig %s --config %s --name %s", kcPath, path, kindName))
|
|
return kcPath
|
|
}
|
|
|
|
func deploySpegel(ctx context.Context, t *testing.T, kindName, imageRef, kcPath string) {
|
|
t.Helper()
|
|
|
|
t.Log("Deploying Spegel")
|
|
command(ctx, t, fmt.Sprintf("kind load docker-image --name %s %s", kindName, imageRef))
|
|
imagesOutput := command(ctx, t, fmt.Sprintf("docker exec %s-worker ctr -n k8s.io images ls name==%s", kindName, imageRef))
|
|
_, imagesOutput, ok := strings.Cut(imagesOutput, "\n")
|
|
require.True(t, ok)
|
|
imageDigest := strings.Split(imagesOutput, " ")[2]
|
|
nodes := getNodes(ctx, t, kindName)
|
|
for _, node := range nodes {
|
|
command(ctx, t, fmt.Sprintf("docker exec %s-%s ctr -n k8s.io image tag %s ghcr.io/spegel-org/spegel@%s", kindName, node, imageRef, imageDigest))
|
|
}
|
|
command(ctx, t, fmt.Sprintf("helm --kubeconfig %s upgrade --timeout 60s --create-namespace --wait --install --namespace=\"spegel\" spegel ../../charts/spegel --set \"image.pullPolicy=Never\" --set \"image.digest=%s\" --set \"nodeSelector.spegel=schedule\"", kcPath, imageDigest))
|
|
}
|
|
|
|
func getNodes(ctx context.Context, t *testing.T, kindName string) []string {
|
|
t.Helper()
|
|
|
|
nodes := []string{}
|
|
nodeOutput := command(ctx, t, fmt.Sprintf("kind get nodes --name %s", kindName))
|
|
for _, node := range strings.Split(nodeOutput, "\n") {
|
|
nodes = append(nodes, strings.TrimPrefix(node, kindName+"-"))
|
|
}
|
|
return nodes
|
|
}
|
|
|
|
func commandWithError(ctx context.Context, t *testing.T, e string) (string, error) {
|
|
t.Helper()
|
|
|
|
cmd := exec.CommandContext(ctx, "bash", "-c", e)
|
|
stdout := bytes.NewBuffer(nil)
|
|
cmd.Stdout = stdout
|
|
stderr := bytes.NewBuffer(nil)
|
|
cmd.Stderr = stderr
|
|
err := cmd.Run()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return strings.TrimSuffix(stdout.String(), "\n"), nil
|
|
}
|
|
|
|
func command(ctx context.Context, t *testing.T, e string) string {
|
|
t.Helper()
|
|
|
|
cmd := exec.CommandContext(ctx, "bash", "-c", e)
|
|
stdout := bytes.NewBuffer(nil)
|
|
cmd.Stdout = stdout
|
|
stderr := bytes.NewBuffer(nil)
|
|
cmd.Stderr = stderr
|
|
err := cmd.Run()
|
|
require.NoError(t, err, "command: %s\nstderr: %s", e, stderr.String())
|
|
return strings.TrimSuffix(stdout.String(), "\n")
|
|
}
|