diff --git a/v1/providers/testkube/images/ubuntu-vm/README.md b/v1/providers/testkube/images/ubuntu-vm/README.md index 97ef6e1..d44ea31 100644 --- a/v1/providers/testkube/images/ubuntu-vm/README.md +++ b/v1/providers/testkube/images/ubuntu-vm/README.md @@ -5,6 +5,8 @@ This image backs the architecture-specific TestKube instance types: ```text test.ok.cpu -> ghcr.io/brevdev/cloud/testkube-ubuntu-vm:multiarch-v2 test.ok.cpu.arm64 -> ghcr.io/brevdev/cloud/testkube-ubuntu-vm:multiarch-v2 +test.ok.cpu.nodeport -> ghcr.io/brevdev/cloud/testkube-ubuntu-vm:multiarch-v2 +test.ok.cpu.arm64.nodeport -> ghcr.io/brevdev/cloud/testkube-ubuntu-vm:multiarch-v2 ``` ## Publish to GHCR diff --git a/v1/providers/testkube/instance.go b/v1/providers/testkube/instance.go index fd4c754..162f318 100644 --- a/v1/providers/testkube/instance.go +++ b/v1/providers/testkube/instance.go @@ -373,12 +373,12 @@ func (c *TestKubeClient) instanceFromResources(pod *corev1.Pod, service *corev1. } instance.InstanceTypeID = cloudv1.MakeGenericInstanceTypeIDFromInstance(*instance) if service != nil { - populateNetwork(service, instance) + populateNetwork(service, pod.Status.HostIP, instance) } return instance } -func populateNetwork(service *corev1.Service, instance *cloudv1.Instance) { +func populateNetwork(service *corev1.Service, hostIP string, instance *cloudv1.Instance) { // Default the private IP to the cluster IP. if service.Spec.ClusterIP != "" && service.Spec.ClusterIP != corev1.ClusterIPNone { instance.PrivateIP = service.Spec.ClusterIP @@ -413,6 +413,10 @@ func populateNetwork(service *corev1.Service, instance *cloudv1.Instance) { } // Set the SSH port to the first node port. instance.SSHPort = int(service.Spec.Ports[0].NodePort) + if hostIP != "" { + instance.PublicIP = hostIP + instance.PublicDNS = hostIP + } case corev1.ServiceTypeClusterIP: // Keep the cluster IP as the private IP. } @@ -441,6 +445,12 @@ func statusFromResources(pod *corev1.Pod, service *corev1.Service) cloudv1.Statu Messages: append(podMessages(*pod), fmt.Sprintf("service %s waiting for load balancer ingress", service.Name)), } } + if service.Spec.Type == corev1.ServiceTypeNodePort && !nodePortReady(pod, service) { + return cloudv1.Status{ + LifecycleStatus: cloudv1.LifecycleStatusPending, + Messages: append(podMessages(*pod), fmt.Sprintf("service %s waiting for node port endpoint", service.Name)), + } + } return cloudv1.Status{ LifecycleStatus: cloudv1.LifecycleStatusRunning, Messages: podMessages(*pod), @@ -452,6 +462,10 @@ func statusFromResources(pod *corev1.Pod, service *corev1.Service) cloudv1.Statu } } +func nodePortReady(pod *corev1.Pod, service *corev1.Service) bool { + return pod.Status.HostIP != "" && len(service.Spec.Ports) > 0 && service.Spec.Ports[0].NodePort != 0 +} + func loadBalancerReady(service *corev1.Service) bool { for _, ingress := range service.Status.LoadBalancer.Ingress { if ingress.IP != "" || ingress.Hostname != "" { diff --git a/v1/providers/testkube/instance_test.go b/v1/providers/testkube/instance_test.go index 906e87c..a7879ca 100644 --- a/v1/providers/testkube/instance_test.go +++ b/v1/providers/testkube/instance_test.go @@ -123,6 +123,41 @@ func TestInstanceLifecycle(t *testing.T) { //nolint:funlen // ok require.True(t, errors.Is(err, cloudv1.ErrInstanceNotFound)) } +func TestNodePortInstanceLifecycle(t *testing.T) { + ctx := context.Background() + client := newTestClient(t) + + instance, err := client.CreateInstance(ctx, cloudv1.CreateInstanceAttrs{ + RefID: "nodeport", + Name: "nodeport", + InstanceType: InstanceTypeOKCPUARM64NodePort, + }) + require.NoError(t, err) + + service, err := client.k8sClient.CoreV1().Services(client.namespace).Get(ctx, string(instance.CloudID), metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, corev1.ServiceTypeNodePort, service.Spec.Type) + require.NotContains(t, service.Annotations, annotationAWSLoadBalancerConnectionIdleTimeout) + service.Spec.Ports[0].NodePort = 32421 + _, err = client.k8sClient.CoreV1().Services(client.namespace).Update(ctx, service, metav1.UpdateOptions{}) + require.NoError(t, err) + + pod, err := client.k8sClient.CoreV1().Pods(client.namespace).Get(ctx, string(instance.CloudID), metav1.GetOptions{}) + require.NoError(t, err) + pod.Status.Phase = corev1.PodRunning + pod.Status.HostIP = "192.168.49.2" + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + _, err = client.k8sClient.CoreV1().Pods(client.namespace).UpdateStatus(ctx, pod, metav1.UpdateOptions{}) + require.NoError(t, err) + + running, err := client.GetInstance(ctx, instance.CloudID) + require.NoError(t, err) + require.Equal(t, cloudv1.LifecycleStatusRunning, running.Status.LifecycleStatus) + require.Equal(t, "192.168.49.2", running.PublicIP) + require.Equal(t, "192.168.49.2", running.PublicDNS) + require.Equal(t, 32421, running.SSHPort) +} + func TestScenarioEnvironment(t *testing.T) { ctx := context.Background() client := newTestClient(t) @@ -255,7 +290,7 @@ func TestPopulateNetworkLoadBalancer(t *testing.T) { }, }, }, - }, instance) + }, "", instance) require.Equal(t, "10.96.119.41", instance.PrivateIP) require.Equal(t, "testkube.example.com", instance.PublicIP) @@ -263,6 +298,22 @@ func TestPopulateNetworkLoadBalancer(t *testing.T) { require.Equal(t, 22, instance.SSHPort) } +func TestPopulateNetworkNodePort(t *testing.T) { + instance := &cloudv1.Instance{} + populateNetwork(&corev1.Service{ + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + ClusterIP: "10.96.119.41", + Ports: []corev1.ServicePort{{Port: 22, NodePort: 32421}}, + }, + }, "192.168.49.2", instance) + + require.Equal(t, "10.96.119.41", instance.PrivateIP) + require.Equal(t, "192.168.49.2", instance.PublicIP) + require.Equal(t, "192.168.49.2", instance.PublicDNS) + require.Equal(t, 32421, instance.SSHPort) +} + func setPodReady(t *testing.T, client *TestKubeClient, instanceID cloudv1.CloudProviderInstanceID) { t.Helper() diff --git a/v1/providers/testkube/instancetype.go b/v1/providers/testkube/instancetype.go index 73b9c3a..855c20d 100644 --- a/v1/providers/testkube/instancetype.go +++ b/v1/providers/testkube/instancetype.go @@ -18,11 +18,13 @@ const ( DefaultPriceCentsPerHour = 1 - InstanceTypeOKCPU = "test.ok.cpu" - InstanceTypeOKCPUARM64 = "test.ok.cpu.arm64" - InstanceTypeFailCapacity = "test.fail.capacity" - InstanceTypeFailQuota = "test.fail.quota" - InstanceTypeFailBuild = "test.fail.build" // TODO: trigger build failure, maybe with a process that monitors build? + InstanceTypeOKCPU = "test.ok.cpu" + InstanceTypeOKCPUARM64 = "test.ok.cpu.arm64" + InstanceTypeOKCPUNodePort = "test.ok.cpu.nodeport" + InstanceTypeOKCPUARM64NodePort = "test.ok.cpu.arm64.nodeport" + InstanceTypeFailCapacity = "test.fail.capacity" + InstanceTypeFailQuota = "test.fail.quota" + InstanceTypeFailBuild = "test.fail.build" // TODO: trigger build failure, maybe with a process that monitors build? ) // instanceTypeSpec is used mainly as a tuple of instance type (from devplane) and service type (from k8s). When a request @@ -38,11 +40,23 @@ type instanceTypeSpec struct { var allInstanceTypeSpecs = []instanceTypeSpec{ makeInstanceTypeSpec(InstanceTypeOKCPU, cloudv1.ArchitectureX86_64, DefaultImageID), makeInstanceTypeSpec(InstanceTypeOKCPUARM64, cloudv1.ArchitectureARM64, ARM64ImageID), + makeNodePortInstanceTypeSpec(InstanceTypeOKCPUNodePort, cloudv1.ArchitectureX86_64, DefaultImageID), + makeNodePortInstanceTypeSpec(InstanceTypeOKCPUARM64NodePort, cloudv1.ArchitectureARM64, ARM64ImageID), makeInstanceTypeSpec(InstanceTypeFailCapacity, cloudv1.ArchitectureX86_64, DefaultImageID), makeInstanceTypeSpec(InstanceTypeFailQuota, cloudv1.ArchitectureX86_64, DefaultImageID), makeInstanceTypeSpec(InstanceTypeFailBuild, cloudv1.ArchitectureX86_64, DefaultImageID), } +func makeNodePortInstanceTypeSpec( + instanceType string, + architecture cloudv1.Architecture, + imageID string, +) instanceTypeSpec { + spec := makeInstanceTypeSpec(instanceType, architecture, imageID) + spec.serviceType = corev1.ServiceTypeNodePort + return spec +} + func makeInstanceTypeSpec( instanceType string, architecture cloudv1.Architecture, diff --git a/v1/providers/testkube/instancetype_test.go b/v1/providers/testkube/instancetype_test.go index 5a5a616..dba051b 100644 --- a/v1/providers/testkube/instancetype_test.go +++ b/v1/providers/testkube/instancetype_test.go @@ -14,7 +14,7 @@ func TestGetInstanceTypes(t *testing.T) { instanceTypes, err := client.GetInstanceTypes(context.Background(), cloudv1.GetInstanceTypeArgs{}) require.NoError(t, err) - require.Len(t, instanceTypes, 5) + require.Len(t, instanceTypes, 7) instanceTypeByName := map[string]cloudv1.InstanceType{} for _, instanceType := range instanceTypes { @@ -23,10 +23,12 @@ func TestGetInstanceTypes(t *testing.T) { for _, expected := range []string{ InstanceTypeOKCPU, + InstanceTypeOKCPUNodePort, InstanceTypeFailCapacity, InstanceTypeFailQuota, InstanceTypeFailBuild, InstanceTypeOKCPUARM64, + InstanceTypeOKCPUARM64NodePort, } { instanceType, ok := instanceTypeByName[expected] require.True(t, ok, "missing instance type %s", expected) @@ -54,6 +56,7 @@ func TestGetInstanceTypesFiltersByArchitecture(t *testing.T) { architecture: cloudv1.ArchitectureX86_64, expected: []string{ InstanceTypeOKCPU, + InstanceTypeOKCPUNodePort, InstanceTypeFailCapacity, InstanceTypeFailQuota, InstanceTypeFailBuild, @@ -62,7 +65,7 @@ func TestGetInstanceTypesFiltersByArchitecture(t *testing.T) { { name: "arm64", architecture: cloudv1.ArchitectureARM64, - expected: []string{InstanceTypeOKCPUARM64}, + expected: []string{InstanceTypeOKCPUARM64, InstanceTypeOKCPUARM64NodePort}, }, } @@ -93,7 +96,7 @@ func TestGetInstanceTypesWithGPUManufacturerFilterIncludesCPU(t *testing.T) { }, }) require.NoError(t, err) - require.Len(t, instanceTypes, 5) + require.Len(t, instanceTypes, 7) } func TestCapabilitiesDoNotAdvertiseImages(t *testing.T) {