diff --git a/e2e/adapter/adapter_with_maestro.go b/e2e/adapter/adapter_with_maestro.go index 5bb3102..30fa395 100644 --- a/e2e/adapter/adapter_with_maestro.go +++ b/e2e/adapter/adapter_with_maestro.go @@ -3,6 +3,7 @@ package adapter import ( "context" "fmt" + "os" "time" "github.com/onsi/ginkgo/v2" @@ -22,8 +23,6 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - var h *helper.Helper var clusterID string var clusterName string - var resourceBundleID string - var namespaceName string ginkgo.BeforeEach(func(ctx context.Context) { h = helper.New() @@ -48,7 +47,7 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - // Define variables for test adapter yaml adapterName := "cl-maestro" maestroConsumerName := "cluster1" - namespaceName = fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) + namespaceName := fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) configmapName := fmt.Sprintf("%s-%s-configmap", clusterID, adapterName) ginkgo.By("Step 1: Verify cluster was created with generation=1") @@ -94,9 +93,6 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - ginkgo.By("Step 3: Verify ManifestWork metadata (labels and annotations)") Expect(resourceBundle).NotTo(BeNil(), "resource bundle should be found") - // Record resource bundle ID for cleanup - resourceBundleID = resourceBundle.ID - // Verify labels Expect(resourceBundle.Metadata.Labels).To(HaveKey(client.KeyClusterID)) Expect(resourceBundle.Metadata.Labels[client.KeyClusterID]).To(Equal(clusterID)) @@ -297,9 +293,7 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - // 3. The Maestro resource version should remain unchanged across multiple Skip operations ginkgo.It("should skip ManifestWork operation when generation is unchanged", func(ctx context.Context) { - // Set namespace name for cleanup adapterName := "cl-maestro" - namespaceName = fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) ginkgo.By("Step 1: Verify cluster was created with generation=1") cluster, err := h.Client.GetCluster(ctx, clusterID) @@ -323,7 +317,6 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) Expect(resourceBundle).NotTo(BeNil(), "resource bundle should be found") - resourceBundleID = resourceBundle.ID initialVersion := resourceBundle.Version ginkgo.By("Step 3: Capture initial adapter status timestamp before skip period") @@ -395,32 +388,654 @@ var _ = ginkgo.Describe("[Suite: adapter][maestro-transport] Adapter Framework - return } - // Note: It will be replaced by API DELETE once HyperFleet API supports DELETE operations for clusters resource type. + // Clean up cluster and all associated resources + if clusterID != "" { + ginkgo.By("Cleanup test cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err) + } + } + }) + }, +) - // Delete resource bundle first - this triggers Maestro agent to clean up K8s resources - // The agent will delete the namespace and configmap according to deleteOption.propagationPolicy - if resourceBundleID != "" { - ginkgo.By("deleting resource bundle " + resourceBundleID) - ginkgo.GinkgoWriter.Printf("Deleting resource bundle ID: %s\n", resourceBundleID) - err := h.GetMaestroClient().DeleteResourceBundle(ctx, resourceBundleID) - if err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to delete resource bundle %s: %v\n", resourceBundleID, err) - } else { - ginkgo.GinkgoWriter.Printf("Successfully deleted resource bundle %s\n", resourceBundleID) +var _ = ginkgo.Describe("[Suite: adapter][maestro-transport][negative] Adapter Framework - Maestro Transport Negative Scenarios", + ginkgo.Label(labels.Tier1), + func() { + var ( + h *helper.Helper + clusterID string + adapterRelease string // Track deployed adapter release name for cleanup + chartPath string + baseDeployOpts helper.AdapterDeploymentOptions + ) + + ginkgo.BeforeEach(func(ctx context.Context) { + h = helper.New() + + // Clone adapter Helm chart repository (shared across negative tests) + ginkgo.By("Clone adapter Helm chart repository for negative tests") + var cleanupChart func() error + var err error + chartPath, cleanupChart, err = h.CloneHelmChart(ctx, helper.HelmChartCloneOptions{ + Component: "adapter", + RepoURL: h.Cfg.AdapterDeployment.ChartRepo, + Ref: h.Cfg.AdapterDeployment.ChartRef, + ChartPath: h.Cfg.AdapterDeployment.ChartPath, + WorkDir: helper.TestWorkDir, + }) + Expect(err).NotTo(HaveOccurred(), "failed to clone adapter Helm chart") + ginkgo.GinkgoWriter.Printf("Cloned adapter chart to: %s\n", chartPath) + + // Ensure chart cleanup after test + ginkgo.DeferCleanup(func(ctx context.Context) { + ginkgo.By("Cleanup cloned Helm chart") + if err := cleanupChart(); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup chart: %v\n", err) } + }) + + // Set up base deployment options with common fields + baseDeployOpts = helper.AdapterDeploymentOptions{ + Namespace: h.Cfg.Namespace, + ChartPath: chartPath, } + }) - // Delete namespace as a safety cleanup (in case Maestro agent didn't clean up) - // This ensures the namespace is removed even if the agent deletion failed - if namespaceName != "" { - ginkgo.By("deleting namespace " + namespaceName) - err := h.K8sClient.DeleteNamespaceAndWait(ctx, namespaceName) - if err != nil { - ginkgo.GinkgoWriter.Printf("Warning: failed to delete namespace %s: %v\n", namespaceName, err) + ginkgo.AfterEach(func(ctx context.Context) { + // Clean up in reverse order: adapter first, then cluster + // This ensures adapter is uninstalled before cluster cleanup + if adapterRelease != "" { + ginkgo.By("Uninstall adapter " + adapterRelease) + if err := h.UninstallAdapter(ctx, adapterRelease, h.Cfg.Namespace); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to uninstall adapter %s: %v\n", adapterRelease, err) } else { - ginkgo.GinkgoWriter.Printf("Successfully deleted namespace %s\n", namespaceName) + ginkgo.GinkgoWriter.Printf("Successfully uninstalled adapter: %s\n", adapterRelease) + } + } + + if clusterID != "" { + ginkgo.By("Cleanup test cluster " + clusterID) + if err := h.CleanupTestCluster(ctx, clusterID); err != nil { + ginkgo.GinkgoWriter.Printf("Warning: failed to cleanup cluster %s: %v\n", clusterID, err) } } }) + + ginkgo.It("should fail when targeting unregistered Maestro consumer and report appropriate error", + func(ctx context.Context) { + // Test-specific adapter configuration + adapterName := "cl-m-unreg-consumer" + err := os.Setenv("ADAPTER_NAME", adapterName) + Expect(err).NotTo(HaveOccurred(), "failed to set ADAPTER_NAME environment variable") + ginkgo.DeferCleanup(func() { + _ = os.Unsetenv("ADAPTER_NAME") + }) + // Generate unique release name for this deployment + releaseName := helper.GenerateAdapterReleaseName(helper.ResourceTypeClusters, adapterName) + + // Deploy the test adapter configured to target unregistered consumer + ginkgo.By("Deploy test adapter with unregistered consumer configuration") + + // Create deployment options from base and add test-specific fields + deployOpts := baseDeployOpts + deployOpts.ReleaseName = releaseName + deployOpts.AdapterName = adapterName + + err = h.DeployAdapter(ctx, deployOpts) + Expect(err).NotTo(HaveOccurred(), "failed to deploy test adapter") + adapterRelease = releaseName + ginkgo.GinkgoWriter.Printf("Successfully deployed adapter: %s (release: %s)\n", adapterName, releaseName) + + // Create cluster after adapter is deployed + ginkgo.By("Create test cluster") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil(), "cluster ID should be generated") + Expect(cluster.Name).NotTo(BeEmpty(), "cluster name should be present") + clusterID = *cluster.Id + ginkgo.GinkgoWriter.Printf("Created cluster ID: %s, Name: %s\n", clusterID, cluster.Name) + + ginkgo.By("Verify adapter reports failure for unregistered consumer") + // Wait for adapter to process the cluster and report failure status + Eventually(func(g Gomega) { + statuses, err := h.Client.GetClusterStatuses(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "failed to get cluster statuses") + g.Expect(statuses.Items).NotTo(BeEmpty(), "adapter should have reported status") + + // Find the test adapter status + var adapterStatus *openapi.AdapterStatus + for i, adapter := range statuses.Items { + if adapter.Adapter == adapterName { + adapterStatus = &statuses.Items[i] + break + } + } + + g.Expect(adapterStatus).NotTo(BeNil(), + "adapter %s should be present in adapter statuses", adapterName) + + // Validate adapter metadata + g.Expect(adapterStatus.ObservedGeneration).To(Equal(int32(1)), + "adapter should have observed_generation=1") + + // Find Health condition + var healthCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeHealth { + healthCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(healthCondition).NotTo(BeNil(), + "adapter should have Health condition") + + // Verify Health condition reports failure + g.Expect(healthCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "adapter Health condition should be False due to unregistered consumer") + + // Verify error details mention consumer not found/registered + g.Expect(healthCondition.Message).NotTo(BeNil(), + "adapter Health condition should have message") + message := *healthCondition.Message + g.Expect(message).To(SatisfyAny( + ContainSubstring("unregistered-consumer"), + ContainSubstring("not found"), + ContainSubstring("not registered"), + ), "error message should mention unregistered consumer") + + // Find Applied condition - should be False + var appliedCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeApplied { + appliedCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(appliedCondition).NotTo(BeNil(), + "adapter should have Applied condition") + g.Expect(appliedCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "adapter Applied condition should be False since ManifestWork was not created") + + ginkgo.GinkgoWriter.Printf("Verified adapter failure for unregistered consumer: Health=%s, Applied=%s\n", + healthCondition.Status, appliedCondition.Status) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("Verify no ManifestWork was created by the test adapter on Maestro") + Eventually(func(g Gomega) { + // Query by cluster ID first to scope to current cluster + rbs, err := h.GetMaestroClient().FindAllResourceBundlesByClusterID(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Maestro for resource bundles") + + // Filter by adapter name using source-id label + var adapterBundles []maestro.ResourceBundle + for _, rb := range rbs { + if rb.Metadata.Labels != nil && rb.Metadata.Labels["maestro.io/source-id"] == adapterName { + adapterBundles = append(adapterBundles, rb) + } + } + + g.Expect(adapterBundles).To(BeEmpty(), + "no ManifestWork should be created by adapter %s for cluster %s with unregistered consumer", adapterName, clusterID) + ginkgo.GinkgoWriter.Printf("Verified no ManifestWork exists from adapter %s for cluster %s\n", + adapterName, clusterID) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("Verify no K8s resources were created by the test adapter") + Eventually(func(g Gomega) { + // Check specifically for namespace that would have been created by THIS adapter + // Expected namespace name pattern: ${clusterID}-${adapterName}-namespace + expectedNamespace := fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) + _, err := h.GetNamespace(ctx, expectedNamespace) + + // We expect the namespace to NOT exist (should get error) + g.Expect(err).To(HaveOccurred(), + "namespace %s should not exist when adapter %s fails to create ManifestWork", + expectedNamespace, adapterName) + + ginkgo.GinkgoWriter.Printf("Verified namespace %s does not exist (adapter %s did not create resources)\n", + expectedNamespace, adapterName) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.GinkgoWriter.Printf("Successfully validated adapter failure for unregistered consumer\n") + }) + + ginkgo.It("should fail to discover ManifestWork when discovery name does not match created resource", + func(ctx context.Context) { + // Test-specific adapter configuration + adapterName := "cl-m-wrong-ds" + // Set environment variable for envsubst expansion in values.yaml + err := os.Setenv("ADAPTER_NAME", adapterName) + Expect(err).NotTo(HaveOccurred(), "failed to set ADAPTER_NAME environment variable") + ginkgo.DeferCleanup(func() { + _ = os.Unsetenv("ADAPTER_NAME") + }) + // Generate unique release name for this deployment + releaseName := helper.GenerateAdapterReleaseName(helper.ResourceTypeClusters, adapterName) + // Deploy the test adapter with wrong main discovery configuration + ginkgo.By("Deploy test adapter with wrong ManifestWork discovery name") + + // Create deployment options from base and add test-specific fields + deployOpts := baseDeployOpts + deployOpts.ReleaseName = releaseName + deployOpts.AdapterName = adapterName + + err = h.DeployAdapter(ctx, deployOpts) + Expect(err).NotTo(HaveOccurred(), "failed to deploy test adapter") + adapterRelease = releaseName + ginkgo.GinkgoWriter.Printf("Successfully deployed adapter: %s (release: %s)\n", adapterName, releaseName) + + // Create cluster after adapter is deployed + ginkgo.By("Create test cluster") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil(), "cluster ID should be generated") + Expect(cluster.Name).NotTo(BeEmpty(), "cluster name should be present") + clusterID = *cluster.Id + ginkgo.GinkgoWriter.Printf("Created cluster ID: %s, Name: %s\n", clusterID, cluster.Name) + + // Verify ManifestWork was created by the test adapter despite wrong discovery config + ginkgo.By("Verify ManifestWork was created by the test adapter on Maestro") + Eventually(func(g Gomega) { + // Query by cluster ID first to scope to current cluster + rbs, err := h.GetMaestroClient().FindAllResourceBundlesByClusterID(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Maestro for resource bundles") + + // Filter by adapter name using source-id label + var adapterBundles []maestro.ResourceBundle + for _, rb := range rbs { + if rb.Metadata.Labels != nil && rb.Metadata.Labels["maestro.io/source-id"] == adapterName { + adapterBundles = append(adapterBundles, rb) + } + } + + g.Expect(adapterBundles).NotTo(BeEmpty(), "ManifestWork should be created by adapter %s for cluster %s despite wrong discovery", adapterName, clusterID) + ginkgo.GinkgoWriter.Printf("Found resource bundle created by adapter %s for cluster %s: ID=%s\n", adapterName, clusterID, adapterBundles[0].ID) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + // Verify K8s resources were created by Maestro agent + ginkgo.By("Verify K8s resources were created by Maestro agent") + namespaceName := fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) + configmapName := fmt.Sprintf("%s-%s-configmap", clusterID, adapterName) + + Eventually(func(g Gomega) { + // Verify namespace exists + _, err := h.GetNamespace(ctx, namespaceName) + g.Expect(err).NotTo(HaveOccurred(), "namespace should be created by Maestro agent") + + // Verify configmap exists in the namespace + _, err = h.GetConfigMap(ctx, namespaceName, configmapName) + g.Expect(err).NotTo(HaveOccurred(), "configmap should be created by Maestro agent") + + ginkgo.GinkgoWriter.Printf("Verified resources exist: namespace=%s, configmap=%s\n", namespaceName, configmapName) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("Verify adapter reports discovery failure with appropriate error") + // Wait for adapter to process the cluster and report discovery failure + Eventually(func(g Gomega) { + statuses, err := h.Client.GetClusterStatuses(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "failed to get cluster statuses") + g.Expect(statuses.Items).NotTo(BeEmpty(), "adapter should have reported status") + + // Find the test adapter status + var adapterStatus *openapi.AdapterStatus + for i, adapter := range statuses.Items { + if adapter.Adapter == adapterName { + adapterStatus = &statuses.Items[i] + break + } + } + + g.Expect(adapterStatus).NotTo(BeNil(), + "adapter %s should be present in adapter statuses", adapterName) + + // Validate adapter metadata + g.Expect(adapterStatus.ObservedGeneration).To(Equal(int32(1)), + "adapter should have observed_generation=1") + + // Find Applied condition - should be False (ManifestWork not discovered) + var appliedCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeApplied { + appliedCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(appliedCondition).NotTo(BeNil(), + "adapter should have Applied condition") + g.Expect(appliedCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "Applied should be False - ManifestWork not discovered") + g.Expect(appliedCondition.Reason).NotTo(BeNil()) + g.Expect(*appliedCondition.Reason).To(Equal("ManifestWorkNotDiscovered"), + "Applied reason should be ManifestWorkNotDiscovered") + + // Find Available condition - should be False + var availableCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeAvailable { + availableCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(availableCondition).NotTo(BeNil(), + "adapter should have Available condition") + g.Expect(availableCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "Available should be False") + g.Expect(availableCondition.Reason).NotTo(BeNil()) + g.Expect(*availableCondition.Reason).To(Equal("NamespaceNotDiscovered"), + "Available reason should be NamespaceNotDiscovered") + + // Find Health condition - should be False (execution failed) + var healthCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeHealth { + healthCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(healthCondition).NotTo(BeNil(), + "adapter should have Health condition") + g.Expect(healthCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "Health should be False - discovery failed") + g.Expect(healthCondition.Reason).NotTo(BeNil()) + g.Expect(*healthCondition.Reason).To(Equal("ExecutionFailed:ResourceFailed"), + "Health reason should be ExecutionFailed:ResourceFailed") + g.Expect(healthCondition.Message).NotTo(BeNil()) + g.Expect(*healthCondition.Message).To(ContainSubstring("not found"), + "Health message should mention ManifestWork not found") + + // Verify data section - all fields empty (main discovery failed) + g.Expect(adapterStatus.Data).NotTo(BeNil(), "adapter status should have data") + + // ManifestWork name should be empty (main discovery failed) + if manifestworkData, ok := (*adapterStatus.Data)["manifestwork"].(map[string]interface{}); ok { + if nameVal, exists := manifestworkData["name"]; exists { + g.Expect(nameVal).To(Or(BeNil(), Equal("")), + "manifestwork name should be empty - main discovery failed") + } + } + + // Namespace name should be empty + if namespaceData, ok := (*adapterStatus.Data)["namespace"].(map[string]interface{}); ok { + if nameVal, exists := namespaceData["name"]; exists { + g.Expect(nameVal).To(Or(BeNil(), Equal("")), + "namespace name should be empty") + } + } + + // ConfigMap name should be empty + if configmapData, ok := (*adapterStatus.Data)["configmap"].(map[string]interface{}); ok { + if nameVal, exists := configmapData["name"]; exists { + g.Expect(nameVal).To(Or(BeNil(), Equal("")), + "configmap name should be empty") + } + } + + ginkgo.GinkgoWriter.Printf("Verified main discovery failure: Applied=%s, Health=%s, Available=%s\n", + appliedCondition.Status, healthCondition.Status, availableCondition.Status) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.GinkgoWriter.Printf("Successfully validated adapter discovery failure reporting\n") + }) + + ginkgo.It("should fail nested discovery when resource names are wrong", + func(ctx context.Context) { + // Test-specific adapter configuration + adapterName := "cl-m-wrong-nest" + // Set environment variable for envsubst expansion in values.yaml + err := os.Setenv("ADAPTER_NAME", adapterName) + Expect(err).NotTo(HaveOccurred(), "failed to set ADAPTER_NAME environment variable") + ginkgo.DeferCleanup(func() { + _ = os.Unsetenv("ADAPTER_NAME") + }) + + // Generate unique release name for this deployment + releaseName := helper.GenerateAdapterReleaseName(helper.ResourceTypeClusters, adapterName) + + // Deploy the test adapter with empty discovery configuration + ginkgo.By("Deploy test adapter with empty nested discovery configuration") + + // Create deployment options from base and add test-specific fields + deployOpts := baseDeployOpts + deployOpts.ReleaseName = releaseName + deployOpts.AdapterName = adapterName + + err = h.DeployAdapter(ctx, deployOpts) + Expect(err).NotTo(HaveOccurred(), "failed to deploy test adapter") + adapterRelease = releaseName + ginkgo.GinkgoWriter.Printf("Successfully deployed adapter: %s (release: %s)\n", adapterName, releaseName) + + // Create cluster after adapter is deployed + ginkgo.By("Create test cluster") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil(), "cluster ID should be generated") + Expect(cluster.Name).NotTo(BeEmpty(), "cluster name should be present") + clusterID = *cluster.Id + ginkgo.GinkgoWriter.Printf("Created cluster ID: %s, Name: %s\n", clusterID, cluster.Name) + + // Construct namespace name AFTER cluster is created + namespaceName := fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) + + // Verify ManifestWork was created by the test adapter + ginkgo.By("Verify ManifestWork was created by the test adapter on Maestro") + Eventually(func(g Gomega) { + // Query by cluster ID first to scope to current cluster + rbs, err := h.GetMaestroClient().FindAllResourceBundlesByClusterID(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Maestro for resource bundles") + + // Filter by adapter name using source-id label + var adapterBundles []maestro.ResourceBundle + for _, rb := range rbs { + if rb.Metadata.Labels != nil && rb.Metadata.Labels["maestro.io/source-id"] == adapterName { + adapterBundles = append(adapterBundles, rb) + } + } + + g.Expect(adapterBundles).NotTo(BeEmpty(), "ManifestWork should be created by adapter %s for cluster %s", adapterName, clusterID) + ginkgo.GinkgoWriter.Printf("Found resource bundle created by adapter %s for cluster %s: ID=%s\n", adapterName, clusterID, adapterBundles[0].ID) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + // Verify K8s resources were created + ginkgo.By("Verify K8s resources were created by Maestro agent") + Eventually(func(g Gomega) { + _, err := h.GetNamespace(ctx, namespaceName) + g.Expect(err).NotTo(HaveOccurred(), "namespace should be created") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("Verify adapter handles empty discovery with fallback values") + // Wait for adapter to process the cluster and report status with fallback values + Eventually(func(g Gomega) { + statuses, err := h.Client.GetClusterStatuses(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "failed to get cluster statuses") + g.Expect(statuses.Items).NotTo(BeEmpty(), "adapter should have reported status") + + // Find the test adapter status + var adapterStatus *openapi.AdapterStatus + for i, adapter := range statuses.Items { + if adapter.Adapter == adapterName { + adapterStatus = &statuses.Items[i] + break + } + } + + g.Expect(adapterStatus).NotTo(BeNil(), + "adapter %s should be present in adapter statuses", adapterName) + + // Validate adapter metadata + g.Expect(adapterStatus.ObservedGeneration).To(Equal(int32(1)), + "adapter should have observed_generation=1") + + // Find Applied condition - should be True (ManifestWork created successfully) + var appliedCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeApplied { + appliedCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(appliedCondition).NotTo(BeNil(), + "adapter should have Applied condition") + g.Expect(appliedCondition.Status).To(Equal(openapi.AdapterConditionStatusTrue), + "adapter Applied condition should be True since ManifestWork was created") + + // Find Available condition - should be False (nested resources not discovered) + var availableCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeAvailable { + availableCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(availableCondition).NotTo(BeNil(), + "adapter should have Available condition") + g.Expect(availableCondition.Status).To(Equal(openapi.AdapterConditionStatusFalse), + "adapter Available condition should be False since nested discovery returned empty") + + // Find Health condition - should be True (adapter executed successfully) + // Note: Nested discovery failure doesn't affect Health - the adapter ran successfully + var healthCondition *openapi.AdapterCondition + for i, condition := range adapterStatus.Conditions { + if condition.Type == client.ConditionTypeHealth { + healthCondition = &adapterStatus.Conditions[i] + break + } + } + + g.Expect(healthCondition).NotTo(BeNil(), + "adapter should have Health condition") + g.Expect(healthCondition.Status).To(Equal(openapi.AdapterConditionStatusTrue), + "adapter Health condition should be True - nested discovery failure doesn't affect health") + g.Expect(healthCondition.Reason).NotTo(BeNil(), + "Health condition should have a reason") + g.Expect(*healthCondition.Reason).To(Equal("Healthy"), + "Health reason should be Healthy") + g.Expect(healthCondition.Message).NotTo(BeNil(), + "Health condition should have a message") + g.Expect(*healthCondition.Message).To(ContainSubstring("completed successfully"), + "Health message should indicate successful execution") + + // Verify data field shows fallback/empty values for nested resources + g.Expect(adapterStatus.Data).NotTo(BeNil(), + "adapter status should have data field") + + // Check that namespace data is either empty or has fallback values + if namespaceData, ok := (*adapterStatus.Data)["namespace"].(map[string]interface{}); ok { + // Namespace name should be empty string or default value + if name, exists := namespaceData["name"]; exists { + g.Expect(name).To(Or(BeEmpty(), Equal("")), + "namespace name should be empty due to empty discovery") + } + } + + ginkgo.GinkgoWriter.Printf("Verified empty discovery handling: Applied=%s, Available=%s, Health=%s\n", + appliedCondition.Status, availableCondition.Status, healthCondition.Status) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.GinkgoWriter.Printf("Successfully validated adapter handling of empty nested discovery\n") + }) + + ginkgo.It("should fail post-action when status API is unreachable", + func(ctx context.Context) { + // Use cl-m-bad-api adapter with overridden API URL + adapterName := "cl-m-bad-api" + // Set environment variable for envsubst expansion in values.yaml + err := os.Setenv("ADAPTER_NAME", adapterName) + Expect(err).NotTo(HaveOccurred(), "failed to set ADAPTER_NAME environment variable") + ginkgo.DeferCleanup(func() { + _ = os.Unsetenv("ADAPTER_NAME") + }) + + // Generate unique release name for this deployment + releaseName := helper.GenerateAdapterReleaseName(helper.ResourceTypeClusters, adapterName) + + // Deploy the test adapter with invalid API URL + ginkgo.By("Deploy test adapter with unreachable API URL configuration") + + // Create deployment options with overridden API URL + deployOpts := baseDeployOpts + deployOpts.ReleaseName = releaseName + deployOpts.AdapterName = adapterName + // Override hyperfleetApi.baseUrl to make it unreachable + deployOpts.SetValues = map[string]string{ + "adapterConfig.hyperfleetApi.baseUrl": "http://invalid-hyperfleet-api-endpoint.local:9999", + } + + err = h.DeployAdapter(ctx, deployOpts) + Expect(err).NotTo(HaveOccurred(), "failed to deploy test adapter") + adapterRelease = releaseName + ginkgo.GinkgoWriter.Printf("Successfully deployed adapter: %s (release: %s)\n", adapterName, releaseName) + + // Create cluster after adapter is deployed + ginkgo.By("Create test cluster") + cluster, err := h.Client.CreateClusterFromPayload(ctx, h.TestDataPath("payloads/clusters/cluster-request.json")) + Expect(err).NotTo(HaveOccurred(), "failed to create cluster") + Expect(cluster.Id).NotTo(BeNil(), "cluster ID should be generated") + Expect(cluster.Name).NotTo(BeEmpty(), "cluster name should be present") + clusterID = *cluster.Id + ginkgo.GinkgoWriter.Printf("Created cluster ID: %s, Name: %s\n", clusterID, cluster.Name) + + // Construct namespace name AFTER cluster is created + namespaceName := fmt.Sprintf("%s-%s-namespace", clusterID, adapterName) + + ginkgo.By("Verify ManifestWork was applied successfully by the test adapter in Maestro") + // Even though post-action failed, the ManifestWork should exist in Maestro + Eventually(func(g Gomega) { + // Query by cluster ID first to scope to current cluster + rbs, err := h.GetMaestroClient().FindAllResourceBundlesByClusterID(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Maestro for resource bundles") + + // Filter by adapter name using source-id label + var adapterBundles []maestro.ResourceBundle + for _, rb := range rbs { + if rb.Metadata.Labels != nil && rb.Metadata.Labels["maestro.io/source-id"] == adapterName { + adapterBundles = append(adapterBundles, rb) + } + } + + g.Expect(adapterBundles).NotTo(BeEmpty(), "ManifestWork should exist in Maestro created by adapter %s for cluster %s", adapterName, clusterID) + g.Expect(adapterBundles[0].ID).NotTo(BeEmpty(), "ManifestWork should have an ID") + g.Expect(adapterBundles[0].ConsumerName).NotTo(BeEmpty(), "ManifestWork should have a consumer") + ginkgo.GinkgoWriter.Printf("ManifestWork verified in Maestro for adapter %s and cluster %s: ID=%s, Consumer=%s\n", + adapterName, clusterID, adapterBundles[0].ID, adapterBundles[0].ConsumerName) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + // Verify K8s resources were created + ginkgo.By("Verify K8s resources were created by Maestro agent despite API being unreachable") + Eventually(func(g Gomega) { + _, err := h.GetNamespace(ctx, namespaceName) + g.Expect(err).NotTo(HaveOccurred(), "namespace should be created") + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.By("Verify adapter cannot report status when API is unreachable") + // Since the adapter has invalid API URL, it should NOT successfully report status + // The status should either not exist or remain empty + Consistently(func(g Gomega) { + statuses, err := h.Client.GetClusterStatuses(ctx, clusterID) + g.Expect(err).NotTo(HaveOccurred(), "failed to get cluster statuses") + + // Find the test adapter status + var adapterStatus *openapi.AdapterStatus + for _, adapter := range statuses.Items { + if adapter.Adapter == adapterName { + adapterStatus = &adapter + break + } + } + + // Adapter should NOT be able to post status + g.Expect(adapterStatus).To(BeNil(), "adapter status should not exist when API is unreachable") + ginkgo.GinkgoWriter.Printf("Verified: no status found for adapter %s (expected)\n", adapterName) + }, h.Cfg.Timeouts.Adapter.Processing, h.Cfg.Polling.Interval).Should(Succeed()) + + ginkgo.GinkgoWriter.Printf("Successfully validated adapter cannot report status with unreachable API\n") + }) }, ) diff --git a/pkg/client/kubernetes/client.go b/pkg/client/kubernetes/client.go index 14eb31b..baee212 100644 --- a/pkg/client/kubernetes/client.go +++ b/pkg/client/kubernetes/client.go @@ -45,8 +45,38 @@ func NewClient() (*Client, error) { // DeleteNamespaceAndWait deletes a namespace and waits for it to be fully removed func (c *Client) DeleteNamespaceAndWait(ctx context.Context, namespace string) error { - // Delete namespace - err := c.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + // Check if namespace exists first + _, err := c.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil // Already deleted + } + + // Delete all resources in the namespace to speed up cleanup + // This helps avoid timeout issues when namespaces have many resources + gracePeriod := int64(0) + propagationPolicy := metav1.DeletePropagationForeground + deleteOpts := metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + PropagationPolicy: &propagationPolicy, + } + + // Delete deployments + _ = c.AppsV1().Deployments(namespace).DeleteCollection(ctx, deleteOpts, metav1.ListOptions{}) + + // Delete jobs + _ = c.BatchV1().Jobs(namespace).DeleteCollection(ctx, deleteOpts, metav1.ListOptions{}) + + // Delete configmaps + _ = c.CoreV1().ConfigMaps(namespace).DeleteCollection(ctx, deleteOpts, metav1.ListOptions{}) + + // Delete pods + _ = c.CoreV1().Pods(namespace).DeleteCollection(ctx, deleteOpts, metav1.ListOptions{}) + + // Delete namespace with foreground propagation to ensure resources are cleaned up + nsDeleteOpts := metav1.DeleteOptions{ + PropagationPolicy: &propagationPolicy, + } + err = c.CoreV1().Namespaces().Delete(ctx, namespace, nsDeleteOpts) if err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete namespace %s: %w", namespace, err) } @@ -56,8 +86,8 @@ func (c *Client) DeleteNamespaceAndWait(ctx context.Context, namespace string) e Duration: 500 * time.Millisecond, Factor: 1.5, Jitter: 0.1, - Steps: 20, - Cap: 10 * time.Second, // Cap individual retries at 10s for ~2.4 min total timeout + Steps: 30, // Increased from 20 to give more time + Cap: 15 * time.Second, // Increased cap for better handling of stuck resources } err = wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { _, err := c.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) @@ -88,6 +118,23 @@ func (c *Client) FetchNamespace(ctx context.Context, name string) (*corev1.Names return ns, nil } +// FindNamespacesByPrefix finds all namespaces whose name starts with the given prefix +func (c *Client) FindNamespacesByPrefix(ctx context.Context, prefix string) ([]string, error) { + namespaces, err := c.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list namespaces: %w", err) + } + + var matchingNamespaces []string + for _, ns := range namespaces.Items { + if len(ns.Name) >= len(prefix) && ns.Name[:len(prefix)] == prefix { + matchingNamespaces = append(matchingNamespaces, ns.Name) + } + } + + return matchingNamespaces, nil +} + // FetchConfigMap gets a configmap by name in the specified namespace func (c *Client) FetchConfigMap(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error) { cm, err := c.CoreV1().ConfigMaps(namespace).Get(ctx, name, metav1.GetOptions{}) diff --git a/pkg/client/maestro/client.go b/pkg/client/maestro/client.go index 40f954e..3461bd3 100644 --- a/pkg/client/maestro/client.go +++ b/pkg/client/maestro/client.go @@ -184,6 +184,96 @@ func (c *Client) FindResourceBundleByClusterID(ctx context.Context, clusterID st return nil, fmt.Errorf("no resource bundle found for cluster ID: %s", clusterID) } +// FindAllResourceBundlesByClusterID finds all resource bundles for a cluster ID +// Returns all matching resource bundles (multiple adapters may create ManifestWorks for the same cluster) +func (c *Client) FindAllResourceBundlesByClusterID(ctx context.Context, clusterID string) ([]ResourceBundle, error) { + // Use labelSelector query parameter to filter server-side + labelSelector := fmt.Sprintf("%s=%s", client.KeyClusterID, clusterID) + apiURL := fmt.Sprintf("%s%s?labelSelector=%s", + c.baseURL, + resourceBundlesBasePath, + url.QueryEscape(labelSelector)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to execute request: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) + } + + var result ResourceBundleList + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Filter and return all matching resource bundles + var bundles []ResourceBundle + for i := range result.Items { + if result.Items[i].Metadata.Labels != nil && + result.Items[i].Metadata.Labels[client.KeyClusterID] == clusterID { + bundles = append(bundles, result.Items[i]) + } + } + + return bundles, nil +} + +// FindResourceBundlesByAdapterName finds all resource bundles created by a specific adapter +// Uses the maestro.io/source-id label to filter by adapter name +func (c *Client) FindResourceBundlesByAdapterName(ctx context.Context, adapterName string) ([]ResourceBundle, error) { + // Use labelSelector query parameter to filter server-side by adapter source-id + labelSelector := fmt.Sprintf("maestro.io/source-id=%s", adapterName) + apiURL := fmt.Sprintf("%s%s?labelSelector=%s", + c.baseURL, + resourceBundlesBasePath, + url.QueryEscape(labelSelector)) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to execute request: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("unexpected status code %d: %s", resp.StatusCode, string(body)) + } + + var result ResourceBundleList + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + + // Filter and return all matching resource bundles + var bundles []ResourceBundle + for i := range result.Items { + if result.Items[i].Metadata.Labels != nil && + result.Items[i].Metadata.Labels["maestro.io/source-id"] == adapterName { + bundles = append(bundles, result.Items[i]) + } + } + + return bundles, nil +} + // ListConsumers retrieves the list of registered Maestro consumers // Returns a list of consumer names func (c *Client) ListConsumers(ctx context.Context) ([]string, error) { diff --git a/pkg/helper/adapter.go b/pkg/helper/adapter.go index e2b8816..1cf4da9 100644 --- a/pkg/helper/adapter.go +++ b/pkg/helper/adapter.go @@ -39,6 +39,7 @@ type AdapterDeploymentOptions struct { ChartPath string AdapterName string Timeout time.Duration + SetValues map[string]string // Additional Helm --set values } // GenerateAdapterReleaseName generates a unique Helm release name for an adapter deployment @@ -152,6 +153,11 @@ func (h *Helper) DeployAdapter(ctx context.Context, opts AdapterDeploymentOption "--set", fmt.Sprintf("fullnameOverride=%s", releaseName), ) + // Add additional --set values if provided + for key, value := range opts.SetValues { + helmArgs = append(helmArgs, "--set", fmt.Sprintf("%s=%s", key, value)) + } + logger.Info("executing Helm command", "args", helmArgs) // Create context with timeout diff --git a/pkg/helper/helper.go b/pkg/helper/helper.go index 8f966cc..ddfa860 100644 --- a/pkg/helper/helper.go +++ b/pkg/helper/helper.go @@ -1,90 +1,195 @@ package helper import ( - "context" - "fmt" - "path/filepath" - - "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" - "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" - k8sclient "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client/kubernetes" - "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client/maestro" - "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/config" - "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/logger" + "context" + "fmt" + "path/filepath" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/api/openapi" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client" + k8sclient "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client/kubernetes" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/client/maestro" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/config" + "github.com/openshift-hyperfleet/hyperfleet-e2e/pkg/logger" ) // Helper provides utility functions for e2e tests type Helper struct { - Cfg *config.Config - Client *client.HyperFleetClient - K8sClient *k8sclient.Client - MaestroClient *maestro.Client + Cfg *config.Config + Client *client.HyperFleetClient + K8sClient *k8sclient.Client + MaestroClient *maestro.Client } // TestDataPath resolves a relative path within the testdata directory // This ensures testdata paths work correctly whether invoked via go test or the e2e binary func (h *Helper) TestDataPath(relativePath string) string { - return filepath.Join(h.Cfg.TestDataDir, relativePath) + return filepath.Join(h.Cfg.TestDataDir, relativePath) } // GetTestCluster creates a new temporary test cluster func (h *Helper) GetTestCluster(ctx context.Context, payloadPath string) (string, error) { - cluster, err := h.Client.CreateClusterFromPayload(ctx, payloadPath) - if err != nil { - return "", err - } - if cluster == nil { - return "", fmt.Errorf("CreateClusterFromPayload returned nil") - } - if cluster.Id == nil { - return "", fmt.Errorf("created cluster has no ID") - } - return *cluster.Id, nil + cluster, err := h.Client.CreateClusterFromPayload(ctx, payloadPath) + if err != nil { + return "", err + } + if cluster == nil { + return "", fmt.Errorf("CreateClusterFromPayload returned nil") + } + if cluster.Id == nil { + return "", fmt.Errorf("created cluster has no ID") + } + return *cluster.Id, nil } -// CleanupTestCluster deletes the temporary test cluster +// CleanupTestCluster deletes the temporary test cluster and resources created by adapters from CLUSTER_TIER0_ADAPTERS_DEPLOYMENT // TODO: Replace this workaround with API DELETE once HyperFleet API supports // DELETE operations for clusters resource type: // -// return h.Client.DeleteCluster(ctx, clusterID) +// return h.Client.DeleteCluster(ctx, clusterID) +// +// Temporary workaround: delete the Kubernetes namespace and adapter resources using client-go // -// Temporary workaround: delete the Kubernetes namespace using client-go (may temporarily hardcode a timeout duration). +// IMPORTANT: This function continues cleanup even if errors occur, to ensure maximum cleanup effort. +// However, all errors are accumulated and returned at the end to avoid hiding failures that could +// cause test pollution (e.g., stale Maestro state being read by subsequent tests). func (h *Helper) CleanupTestCluster(ctx context.Context, clusterID string) error { - logger.Info("deleting cluster namespace (workaround)", "cluster_id", clusterID, "namespace", clusterID) - - // Guard against nil K8sClient - if h == nil || h.K8sClient == nil { - err := fmt.Errorf("K8sClient is nil, cannot delete namespace") - logger.Error("K8sClient is nil", "cluster_id", clusterID) - return err - } - - // Delete namespace and wait for deletion to complete - err := h.K8sClient.DeleteNamespaceAndWait(ctx, clusterID) - if err != nil { - logger.Error("failed to delete cluster namespace", "cluster_id", clusterID, "error", err) - return err - } - - logger.Info("successfully deleted cluster namespace", "cluster_id", clusterID) - return nil + logger.Info("cleaning up cluster resources", "cluster_id", clusterID) + + // Guard against nil K8sClient + if h == nil || h.K8sClient == nil { + err := fmt.Errorf("K8sClient is nil, cannot delete resources") + logger.Error("K8sClient is nil", "cluster_id", clusterID) + return err + } + + // Accumulate errors but continue cleanup to maximize cleanup effort + var cleanupErrors []error + + // Step 1: Delete all Maestro resource bundles (ManifestWorks) for this cluster + // Multiple adapters may create ManifestWorks for the same cluster (e.g., cl-maestro, cl-m-wrong-ds) + maestroClient := h.GetMaestroClient() + if maestroClient != nil { + logger.Info("attempting to delete all maestro resource bundles for cluster", "cluster_id", clusterID) + + rbs, err := maestroClient.FindAllResourceBundlesByClusterID(ctx, clusterID) + if err != nil { + logger.Error("failed to find maestro resource bundles", "cluster_id", clusterID, "error", err) + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to find maestro resource bundles: %w", err)) + } else if len(rbs) > 0 { + logger.Info("found maestro resource bundles", "cluster_id", clusterID, "count", len(rbs)) + + // Delete all resource bundles - this triggers Maestro agent to clean up K8s resources + for _, rb := range rbs { + if err := maestroClient.DeleteResourceBundle(ctx, rb.ID); err != nil { + logger.Error("failed to delete maestro resource bundle", "cluster_id", clusterID, "resource_bundle_id", rb.ID, "error", err) + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to delete maestro resource bundle %s: %w", rb.ID, err)) + } else { + logger.Info("successfully deleted maestro resource bundle", "cluster_id", clusterID, "resource_bundle_id", rb.ID) + } + } + // Wait for Maestro agent to clean up K8s resources + // The agent watches ManifestWork deletions and removes applied resources + logger.Info("waiting for Maestro agent to clean up K8s resources", "cluster_id", clusterID) + + // Poll for up to 30 seconds to verify resources are being cleaned up + maxWait := 30 * time.Second + pollInterval := 2 * time.Second + startTime := time.Now() + + for time.Since(startTime) < maxWait { + // Check if any namespaces still exist + namespaces, err := h.K8sClient.FindNamespacesByPrefix(ctx, clusterID) + if err == nil && len(namespaces) == 0 { + logger.Info("all namespaces cleaned up by Maestro agent", "cluster_id", clusterID) + break + } + + // Check if resources are being deleted (pods/deployments going away) + // This is a good indicator that Maestro agent is working + allClean := true + for _, ns := range namespaces { + // Quick check: if namespace has minimal resources, it's likely clean + pods, _ := h.K8sClient.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) + deployments, _ := h.K8sClient.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{}) + jobs, _ := h.K8sClient.BatchV1().Jobs(ns).List(ctx, metav1.ListOptions{}) + + if (pods != nil && len(pods.Items) > 0) || + (deployments != nil && len(deployments.Items) > 0) || + (jobs != nil && len(jobs.Items) > 0) { + allClean = false + break + } + } + + if allClean { + logger.Info("resources cleaned up by Maestro agent", "cluster_id", clusterID) + break + } + + time.Sleep(pollInterval) + } + + logger.Info("finished waiting for Maestro agent cleanup", "cluster_id", clusterID, "elapsed", time.Since(startTime)) + } + } + + // Step 2: Find and delete all namespaces associated with this cluster + // Adapters create namespaces with pattern: {clusterId}-{adapterName}-namespace or just {clusterId} + logger.Info("finding all namespaces for cluster", "cluster_id", clusterID) + namespaces, err := h.K8sClient.FindNamespacesByPrefix(ctx, clusterID) + if err != nil { + logger.Error("failed to find namespaces for cluster", "cluster_id", clusterID, "error", err) + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to find namespaces: %w", err)) + } else { + logger.Info("found namespaces for cluster", "cluster_id", clusterID, "count", len(namespaces)) + for _, ns := range namespaces { + logger.Info("deleting namespace", "cluster_id", clusterID, "namespace", ns) + if err := h.K8sClient.DeleteNamespaceAndWait(ctx, ns); err != nil { + logger.Error("failed to delete namespace", "cluster_id", clusterID, "namespace", ns, "error", err) + cleanupErrors = append(cleanupErrors, fmt.Errorf("failed to delete namespace %s: %w", ns, err)) + } else { + logger.Info("successfully deleted namespace", "cluster_id", clusterID, "namespace", ns) + } + } + } + + // Return accumulated errors if any occurred + if len(cleanupErrors) > 0 { + logger.Error("cleanup completed with errors", "cluster_id", clusterID, "error_count", len(cleanupErrors)) + // Combine all errors into a single error message + var errMsg string + for i, err := range cleanupErrors { + if i > 0 { + errMsg += "; " + } + errMsg += err.Error() + } + return fmt.Errorf("cleanup errors: %s", errMsg) + } + + logger.Info("successfully cleaned up cluster resources", "cluster_id", clusterID) + return nil } // GetTestNodePool creates a nodepool on the specified cluster from a payload file func (h *Helper) GetTestNodePool(ctx context.Context, clusterID, payloadPath string) (*openapi.NodePool, error) { - return h.Client.CreateNodePoolFromPayload(ctx, clusterID, payloadPath) + return h.Client.CreateNodePoolFromPayload(ctx, clusterID, payloadPath) } // CleanupTestNodePool cleans up test nodepool func (h *Helper) CleanupTestNodePool(ctx context.Context, clusterID, nodepoolID string) error { - return h.Client.DeleteNodePool(ctx, clusterID, nodepoolID) + return h.Client.DeleteNodePool(ctx, clusterID, nodepoolID) } // GetMaestroClient returns the Maestro client, initializing it lazily on first access // This avoids the overhead of K8s service discovery for test suites that don't use Maestro func (h *Helper) GetMaestroClient() *maestro.Client { - if h.MaestroClient == nil { - h.MaestroClient = maestro.NewClient("") - } - return h.MaestroClient + if h.MaestroClient == nil { + h.MaestroClient = maestro.NewClient("") + } + return h.MaestroClient } diff --git a/test-design/testcases/adapter-failover.md b/test-design/testcases/adapter-failover.md index 1b683e8..1ec5710 100644 --- a/test-design/testcases/adapter-failover.md +++ b/test-design/testcases/adapter-failover.md @@ -3,7 +3,6 @@ ## Table of Contents 1. [Adapter can detect and report failures to cluster API endpoints](#test-title-adapter-can-detect-and-report-failures-to-cluster-api-endpoints) -2. [Adapter can detect and handle resource timeouts to cluster API endpoints](#test-title-adapter-can-detect-and-handle-resource-timeouts-to-cluster-api-endpoints) --- @@ -79,74 +78,3 @@ curl -X POST ${API_URL}/api/hyperfleet/v1/clusters \ ``` --- - - -## Test Title: Adapter can detect and handle resource timeouts to cluster API endpoints - -### Description - -This test validates that the adapter correctly detects and handles resource timeouts when adapter Jobs exceed configured timeout limits. - ---- - -| **Field** | **Value** | -|-----------|-----------| -| **Pos/Neg** | Negative | -| **Priority** | Tier2 | -| **Status** | Draft | -| **Automation** | Not Automated | -| **Version** | MVP | -| **Created** | 2026-01-30 | -| **Updated** | 2026-01-30 | - - ---- - -### Preconditions -1. Environment is prepared using [hyperfleet-infra](https://github.com/openshift-hyperfleet/hyperfleet-infra) with all required platform resources -2. HyperFleet API and HyperFleet Sentinel services are deployed and running successfully - ---- - -### Test Steps - -#### Step 1: Configure adapter with timeout setting -**Action:** -- Simulate a scenario where the adapter will be stuck -- Deploy the test adapter - -**Expected Result:** -- Adapter pods are running successfully - -#### Step 2: Send POST request to create a new cluster -**Action:** -- Execute cluster creation request: -```bash -curl -X POST ${API_URL}/api/hyperfleet/v1/clusters \ - -H "Content-Type: application/json" \ - -d -``` - -**Expected Result:** -- API returns successful response - -#### Step 3: Wait for timeout and Verify Timeout Handling -**Action:** -- Wait for some minutes -- Verify adapter status - -**Expected Result:** -```bash - curl -X GET ${API_URL}/api/hyperfleet/v1/clusters//statuses \ - | jq -r '.items[] | select(.adapter=="") | .conditions[] | select(.type=="Available")' - - # Example: - # { - # "type": "Available", - # "status": "False", - # "reason": "JobTimeout", - # "message": "Validation job did not complete within 30 seconds" - # } -``` - ---- diff --git a/test-design/testcases/adapter-with-maestro-transport.md b/test-design/testcases/adapter-with-maestro-transport.md index 8238774..82dbee2 100644 --- a/test-design/testcases/adapter-with-maestro-transport.md +++ b/test-design/testcases/adapter-with-maestro-transport.md @@ -6,7 +6,10 @@ 2. [Adapter can skip ManifestWork operation when generation is unchanged](#test-title-adapter-can-skip-manifestwork-operation-when-generation-is-unchanged) 3. [Adapter can route ManifestWork to correct consumer based on targetCluster](#test-title-adapter-can-route-manifestwork-to-correct-consumer-based-on-targetcluster) 4. [Adapter can handle Maestro server unavailability gracefully](#test-title-adapter-can-handle-maestro-server-unavailability-gracefully) -5. [Adapter can handle invalid targetCluster (consumer not found) gracefully](#test-title-adapter-can-handle-invalid-targetcluster-consumer-not-found-gracefully) +5. [ManifestWork apply fails when targeting unregistered consumer](#test-title-manifestwork-apply-fails-when-targeting-unregistered-consumer) +6. [Main discovery fails when ManifestWork name is wrong](#test-title-main-discovery-fails-when-manifestwork-name-is-wrong) +7. [Nested discovery returns empty when criteria match nothing in manifests](#test-title-nested-discovery-returns-empty-when-criteria-match-nothing-in-manifests) +8. [Post-action fails when status API is unreachable or returns error](#test-title-post-action-fails-when-status-api-is-unreachable-or-returns-error) --- @@ -542,7 +545,7 @@ This test validates the adapter's behavior when the Maestro server is unreachabl | **Field** | **Value** | |-----------|-----------| | **Pos/Neg** | Negative | -| **Priority** | Tier1 | +| **Priority** | Tier2 | | **Status** | Draft | | **Automation** | Not Automated | | **Version** | MVP | @@ -675,11 +678,11 @@ kubectl get pods -n maestro --no-headers --- -## Test Title: Adapter can handle invalid targetCluster (consumer not found) gracefully +## Test Title: ManifestWork apply fails when targeting unregistered consumer ### Description -This test validates the adapter's behavior when the configured `targetCluster` resolves to a Maestro consumer that does not exist. The adapter should detect the error, report it properly, and not crash. +This test validates the adapter's behavior when a cluster event targets a Maestro consumer that is not registered. The ManifestWork apply operation should fail with "not registered in Maestro" error, and the adapter should report appropriate failure status via the Health condition without crashing. --- @@ -688,7 +691,7 @@ This test validates the adapter's behavior when the configured `targetCluster` r | **Pos/Neg** | Negative | | **Priority** | Tier1 | | **Status** | Draft | -| **Automation** | Not Automated | +| **Automation** | Automated | | **Version** | MVP | | **Created** | 2026-02-12 | | **Updated** | 2026-02-26 | @@ -697,41 +700,86 @@ This test validates the adapter's behavior when the configured `targetCluster` r ### Preconditions -1. HyperFleet environment deployed with Maestro transport adapter -2. Maestro server is accessible -3. Adapter task config backup saved for restoration after test +1. HyperFleet API, Sentinel, and Maestro are deployed and running successfully +2. Adapter is deployed in Maestro transport mode (`transport.client: "maestro"`) +3. Adapter task config is configured to target a consumer named "unregistered-consumer" which does NOT exist in Maestro +4. At least one valid Maestro consumer exists for comparison (e.g., `cluster1`) +5. **Option 1**: Use the pre-configured adapter config: `testdata/adapter-configs/cl-m-unreg-consumer/` +6. **Option 2**: Temporarily modify an existing adapter's task config to point to "unregistered-consumer" --- ### Test Steps -#### Step 1: Backup and modify adapter task config to target a non-existent consumer +#### Step 1: Verify Maestro is healthy and "unregistered-consumer" does not exist +**Action:** +```bash +# Verify Maestro is running +kubectl get pods -n maestro -l app=maestro --no-headers + +# List all registered consumers to confirm "unregistered-consumer" is NOT present +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/consumers \ + | jq '.items[].name' +``` + +**Expected Result:** +- Maestro pod is `Running` +- "unregistered-consumer" does NOT appear in the consumer list +- Other consumers (e.g., "cluster1") exist for comparison + +#### Step 2: Deploy or verify adapter with unregistered consumer configuration **Action:** + +**Option A: Using pre-configured adapter (recommended)** +```bash +export ADAPTER_NAME='cl-m-unreg-consumer' + +# Deploy the adapter using the pre-configured adapter config + - name: "placementClusterName" + expression: "\"unregistered-consumer\"" # Points to non-existent consumer to test apply failure +# Use helm install cmd to deploy + helm install {release_name} {adapter_charts_folder} --namespace {namespace_name} --create-namespace -f testdata/adapter-configs/cl-m-unreg-consumer/values.yaml +``` + +**Option B: Modify existing adapter config** ```bash +export ADAPTER_NAME='test-adapter' # or your existing adapter name + # Backup original config kubectl get configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ - -o jsonpath='{.data.task-config\.yaml}' > /tmp/adapter2-task-original.yaml + -o jsonpath='{.data.task-config\.yaml}' > /tmp/adapter-task-backup.yaml -# Modify: change placementClusterName from "${MAESTRO_CONSUMER}" to "non-existent-cluster" -# In the task config, change: -# expression: "\"${MAESTRO_CONSUMER}\"" +# Modify task config: change placementClusterName to "unregistered-consumer" +# Edit the file to change: +# expression: "\"cluster1\"" # To: -# expression: "\"non-existent-cluster\"" +# expression: "\"unregistered-consumer\"" # Apply modified config kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ - --from-file=task-config.yaml=/tmp/adapter2-task-modified.yaml \ + --from-file=task-config.yaml=/tmp/adapter-task-modified.yaml \ --dry-run=client -o yaml | kubectl apply -f - -# Restart adapter +# Restart adapter to pick up new config kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s ``` **Expected Result:** -- Adapter restarts with `placementClusterName` = `"non-existent-cluster"` +- Adapter pod restarts successfully +- Adapter task config now targets "unregistered-consumer" -#### Step 2: Create a cluster to trigger adapter processing +#### Step 3: Verify adapter is running and ready +**Action:** +```bash +kubectl get pods -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --no-headers +``` + +**Expected Result:** +- Adapter pod is `Running` with `1/1 Ready` + +#### Step 4: Create a cluster to trigger adapter processing **Action:** ```bash CLUSTER_ID=$(curl -s -X POST ${API_URL}/api/hyperfleet/v1/clusters \ @@ -749,50 +797,785 @@ echo "CLUSTER_ID=${CLUSTER_ID}" **Expected Result:** - API returns HTTP 201 with a valid cluster ID +- Cluster has `generation: 1` -#### Step 3: Verify error handling for invalid consumer (check logs after ~15 seconds) +#### Step 5: Verify error status reported to HyperFleet API **Action:** ```bash -kubectl logs -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --tail=30 \ - | grep -E "FAILED|error|non-existent" | head -5 +curl -s ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID}/statuses \ + | jq '.items[] | select(.adapter == "'"${ADAPTER_NAME}"'")' +``` + +**Expected Result:** +- Adapter status entry exists with `adapter: "${ADAPTER_NAME}"` +- `observed_generation: 1` (adapter processed the event) +- `last_report_time` is present and recent +- **Condition validation**: + - `Applied: False` - ManifestWork was not created (consumer not registered) + - `Available: False` - Resources not available (ManifestWork not applied) + - `Health: False` - Adapter execution failed at ResourceFailed phase + - Health reason: `ExecutionFailed:ResourceFailed` + - Health message contains: "consumer \"xxxxxx\" is not registered in Maestro" + + +#### Step 6: Verify no ManifestWork was created on Maestro +**Action:** +```bash +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles \ + | jq '.items[] | select(.metadata.labels["hyperfleet.io/cluster-id"] == "'"${CLUSTER_ID}"'")' ``` **Expected Result:** -- Adapter logs show error related to consumer not found -- Error message includes the invalid consumer name -- Adapter does NOT crash +- No resource bundle (ManifestWork) exists for the cluster ID +- Query returns empty result or null +- This confirms the apply operation failed before creating the ManifestWork -#### Step 4: Verify adapter pod is still running (no crash) +#### Step 7: Verify no Kubernetes resources were created +**Action:** +```bash +# Attempt to find namespace that would have been created +kubectl get ns | grep ${CLUSTER_ID} +``` + +**Expected Result:** +- No namespace exists with the cluster ID +- This confirms that Maestro agent did not apply any resources (because ManifestWork was never created) + +#### Step 8: Cleanup +**Action:** + +**If using Option A (pre-configured adapter):** +```bash +# Delete the test adapter deployment +helm uninstall {release_name} -n {namespace} + +# Note: Cluster will remain in API until DELETE endpoint is available +``` + +**If using Option B (modified existing adapter):** +```bash +# Restore original adapter config +kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + --from-file=task-config.yaml=/tmp/adapter-task-backup.yaml \ + --dry-run=client -o yaml | kubectl apply -f - + +# Restart adapter with restored config +kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet +kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s + +echo "Adapter config restored successfully" +``` + +> **Note:** Once the HyperFleet API supports DELETE operations for clusters, this step should be added with: +> ```bash +> curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID} +> ``` + +--- + +## Test Title: Main discovery fails when ManifestWork name is wrong + +### Description + +This test validates the adapter's behavior when the main discovery configuration uses the wrong ManifestWork name. The adapter creates a ManifestWork on Maestro with the correct name, but then tries to discover it using a wrong name (with `-wrong` suffix). This simulates a misconfiguration where the discovery name doesn't match the created resource name. The adapter should fail at the discovery phase and report the error appropriately. + +--- + +| **Field** | **Value** | +|-----------|-----------| +| **Pos/Neg** | Negative | +| **Priority** | Tier1 | +| **Status** | Draft | +| **Automation** | Automated | +| **Version** | MVP | +| **Created** | 2026-03-20 | +| **Updated** | 2026-03-20 | + +--- + +### Preconditions + +1. HyperFleet API, Sentinel, and Maestro are deployed and running successfully +2. At least one Maestro consumer is registered (e.g., `cluster1`) +3. Adapter is deployed in Maestro transport mode +4. Adapter task config has discovery names that DO NOT match the actual resource names created +5. **Option 1**: Use the pre-configured adapter config: `testdata/adapter-configs/cl-m-wrong-ds/` +6. **Option 2**: Temporarily modify an existing adapter's task config discovery names to be incorrect + +--- + +### Test Steps + +#### Step 1: Verify Maestro is healthy and consumer is registered +**Action:** +```bash +# Verify Maestro is running +kubectl get pods -n maestro -l app=maestro --no-headers + +# Verify consumer exists +export MAESTRO_CONSUMER='cluster1' # or your registered consumer +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/consumers \ + | jq -r '.items[] | select(.name == "'"${MAESTRO_CONSUMER}"'") | .name' +``` + +**Expected Result:** +- Maestro pod is `Running` +- Consumer `${MAESTRO_CONSUMER}` exists + +#### Step 2: Deploy or verify adapter with wrong discovery configuration +**Action:** + +**Option A: Using pre-configured adapter (recommended)** +```bash +export ADAPTER_NAME='cl-m-wrong-ds' + +# Deploy the test adapter deployment + helm install {release_name} {adapter_charts_folder} --namespace {namespace_name} --create-namespace -f testdata/adapter-configs/cl-m-wrong-ds/values.yaml + +OR + +# Deploy the adapter using the pre-configured adapter config supported in hyperfleet-infra +# The config has discovery names with "-wrong" suffix that don't match actual resources +make install-adapter-custom ADAPTER_CONFIG_PATH=testdata/adapter-configs/cl-m-wrong-ds +``` + +**Option B: Modify existing adapter config** +```bash +export ADAPTER_NAME='cl-maestro' # or your existing adapter name + +# Backup original config +kubectl get configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + -o jsonpath='{.data.task-config\.yaml}' > /tmp/adapter-task-backup.yaml + +# Modify task config nested_discoveries section: +# Change: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" +# To: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace-wrong" +# And: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" +# To: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap-wrong" + +# Apply modified config +kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + --from-file=task-config.yaml=/tmp/adapter-task-modified.yaml \ + --dry-run=client -o yaml | kubectl apply -f - + +# Restart adapter to pick up new config +kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet +kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s +``` + +**Expected Result:** +- Adapter pod restarts successfully +- Adapter task config now has wrong discovery names (with "-wrong" suffix) + +#### Step 3: Verify adapter is running **Action:** ```bash kubectl get pods -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --no-headers ``` **Expected Result:** -- Pod is `Running` with 0 restarts +- Adapter pod is `Running` with `1/1 Ready` + +#### Step 4: Create a cluster to trigger adapter processing +**Action:** +```bash +export API_URL='http://localhost:8000' # Adjust if different + +CLUSTER_ID=$(curl -s -X POST ${API_URL}/api/hyperfleet/v1/clusters \ + -H "Content-Type: application/json" \ + -d '{ + "kind": "Cluster", + "name": "maestro-discovery-fail-'$(date +%Y%m%d-%H%M%S)'", + "spec": { + "platform": { + "type": "gcp", + "gcp": {"projectID": "test-project", "region": "us-central1"} + }, + "release": {"version": "4.14.0"} + } + }' | jq -r '.id') +echo "CLUSTER_ID=${CLUSTER_ID}" +``` + +**Expected Result:** +- API returns HTTP 201 with a valid cluster ID +- Cluster has `generation: 1` + +#### Step 5: Verify ManifestWork was created successfully on Maestro +**Action:** +```bash +# Wait for ManifestWork creation +sleep 10 + +# Capture resource bundle ID +RESOURCE_BUNDLE_ID=$(kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles \ + | jq -r --arg cid "${CLUSTER_ID}" \ + '.items[] | select(.metadata.labels["hyperfleet.io/cluster-id"] == $cid) | .id') +echo "RESOURCE_BUNDLE_ID=${RESOURCE_BUNDLE_ID}" + +# Display resource bundle details +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} \ + | jq '{id: .id, consumer_name: .consumer_name, version: .version, + manifest_names: [.manifests[].metadata.name]}' +``` + +**Expected Result:** +- ManifestWork (resource bundle) was created successfully +- Resource bundle has correct consumer name (e.g., `cluster1`) +- Manifests include namespace and configmap with correct actual names: + - `${CLUSTER_ID}-${ADAPTER_NAME}-namespace` + - `${CLUSTER_ID}-${ADAPTER_NAME}-configmap` -#### Step 5: Verify error status reported to API +#### Step 6: Verify Kubernetes resources were created by Maestro agent +**Action:** +```bash +# Wait for Maestro agent to apply resources +sleep 15 + +# Verify namespace exists +kubectl get ns | grep ${CLUSTER_ID}-${ADAPTER_NAME} + +# Verify configmap exists +kubectl get configmap -n ${CLUSTER_ID}-${ADAPTER_NAME}-namespace | grep ${CLUSTER_ID}-${ADAPTER_NAME} +``` + +**Expected Result:** +- Namespace `${CLUSTER_ID}-${ADAPTER_NAME}-namespace` exists and is `Active` +- ConfigMap `${CLUSTER_ID}-${ADAPTER_NAME}-configmap` exists in the namespace +- Resources were successfully applied by Maestro agent + +#### Step 7: Verify error status reported to HyperFleet API **Action:** ```bash curl -s ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID}/statuses \ - | jq '.items[] | select(.adapter == "'"${ADAPTER_NAME}"'") | .conditions' + | jq '.items[] | select(.adapter == "'"${ADAPTER_NAME}"'")' +``` + +**Expected Result:** +- Adapter status entry exists with `adapter: "${ADAPTER_NAME}"` +- `observed_generation: 1` (adapter processed the event) +- `last_report_time` is present and recent +- **Condition validation**: + - `Applied: False` - ManifestWork not discovered (main discovery failed) + - Reason: `ManifestWorkNotDiscovered` + - `Available: False` - Resources not available (ManifestWork not found) + - Reason: `NamespaceNotDiscovered` + - `Health: False` - Adapter execution failed + - Reason: `ExecutionFailed:ResourceFailed` + - Message contains: "failed to discover resource after apply: manifestworks...not found" +- **Data validation**: + - `data.manifestwork.name` is empty (main discovery failed) + - `data.namespace.name` is empty (cannot discover nested resources) + - `data.configmap.name` is empty (cannot discover nested resources) + +#### Step 8: Verify ManifestWork was created but cannot be discovered +**Action:** +```bash +# Search for ManifestWork with correct name (without -wrong suffix) +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles \ + | jq '.items[] | select(.metadata.labels["hyperfleet.io/cluster-id"] == "'"${CLUSTER_ID}"'")' + +# Try to find ManifestWork with wrong name (what adapter is looking for) +kubectl exec -n maestro deployment/maestro -- \ + curl -s "http://localhost:8000/api/maestro/v1/resource-bundles/${CLUSTER_ID}-${ADAPTER_NAME}-wrong" ``` **Expected Result:** -- Health: `status: "False"`, error message should contain key points like `non-existent-cluster` or `consumer` not found -- Applied: `status: "False"` +- ManifestWork with correct name `${CLUSTER_ID}-${ADAPTER_NAME}` exists on Maestro +- ManifestWork with wrong name `${CLUSTER_ID}-${ADAPTER_NAME}-wrong` does NOT exist (404) +- Adapter created the ManifestWork correctly but cannot discover it due to wrong discovery name +- K8s resources (namespace, configmap) were created by Maestro agent -#### Step 6: Restore and cleanup +#### Step 9: Cleanup **Action:** + +**Common cleanup steps:** ```bash -# Restore original config +# Delete the resource bundle on Maestro (triggers agent to clean up K8s resources) +kubectl exec -n maestro deployment/maestro -- \ + curl -s -X DELETE http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} + +# Delete namespace as safety cleanup +kubectl delete ns ${CLUSTER_ID}-${ADAPTER_NAME}-namespace --ignore-not-found + +# Wait for cleanup +sleep 5 +``` +> **Note:** Once the HyperFleet API supports DELETE operations for clusters, it can be replaced via this cleanup step: +> ```bash +> curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID} +> ``` + +**If using Option A (pre-configured adapter):** +```bash +# Delete the test adapter deployment +helm uninstall hyperfleet-${ADAPTER_NAME} -n hyperfleet + +# Or using make target supported in hyperfleet-infra +make uninstall-adapter ADAPTER_NAME=cl-maestro-wrong-discovery +``` + +**If using Option B (modified existing adapter):** +```bash +# Restore original adapter config kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ - --from-file=task-config.yaml=/tmp/adapter2-task-original.yaml \ + --from-file=task-config.yaml=/tmp/adapter-task-backup.yaml \ --dry-run=client -o yaml | kubectl apply -f - +# Restart adapter with restored config kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s + +echo "Adapter config restored successfully" +``` +--- + +## Test Title: Nested discovery returns empty when criteria match nothing in manifests + +### Description + +This test validates the adapter's behavior when a ManifestWork is successfully created and discovered, but the nested discovery criteria match nothing in the `spec.workload.manifests` array. The ManifestWork apply and primary discovery succeed, but nested discovery returns empty results. This is not a hard failure - it's logged as debug information, and CEL expressions using `orValue("")` fallbacks handle the missing data gracefully. The adapter reports status with conditions showing pending/unknown state due to unavailable nested resource data. + +--- + +| **Field** | **Value** | +|-----------|-----------| +| **Pos/Neg** | Negative | +| **Priority** | Tier1 | +| **Status** | Draft | +| **Automation** | Automated | +| **Version** | MVP | +| **Created** | 2026-03-20 | +| **Updated** | 2026-03-20 | + +--- + +### Preconditions + +1. HyperFleet API, Sentinel, and Maestro are deployed and running successfully +2. At least one Maestro consumer is registered (e.g., `cluster1`) +3. Adapter is deployed in Maestro transport mode +4. Adapter task config has nested discovery criteria that look for resources NOT present in the ManifestWork manifests +5. **Option 1**: Use the pre-configured adapter config: `testdata/adapter-configs/cl-m-wrong-nest/` +6. **Option 2**: Temporarily modify an existing adapter's task config to have mismatched nested discovery criteria + +--- + +### Test Steps + +#### Step 1: Verify Maestro is healthy and consumer is registered +**Action:** +```bash +# Verify Maestro is running +kubectl get pods -n maestro -l app=maestro --no-headers + +# Verify consumer exists +export MAESTRO_CONSUMER='cluster1' # or your registered consumer +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/consumers \ + | jq -r '.items[] | select(.name == "'"${MAESTRO_CONSUMER}"'") | .name' ``` -> **Important:** Always restore the adapter config after this test to avoid impacting other tests. +**Expected Result:** +- Maestro pod is `Running` +- Consumer `${MAESTRO_CONSUMER}` exists + +#### Step 2: Deploy or verify adapter with empty nested discovery configuration +**Action:** + +**Option A: Using pre-configured adapter (recommended)** +```bash +export ADAPTER_NAME='cl-m-wrong-nest' + +# Deploy the test adapter deployment +helm install {release_name} {adapter_charts_folder} --namespace {namespace_name} --create-namespace -f testdata/adapter-configs/cl-m-wrong-nest/values.yaml + +# OR using make target supported in hyperfleet-infra +make install-adapter-custom ADAPTER_CONFIG_PATH=testdata/adapter-configs/cl-m-wrong-nest +``` + +**Option B: Modify existing adapter config** +```bash +export ADAPTER_NAME='cl-maestro' # or your existing adapter name + +# Backup original config +kubectl get configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + -o jsonpath='{.data.task-config\.yaml}' > /tmp/adapter-task-backup.yaml + +# Modify task config nested_discoveries section to look for non-existent resources: +# Change: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" +# To: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-deployment" +# And: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" +# To: +# by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-service" + +# Apply modified config +kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + --from-file=task-config.yaml=/tmp/adapter-task-modified.yaml \ + --dry-run=client -o yaml | kubectl apply -f - + +# Restart adapter to pick up new config +kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet +kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s +``` + +**Expected Result:** +- Adapter pod restarts successfully +- Adapter task config now has nested discovery criteria that won't match any manifests + +#### Step 3: Verify adapter is running +**Action:** +```bash +kubectl get pods -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --no-headers +``` + +**Expected Result:** +- Adapter pod is `Running` with `1/1 Ready` + +#### Step 4: Create a cluster to trigger adapter processing +**Action:** +```bash +export API_URL='http://localhost:8000' # Adjust if different + +CLUSTER_ID=$(curl -s -X POST ${API_URL}/api/hyperfleet/v1/clusters \ + -H "Content-Type: application/json" \ + -d '{ + "kind": "Cluster", + "name": "maestro-empty-discovery-'$(date +%Y%m%d-%H%M%S)'", + "spec": { + "platform": { + "type": "gcp", + "gcp": {"projectID": "test-project", "region": "us-central1"} + }, + "release": {"version": "4.14.0"} + } + }' | jq -r '.id') +echo "CLUSTER_ID=${CLUSTER_ID}" +``` + +**Expected Result:** +- API returns HTTP 201 with a valid cluster ID +- Cluster has `generation: 1` + +#### Step 5: Verify ManifestWork was created successfully on Maestro +**Action:** +```bash +# Wait for ManifestWork creation +sleep 10 + +# Capture resource bundle ID +RESOURCE_BUNDLE_ID=$(kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles \ + | jq -r --arg cid "${CLUSTER_ID}" \ + '.items[] | select(.metadata.labels["hyperfleet.io/cluster-id"] == $cid) | .id') +echo "RESOURCE_BUNDLE_ID=${RESOURCE_BUNDLE_ID}" + +# Display resource bundle details +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} \ + | jq '{id: .id, consumer_name: .consumer_name, version: .version, + manifest_names: [.manifests[].metadata.name]}' +``` + +**Expected Result:** +- ManifestWork (resource bundle) was created successfully +- Resource bundle has correct consumer name (e.g., `cluster1`) +- Manifests include the actual resources (namespace and configmap): + - `${CLUSTER_ID}-${ADAPTER_NAME}-namespace` + - `${CLUSTER_ID}-${ADAPTER_NAME}-configmap` +- Note: Nested discovery is looking for deployment and service which don't exist + +#### Step 6: Verify Kubernetes resources were created by Maestro agent +**Action:** +```bash +# Wait for Maestro agent to apply resources +sleep 15 + +# Verify namespace exists +kubectl get ns | grep ${CLUSTER_ID}-${ADAPTER_NAME} + +# Verify configmap exists +kubectl get configmap -n ${CLUSTER_ID}-${ADAPTER_NAME}-namespace | grep ${CLUSTER_ID}-${ADAPTER_NAME} +``` + +**Expected Result:** +- Namespace `${CLUSTER_ID}-${ADAPTER_NAME}-namespace` exists and is `Active` +- ConfigMap `${CLUSTER_ID}-${ADAPTER_NAME}-configmap` exists in the namespace +- Resources were successfully applied by Maestro agent + +#### Step 7: Verify status reported with pending/unknown conditions +**Action:** +```bash +curl -s ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID}/statuses \ + | jq '.items[] | select(.adapter == "'"${ADAPTER_NAME}"'")' +``` + +**Expected Result:** +- Adapter status entry exists with `adapter: "${ADAPTER_NAME}"` +- `observed_generation: 1` (adapter processed the event) +- `last_report_time` is present and recent +- **Condition validation**: + - `Applied: True` with `reason: "AppliedManifestWorkComplete"` - ManifestWork was created successfully + - `Available: False` with `reason: "NamespaceNotDiscovered"` - Nested resources not discovered + - `Health: True` with `reason: "Healthy"` - Adapter executed successfully (nested discovery failure doesn't affect health) +- **Data field validation**: + - `manifestwork.name`: `"${CLUSTER_ID}-${ADAPTER_NAME}"` (main discovery succeeded) + - `namespace.name`: `""` (empty - nested discovery failed) + - `namespace.phase`: `"Unknown"` (nested discovery failed) + - `configmap.name`: `""` (empty - nested discovery failed) + + +#### Step 8: Verify ManifestWork and actual resources exist +**Action:** +```bash +# Verify ManifestWork exists on Maestro +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} \ + | jq '{id: .id, version: .version, manifests: [.manifests[].metadata.name]}' + +# Verify actual K8s resources exist (namespace and configmap) +kubectl get ns ${CLUSTER_ID}-${ADAPTER_NAME}-namespace +kubectl get configmap ${CLUSTER_ID}-${ADAPTER_NAME}-configmap \ + -n ${CLUSTER_ID}-${ADAPTER_NAME}-namespace +``` + +**Expected Result:** +- ManifestWork exists with correct manifests (namespace and configmap) +- Kubernetes namespace and configmap exist and are functional +- Nested discovery failure doesn't affect the actual resources + +#### Step 9: Cleanup +**Action:** + +**Common cleanup steps:** +```bash +# Delete the resource bundle on Maestro (triggers agent to clean up K8s resources) +kubectl exec -n maestro deployment/maestro -- \ + curl -s -X DELETE http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} + +# Delete namespace as safety cleanup +kubectl delete ns ${CLUSTER_ID}-${ADAPTER_NAME}-namespace --ignore-not-found + +# Wait for cleanup +sleep 5 +``` + +> **Note:** Once the HyperFleet API supports DELETE operations for clusters, it can be replaced via this cleanup step: +> ```bash +> curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID} +> ``` + +**If using Option A (pre-configured adapter):** +```bash +# Delete the test adapter deployment +helm uninstall hyperfleet-${ADAPTER_NAME} -n hyperfleet + +# Or using make target supported in hyperfleet-infra +make uninstall-adapter ADAPTER_NAME=cl-m-wrong-nest +``` + +**If using Option B (modified existing adapter):** +```bash +# Restore original adapter config +kubectl create configmap hyperfleet-${ADAPTER_NAME}-task -n hyperfleet \ + --from-file=task-config.yaml=/tmp/adapter-task-backup.yaml \ + --dry-run=client -o yaml | kubectl apply -f - + +# Restart adapter with restored config +kubectl rollout restart deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet +kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s + +echo "Adapter config restored successfully" +``` + +--- + +## Test Title: Post-action fails when status API is unreachable or returns error + +### Description + +This test validates the adapter's behavior when ManifestWork creation and discovery succeed, but the POST to `/clusters/{clusterId}/statuses` endpoint fails (returns 500 error or is unreachable). The adapter should handle the API failure gracefully, record the post-action failure in execution metadata, log the error appropriately, and continue running without crashing. + +--- + +| **Field** | **Value** | +|-----------|-----------| +| **Pos/Neg** | Negative | +| **Priority** | Tier1 | +| **Status** | Draft | +| **Automation** | Automated | +| **Version** | MVP | +| **Created** | 2026-03-20 | +| **Updated** | 2026-03-20 | + +--- + +### Preconditions + +1. HyperFleet API, Sentinel, and Maestro are deployed and running successfully +2. At least one Maestro consumer is registered (e.g., `cluster1`) +3. Pre-configured test adapter available: `testdata/adapter-configs/cl-m-bad-api/` + - This adapter has an invalid API URL configured to simulate unreachable API + - Clean approach that doesn't affect test environment or existing adapters + +--- + +### Test Steps + +#### Step 1: Deploy test adapter with invalid API URL +**Action:** +```bash +export ADAPTER_NAME='cl-m-bad-api' + +# Deploy the test adapter with pre-configured invalid API URL +# This adapter will successfully connect to Maestro but fail when POSTing to status API +helm install hyperfleet-${ADAPTER_NAME} {adapter_charts_folder} \ + --namespace hyperfleet \ + --create-namespace \ + -f testdata/adapter-configs/cl-m-bad-api/values.yaml + +# OR using make target supported in hyperfleet-infra +make install-adapter-custom ADAPTER_CONFIG_PATH=testdata/adapter-configs/cl-m-bad-api + +# Wait for adapter to be ready +kubectl rollout status deployment/hyperfleet-${ADAPTER_NAME} -n hyperfleet --timeout=60s + +# Verify adapter is running +kubectl get pods -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --no-headers +``` + +**Expected Result:** +- Test adapter pod is `Running` with `1/1 Ready` +- Adapter is configured with invalid API URL: `http://invalid-hyperfleet-api-endpoint.local:9999` + +#### Step 2: Create a cluster to trigger adapter processing +**Action:** +```bash +export API_URL='http://localhost:8000' # Adjust if different + +CLUSTER_ID=$(curl -s -X POST ${API_URL}/api/hyperfleet/v1/clusters \ + -H "Content-Type: application/json" \ + -d '{ + "kind": "Cluster", + "name": "maestro-api-fail-test-'$(date +%Y%m%d-%H%M%S)'", + "spec": { + "platform": { + "type": "gcp", + "gcp": {"projectID": "test-project", "region": "us-central1"} + }, + "release": {"version": "4.14.0"} + } + }' | jq -r '.id') +echo "CLUSTER_ID=${CLUSTER_ID}" +``` + +**Expected Result:** +- API returns HTTP 201 with a valid cluster ID +- Cluster has `generation: 1` + +#### Step 3: Verify ManifestWork was created successfully despite API failure +**Action:** +```bash +# Capture resource bundle ID +RESOURCE_BUNDLE_ID=$(kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles \ + | jq -r --arg cid "${CLUSTER_ID}" \ + '.items[] | select(.metadata.labels["hyperfleet.io/cluster-id"] == $cid) | .id') +echo "RESOURCE_BUNDLE_ID=${RESOURCE_BUNDLE_ID}" + +# Display resource bundle details +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} \ + | jq '{id: .id, consumer_name: .consumer_name, version: .version, + manifest_names: [.manifests[].metadata.name]}' +``` + +**Expected Result:** +- ManifestWork (resource bundle) was created successfully on Maestro + +#### Step 4: Verify Kubernetes resources were created by Maestro agent +**Action:** +```bash +# Wait for Maestro agent to apply resources +sleep 15 + +# Verify namespace exists +kubectl get ns | grep ${CLUSTER_ID}-${ADAPTER_NAME} + +# Verify configmap exists +kubectl get configmap -n ${CLUSTER_ID}-${ADAPTER_NAME}-namespace | grep ${CLUSTER_ID}-${ADAPTER_NAME} +``` + +**Expected Result:** +- Namespace `${CLUSTER_ID}-${ADAPTER_NAME}-namespace` exists and is `Active` +- ConfigMap `${CLUSTER_ID}-${ADAPTER_NAME}-configmap` exists +- Resources were successfully applied despite post-action failure + +#### Step 5: Verify post-action failure via indirect evidence (beyond logs) +**Action:** +```bash +# Method: Check ManifestWork status in Maestro (should be healthy) +kubectl exec -n maestro deployment/maestro -- \ + curl -s http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} \ + | jq '{ + id: .id, + status: .status, + conditions: [.status.conditions[] | {type: .type, status: .status}] + }' +``` +**Expected Result:** +- **ManifestWork in Maestro:** + - Status shows `Applied` and `Available` conditions are `True` (from Maestro agent's perspective) + - This confirms apply and discovery phases succeeded + +#### Step 6: Verify no status was reported to API (expected behavior) +**Action:** +```bash +# Since the adapter has invalid API URL, status should NOT be in the API +curl -s ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID}/statuses \ + | jq '.items[] | select(.adapter == "'"${ADAPTER_NAME}"'")' +``` + +**Expected Result:** +- No status entry for this adapter (empty result) +- This confirms that POST to /statuses failed as expected + +#### Step 9: Cleanup +**Action:** +```bash +# Delete the resource bundle on Maestro +kubectl exec -n maestro deployment/maestro -- \ + curl -s -X DELETE http://localhost:8000/api/maestro/v1/resource-bundles/${RESOURCE_BUNDLE_ID} + +# Delete namespace +kubectl delete ns ${CLUSTER_ID}-${ADAPTER_NAME}-namespace --ignore-not-found + +> **Note:** Once the HyperFleet API supports DELETE operations for clusters, add this cleanup step: +> ```bash +> curl -X DELETE ${API_URL}/api/hyperfleet/v1/clusters/${CLUSTER_ID} +> ``` + +# Delete the test adapter deployment +helm uninstall hyperfleet-${ADAPTER_NAME} -n hyperfleet + +# OR using make target supported in hyperfleet-infra +make uninstall-adapter ADAPTER_NAME=cl-m-bad-api + +# Verify adapter is deleted +kubectl get pods -n hyperfleet -l app.kubernetes.io/instance=hyperfleet-${ADAPTER_NAME} --no-headers +``` + +--- diff --git a/testdata/adapter-configs/cl-m-bad-api/adapter-config.yaml b/testdata/adapter-configs/cl-m-bad-api/adapter-config.yaml new file mode 100644 index 0000000..24d94aa --- /dev/null +++ b/testdata/adapter-configs/cl-m-bad-api/adapter-config.yaml @@ -0,0 +1,65 @@ +# Example HyperFleet Adapter deployment configuration +# This configuration is for testing post-action failure when API is unreachable +# The API URL will be overridden at deployment time to test status reporting failure +adapter: + name: cl-m-bad-api + #version: "0.1.0" + +# Log the full merged configuration after load (default: false) +debug_config: true +log: + level: debug + +clients: + hyperfleet_api: + base_url: http://hyperfleet-api:8000 + version: v1 + timeout: 2s + retry_attempts: 3 + retry_backoff: exponential + + broker: + # These values are overridden at deploy time via env vars from Helm values + subscription_id: CHANGE_ME + topic: CHANGE_ME + + maestro: + grpc_server_address: "maestro-grpc.maestro.svc.cluster.local:8090" + + # HTTPS server address for REST API operations (optional) + # Environment variable: HYPERFLEET_MAESTRO_HTTP_SERVER_ADDRESS + http_server_address: "http://maestro.maestro.svc.cluster.local:8000" + + # Source identifier for CloudEvents routing (must be unique across adapters) + # Environment variable: HYPERFLEET_MAESTRO_SOURCE_ID + source_id: "cl-m-bad-api" + + # Client identifier (defaults to source_id if not specified) + # Environment variable: HYPERFLEET_MAESTRO_CLIENT_ID + client_id: "cl-m-bad-api-client" + insecure: true + + # Authentication configuration + #auth: + # type: "tls" # TLS certificate-based mTLS + # + # tls_config: + # # gRPC TLS configuration + # # Certificate paths (mounted from Kubernetes secrets) + # # Environment variable: HYPERFLEET_MAESTRO_CA_FILE + # ca_file: "/etc/maestro/certs/grpc/ca.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_CERT_FILE + # cert_file: "/etc/maestro/certs/grpc/client.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_KEY_FILE + # key_file: "/etc/maestro/certs/grpc/client.key" + # + # # Server name for TLS verification + # # Environment variable: HYPERFLEET_MAESTRO_SERVER_NAME + # server_name: "maestro-grpc.maestro.svc.cluster.local" + # + # # HTTP API TLS configuration (may use different CA than gRPC) + # # If not set, falls back to ca_file for backwards compatibility + # # Environment variable: HYPERFLEET_MAESTRO_HTTP_CA_FILE + # http_ca_file: "/etc/maestro/certs/https/ca.crt" diff --git a/testdata/adapter-configs/cl-m-bad-api/adapter-task-config.yaml b/testdata/adapter-configs/cl-m-bad-api/adapter-task-config.yaml new file mode 100644 index 0000000..4ee3d21 --- /dev/null +++ b/testdata/adapter-configs/cl-m-bad-api/adapter-task-config.yaml @@ -0,0 +1,311 @@ +# Example HyperFleet Adapter task configuration + +# Parameters with all required variables +params: + + - name: "clusterId" + source: "event.id" + type: "string" + required: true + + - name: "generation" + source: "event.generation" + type: "int" + required: true + + - name: "namespace" + source: "env.NAMESPACE" + type: "string" + + +# Preconditions without API calls (API URL is unreachable in this test) +# Use only parameter-based values to avoid API dependency +preconditions: + - name: "clusterReady" + # Simple expression that always passes + expression: "true" + +# Resources with valid K8s manifests +resources: + - name: "resource0" + transport: + client: "maestro" + maestro: + target_cluster: "cluster1" + + # ManifestWork is a kind of manifest that can be used to create resources on the cluster. + # It is a collection of resources that are created together. + manifest: + apiVersion: work.open-cluster-management.io/v1 + kind: ManifestWork + metadata: + # ManifestWork name - must be unique within consumer namespace + name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Labels for identification, filtering, and management + labels: + # HyperFleet tracking labels + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/adapter: "{{ .adapter.name }}" + hyperfleet.io/component: "infrastructure" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/resource-group: "cluster-setup" + + # Maestro-specific labels + maestro.io/source-id: "{{ .adapter.name }}" + maestro.io/resource-type: "manifestwork" + maestro.io/priority: "normal" + + # Standard Kubernetes application labels + app.kubernetes.io/name: "aro-hcp-cluster" + app.kubernetes.io/instance: "{{ .clusterId }}" + app.kubernetes.io/version: "v1.0.0" + app.kubernetes.io/component: "infrastructure" + app.kubernetes.io/part-of: "hyperfleet" + app.kubernetes.io/managed-by: "cl-maestro" + app.kubernetes.io/created-by: "{{ .adapter.name }}" + + # Annotations for metadata and operational information + annotations: + # Tracking and lifecycle + hyperfleet.io/created-by: "cl-maestro-framework" + hyperfleet.io/managed-by: "{{ .adapter.name }}" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/cluster-name: "{{ .clusterId }}" + hyperfleet.io/deployment-time: "2024-01-01T00:00:00Z" + + # Maestro-specific annotations + maestro.io/applied-time: "2024-01-01T00:00:00Z" + maestro.io/source-adapter: "{{ .adapter.name }}" + + # Documentation + description: "Complete cluster setup including namespace, configuration, and RBAC" + + # ManifestWork specification + spec: + # ============================================================================ + # Workload - Contains the Kubernetes manifests to deploy + # ============================================================================ + workload: + # Kubernetes manifests array - injected by framework from business logic config + manifests: + - apiVersion: v1 + kind: Namespace + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + - apiVersion: v1 + kind: ConfigMap + data: + cluster_id: "{{ .clusterId }}" + cluster_name: "{{ .clusterId }}" + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/version: 1.0.0 + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + + # ============================================================================ + # Delete Options - How resources should be removed + # ============================================================================ + deleteOption: + # Propagation policy for resource deletion + # - "Foreground": Wait for dependent resources to be deleted first + # - "Background": Delete immediately, let cluster handle dependents + # - "Orphan": Leave resources on cluster when ManifestWork is deleted + propagationPolicy: "Foreground" + + # Grace period for graceful deletion (seconds) + gracePeriodSeconds: 30 + + # ============================================================================ + # Manifest Configurations - Per-resource settings for update and feedback + # ============================================================================ + manifestConfigs: + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "namespaces" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" # Specific resource name + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "phase" + path: ".status.phase" + # ======================================================================== + # Configuration for Namespace resources + # ======================================================================== + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "configmaps" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" # Specific resource name + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + serverSideApply: + fieldManager: "cl-maestro" # Field manager name for conflict resolution + force: false # Don't force conflicts (fail on conflicts) + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "data" + path: ".data" + - name: "resourceVersion" + path: ".metadata.resourceVersion" + # Discover the ResourceBundle (ManifestWork) by name from Maestro + discovery: + by_name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Discover nested resources deployed by the ManifestWork + nested_discoveries: + - name: "namespace0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + - name: "configmap0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + +post: + payloads: + - name: "statusPayload" + build: + adapter: "{{ .adapter.name }}" + conditions: + # Applied: Check if ManifestWork exists and has type="Applied", status="True" + - type: "Applied" + status: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].status : "False" + reason: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].reason : "ManifestWorkNotDiscovered" + message: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].message : "ManifestWork not discovered from Maestro or no Applied condition" + + # Available: Check if nested discovered manifests are available on the spoke cluster + # Each nested discovery is enriched with top-level "conditions" from status.resourceStatus.manifests[] + - type: "Available" + status: + expression: | + has(resources.namespace0) && has(resources.namespace0.conditions) + && resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + && has(resources.configmap0) && has(resources.configmap0.conditions) + && resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "True" + : "False" + reason: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "NamespaceNotDiscovered" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "NamespaceNotAvailable" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMapNotDiscovered" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMapNotAvailable" + : "AllResourcesAvailable" + message: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "Namespace not discovered from ManifestWork" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "Namespace not yet available on spoke cluster" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMap not discovered from ManifestWork" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMap not yet available on spoke cluster" + : "All manifests (namespace, configmap) are available on spoke cluster" + + # Health: Adapter execution status — surfaces errors from any phase + - type: "Health" + status: + expression: | + adapter.?executionStatus.orValue("") == "success" + && !adapter.?resourcesSkipped.orValue(false) + ? "True" + : "False" + reason: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "ExecutionFailed:" + adapter.?executionError.?phase.orValue("unknown") + : adapter.?resourcesSkipped.orValue(false) + ? "ResourcesSkipped" + : "Healthy" + message: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "Adapter failed at phase [" + + adapter.?executionError.?phase.orValue("unknown") + + "] step [" + + adapter.?executionError.?step.orValue("unknown") + + "]: " + + adapter.?executionError.?message.orValue(adapter.?errorMessage.orValue("no details")) + : adapter.?resourcesSkipped.orValue(false) + ? "Resources skipped: " + adapter.?skipReason.orValue("unknown reason") + : "Adapter execution completed successfully" + + observed_generation: + expression: "generation" + observed_time: "{{ now | date \"2006-01-02T15:04:05Z07:00\" }}" + + # Extract data from discovered ManifestWork from Maestro + data: + manifestwork: + name: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.name + : "" + consumer: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.namespace + : "cluster1" + configmap: + name: + expression: | + has(resources.configmap0) && has(resources.configmap0.metadata) + ? resources.configmap0.metadata.name + : "" + clusterId: + expression: | + has(resources.configmap0) && has(resources.configmap0.data) && has(resources.configmap0.data.cluster_id) + ? resources.configmap0.data.cluster_id + : clusterId + namespace: + name: + expression: | + has(resources.namespace0) && has(resources.namespace0.metadata) + ? resources.namespace0.metadata.name + : "" + phase: + expression: | + has(resources.namespace0) && has(resources.namespace0.statusFeedback) && has(resources.namespace0.statusFeedback.values) + && resources.namespace0.statusFeedback.values.exists(v, has(v.name) && v.name == "phase" && has(v.fieldValue)) + ? resources.namespace0.statusFeedback.values.filter(v, v.name == "phase")[0].fieldValue.string + : "Unknown" + + post_actions: + - name: "reportClusterStatus" + api_call: + method: "POST" + url: "/clusters/{{ .clusterId }}/statuses" + headers: + - name: "Content-Type" + value: "application/json" + body: "{{ .statusPayload }}" diff --git a/testdata/adapter-configs/cl-m-bad-api/values.yaml b/testdata/adapter-configs/cl-m-bad-api/values.yaml new file mode 100644 index 0000000..d235e97 --- /dev/null +++ b/testdata/adapter-configs/cl-m-bad-api/values.yaml @@ -0,0 +1,33 @@ +adapterConfig: + create: true + files: + adapter-config.yaml: cl-m-bad-api/adapter-config.yaml + log: + level: debug + +adapterTaskConfig: + create: true + files: + task-config.yaml: cl-m-bad-api/adapter-task-config.yaml + +broker: + create: true + googlepubsub: + projectId: ${GCP_PROJECT_ID} + subscriptionId: ${NAMESPACE}-clusters-${ADAPTER_NAME} + topic: ${NAMESPACE}-clusters + deadLetterTopic: ${NAMESPACE}-clusters-dlq + createTopicIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_TOPIC_IF_MISSING} + createSubscriptionIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_SUBSCRIPTION_IF_MISSING} + +image: + registry: ${IMAGE_REGISTRY} + repository: ci/hyperfleet-adapter + pullPolicy: Always + tag: latest + +rbac: + resources: + - namespaces + - configmaps + - configmaps/status diff --git a/testdata/adapter-configs/cl-m-unreg-consumer/adapter-config.yaml b/testdata/adapter-configs/cl-m-unreg-consumer/adapter-config.yaml new file mode 100644 index 0000000..427c353 --- /dev/null +++ b/testdata/adapter-configs/cl-m-unreg-consumer/adapter-config.yaml @@ -0,0 +1,65 @@ +# Example HyperFleet Adapter deployment configuration +# This configuration is for testing Maestro transport with an UNREGISTERED consumer +# to validate error handling when ManifestWork apply fails +adapter: + name: cl-m-unreg-consumer + #version: "0.1.0" + +# Log the full merged configuration after load (default: false) +debug_config: true +log: + level: debug + +clients: + hyperfleet_api: + base_url: http://hyperfleet-api:8000 + version: v1 + timeout: 2s + retry_attempts: 3 + retry_backoff: exponential + + broker: + # These values are overridden at deploy time via env vars from Helm values + subscription_id: CHANGE_ME + topic: CHANGE_ME + + maestro: + grpc_server_address: "maestro-grpc.maestro.svc.cluster.local:8090" + + # HTTPS server address for REST API operations (optional) + # Environment variable: HYPERFLEET_MAESTRO_HTTP_SERVER_ADDRESS + http_server_address: "http://maestro.maestro.svc.cluster.local:8000" + + # Source identifier for CloudEvents routing (must be unique across adapters) + # Environment variable: HYPERFLEET_MAESTRO_SOURCE_ID + source_id: "cl-m-unreg-consumer" + + # Client identifier (defaults to source_id if not specified) + # Environment variable: HYPERFLEET_MAESTRO_CLIENT_ID + client_id: "cl-m-unreg-consumer-client" + insecure: true + + # Authentication configuration + #auth: + # type: "tls" # TLS certificate-based mTLS + # + # tls_config: + # # gRPC TLS configuration + # # Certificate paths (mounted from Kubernetes secrets) + # # Environment variable: HYPERFLEET_MAESTRO_CA_FILE + # ca_file: "/etc/maestro/certs/grpc/ca.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_CERT_FILE + # cert_file: "/etc/maestro/certs/grpc/client.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_KEY_FILE + # key_file: "/etc/maestro/certs/grpc/client.key" + # + # # Server name for TLS verification + # # Environment variable: HYPERFLEET_MAESTRO_SERVER_NAME + # server_name: "maestro-grpc.maestro.svc.cluster.local" + # + # # HTTP API TLS configuration (may use different CA than gRPC) + # # If not set, falls back to ca_file for backwards compatibility + # # Environment variable: HYPERFLEET_MAESTRO_HTTP_CA_FILE + # http_ca_file: "/etc/maestro/certs/https/ca.crt" diff --git a/testdata/adapter-configs/cl-m-unreg-consumer/adapter-task-config.yaml b/testdata/adapter-configs/cl-m-unreg-consumer/adapter-task-config.yaml new file mode 100644 index 0000000..3feba6b --- /dev/null +++ b/testdata/adapter-configs/cl-m-unreg-consumer/adapter-task-config.yaml @@ -0,0 +1,340 @@ +# Example HyperFleet Adapter task configuration + +# Parameters with all required variables +params: + + - name: "clusterId" + source: "event.id" + type: "string" + required: true + + - name: "generation" + source: "event.generation" + type: "int" + required: true + + - name: "namespace" + source: "env.NAMESPACE" + type: "string" + + +# Preconditions with valid operators and CEL expressions +preconditions: + - name: "clusterStatus" + api_call: + method: "GET" + url: "/clusters/{{ .clusterId }}" + timeout: 10s + retry_attempts: 3 + retry_backoff: "exponential" + capture: + - name: "clusterName" + field: "name" + - name: "generation" + field: "generation" + - name: "timestamp" + field: "created_time" + - name: "readyConditionStatus" + expression: | + status.conditions.filter(c, c.type == "Ready").size() > 0 + ? status.conditions.filter(c, c.type == "Ready")[0].status + : "False" + - name: "placementClusterName" + expression: "\"unregistered-consumer\"" # Points to non-existent consumer to test apply failure + + + # Structured conditions with valid operators + conditions: + - field: "readyConditionStatus" + operator: "equals" + value: "False" + + - name: "validationCheck" + # Valid CEL expression + expression: | + readyConditionStatus == "False" + +# Resources with valid K8s manifests +resources: + - name: "resource0" + transport: + client: "maestro" + maestro: + target_cluster: "{{ .placementClusterName }}" + + # ManifestWork is a kind of manifest that can be used to create resources on the cluster. + # It is a collection of resources that are created together. + manifest: + apiVersion: work.open-cluster-management.io/v1 + kind: ManifestWork + metadata: + # ManifestWork name - must be unique within consumer namespace + name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Labels for identification, filtering, and management + labels: + # HyperFleet tracking labels + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/adapter: "{{ .adapter.name }}" + hyperfleet.io/component: "infrastructure" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/resource-group: "cluster-setup" + + # Maestro-specific labels + maestro.io/source-id: "{{ .adapter.name }}" + maestro.io/resource-type: "manifestwork" + maestro.io/priority: "normal" + + # Standard Kubernetes application labels + app.kubernetes.io/name: "aro-hcp-cluster" + app.kubernetes.io/instance: "{{ .clusterId }}" + app.kubernetes.io/version: "v1.0.0" + app.kubernetes.io/component: "infrastructure" + app.kubernetes.io/part-of: "hyperfleet" + app.kubernetes.io/managed-by: "cl-maestro" + app.kubernetes.io/created-by: "{{ .adapter.name }}" + + # Annotations for metadata and operational information + annotations: + # Tracking and lifecycle + hyperfleet.io/created-by: "cl-maestro-framework" + hyperfleet.io/managed-by: "{{ .adapter.name }}" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/cluster-name: "{{ .clusterName }}" + hyperfleet.io/deployment-time: "{{ .timestamp }}" + + # Maestro-specific annotations + maestro.io/applied-time: "{{ .timestamp }}" + maestro.io/source-adapter: "{{ .adapter.name }}" + + # Documentation + description: "Complete cluster setup including namespace, configuration, and RBAC" + + # ManifestWork specification + spec: + # ============================================================================ + # Workload - Contains the Kubernetes manifests to deploy + # ============================================================================ + workload: + # Kubernetes manifests array - injected by framework from business logic config + manifests: + - apiVersion: v1 + kind: Namespace + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + - apiVersion: v1 + kind: ConfigMap + data: + cluster_id: "{{ .clusterId }}" + cluster_name: "{{ .clusterName }}" + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/version: 1.0.0 + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + + # ============================================================================ + # Delete Options - How resources should be removed + # ============================================================================ + deleteOption: + # Propagation policy for resource deletion + # - "Foreground": Wait for dependent resources to be deleted first + # - "Background": Delete immediately, let cluster handle dependents + # - "Orphan": Leave resources on cluster when ManifestWork is deleted + propagationPolicy: "Foreground" + + # Grace period for graceful deletion (seconds) + gracePeriodSeconds: 30 + + # ============================================================================ + # Manifest Configurations - Per-resource settings for update and feedback + # ============================================================================ + manifestConfigs: + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "namespaces" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" # Specific resource name + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "phase" + path: ".status.phase" + # ======================================================================== + # Configuration for Namespace resources + # ======================================================================== + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "configmaps" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" # Specific resource name + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + serverSideApply: + fieldManager: "cl-maestro" # Field manager name for conflict resolution + force: false # Don't force conflicts (fail on conflicts) + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "data" + path: ".data" + - name: "resourceVersion" + path: ".metadata.resourceVersion" + # Discover the ResourceBundle (ManifestWork) by name from Maestro + discovery: + by_name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Discover nested resources deployed by the ManifestWork + nested_discoveries: + - name: "namespace0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + - name: "configmap0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + +post: + payloads: + - name: "statusPayload" + build: + adapter: "{{ .adapter.name }}" + conditions: + # Applied: Check if ManifestWork exists and has type="Applied", status="True" + - type: "Applied" + status: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].status : "False" + reason: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].reason : "ManifestWorkNotDiscovered" + message: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].message : "ManifestWork not discovered from Maestro or no Applied condition" + + # Available: Check if nested discovered manifests are available on the spoke cluster + # Each nested discovery is enriched with top-level "conditions" from status.resourceStatus.manifests[] + - type: "Available" + status: + expression: | + has(resources.namespace0) && has(resources.namespace0.conditions) + && resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + && has(resources.configmap0) && has(resources.configmap0.conditions) + && resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "True" + : "False" + reason: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "NamespaceNotDiscovered" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "NamespaceNotAvailable" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMapNotDiscovered" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMapNotAvailable" + : "AllResourcesAvailable" + message: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "Namespace not discovered from ManifestWork" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "Namespace not yet available on spoke cluster" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMap not discovered from ManifestWork" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMap not yet available on spoke cluster" + : "All manifests (namespace, configmap) are available on spoke cluster" + + # Health: Adapter execution status — surfaces errors from any phase + - type: "Health" + status: + expression: | + adapter.?executionStatus.orValue("") == "success" + && !adapter.?resourcesSkipped.orValue(false) + ? "True" + : "False" + reason: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "ExecutionFailed:" + adapter.?executionError.?phase.orValue("unknown") + : adapter.?resourcesSkipped.orValue(false) + ? "ResourcesSkipped" + : "Healthy" + message: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "Adapter failed at phase [" + + adapter.?executionError.?phase.orValue("unknown") + + "] step [" + + adapter.?executionError.?step.orValue("unknown") + + "]: " + + adapter.?executionError.?message.orValue(adapter.?errorMessage.orValue("no details")) + : adapter.?resourcesSkipped.orValue(false) + ? "Resources skipped: " + adapter.?skipReason.orValue("unknown reason") + : "Adapter execution completed successfully" + + observed_generation: + expression: "generation" + observed_time: "{{ now | date \"2006-01-02T15:04:05Z07:00\" }}" + + # Extract data from discovered ManifestWork from Maestro + data: + manifestwork: + name: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.name + : "" + consumer: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.namespace + : placementClusterName + configmap: + name: + expression: | + has(resources.configmap0) && has(resources.configmap0.metadata) + ? resources.configmap0.metadata.name + : "" + clusterId: + expression: | + has(resources.configmap0) && has(resources.configmap0.data) && has(resources.configmap0.data.cluster_id) + ? resources.configmap0.data.cluster_id + : clusterId + namespace: + name: + expression: | + has(resources.namespace0) && has(resources.namespace0.metadata) + ? resources.namespace0.metadata.name + : "" + phase: + expression: | + has(resources.namespace0) && has(resources.namespace0.statusFeedback) && has(resources.namespace0.statusFeedback.values) + && resources.namespace0.statusFeedback.values.exists(v, has(v.name) && v.name == "phase" && has(v.fieldValue)) + ? resources.namespace0.statusFeedback.values.filter(v, v.name == "phase")[0].fieldValue.string + : "Unknown" + + post_actions: + - name: "reportClusterStatus" + api_call: + method: "POST" + url: "/clusters/{{ .clusterId }}/statuses" + headers: + - name: "Content-Type" + value: "application/json" + body: "{{ .statusPayload }}" diff --git a/testdata/adapter-configs/cl-m-unreg-consumer/values.yaml b/testdata/adapter-configs/cl-m-unreg-consumer/values.yaml new file mode 100644 index 0000000..829c67a --- /dev/null +++ b/testdata/adapter-configs/cl-m-unreg-consumer/values.yaml @@ -0,0 +1,33 @@ +adapterConfig: + create: true + files: + adapter-config.yaml: cl-m-unreg-consumer/adapter-config.yaml + log: + level: debug + +adapterTaskConfig: + create: true + files: + task-config.yaml: cl-m-unreg-consumer/adapter-task-config.yaml + +broker: + create: true + googlepubsub: + projectId: ${GCP_PROJECT_ID} + subscriptionId: ${NAMESPACE}-clusters-${ADAPTER_NAME} + topic: ${NAMESPACE}-clusters + deadLetterTopic: ${NAMESPACE}-clusters-dlq + createTopicIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_TOPIC_IF_MISSING} + createSubscriptionIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_SUBSCRIPTION_IF_MISSING} + +image: + registry: ${IMAGE_REGISTRY} + repository: ci/hyperfleet-adapter + pullPolicy: Always + tag: latest + +rbac: + resources: + - namespaces + - configmaps + - configmaps/status diff --git a/testdata/adapter-configs/cl-m-wrong-ds/adapter-config.yaml b/testdata/adapter-configs/cl-m-wrong-ds/adapter-config.yaml new file mode 100644 index 0000000..ed15355 --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-ds/adapter-config.yaml @@ -0,0 +1,65 @@ +# Example HyperFleet Adapter deployment configuration +# This configuration is for testing Maestro transport with WRONG main discovery name +# to validate that adapter fails when it cannot find the ManifestWork it created +adapter: + name: cl-m-wrong-ds + #version: "0.1.0" + +# Log the full merged configuration after load (default: false) +debug_config: true +log: + level: debug + +clients: + hyperfleet_api: + base_url: http://hyperfleet-api:8000 + version: v1 + timeout: 2s + retry_attempts: 3 + retry_backoff: exponential + + broker: + # These values are overridden at deploy time via env vars from Helm values + subscription_id: CHANGE_ME + topic: CHANGE_ME + + maestro: + grpc_server_address: "maestro-grpc.maestro.svc.cluster.local:8090" + + # HTTPS server address for REST API operations (optional) + # Environment variable: HYPERFLEET_MAESTRO_HTTP_SERVER_ADDRESS + http_server_address: "http://maestro.maestro.svc.cluster.local:8000" + + # Source identifier for CloudEvents routing (must be unique across adapters) + # Environment variable: HYPERFLEET_MAESTRO_SOURCE_ID + source_id: "cl-m-wrong-ds" + + # Client identifier (defaults to source_id if not specified) + # Environment variable: HYPERFLEET_MAESTRO_CLIENT_ID + client_id: "cl-m-wrong-ds-client" + insecure: true + + # Authentication configuration + #auth: + # type: "tls" # TLS certificate-based mTLS + # + # tls_config: + # # gRPC TLS configuration + # # Certificate paths (mounted from Kubernetes secrets) + # # Environment variable: HYPERFLEET_MAESTRO_CA_FILE + # ca_file: "/etc/maestro/certs/grpc/ca.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_CERT_FILE + # cert_file: "/etc/maestro/certs/grpc/client.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_KEY_FILE + # key_file: "/etc/maestro/certs/grpc/client.key" + # + # # Server name for TLS verification + # # Environment variable: HYPERFLEET_MAESTRO_SERVER_NAME + # server_name: "maestro-grpc.maestro.svc.cluster.local" + # + # # HTTP API TLS configuration (may use different CA than gRPC) + # # If not set, falls back to ca_file for backwards compatibility + # # Environment variable: HYPERFLEET_MAESTRO_HTTP_CA_FILE + # http_ca_file: "/etc/maestro/certs/https/ca.crt" diff --git a/testdata/adapter-configs/cl-m-wrong-ds/adapter-task-config.yaml b/testdata/adapter-configs/cl-m-wrong-ds/adapter-task-config.yaml new file mode 100644 index 0000000..10bd509 --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-ds/adapter-task-config.yaml @@ -0,0 +1,342 @@ +# Example HyperFleet Adapter task configuration + +# Parameters with all required variables +params: + + - name: "clusterId" + source: "event.id" + type: "string" + required: true + + - name: "generation" + source: "event.generation" + type: "int" + required: true + + - name: "namespace" + source: "env.NAMESPACE" + type: "string" + + +# Preconditions with valid operators and CEL expressions +preconditions: + - name: "clusterStatus" + api_call: + method: "GET" + url: "/clusters/{{ .clusterId }}" + timeout: 10s + retry_attempts: 3 + retry_backoff: "exponential" + capture: + - name: "clusterName" + field: "name" + - name: "generation" + field: "generation" + - name: "timestamp" + field: "created_time" + - name: "readyConditionStatus" + expression: | + status.conditions.filter(c, c.type == "Ready").size() > 0 + ? status.conditions.filter(c, c.type == "Ready")[0].status + : "False" + - name: "placementClusterName" + expression: "\"cluster1\"" # TBC coming from placement adapter + + + # Structured conditions with valid operators + conditions: + - field: "readyConditionStatus" + operator: "equals" + value: "False" + + - name: "validationCheck" + # Valid CEL expression + expression: | + readyConditionStatus == "False" + +# Resources with valid K8s manifests +resources: + - name: "resource0" + transport: + client: "maestro" + maestro: + target_cluster: "{{ .placementClusterName }}" + + # ManifestWork is a kind of manifest that can be used to create resources on the cluster. + # It is a collection of resources that are created together. + manifest: + apiVersion: work.open-cluster-management.io/v1 + kind: ManifestWork + metadata: + # ManifestWork name - must be unique within consumer namespace + name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Labels for identification, filtering, and management + labels: + # HyperFleet tracking labels + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/adapter: "{{ .adapter.name }}" + hyperfleet.io/component: "infrastructure" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/resource-group: "cluster-setup" + + # Maestro-specific labels + maestro.io/source-id: "{{ .adapter.name }}" + maestro.io/resource-type: "manifestwork" + maestro.io/priority: "normal" + + # Standard Kubernetes application labels + app.kubernetes.io/name: "aro-hcp-cluster" + app.kubernetes.io/instance: "{{ .clusterId }}" + app.kubernetes.io/version: "v1.0.0" + app.kubernetes.io/component: "infrastructure" + app.kubernetes.io/part-of: "hyperfleet" + app.kubernetes.io/managed-by: "cl-maestro" + app.kubernetes.io/created-by: "{{ .adapter.name }}" + + # Annotations for metadata and operational information + annotations: + # Tracking and lifecycle + hyperfleet.io/created-by: "cl-maestro-framework" + hyperfleet.io/managed-by: "{{ .adapter.name }}" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/cluster-name: "{{ .clusterName }}" + hyperfleet.io/deployment-time: "{{ .timestamp }}" + + # Maestro-specific annotations + maestro.io/applied-time: "{{ .timestamp }}" + maestro.io/source-adapter: "{{ .adapter.name }}" + + # Documentation + description: "Complete cluster setup including namespace, configuration, and RBAC" + + # ManifestWork specification + spec: + # ============================================================================ + # Workload - Contains the Kubernetes manifests to deploy + # ============================================================================ + workload: + # Kubernetes manifests array - injected by framework from business logic config + manifests: + - apiVersion: v1 + kind: Namespace + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + - apiVersion: v1 + kind: ConfigMap + data: + cluster_id: "{{ .clusterId }}" + cluster_name: "{{ .clusterName }}" + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/version: 1.0.0 + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + + # ============================================================================ + # Delete Options - How resources should be removed + # ============================================================================ + deleteOption: + # Propagation policy for resource deletion + # - "Foreground": Wait for dependent resources to be deleted first + # - "Background": Delete immediately, let cluster handle dependents + # - "Orphan": Leave resources on cluster when ManifestWork is deleted + propagationPolicy: "Foreground" + + # Grace period for graceful deletion (seconds) + gracePeriodSeconds: 30 + + # ============================================================================ + # Manifest Configurations - Per-resource settings for update and feedback + # ============================================================================ + manifestConfigs: + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "namespaces" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" # Specific resource name + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "phase" + path: ".status.phase" + # ======================================================================== + # Configuration for Namespace resources + # ======================================================================== + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "configmaps" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" # Specific resource name + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + serverSideApply: + fieldManager: "cl-maestro" # Field manager name for conflict resolution + force: false # Don't force conflicts (fail on conflicts) + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "data" + path: ".data" + - name: "resourceVersion" + path: ".metadata.resourceVersion" + # Discover the ResourceBundle (ManifestWork) by name from Maestro + # NOTE: This discovery name is intentionally WRONG to test main discovery failure + discovery: + by_name: "{{ .clusterId }}-{{ .adapter.name }}-wrong" + + # Discover nested resources deployed by the ManifestWork + # These are correct, but won't be reached if main discovery fails + nested_discoveries: + - name: "namespace0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + - name: "configmap0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + +post: + payloads: + - name: "statusPayload" + build: + adapter: "{{ .adapter.name }}" + conditions: + # Applied: Check if ManifestWork exists and has type="Applied", status="True" + - type: "Applied" + status: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].status : "False" + reason: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].reason : "ManifestWorkNotDiscovered" + message: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].message : "ManifestWork not discovered from Maestro or no Applied condition" + + # Available: Check if nested discovered manifests are available on the spoke cluster + # Each nested discovery is enriched with top-level "conditions" from status.resourceStatus.manifests[] + - type: "Available" + status: + expression: | + has(resources.namespace0) && has(resources.namespace0.conditions) + && resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + && has(resources.configmap0) && has(resources.configmap0.conditions) + && resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "True" + : "False" + reason: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "NamespaceNotDiscovered" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "NamespaceNotAvailable" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMapNotDiscovered" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMapNotAvailable" + : "AllResourcesAvailable" + message: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "Namespace not discovered from ManifestWork" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "Namespace not yet available on spoke cluster" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMap not discovered from ManifestWork" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMap not yet available on spoke cluster" + : "All manifests (namespace, configmap) are available on spoke cluster" + + # Health: Adapter execution status — surfaces errors from any phase + - type: "Health" + status: + expression: | + adapter.?executionStatus.orValue("") == "success" + && !adapter.?resourcesSkipped.orValue(false) + ? "True" + : "False" + reason: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "ExecutionFailed:" + adapter.?executionError.?phase.orValue("unknown") + : adapter.?resourcesSkipped.orValue(false) + ? "ResourcesSkipped" + : "Healthy" + message: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "Adapter failed at phase [" + + adapter.?executionError.?phase.orValue("unknown") + + "] step [" + + adapter.?executionError.?step.orValue("unknown") + + "]: " + + adapter.?executionError.?message.orValue(adapter.?errorMessage.orValue("no details")) + : adapter.?resourcesSkipped.orValue(false) + ? "Resources skipped: " + adapter.?skipReason.orValue("unknown reason") + : "Adapter execution completed successfully" + + observed_generation: + expression: "generation" + observed_time: "{{ now | date \"2006-01-02T15:04:05Z07:00\" }}" + + # Extract data from discovered ManifestWork from Maestro + data: + manifestwork: + name: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.name + : "" + consumer: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.namespace + : placementClusterName + configmap: + name: + expression: | + has(resources.configmap0) && has(resources.configmap0.metadata) + ? resources.configmap0.metadata.name + : "" + clusterId: + expression: | + has(resources.configmap0) && has(resources.configmap0.data) && has(resources.configmap0.data.cluster_id) + ? resources.configmap0.data.cluster_id + : clusterId + namespace: + name: + expression: | + has(resources.namespace0) && has(resources.namespace0.metadata) + ? resources.namespace0.metadata.name + : "" + phase: + expression: | + has(resources.namespace0) && has(resources.namespace0.statusFeedback) && has(resources.namespace0.statusFeedback.values) + && resources.namespace0.statusFeedback.values.exists(v, has(v.name) && v.name == "phase" && has(v.fieldValue)) + ? resources.namespace0.statusFeedback.values.filter(v, v.name == "phase")[0].fieldValue.string + : "Unknown" + + post_actions: + - name: "reportClusterStatus" + api_call: + method: "POST" + url: "/clusters/{{ .clusterId }}/statuses" + headers: + - name: "Content-Type" + value: "application/json" + body: "{{ .statusPayload }}" diff --git a/testdata/adapter-configs/cl-m-wrong-ds/values.yaml b/testdata/adapter-configs/cl-m-wrong-ds/values.yaml new file mode 100644 index 0000000..c677b81 --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-ds/values.yaml @@ -0,0 +1,33 @@ +adapterConfig: + create: true + files: + adapter-config.yaml: cl-m-wrong-ds/adapter-config.yaml + log: + level: debug + +adapterTaskConfig: + create: true + files: + task-config.yaml: cl-m-wrong-ds/adapter-task-config.yaml + +broker: + create: true + googlepubsub: + projectId: ${GCP_PROJECT_ID} + subscriptionId: ${NAMESPACE}-clusters-${ADAPTER_NAME} + topic: ${NAMESPACE}-clusters + deadLetterTopic: ${NAMESPACE}-clusters-dlq + createTopicIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_TOPIC_IF_MISSING} + createSubscriptionIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_SUBSCRIPTION_IF_MISSING} + +image: + registry: ${IMAGE_REGISTRY} + repository: ci/hyperfleet-adapter + pullPolicy: Always + tag: latest + +rbac: + resources: + - namespaces + - configmaps + - configmaps/status diff --git a/testdata/adapter-configs/cl-m-wrong-nest/adapter-config.yaml b/testdata/adapter-configs/cl-m-wrong-nest/adapter-config.yaml new file mode 100644 index 0000000..596fc3f --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-nest/adapter-config.yaml @@ -0,0 +1,65 @@ +# Example HyperFleet Adapter deployment configuration +# This configuration is for testing Maestro transport with WRONG nested discovery names +# to validate error handling when ManifestWork is found but nested resources cannot be discovered +adapter: + name: cl-m-wrong-nest + #version: "0.1.0" + +# Log the full merged configuration after load (default: false) +debug_config: true +log: + level: debug + +clients: + hyperfleet_api: + base_url: http://hyperfleet-api:8000 + version: v1 + timeout: 2s + retry_attempts: 3 + retry_backoff: exponential + + broker: + # These values are overridden at deploy time via env vars from Helm values + subscription_id: CHANGE_ME + topic: CHANGE_ME + + maestro: + grpc_server_address: "maestro-grpc.maestro.svc.cluster.local:8090" + + # HTTPS server address for REST API operations (optional) + # Environment variable: HYPERFLEET_MAESTRO_HTTP_SERVER_ADDRESS + http_server_address: "http://maestro.maestro.svc.cluster.local:8000" + + # Source identifier for CloudEvents routing (must be unique across adapters) + # Environment variable: HYPERFLEET_MAESTRO_SOURCE_ID + source_id: "cl-m-wrong-nest" + + # Client identifier (defaults to source_id if not specified) + # Environment variable: HYPERFLEET_MAESTRO_CLIENT_ID + client_id: "cl-m-wrong-nest-client" + insecure: true + + # Authentication configuration + #auth: + # type: "tls" # TLS certificate-based mTLS + # + # tls_config: + # # gRPC TLS configuration + # # Certificate paths (mounted from Kubernetes secrets) + # # Environment variable: HYPERFLEET_MAESTRO_CA_FILE + # ca_file: "/etc/maestro/certs/grpc/ca.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_CERT_FILE + # cert_file: "/etc/maestro/certs/grpc/client.crt" + # + # # Environment variable: HYPERFLEET_MAESTRO_KEY_FILE + # key_file: "/etc/maestro/certs/grpc/client.key" + # + # # Server name for TLS verification + # # Environment variable: HYPERFLEET_MAESTRO_SERVER_NAME + # server_name: "maestro-grpc.maestro.svc.cluster.local" + # + # # HTTP API TLS configuration (may use different CA than gRPC) + # # If not set, falls back to ca_file for backwards compatibility + # # Environment variable: HYPERFLEET_MAESTRO_HTTP_CA_FILE + # http_ca_file: "/etc/maestro/certs/https/ca.crt" diff --git a/testdata/adapter-configs/cl-m-wrong-nest/adapter-task-config.yaml b/testdata/adapter-configs/cl-m-wrong-nest/adapter-task-config.yaml new file mode 100644 index 0000000..8db12c6 --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-nest/adapter-task-config.yaml @@ -0,0 +1,341 @@ +# Example HyperFleet Adapter task configuration + +# Parameters with all required variables +params: + + - name: "clusterId" + source: "event.id" + type: "string" + required: true + + - name: "generation" + source: "event.generation" + type: "int" + required: true + + - name: "namespace" + source: "env.NAMESPACE" + type: "string" + + +# Preconditions with valid operators and CEL expressions +preconditions: + - name: "clusterStatus" + api_call: + method: "GET" + url: "/clusters/{{ .clusterId }}" + timeout: 10s + retry_attempts: 3 + retry_backoff: "exponential" + capture: + - name: "clusterName" + field: "name" + - name: "generation" + field: "generation" + - name: "timestamp" + field: "created_time" + - name: "readyConditionStatus" + expression: | + status.conditions.filter(c, c.type == "Ready").size() > 0 + ? status.conditions.filter(c, c.type == "Ready")[0].status + : "False" + - name: "placementClusterName" + expression: "\"cluster1\"" # TBC coming from placement adapter + + + # Structured conditions with valid operators + conditions: + - field: "readyConditionStatus" + operator: "equals" + value: "False" + + - name: "validationCheck" + # Valid CEL expression + expression: | + readyConditionStatus == "False" + +# Resources with valid K8s manifests +resources: + - name: "resource0" + transport: + client: "maestro" + maestro: + target_cluster: "{{ .placementClusterName }}" + + # ManifestWork is a kind of manifest that can be used to create resources on the cluster. + # It is a collection of resources that are created together. + manifest: + apiVersion: work.open-cluster-management.io/v1 + kind: ManifestWork + metadata: + # ManifestWork name - must be unique within consumer namespace + name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Labels for identification, filtering, and management + labels: + # HyperFleet tracking labels + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/adapter: "{{ .adapter.name }}" + hyperfleet.io/component: "infrastructure" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/resource-group: "cluster-setup" + + # Maestro-specific labels + maestro.io/source-id: "{{ .adapter.name }}" + maestro.io/resource-type: "manifestwork" + maestro.io/priority: "normal" + + # Standard Kubernetes application labels + app.kubernetes.io/name: "aro-hcp-cluster" + app.kubernetes.io/instance: "{{ .clusterId }}" + app.kubernetes.io/version: "v1.0.0" + app.kubernetes.io/component: "infrastructure" + app.kubernetes.io/part-of: "hyperfleet" + app.kubernetes.io/managed-by: "cl-maestro" + app.kubernetes.io/created-by: "{{ .adapter.name }}" + + # Annotations for metadata and operational information + annotations: + # Tracking and lifecycle + hyperfleet.io/created-by: "cl-maestro-framework" + hyperfleet.io/managed-by: "{{ .adapter.name }}" + hyperfleet.io/generation: "{{ .generation }}" + hyperfleet.io/cluster-id: "{{ .clusterId }}" + hyperfleet.io/cluster-name: "{{ .clusterName }}" + hyperfleet.io/deployment-time: "{{ .timestamp }}" + + # Maestro-specific annotations + maestro.io/applied-time: "{{ .timestamp }}" + maestro.io/source-adapter: "{{ .adapter.name }}" + + # Documentation + description: "Complete cluster setup including namespace, configuration, and RBAC" + + # ManifestWork specification + spec: + # ============================================================================ + # Workload - Contains the Kubernetes manifests to deploy + # ============================================================================ + workload: + # Kubernetes manifests array - injected by framework from business logic config + manifests: + - apiVersion: v1 + kind: Namespace + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + - apiVersion: v1 + kind: ConfigMap + data: + cluster_id: "{{ .clusterId }}" + cluster_name: "{{ .clusterName }}" + metadata: + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + labels: + app.kubernetes.io/component: adapter-task-config + app.kubernetes.io/instance: "{{ .adapter.name }}" + app.kubernetes.io/name: cl-maestro + app.kubernetes.io/version: 1.0.0 + app.kubernetes.io/transport: maestro + annotations: + hyperfleet.io/generation: "{{ .generation }}" + + # ============================================================================ + # Delete Options - How resources should be removed + # ============================================================================ + deleteOption: + # Propagation policy for resource deletion + # - "Foreground": Wait for dependent resources to be deleted first + # - "Background": Delete immediately, let cluster handle dependents + # - "Orphan": Leave resources on cluster when ManifestWork is deleted + propagationPolicy: "Foreground" + + # Grace period for graceful deletion (seconds) + gracePeriodSeconds: 30 + + # ============================================================================ + # Manifest Configurations - Per-resource settings for update and feedback + # ============================================================================ + manifestConfigs: + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "namespaces" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" # Specific resource name + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "phase" + path: ".status.phase" + # ======================================================================== + # Configuration for Namespace resources + # ======================================================================== + - resourceIdentifier: + group: "" # Core API group (empty for v1 resources) + resource: "configmaps" # Resource type + name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap" # Specific resource name + namespace: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace" + updateStrategy: + type: "ServerSideApply" # Use server-side apply for namespaces + serverSideApply: + fieldManager: "cl-maestro" # Field manager name for conflict resolution + force: false # Don't force conflicts (fail on conflicts) + feedbackRules: + - type: "JSONPaths" # Use JSON path expressions for status feedback + jsonPaths: + - name: "data" + path: ".data" + - name: "resourceVersion" + path: ".metadata.resourceVersion" + # Discover the ResourceBundle (ManifestWork) by name from Maestro + discovery: + by_name: "{{ .clusterId }}-{{ .adapter.name }}" + + # Discover nested resources deployed by the ManifestWork + # NOTE: These discovery names are intentionally WRONG for testing discovery failure + nested_discoveries: + - name: "namespace0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-namespace-wrong" + - name: "configmap0" + discovery: + by_name: "{{ .clusterId | lower }}-{{ .adapter.name }}-configmap-wrong" + +post: + payloads: + - name: "statusPayload" + build: + adapter: "{{ .adapter.name }}" + conditions: + # Applied: Check if ManifestWork exists and has type="Applied", status="True" + - type: "Applied" + status: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].status : "False" + reason: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].reason : "ManifestWorkNotDiscovered" + message: + expression: | + has(resources.resource0) && has(resources.resource0.status) && has(resources.resource0.status.conditions) && resources.resource0.status.conditions.filter(c, has(c.type) && c.type == "Applied").size() > 0 ? resources.resource0.status.conditions.filter(c, c.type == "Applied")[0].message : "ManifestWork not discovered from Maestro or no Applied condition" + + # Available: Check if nested discovered manifests are available on the spoke cluster + # Each nested discovery is enriched with top-level "conditions" from status.resourceStatus.manifests[] + - type: "Available" + status: + expression: | + has(resources.namespace0) && has(resources.namespace0.conditions) + && resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + && has(resources.configmap0) && has(resources.configmap0.conditions) + && resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "True" + : "False" + reason: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "NamespaceNotDiscovered" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "NamespaceNotAvailable" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMapNotDiscovered" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMapNotAvailable" + : "AllResourcesAvailable" + message: + expression: | + !(has(resources.namespace0) && has(resources.namespace0.conditions)) + ? "Namespace not discovered from ManifestWork" + : !resources.namespace0.conditions.exists(c, has(c.type) && c.type == "Available" && has(c.status) && c.status == "True") + ? "Namespace not yet available on spoke cluster" + : !(has(resources.configmap0) && has(resources.configmap0.conditions)) + ? "ConfigMap not discovered from ManifestWork" + : !resources.configmap0.conditions.exists(c, c.type == "Available" && has(c.status) && c.status == "True") + ? "ConfigMap not yet available on spoke cluster" + : "All manifests (namespace, configmap) are available on spoke cluster" + + # Health: Adapter execution status — surfaces errors from any phase + - type: "Health" + status: + expression: | + adapter.?executionStatus.orValue("") == "success" + && !adapter.?resourcesSkipped.orValue(false) + ? "True" + : "False" + reason: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "ExecutionFailed:" + adapter.?executionError.?phase.orValue("unknown") + : adapter.?resourcesSkipped.orValue(false) + ? "ResourcesSkipped" + : "Healthy" + message: + expression: | + adapter.?executionStatus.orValue("") != "success" + ? "Adapter failed at phase [" + + adapter.?executionError.?phase.orValue("unknown") + + "] step [" + + adapter.?executionError.?step.orValue("unknown") + + "]: " + + adapter.?executionError.?message.orValue(adapter.?errorMessage.orValue("no details")) + : adapter.?resourcesSkipped.orValue(false) + ? "Resources skipped: " + adapter.?skipReason.orValue("unknown reason") + : "Adapter execution completed successfully" + + observed_generation: + expression: "generation" + observed_time: "{{ now | date \"2006-01-02T15:04:05Z07:00\" }}" + + # Extract data from discovered ManifestWork from Maestro + data: + manifestwork: + name: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.name + : "" + consumer: + expression: | + has(resources.resource0) && has(resources.resource0.metadata) + ? resources.resource0.metadata.namespace + : placementClusterName + configmap: + name: + expression: | + has(resources.configmap0) && has(resources.configmap0.metadata) + ? resources.configmap0.metadata.name + : "" + clusterId: + expression: | + has(resources.configmap0) && has(resources.configmap0.data) && has(resources.configmap0.data.cluster_id) + ? resources.configmap0.data.cluster_id + : clusterId + namespace: + name: + expression: | + has(resources.namespace0) && has(resources.namespace0.metadata) + ? resources.namespace0.metadata.name + : "" + phase: + expression: | + has(resources.namespace0) && has(resources.namespace0.statusFeedback) && has(resources.namespace0.statusFeedback.values) + && resources.namespace0.statusFeedback.values.exists(v, has(v.name) && v.name == "phase" && has(v.fieldValue)) + ? resources.namespace0.statusFeedback.values.filter(v, v.name == "phase")[0].fieldValue.string + : "Unknown" + + post_actions: + - name: "reportClusterStatus" + api_call: + method: "POST" + url: "/clusters/{{ .clusterId }}/statuses" + headers: + - name: "Content-Type" + value: "application/json" + body: "{{ .statusPayload }}" diff --git a/testdata/adapter-configs/cl-m-wrong-nest/values.yaml b/testdata/adapter-configs/cl-m-wrong-nest/values.yaml new file mode 100644 index 0000000..47c7a0f --- /dev/null +++ b/testdata/adapter-configs/cl-m-wrong-nest/values.yaml @@ -0,0 +1,33 @@ +adapterConfig: + create: true + files: + adapter-config.yaml: cl-m-wrong-nest/adapter-config.yaml + log: + level: debug + +adapterTaskConfig: + create: true + files: + task-config.yaml: cl-m-wrong-nest/adapter-task-config.yaml + +broker: + create: true + googlepubsub: + projectId: ${GCP_PROJECT_ID} + subscriptionId: ${NAMESPACE}-clusters-${ADAPTER_NAME} + topic: ${NAMESPACE}-clusters + deadLetterTopic: ${NAMESPACE}-clusters-dlq + createTopicIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_TOPIC_IF_MISSING} + createSubscriptionIfMissing: ${ADAPTER_GOOGLEPUBSUB_CREATE_SUBSCRIPTION_IF_MISSING} + +image: + registry: ${IMAGE_REGISTRY} + repository: ci/hyperfleet-adapter + pullPolicy: Always + tag: latest + +rbac: + resources: + - namespaces + - configmaps + - configmaps/status