diff --git a/Makefile b/Makefile index 2aa1be2..e3f4a9f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ all: build-cli build-service build-cli: cd cli && go build -v && mv cli ../oms-cli +build-cli-linux: + GOOS=linux GOARCH=amd64 go build -C cli -o ../oms-cli + build-service: cd service && go build -v && mv service ../oms-service diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index 301b461..8569f48 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -114,7 +114,7 @@ func (c *BootstrapGcpCmd) BootstrapGcp() error { return fmt.Errorf("failed to bootstrap GCP: %w, env: %s", err, envString) } - log.Println("\nšŸŽ‰šŸŽ‰šŸŽ‰ GCP infrastructure bootstrapped successfully!") + log.Println("\nGCP infrastructure bootstrapped successfully!") log.Println(envString) log.Printf("Start the Codesphere installation using OMS from the jumpbox host:\nssh-add $SSH_KEY_PATH; ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o SendEnv=OMS_PORTAL_API_KEY root@%s", bs.Env.Jumpbox.GetExternalIP()) log.Printf("When the installation is done, run the k0s configuration script generated at the k0s-1 host %s /root/configure-k0s.sh.", bs.Env.ControlPlaneNodes[0].GetInternalIP()) diff --git a/cli/cmd/install_k0s.go b/cli/cmd/install_k0s.go index a513933..da845a7 100644 --- a/cli/cmd/install_k0s.go +++ b/cli/cmd/install_k0s.go @@ -5,6 +5,8 @@ package cmd import ( "fmt" + "log" + "path/filepath" packageio "github.com/codesphere-cloud/cs-go/pkg/io" "github.com/spf13/cobra" @@ -25,10 +27,13 @@ type InstallK0sCmd struct { type InstallK0sOpts struct { *GlobalOptions - Version string - Package string - Config string - Force bool + Version string + K0sctlVersion string + Package string + InstallConfig string + SSHKeyPath string + Force bool + NoDownload bool } func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { @@ -36,13 +41,9 @@ func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { env := c.Env pm := installer.NewPackage(env.GetOmsWorkdir(), c.Opts.Package) k0s := installer.NewK0s(hw, env, c.FileWriter) + k0sctl := installer.NewK0sctl(hw, env, c.FileWriter) - err := c.InstallK0s(pm, k0s) - if err != nil { - return fmt.Errorf("failed to install k0s: %w", err) - } - - return nil + return c.InstallK0s(pm, k0s, k0sctl) } func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { @@ -51,16 +52,20 @@ func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { Use: "k0s", Short: "Install k0s Kubernetes distribution", Long: packageio.Long(`Install k0s either from the package or by downloading it. - This will either download the k0s binary directly to the OMS workdir, if not already present, and install it - or load the k0s binary from the provided package file and install it. - If no version is specified, the latest version will be downloaded. - If no install config is provided, k0s will be installed with the '--single' flag.`), + This command uses k0sctl to deploy k0s clusters from a Codesphere install-config. + + You must provide a Codesphere install-config file, which will: + - Generate a k0s configuration from the install-config + - Generate a k0sctl configuration for cluster deployment + - Deploy k0s to all nodes defined in the install-config using k0sctl`), Example: formatExamplesWithBinary("install k0s", []packageio.Example{ - {Cmd: "", Desc: "Install k0s using the Go-native implementation"}, - {Cmd: "--version ", Desc: "Version of k0s to install"}, + {Cmd: "--install-config ", Desc: "Path to Codesphere install-config file to generate k0s config from"}, + {Cmd: "--version ", Desc: "Version of k0s to install (e.g., v1.30.0+k0s.0)"}, + {Cmd: "--k0sctl-version ", Desc: "Version of k0sctl to use (e.g., v0.17.4)"}, {Cmd: "--package ", Desc: "Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from"}, - {Cmd: "--k0s-config ", Desc: "Path to k0s configuration file, if not set k0s will be installed with the '--single' flag"}, - {Cmd: "--force", Desc: "Force new download and installation even if k0s binary exists or is already installed"}, + {Cmd: "--ssh-key-path ", Desc: "SSH private key path for remote installation"}, + {Cmd: "--force", Desc: "Force new download and installation"}, + {Cmd: "--no-download", Desc: "Skip downloading k0s binary (expects it to be on remote nodes)"}, }, "oms-cli"), }, Opts: InstallK0sOpts{GlobalOptions: opts}, @@ -68,33 +73,102 @@ func AddInstallK0sCmd(install *cobra.Command, opts *GlobalOptions) { FileWriter: util.NewFilesystemWriter(), } k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to install") + k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", "", "Version of k0sctl to use") k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from") - k0s.cmd.Flags().StringVar(&k0s.Opts.Config, "k0s-config", "", "Path to k0s configuration file") + k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)") + k0s.cmd.Flags().StringVar(&k0s.Opts.SSHKeyPath, "ssh-key-path", "", "SSH private key path for remote installation") k0s.cmd.Flags().BoolVarP(&k0s.Opts.Force, "force", "f", false, "Force new download and installation") + k0s.cmd.Flags().BoolVar(&k0s.Opts.NoDownload, "no-download", false, "Skip downloading k0s binary") + + _ = k0s.cmd.MarkFlagRequired("install-config") install.AddCommand(k0s.cmd) k0s.cmd.RunE = k0s.RunE } -const defaultK0sPath = "kubernetes/files/k0s" +const ( + defaultK0sPath = "kubernetes/files/k0s" + k0sctlConfigFile = "k0sctl-config.yaml" +) -func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0sManager) error { - // Default dependency path for k0s binary within package - k0sPath := pm.GetDependencyPath(defaultK0sPath) +func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0sManager, k0sctl installer.K0sctlManager) error { + // Load install-config + icg := installer.NewInstallConfigManager() + if err := icg.LoadInstallConfigFromFile(c.Opts.InstallConfig); err != nil { + return fmt.Errorf("failed to load install-config: %w", err) + } - var err error - if c.Opts.Package == "" { - k0sPath, err = k0s.Download(c.Opts.Version, c.Opts.Force, false) + config := icg.GetInstallConfig() + + if !config.Kubernetes.ManagedByCodesphere { + return fmt.Errorf("install-config specifies external Kubernetes, k0s installation is only supported for Codesphere-managed Kubernetes") + } + + // Determine k0s version + k0sVersion := c.Opts.Version + if k0sVersion == "" { + var err error + k0sVersion, err = k0s.GetLatestVersion() if err != nil { - return fmt.Errorf("failed to download k0s: %w", err) + return fmt.Errorf("failed to get latest k0s version: %w", err) + } + log.Printf("Using latest k0s version: %s", k0sVersion) + } + + // Download or get k0s binary path + var k0sBinaryPath string + if !c.Opts.NoDownload { + if c.Opts.Package != "" { + // Extract the k0s binary from the package first + if err := pm.ExtractDependency(defaultK0sPath, c.Opts.Force); err != nil { + return fmt.Errorf("failed to extract k0s from package: %w", err) + } + k0sBinaryPath = pm.GetDependencyPath(defaultK0sPath) + } else { + var err error + k0sBinaryPath, err = k0s.Download(k0sVersion, c.Opts.Force, false) + if err != nil { + return fmt.Errorf("failed to download k0s: %w", err) + } } } - err = k0s.Install(c.Opts.Config, k0sPath, c.Opts.Force) + // Download k0sctl + log.Println("Downloading k0sctl...") + k0sctlPath, err := k0sctl.Download(c.Opts.K0sctlVersion, c.Opts.Force, false) if err != nil { - return fmt.Errorf("failed to install k0s: %w", err) + return fmt.Errorf("failed to download k0sctl: %w", err) } + // Generate k0sctl configuration + log.Println("Generating k0sctl configuration from install-config...") + k0sctlConfig, err := installer.GenerateK0sctlConfig(config, k0sVersion, c.Opts.SSHKeyPath, k0sBinaryPath) + if err != nil { + return fmt.Errorf("failed to generate k0sctl config: %w", err) + } + + // Write k0sctl config to file + k0sctlConfigData, err := k0sctlConfig.Marshal() + if err != nil { + return fmt.Errorf("failed to marshal k0sctl config: %w", err) + } + + k0sctlConfigPath := filepath.Join(c.Env.GetOmsWorkdir(), k0sctlConfigFile) + if err := c.FileWriter.WriteFile(k0sctlConfigPath, k0sctlConfigData, 0644); err != nil { + return fmt.Errorf("failed to write k0sctl config: %w", err) + } + + log.Printf("Generated k0sctl configuration at %s", k0sctlConfigPath) + + // Apply k0sctl configuration + log.Println("Applying k0sctl configuration to deploy k0s cluster...") + if err := k0sctl.Apply(k0sctlConfigPath, k0sctlPath, c.Opts.Force); err != nil { + return fmt.Errorf("failed to apply k0sctl config: %w", err) + } + + log.Println("k0s cluster deployed successfully!") + log.Printf("To manage your cluster, use: %s kubeconfig --config %s", k0sctlPath, k0sctlConfigPath) + return nil } diff --git a/cli/cmd/install_k0s_integration_test.go b/cli/cmd/install_k0s_integration_test.go new file mode 100644 index 0000000..28a5523 --- /dev/null +++ b/cli/cmd/install_k0s_integration_test.go @@ -0,0 +1,409 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build integration +// +build integration + +package cmd_test + +import ( + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("K0s Install-Config Integration", func() { + var ( + tempDir string + configPath string + k0sConfigOut string + ) + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "k0s-integration-test-*") + Expect(err).NotTo(HaveOccurred()) + + configPath = filepath.Join(tempDir, "install-config.yaml") + k0sConfigOut = filepath.Join(tempDir, "k0s-config.yaml") + }) + + AfterEach(func() { + if tempDir != "" { + os.RemoveAll(tempDir) + } + }) + + createBaseConfig := func(name string, ip string) *files.RootConfig { + return &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: name, + City: "Test City", + CountryCode: "US", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: ip}, + }, + APIServerHost: "api.test.example.com", + }, + Codesphere: files.CodesphereConfig{ + Domain: "test.example.com", + PublicIP: ip, + DeployConfig: files.DeployConfig{ + Images: map[string]files.ImageConfig{}, + }, + Plans: files.PlansConfig{ + HostingPlans: map[int]files.HostingPlan{}, + WorkspacePlans: map[int]files.WorkspacePlan{}, + }, + }, + } + } + + Describe("Complete Workflow", func() { + It("should generate valid k0s config from install-config file", func() { + installConfig := createBaseConfig("test-dc", "192.168.1.100") + + // Write and load install-config + configData, err := yaml.Marshal(installConfig) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).NotTo(HaveOccurred()) + + loadedConfig := icg.GetInstallConfig() + Expect(loadedConfig).NotTo(BeNil()) + Expect(loadedConfig.Kubernetes.ManagedByCodesphere).To(BeTrue()) + + // Generate k0s config + k0sConfig, err := installer.GenerateK0sConfig(loadedConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig).NotTo(BeNil()) + + // Verify k0s config structure + Expect(k0sConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(k0sConfig.Kind).To(Equal("ClusterConfig")) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + Expect(k0sConfig.Spec.API.Address).To(Equal("192.168.1.100")) + Expect(k0sConfig.Spec.API.ExternalAddress).To(Equal("api.test.example.com")) + + // Write k0s config to file and verify + k0sData, err := k0sConfig.Marshal() + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(k0sConfigOut, k0sData, 0644) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfigOut).To(BeAnExistingFile()) + data, err := os.ReadFile(k0sConfigOut) + Expect(err).NotTo(HaveOccurred()) + + var verifyConfig installer.K0sConfig + err = yaml.Unmarshal(data, &verifyConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(verifyConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(verifyConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + }) + }) + + Describe("Configuration Features", func() { + It("should handle multi-control-plane configuration", func() { + installConfig := createBaseConfig("multi-dc", "10.0.0.10") + installConfig.Kubernetes.ControlPlanes = []files.K8sNode{ + {IPAddress: "10.0.0.10"}, + {IPAddress: "10.0.0.11"}, + {IPAddress: "10.0.0.12"}, + } + installConfig.Kubernetes.APIServerHost = "api.cluster.test" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify primary IP is used + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.0.10")) + // Verify all IPs are in SANs + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.10")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.11")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("10.0.0.12")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("api.cluster.test")) + }) + + It("should preserve custom network configuration", func() { + installConfig := createBaseConfig("network-test", "192.168.1.100") + installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" + installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + Expect(k0sConfig.Spec.Network.PodCIDR).To(Equal("10.244.0.0/16")) + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(Equal("10.96.0.0/12")) + }) + + It("should configure etcd storage correctly", func() { + installConfig := createBaseConfig("storage-test", "192.168.1.100") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.Storage).NotTo(BeNil()) + Expect(k0sConfig.Spec.Storage.Type).To(Equal("etcd")) + Expect(k0sConfig.Spec.Storage.Etcd).NotTo(BeNil()) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("192.168.1.100")) + }) + + It("should generate correct cluster name from datacenter", func() { + installConfig := createBaseConfig("prod-us-east", "10.1.2.3") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-prod-us-east")) + }) + + It("should handle empty control plane list", func() { + installConfig := createBaseConfig("empty-cp", "10.0.0.1") + installConfig.Kubernetes.ControlPlanes = []files.K8sNode{} + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + // Should either handle gracefully or error + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should use default network values when not specified", func() { + installConfig := createBaseConfig("defaults-test", "10.0.0.1") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify defaults are applied or fields are present + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + if k0sConfig.Spec.Network.PodCIDR != "" { + Expect(k0sConfig.Spec.Network.PodCIDR).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+/\d+$`)) + } + if k0sConfig.Spec.Network.ServiceCIDR != "" { + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(MatchRegexp(`^\d+\.\d+\.\d+\.\d+/\d+$`)) + } + }) + + It("should handle special characters in datacenter names", func() { + testCases := []struct { + name string + expected string + }{ + {"test-dc-01", "codesphere-test-dc-01"}, + {"test_dc_02", "codesphere-test_dc_02"}, + {"TestDC03", "codesphere-TestDC03"}, + } + + for _, tc := range testCases { + installConfig := createBaseConfig(tc.name, "10.0.0.1") + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal(tc.expected)) + } + }) + + It("should handle large multi-control-plane setup", func() { + installConfig := createBaseConfig("large-cluster", "10.0.1.1") + controlPlanes := make([]files.K8sNode, 7) + for i := 0; i < 7; i++ { + controlPlanes[i] = files.K8sNode{ + IPAddress: fmt.Sprintf("10.0.1.%d", i+1), + } + } + installConfig.Kubernetes.ControlPlanes = controlPlanes + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.1.1")) + Expect(len(k0sConfig.Spec.API.SANs)).To(BeNumerically(">=", 7)) + }) + + It("should properly configure certificate SANs", func() { + installConfig := createBaseConfig("san-test", "192.168.100.50") + installConfig.Kubernetes.APIServerHost = "k8s.example.com" + installConfig.Codesphere.Domain = "app.example.com" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("192.168.100.50")) + Expect(k0sConfig.Spec.API.SANs).To(ContainElement("k8s.example.com")) + Expect(len(k0sConfig.Spec.API.SANs)).To(BeNumerically(">=", 2)) + }) + }) + + Describe("Error Handling", func() { + It("should fail when loading non-existent file", func() { + nonExistentPath := filepath.Join(tempDir, "does-not-exist.yaml") + icg := installer.NewInstallConfigManager() + err := icg.LoadInstallConfigFromFile(nonExistentPath) + Expect(err).To(HaveOccurred()) + }) + + It("should fail when generating config from nil", func() { + _, err := installer.GenerateK0sConfig(nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot be nil")) + }) + + It("should fail when loading invalid YAML", func() { + invalidYAML := []byte("invalid: [unclosed bracket") + err := os.WriteFile(configPath, invalidYAML, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).To(HaveOccurred()) + }) + + It("should handle empty file gracefully", func() { + err := os.WriteFile(configPath, []byte{}, 0644) + Expect(err).NotTo(HaveOccurred()) + + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + // Empty file loads successfully but returns empty config + Expect(err).NotTo(HaveOccurred()) + config := icg.GetInstallConfig() + Expect(config).NotTo(BeNil()) + }) + + It("should handle external Kubernetes cluster config", func() { + installConfig := createBaseConfig("external-k8s", "10.0.0.1") + installConfig.Kubernetes.ManagedByCodesphere = false + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig).NotTo(BeNil()) + }) + + It("should handle missing APIServerHost", func() { + installConfig := createBaseConfig("missing-host", "10.0.0.1") + installConfig.Kubernetes.APIServerHost = "" + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.0.1")) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should handle missing datacenter name", func() { + installConfig := createBaseConfig("", "10.0.0.1") + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + if err == nil { + Expect(k0sConfig).NotTo(BeNil()) + } else { + Expect(err).To(HaveOccurred()) + } + }) + + It("should fail when writing to read-only directory", func() { + readOnlyDir := filepath.Join(tempDir, "readonly") + err := os.Mkdir(readOnlyDir, 0444) + Expect(err).NotTo(HaveOccurred()) + + readOnlyPath := filepath.Join(readOnlyDir, "config.yaml") + installConfig := createBaseConfig("test", "10.0.0.1") + configData, err := yaml.Marshal(installConfig) + Expect(err).NotTo(HaveOccurred()) + + err = os.WriteFile(readOnlyPath, configData, 0644) + Expect(err).To(HaveOccurred()) + + os.Chmod(readOnlyDir, 0755) + }) + }) + + Describe("YAML Serialization", func() { + It("should marshal and unmarshal k0s config correctly", func() { + installConfig := createBaseConfig("roundtrip-test", "172.16.0.1") + installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" + installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" + + original, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + + yamlData, err := original.Marshal() + Expect(err).NotTo(HaveOccurred()) + Expect(string(yamlData)).To(ContainSubstring("k0s.k0sproject.io/v1beta1")) + Expect(string(yamlData)).To(ContainSubstring("ClusterConfig")) + + var restored installer.K0sConfig + err = yaml.Unmarshal(yamlData, &restored) + Expect(err).NotTo(HaveOccurred()) + + // Verify critical fields match + Expect(restored.APIVersion).To(Equal(original.APIVersion)) + Expect(restored.Kind).To(Equal(original.Kind)) + Expect(restored.Metadata.Name).To(Equal(original.Metadata.Name)) + Expect(restored.Spec.API.Address).To(Equal(original.Spec.API.Address)) + Expect(restored.Spec.Network.PodCIDR).To(Equal(original.Spec.Network.PodCIDR)) + Expect(restored.Spec.Network.ServiceCIDR).To(Equal(original.Spec.Network.ServiceCIDR)) + }) + }) + + Describe("Config Persistence", func() { + It("should persist and reload config correctly", func() { + originalConfig := createBaseConfig("persist-test", "172.20.30.40") + originalConfig.Kubernetes.PodCIDR = "10.100.0.0/16" + originalConfig.Kubernetes.ServiceCIDR = "10.200.0.0/16" + + // Save install-config + configData, err := yaml.Marshal(originalConfig) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Generate and save k0s config + k0sConfig, err := installer.GenerateK0sConfig(originalConfig) + Expect(err).NotTo(HaveOccurred()) + k0sData, err := k0sConfig.Marshal() + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(k0sConfigOut, k0sData, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Reload install-config + icg := installer.NewInstallConfigManager() + err = icg.LoadInstallConfigFromFile(configPath) + Expect(err).NotTo(HaveOccurred()) + reloadedInstallConfig := icg.GetInstallConfig() + + // Reload k0s config + reloadedK0sData, err := os.ReadFile(k0sConfigOut) + Expect(err).NotTo(HaveOccurred()) + var reloadedK0sConfig installer.K0sConfig + err = yaml.Unmarshal(reloadedK0sData, &reloadedK0sConfig) + Expect(err).NotTo(HaveOccurred()) + + // Verify both configs match original + Expect(reloadedInstallConfig.Datacenter.Name).To(Equal(originalConfig.Datacenter.Name)) + Expect(reloadedInstallConfig.Kubernetes.PodCIDR).To(Equal(originalConfig.Kubernetes.PodCIDR)) + Expect(reloadedK0sConfig.Metadata.Name).To(Equal(k0sConfig.Metadata.Name)) + Expect(reloadedK0sConfig.Spec.API.Address).To(Equal(k0sConfig.Spec.API.Address)) + Expect(reloadedK0sConfig.Spec.Network.PodCIDR).To(Equal(k0sConfig.Spec.Network.PodCIDR)) + }) + }) +}) diff --git a/cli/cmd/install_k0s_test.go b/cli/cmd/install_k0s_test.go index 8c59b54..32de387 100644 --- a/cli/cmd/install_k0s_test.go +++ b/cli/cmd/install_k0s_test.go @@ -4,14 +4,18 @@ package cmd_test import ( - "errors" + "os" + "path/filepath" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "gopkg.in/yaml.v3" "github.com/codesphere-cloud/oms/cli/cmd" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" "github.com/codesphere-cloud/oms/internal/util" ) @@ -32,7 +36,7 @@ var _ = Describe("InstallK0sCmd", func() { GlobalOptions: globalOpts, Version: "", Package: "", - Config: "", + InstallConfig: "", Force: false, } c = cmd.InstallK0sCmd{ @@ -47,63 +51,203 @@ var _ = Describe("InstallK0sCmd", func() { mockFileWriter.AssertExpectations(GinkgoT()) }) + Context("RunE method", func() { + It("fails when install-config is not provided", func() { + c.Opts.InstallConfig = "" + mockEnv.EXPECT().GetOmsWorkdir().Return("/test/workdir") + + err := c.RunE(nil, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("install-config")) + }) + }) + Context("InstallK0s method", func() { - It("fails when package is not specified and k0s download fails", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + var ( + mockPM *installer.MockPackageManager + mockK0s *installer.MockK0sManager + mockK0sctl *installer.MockK0sctlManager + tempDir string + ) + + BeforeEach(func() { + mockPM = installer.NewMockPackageManager(GinkgoT()) + mockK0s = installer.NewMockK0sManager(GinkgoT()) + mockK0sctl = installer.NewMockK0sctlManager(GinkgoT()) + var err error + tempDir, err = os.MkdirTemp("", "install-k0s-test-*") + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + mockPM.AssertExpectations(GinkgoT()) + mockK0s.AssertExpectations(GinkgoT()) + mockK0sctl.AssertExpectations(GinkgoT()) + if tempDir != "" { + _ = os.RemoveAll(tempDir) + } + }) + + createTestConfig := func(managedByCodesphere bool) *files.RootConfig { + return &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + City: "Test City", + CountryCode: "US", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: managedByCodesphere, + ControlPlanes: []files.K8sNode{ + {IPAddress: "192.168.1.100"}, + }, + APIServerHost: "api.test.example.com", + }, + Codesphere: files.CodesphereConfig{ + Domain: "test.example.com", + PublicIP: "192.168.1.100", + DeployConfig: files.DeployConfig{ + Images: map[string]files.ImageConfig{}, + }, + Plans: files.PlansConfig{ + HostingPlans: map[int]files.HostingPlan{}, + WorkspacePlans: map[int]files.WorkspacePlan{}, + }, + }, + } + } - c.Opts.Package = "" // No package specified, should download - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", false, false).Return("", errors.New("download failed")) + It("fails when install-config file does not exist", func() { + c.Opts.InstallConfig = "/nonexistent/install-config.yaml" - err := c.InstallK0s(mockPackageManager, mockK0sManager) + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to download k0s")) - Expect(err.Error()).To(ContainSubstring("download failed")) + Expect(err.Error()).To(ContainSubstring("failed to load install-config")) }) - It("fails when k0s install fails", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + It("fails when install-config specifies external Kubernetes", func() { + config := createTestConfig(false) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) - c.Opts.Package = "" // No package specified, should download - c.Opts.Config = "/path/to/config.yaml" - c.Opts.Force = true - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", true, false).Return("/test/workdir/k0s", nil) - mockK0sManager.EXPECT().Install("/path/to/config.yaml", "/test/workdir/k0s", true).Return(errors.New("install failed")) + c.Opts.InstallConfig = configPath - err := c.InstallK0s(mockPackageManager, mockK0sManager) + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to install k0s")) - Expect(err.Error()).To(ContainSubstring("install failed")) + Expect(err.Error()).To(ContainSubstring("external Kubernetes")) }) - It("succeeds when package is not specified and k0s download and install work", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + It("successfully installs k0s with valid config using k0sctl", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Force = true - c.Opts.Package = "" // No package specified, should download - c.Opts.Config = "" // No config, will use single mode - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Download("", false, false).Return("/test/workdir/k0s", nil) - mockK0sManager.EXPECT().Install("", "/test/workdir/k0s", false).Return(nil) + mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) + mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", true).Return(nil) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0sctl.EXPECT().Download("", true, false).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", true).Return(nil) - err := c.InstallK0s(mockPackageManager, mockK0sManager) - Expect(err).ToNot(HaveOccurred()) + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) }) - It("succeeds when package is specified and k0s install works", func() { - mockPackageManager := installer.NewMockPackageManager(GinkgoT()) - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + It("downloads k0s when package is not specified", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "" + c.Opts.Version = "v1.29.0+k0s.0" + + mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) + mockK0s.EXPECT().Download("v1.29.0+k0s.0", false, false).Return("/downloaded/k0s", nil) + mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) + + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) - c.Opts.Package = "test-package.tar.gz" // Package specified, should use k0s from package - c.Opts.Config = "/path/to/config.yaml" - mockPackageManager.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/workdir/test-package/deps/kubernetes/files/k0s") - mockK0sManager.EXPECT().Install("/path/to/config.yaml", "/test/workdir/test-package/deps/kubernetes/files/k0s", false).Return(nil) + It("fails when k0s download fails", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) - err := c.InstallK0s(mockPackageManager, mockK0sManager) - Expect(err).ToNot(HaveOccurred()) + c.Opts.InstallConfig = configPath + c.Opts.Package = "" + + mockK0s.EXPECT().GetLatestVersion().Return("v1.30.0+k0s.0", nil) + mockK0s.EXPECT().Download("v1.30.0+k0s.0", false, false).Return("", os.ErrNotExist) + + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to download k0s")) + }) + + It("fails when k0sctl download fails", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" + c.Opts.Version = "v1.30.0+k0s.0" + + mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false).Return(nil) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0sctl.EXPECT().Download("", false, false).Return("", os.ErrPermission) + + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to download k0sctl")) + }) + + It("fails when k0sctl apply fails", func() { + config := createTestConfig(true) + configPath := filepath.Join(tempDir, "install-config.yaml") + configData, err := yaml.Marshal(config) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(configPath, configData, 0644) + Expect(err).NotTo(HaveOccurred()) + + c.Opts.InstallConfig = configPath + c.Opts.Package = "test-package.tar.gz" + c.Opts.Version = "v1.30.0+k0s.0" + + mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) + mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false).Return(nil) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(os.ErrPermission) + + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to apply k0sctl config")) }) }) }) diff --git a/docs/oms-cli_install_k0s.md b/docs/oms-cli_install_k0s.md index 785e4f8..023d66d 100644 --- a/docs/oms-cli_install_k0s.md +++ b/docs/oms-cli_install_k0s.md @@ -5,10 +5,12 @@ Install k0s Kubernetes distribution ### Synopsis Install k0s either from the package or by downloading it. -This will either download the k0s binary directly to the OMS workdir, if not already present, and install it -or load the k0s binary from the provided package file and install it. -If no version is specified, the latest version will be downloaded. -If no install config is provided, k0s will be installed with the '--single' flag. +This command uses k0sctl to deploy k0s clusters from a Codesphere install-config. + +You must provide a Codesphere install-config file, which will: +- Generate a k0s configuration from the install-config +- Generate a k0sctl configuration for cluster deployment +- Deploy k0s to all nodes defined in the install-config using k0sctl ``` oms-cli install k0s [flags] @@ -17,31 +19,40 @@ oms-cli install k0s [flags] ### Examples ``` -# Install k0s using the Go-native implementation -$ oms-cli install k0s +# Path to Codesphere install-config file to generate k0s config from +$ oms-cli install k0s --install-config -# Version of k0s to install +# Version of k0s to install (e.g., v1.30.0+k0s.0) $ oms-cli install k0s --version +# Version of k0sctl to use (e.g., v0.17.4) +$ oms-cli install k0s --k0sctl-version + # Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from $ oms-cli install k0s --package -# Path to k0s configuration file, if not set k0s will be installed with the '--single' flag -$ oms-cli install k0s --k0s-config +# SSH private key path for remote installation +$ oms-cli install k0s --ssh-key-path -# Force new download and installation even if k0s binary exists or is already installed +# Force new download and installation $ oms-cli install k0s --force +# Skip downloading k0s binary (expects it to be on remote nodes) +$ oms-cli install k0s --no-download + ``` ### Options ``` - -f, --force Force new download and installation - -h, --help help for k0s - --k0s-config string Path to k0s configuration file - -p, --package string Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from - -v, --version string Version of k0s to install + -f, --force Force new download and installation + -h, --help help for k0s + --install-config string Path to Codesphere install-config file (required) + --k0sctl-version string Version of k0sctl to use + --no-download Skip downloading k0s binary + -p, --package string Package file (e.g. codesphere-v1.2.3-installer.tar.gz) to load k0s from + --ssh-key-path string SSH private key path for remote installation + -v, --version string Version of k0s to install ``` ### SEE ALSO diff --git a/go.mod b/go.mod index 3ca1a2a..3242b74 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( golang.org/x/term v0.39.0 google.golang.org/api v0.264.0 google.golang.org/grpc v1.78.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -538,7 +539,6 @@ require ( golang.org/x/text v0.33.0 // indirect golang.org/x/tools v0.41.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) tool ( diff --git a/hack/lima-oms.yaml b/hack/lima-oms.yaml index 8dabfd2..e0748fd 100644 --- a/hack/lima-oms.yaml +++ b/hack/lima-oms.yaml @@ -18,8 +18,8 @@ disk: "60GiB" # Mount your OMS project mounts: -- location: "." - mountPoint: "/home/user/oms" +- location: "~" + mountPoint: "/home/user/host-home" writable: true # Ports and SSH @@ -61,27 +61,110 @@ provision: set -eux -o pipefail # Install Docker in rootless mode - dockerd-rootless-setuptool.sh install + dockerd-rootless-setuptool.sh install || true - # Set up the OMS project - cd /home/user/oms - export PATH=$PATH:/usr/local/go/bin - go mod download - cd cli && go build -a -buildvcs=false && mv cli ../oms-cli + # Clone OMS repository locally for better build performance + if [ ! -d ~/oms ]; then + git clone https://github.com/codesphere-cloud/oms.git ~/oms + fi + + # Create a test install-config for k0s testing + VM_IP=$(hostname -I | awk '{print $1}') + cat > ~/oms/test-install-config.yaml << EOFCONFIG + datacenter: + id: 1 + name: test-dc + city: Test City + countryCode: US + kubernetes: + managedByCodesphere: true + controlPlanes: + - ipAddress: ${VM_IP} + apiServerHost: api.test.local + codesphere: + domain: test.local + publicIP: ${VM_IP} + deployConfig: + images: {} + plans: + hostingPlans: {} + workspacePlans: {} + EOFCONFIG message: | Your OMS development environment is ready! + VM IP: Run "hostname -I | awk '{print $1}'" to get your VM's IP address - To access it: + Quick Start: ------ limactl shell lima-oms - cd /home/user/oms - ./oms-cli --help + cd oms + make build-cli ------ - - To install Codesphere eg.: + + Test k0s installation locally: ------ - ./oms-cli install codesphere --package codesphere-v1.66.0-installer --config config.yaml --priv-key ./path-to-private-key + limactl shell lima-oms + cd oms + + # Build the CLI + make build-cli + + # Install k0s (uses test-install-config.yaml which is auto-generated) + sudo ./oms-cli install k0s --install-config test-install-config.yaml --version v1.30.0+k0s.0 --force + + # Check k0s installation + sudo systemctl status k0scontroller + sudo oms-workdir/k0s kubectl get nodes + + # Stop and cleanup k0s + sudo systemctl stop k0scontroller + sudo systemctl disable k0scontroller + sudo oms-workdir/k0s reset + ------ + + Check k0s config that was generated: + ------ + sudo cat /etc/k0s/k0s.yaml + ------ + + Test with package instead of download: + ------ + # First, create a package (run on your host machine with access to real install-config) + ./oms-cli download package --version v1.x.x + + # Then in Lima: + limactl shell lima-oms + cd oms + sudo ./oms-cli install k0s --install-config test-install-config.yaml \ + --package codesphere-vX.X.X-installer.tar.gz --force + ------ + + Run tests: + ------ + limactl shell lima-oms + cd oms + + # Run all tests + go test ./... + + # Run specific k0s tests + go test ./cli/cmd -ginkgo.v -ginkgo.focus "InstallK0s" + go test ./internal/installer -ginkgo.v -ginkgo.focus "k0s" + ------ + + Build for different platforms: + ------ + # From your Mac (cross-compile to Linux) + make build-cli-linux + + # Inside Lima (native Linux build) + limactl shell lima-oms + cd oms + make build-cli + ------ + + Notes: + - test-install-config.yaml is automatically created in ~/oms with your VM's IP + - Remote installation requires proper SSH setup ------ - - Go 1.24 and Docker are installed and ready to use. diff --git a/internal/installer/files/config_yaml.go b/internal/installer/files/config_yaml.go index 85cffa4..e4fffa5 100644 --- a/internal/installer/files/config_yaml.go +++ b/internal/installer/files/config_yaml.go @@ -160,7 +160,9 @@ type KubernetesConfig struct { } type K8sNode struct { - IPAddress string `yaml:"ipAddress"` + IPAddress string `yaml:"ipAddress"` + SSHAddress string `yaml:"sshAddress,omitempty"` + SSHPort int `yaml:"sshPort,omitempty"` } type ClusterConfig struct { diff --git a/internal/installer/k0s.go b/internal/installer/k0s.go index 1c350b3..0ef8b83 100644 --- a/internal/installer/k0s.go +++ b/internal/installer/k0s.go @@ -6,6 +6,7 @@ package installer import ( "fmt" "log" + "net" "os" "path/filepath" "runtime" @@ -19,7 +20,6 @@ import ( type K0sManager interface { GetLatestVersion() (string, error) Download(version string, force bool, quiet bool) (string, error) - Install(configPath string, k0sPath string, force bool) error } type K0s struct { @@ -62,6 +62,12 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { // Check if k0s binary already exists and create destination file workdir := k.Env.GetOmsWorkdir() + + // Ensure workdir exists + if err := os.MkdirAll(workdir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + k0sPath := filepath.Join(workdir, "k0s") if k.FileWriter.Exists(k0sPath) && !force { return "", fmt.Errorf("k0s binary already exists at %s. Use --force to overwrite", k0sPath) @@ -93,38 +99,40 @@ func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { return k0sPath, nil } -func (k *K0s) Install(configPath string, k0sPath string, force bool) error { - if k.Goos != "linux" || k.Goarch != "amd64" { - return fmt.Errorf("k0s installation is only supported on Linux amd64. Current platform: %s/%s", k.Goos, k.Goarch) - } - - if !k.FileWriter.Exists(k0sPath) { - return fmt.Errorf("k0s binary does not exist in '%s', please download first", k0sPath) - } - - args := []string{k0sPath, "install", "controller"} - if configPath != "" { - args = append(args, "--config", configPath) - } else { - args = append(args, "--single") +// GetNodeIPAddress finds the IP address of the current node by matching +// against the control plane IPs in the config. Returns matching control plane IP +// if found, otherwise returns the first non-loopback IPv4 address. +func GetNodeIPAddress(controlPlanes []string) (string, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return "", fmt.Errorf("failed to get network interfaces: %w", err) } - if force { - args = append(args, "--force") + // Build a set of control plane IPs for O(1) lookup + cpSet := make(map[string]bool, len(controlPlanes)) + for _, ip := range controlPlanes { + cpSet[ip] = true } - err := util.RunCommand("sudo", args, "") - if err != nil { - return fmt.Errorf("failed to install k0s: %w", err) + var fallbackIP string + for _, addr := range addrs { + ipnet, ok := addr.(*net.IPNet) + if !ok || ipnet.IP.IsLoopback() || ipnet.IP.To4() == nil { + continue + } + + ip := ipnet.IP.String() + if cpSet[ip] { + return ip, nil + } + if fallbackIP == "" { + fallbackIP = ip + } } - if configPath != "" { - log.Println("k0s installed successfully with provided configuration.") - } else { - log.Println("k0s installed successfully in single-node mode.") + if fallbackIP != "" { + return fallbackIP, nil } - log.Printf("You can start it using 'sudo %v start'", k0sPath) - log.Printf("You can check the status using 'sudo %v status'", k0sPath) - return nil + return "", fmt.Errorf("no suitable IP address found") } diff --git a/internal/installer/k0s_config.go b/internal/installer/k0s_config.go new file mode 100644 index 0000000..29b99b0 --- /dev/null +++ b/internal/installer/k0s_config.go @@ -0,0 +1,145 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "fmt" + + "github.com/codesphere-cloud/oms/internal/installer/files" + "gopkg.in/yaml.v3" +) + +type K0sConfig struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata K0sMetadata `yaml:"metadata"` + Spec K0sSpec `yaml:"spec"` +} + +type K0sMetadata struct { + Name string `yaml:"name"` +} + +type K0sSpec struct { + API *K0sAPI `yaml:"api,omitempty"` + Network *K0sNetwork `yaml:"network,omitempty"` + Storage *K0sStorage `yaml:"storage,omitempty"` + Images *K0sImages `yaml:"images,omitempty"` + Telemetry *K0sTelemetry `yaml:"telemetry,omitempty"` + Konnectivity *K0sKonnectivity `yaml:"konnectivity,omitempty"` +} + +type K0sAPI struct { + Address string `yaml:"address,omitempty"` + ExternalAddress string `yaml:"externalAddress,omitempty"` + SANs []string `yaml:"sans,omitempty"` + Port int `yaml:"port,omitempty"` +} + +type K0sNetwork struct { + PodCIDR string `yaml:"podCIDR,omitempty"` + ServiceCIDR string `yaml:"serviceCIDR,omitempty"` + Provider string `yaml:"provider,omitempty"` +} + +type K0sStorage struct { + Type string `yaml:"type,omitempty"` + Etcd *K0sEtcd `yaml:"etcd,omitempty"` +} + +type K0sEtcd struct { + PeerAddress string `yaml:"peerAddress,omitempty"` +} + +type K0sImages struct { + DefaultPullPolicy string `yaml:"default_pull_policy,omitempty"` +} + +type K0sTelemetry struct { + Enabled bool `yaml:"enabled"` +} + +type K0sKonnectivity struct { + AdminPort int `yaml:"adminPort,omitempty"` + AgentPort int `yaml:"agentPort,omitempty"` +} + +func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { + if installConfig == nil { + return nil, fmt.Errorf("installConfig cannot be nil") + } + + k0sConfig := &K0sConfig{ + APIVersion: "k0s.k0sproject.io/v1beta1", + Kind: "ClusterConfig", + Metadata: K0sMetadata{ + Name: fmt.Sprintf("codesphere-%s", installConfig.Datacenter.Name), + }, + Spec: K0sSpec{}, + } + + if installConfig.Kubernetes.ManagedByCodesphere { + if len(installConfig.Kubernetes.ControlPlanes) > 0 { + firstControlPlaneIP := installConfig.Kubernetes.ControlPlanes[0].IPAddress + + sans := make([]string, 0, len(installConfig.Kubernetes.ControlPlanes)+1) + for _, cp := range installConfig.Kubernetes.ControlPlanes { + sans = append(sans, cp.IPAddress) + } + if installConfig.Kubernetes.APIServerHost != "" { + sans = append(sans, installConfig.Kubernetes.APIServerHost) + } + + k0sConfig.Spec.API = &K0sAPI{ + Address: firstControlPlaneIP, + ExternalAddress: installConfig.Kubernetes.APIServerHost, + SANs: sans, + Port: 6443, + } + + k0sConfig.Spec.Storage = &K0sStorage{ + Type: "etcd", + Etcd: &K0sEtcd{ + PeerAddress: firstControlPlaneIP, + }, + } + } + + k0sConfig.Spec.Network = &K0sNetwork{ + Provider: "calico", + PodCIDR: defaultIfEmpty(installConfig.Kubernetes.PodCIDR, "100.96.0.0/11"), + ServiceCIDR: defaultIfEmpty(installConfig.Kubernetes.ServiceCIDR, "100.64.0.0/13"), + } + + k0sConfig.Spec.Images = &K0sImages{ + DefaultPullPolicy: "Never", + } + + k0sConfig.Spec.Telemetry = &K0sTelemetry{ + Enabled: false, + } + + k0sConfig.Spec.Konnectivity = &K0sKonnectivity{ + AdminPort: 8133, + AgentPort: 8132, + } + } + + return k0sConfig, nil +} + +func defaultIfEmpty(value, defaultValue string) string { + if value != "" { + return value + } + return defaultValue +} + +func (c *K0sConfig) Marshal() ([]byte, error) { + return yaml.Marshal(c) +} + +func (c *K0sConfig) Unmarshal(data []byte) error { + return yaml.Unmarshal(data, c) +} diff --git a/internal/installer/k0s_config_test.go b/internal/installer/k0s_config_test.go new file mode 100644 index 0000000..89efeae --- /dev/null +++ b/internal/installer/k0s_config_test.go @@ -0,0 +1,348 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("K0sConfig", func() { + Describe("GenerateK0sConfig", func() { + Context("with valid install-config", func() { + It("should generate k0s config with control plane settings", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + APIServerHost: "k8s.example.com", + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + {IPAddress: "10.0.1.11"}, + {IPAddress: "10.0.1.12"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + + // Check basic structure + Expect(k0sConfig.APIVersion).To(Equal("k0s.k0sproject.io/v1beta1")) + Expect(k0sConfig.Kind).To(Equal("ClusterConfig")) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + + // Check API configuration + Expect(k0sConfig.Spec.API).ToNot(BeNil()) + Expect(k0sConfig.Spec.API.Address).To(Equal("10.0.1.10")) + Expect(k0sConfig.Spec.API.ExternalAddress).To(Equal("k8s.example.com")) + Expect(k0sConfig.Spec.API.Port).To(Equal(6443)) + Expect(k0sConfig.Spec.API.SANs).To(ContainElements("10.0.1.10", "10.0.1.11", "10.0.1.12", "k8s.example.com")) + + // Check Network configuration + Expect(k0sConfig.Spec.Network).ToNot(BeNil()) + Expect(k0sConfig.Spec.Network.PodCIDR).To(Equal("10.244.0.0/16")) + Expect(k0sConfig.Spec.Network.ServiceCIDR).To(Equal("10.96.0.0/12")) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + Expect(k0sConfig.Spec.Storage.Etcd).ToNot(BeNil()) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("10.0.1.10")) + + // Check Konnectivity configuration (prevents webhook failures) + Expect(k0sConfig.Spec.Konnectivity).ToNot(BeNil()) + Expect(k0sConfig.Spec.Konnectivity.AdminPort).To(Equal(8133)) + Expect(k0sConfig.Spec.Konnectivity.AgentPort).To(Equal(8132)) + }) + + It("should handle minimal configuration", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "minimal", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "192.168.1.100"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-minimal")) + }) + + It("should generate valid YAML", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + + yamlData, err := k0sConfig.Marshal() + Expect(err).ToNot(HaveOccurred()) + Expect(yamlData).ToNot(BeEmpty()) + + // Verify it can be unmarshalled back + var parsedConfig installer.K0sConfig + err = yaml.Unmarshal(yamlData, &parsedConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + }) + }) + + Context("with invalid input", func() { + It("should return error for nil install-config", func() { + k0sConfig, err := installer.GenerateK0sConfig(nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("installConfig cannot be nil")) + Expect(k0sConfig).To(BeNil()) + }) + }) + + Context("with non-managed Kubernetes", func() { + It("should not configure k0s for external kubernetes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "external", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: false, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sConfig).ToNot(BeNil()) + // Should still have basic structure but no specific config + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-external")) + }) + }) + + Context("edge cases and validation", func() { + It("should handle empty datacenter name", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-")) + }) + + It("should handle empty control plane list", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{}, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + // Should have basic structure but no API/Storage config + Expect(k0sConfig.Spec.API).To(BeNil()) + Expect(k0sConfig.Spec.Storage).To(BeNil()) + }) + + It("should handle nil control plane addresses", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: nil, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API).To(BeNil()) + }) + + It("should handle missing APIServerHost", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + APIServerHost: "", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.ExternalAddress).To(BeEmpty()) + Expect(k0sConfig.Spec.API.SANs).To(ConsistOf("10.0.0.1")) + }) + + It("should handle missing network CIDRs", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + PodCIDR: "", + ServiceCIDR: "", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Network).NotTo(BeNil()) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + }) + + It("should use default network provider", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Network.Provider).To(Equal("calico")) + }) + + It("should generate correct SANs with single control plane", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + APIServerHost: "api.example.com", + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.SANs).To(HaveLen(2)) + Expect(k0sConfig.Spec.API.SANs).To(ContainElements("10.0.0.1", "api.example.com")) + }) + + It("should handle special characters in datacenter name", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc_01.prod", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Metadata.Name).To(Equal("codesphere-test-dc_01.prod")) + }) + + It("should set correct API port", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.API.Port).To(Equal(6443)) + }) + + It("should configure etcd with first control plane IP", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.0.1"}, + {IPAddress: "10.0.0.2"}, + }, + }, + } + + k0sConfig, err := installer.GenerateK0sConfig(installConfig) + Expect(err).NotTo(HaveOccurred()) + Expect(k0sConfig.Spec.Storage.Type).To(Equal("etcd")) + Expect(k0sConfig.Spec.Storage.Etcd.PeerAddress).To(Equal("10.0.0.1")) + }) + }) + }) +}) diff --git a/internal/installer/k0s_nodeip_test.go b/internal/installer/k0s_nodeip_test.go new file mode 100644 index 0000000..06a3bcd --- /dev/null +++ b/internal/installer/k0s_nodeip_test.go @@ -0,0 +1,137 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "net" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("GetNodeIPAddress", func() { + Context("with valid network setup", func() { + It("should return a valid IPv4 address", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err != nil { + Skip("No non-loopback network interfaces available on this system") + } + + Expect(ip).NotTo(BeEmpty()) + parsedIP := net.ParseIP(ip) + Expect(parsedIP).NotTo(BeNil()) + Expect(parsedIP.To4()).NotTo(BeNil()) + }) + + It("should not return loopback address", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err != nil { + Skip("No non-loopback network interfaces available") + } + + Expect(ip).NotTo(Equal("127.0.0.1")) + Expect(ip).NotTo(HavePrefix("127.")) + }) + }) + + Context("with control plane addresses provided", func() { + It("should prioritize control plane IP if available", func() { + interfaces, err := net.Interfaces() + if err != nil { + Skip("Cannot list network interfaces") + } + + var testIP string + for _, iface := range interfaces { + if iface.Flags&net.FlagLoopback != 0 { + continue + } + + addrs, err := iface.Addrs() + if err != nil { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip == nil || ip.IsLoopback() { + continue + } + + if ip.To4() != nil { + testIP = ip.String() + break + } + } + + if testIP != "" { + break + } + } + + if testIP == "" { + Skip("No suitable test IP found") + } + + result, err := GetNodeIPAddress([]string{testIP, "10.0.0.1"}) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(testIP)) + }) + + It("should fallback if control plane IPs don't match", func() { + result, err := GetNodeIPAddress([]string{"10.254.254.1", "192.168.254.254"}) + + if err == nil { + Expect(result).NotTo(BeEmpty()) + parsedIP := net.ParseIP(result) + Expect(parsedIP).NotTo(BeNil()) + Expect(parsedIP.To4()).NotTo(BeNil()) + } + }) + }) + + Context("with edge cases", func() { + It("should handle empty control plane list", func() { + ip, err := GetNodeIPAddress([]string{}) + + if err == nil { + Expect(ip).NotTo(BeEmpty()) + } else { + Expect(err.Error()).To(ContainSubstring("no suitable")) + } + }) + + It("should handle nil control plane list", func() { + ip, err := GetNodeIPAddress(nil) + + if err == nil { + Expect(ip).NotTo(BeEmpty()) + } else { + Expect(err.Error()).To(ContainSubstring("no suitable")) + } + }) + + It("should return error when no interfaces are available", func() { + ip, err := GetNodeIPAddress([]string{"invalid"}) + + if err != nil { + Expect(err.Error()).To(Or( + ContainSubstring("no suitable"), + ContainSubstring("network"), + )) + } else { + Expect(ip).NotTo(BeEmpty()) + } + }) + }) +}) diff --git a/internal/installer/k0s_test.go b/internal/installer/k0s_test.go index 83b9109..dbd36d8 100644 --- a/internal/installer/k0s_test.go +++ b/internal/installer/k0s_test.go @@ -281,44 +281,4 @@ var _ = Describe("K0s", func() { }) }) }) - - Describe("Install", func() { - Context("Platform support", func() { - It("should fail on non-Linux platforms", func() { - k0sImpl.Goos = "windows" - k0sImpl.Goarch = "amd64" - - err := k0s.Install("", k0sPath, false) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("k0s installation is only supported on Linux amd64")) - Expect(err.Error()).To(ContainSubstring("windows/amd64")) - }) - - It("should fail on non-amd64 architectures", func() { - k0sImpl.Goos = "linux" - k0sImpl.Goarch = "arm64" - - err := k0s.Install("", k0sPath, false) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("k0s installation is only supported on Linux amd64")) - Expect(err.Error()).To(ContainSubstring("linux/arm64")) - }) - }) - - Context("Binary existence checks", func() { - BeforeEach(func() { - k0sImpl.Goos = "linux" - k0sImpl.Goarch = "amd64" - }) - - It("should fail when k0s binary doesn't exist", func() { - mockFileWriter.EXPECT().Exists(k0sPath).Return(false) - - err := k0s.Install("", k0sPath, false) - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("k0s binary does not exist")) - Expect(err.Error()).To(ContainSubstring("please download first")) - }) - }) - }) }) diff --git a/internal/installer/k0sctl.go b/internal/installer/k0sctl.go new file mode 100644 index 0000000..ca9b5de --- /dev/null +++ b/internal/installer/k0sctl.go @@ -0,0 +1,177 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "encoding/json" + "fmt" + "log" + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" +) + +type K0sctlManager interface { + GetLatestVersion() (string, error) + Download(version string, force bool, quiet bool) (string, error) + Apply(configPath string, k0sctlPath string, force bool) error + Reset(configPath string, k0sctlPath string) error +} + +type K0sctl struct { + Env env.Env + Http portal.Http + FileWriter util.FileIO + Goos string + Goarch string +} + +func NewK0sctl(hw portal.Http, env env.Env, fw util.FileIO) K0sctlManager { + return &K0sctl{ + Env: env, + Http: hw, + FileWriter: fw, + Goos: runtime.GOOS, + Goarch: runtime.GOARCH, + } +} + +// githubRelease represents the minimal GitHub release API response +type githubRelease struct { + TagName string `json:"tag_name"` +} + +func (k *K0sctl) GetLatestVersion() (string, error) { + // k0sctl uses GitHub releases - fetch latest from API + releaseURL := "https://api.github.com/repos/k0sproject/k0sctl/releases/latest" + responseBody, err := k.Http.Get(releaseURL) + if err != nil { + return "", fmt.Errorf("failed to fetch latest k0sctl release: %w", err) + } + + var release githubRelease + if err := json.Unmarshal(responseBody, &release); err != nil { + return "", fmt.Errorf("failed to parse GitHub API response: %w", err) + } + + if release.TagName == "" { + return "", fmt.Errorf("no tag_name found in GitHub API response") + } + + return release.TagName, nil +} + +func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error) { + if version == "" { + var err error + version, err = k.GetLatestVersion() + if err != nil { + return "", fmt.Errorf("failed to get latest version: %w", err) + } + if !quiet { + log.Printf("Using latest k0sctl version: %s", version) + } + } + + // Ensure workdir exists + workdir := k.Env.GetOmsWorkdir() + if err := os.MkdirAll(workdir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + + k0sctlPath := filepath.Join(workdir, "k0sctl") + if k.FileWriter.Exists(k0sctlPath) && !force { + return "", fmt.Errorf("k0sctl binary already exists at %s. Use --force to overwrite", k0sctlPath) + } + + // Construct download URL + // Format: https://github.com/k0sproject/k0sctl/releases/download/v0.17.4/k0sctl-linux-amd64 + binaryName := fmt.Sprintf("k0sctl-%s-%s", k.Goos, k.Goarch) + // Ensure version has v prefix for GitHub URL + if !strings.HasPrefix(version, "v") { + version = "v" + version + } + downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0sctl/releases/download/%s/%s", version, binaryName) + + if !quiet { + log.Printf("Downloading k0sctl %s from %s", version, downloadURL) + } + + dstFile, err := k.FileWriter.Create(k0sctlPath) + if err != nil { + return "", fmt.Errorf("failed to create destination file: %w", err) + } + defer util.IgnoreError(dstFile.Close) + + if err := k.Http.Download(downloadURL, dstFile, quiet); err != nil { + return "", fmt.Errorf("failed to download k0sctl: %w", err) + } + + // Make binary executable + if err := os.Chmod(k0sctlPath, 0755); err != nil { + return "", fmt.Errorf("failed to make k0sctl executable: %w", err) + } + + if !quiet { + log.Printf("k0sctl downloaded successfully to %s", k0sctlPath) + } + + return k0sctlPath, nil +} + +func (k *K0sctl) Apply(configPath string, k0sctlPath string, force bool) error { + if !k.FileWriter.Exists(k0sctlPath) { + return fmt.Errorf("k0sctl binary does not exist at '%s', please download first", k0sctlPath) + } + + if !k.FileWriter.Exists(configPath) { + return fmt.Errorf("k0sctl config does not exist at '%s'", configPath) + } + + args := []string{"apply", "--config", configPath} + + if force { + args = append(args, "--force") + } + + // Add debug flag for more verbose output + args = append(args, "--debug") + + log.Printf("Running k0sctl apply with config: %s", configPath) + + err := util.RunCommand(k0sctlPath, args, "") + if err != nil { + return fmt.Errorf("k0sctl apply failed: %w", err) + } + + log.Println("k0sctl apply completed successfully") + return nil +} + +func (k *K0sctl) Reset(configPath string, k0sctlPath string) error { + if !k.FileWriter.Exists(k0sctlPath) { + return nil + } + + if !k.FileWriter.Exists(configPath) { + return fmt.Errorf("k0sctl config does not exist at '%s'", configPath) + } + + log.Println("Resetting k0s cluster using k0sctl...") + + args := []string{"reset", "--config", configPath, "--force"} + + err := util.RunCommand(k0sctlPath, args, "") + if err != nil { + return fmt.Errorf("k0sctl reset failed: %w", err) + } + + log.Println("k0sctl reset completed successfully") + return nil +} diff --git a/internal/installer/k0sctl_config.go b/internal/installer/k0sctl_config.go new file mode 100644 index 0000000..19c3757 --- /dev/null +++ b/internal/installer/k0sctl_config.go @@ -0,0 +1,179 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "fmt" + + "github.com/codesphere-cloud/oms/internal/installer/files" + "gopkg.in/yaml.v3" +) + +// K0sctlConfig represents the k0sctl configuration file structure +type K0sctlConfig struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata K0sctlMeta `yaml:"metadata,omitempty"` + Spec K0sctlSpec `yaml:"spec"` +} + +type K0sctlMeta struct { + Name string `yaml:"name"` +} + +type K0sctlSpec struct { + Hosts []K0sctlHost `yaml:"hosts"` + K0s K0sctlK0s `yaml:"k0s"` +} + +type K0sctlHost struct { + Role string `yaml:"role"` + SSH K0sctlSSH `yaml:"ssh"` + InstallFlags []string `yaml:"installFlags,omitempty"` + PrivateInterface string `yaml:"privateInterface,omitempty"` + PrivateAddress string `yaml:"privateAddress,omitempty"` + Environment map[string]string `yaml:"environment,omitempty"` + UploadBinary bool `yaml:"uploadBinary,omitempty"` + K0sBinaryPath string `yaml:"k0sBinaryPath,omitempty"` + Hooks *K0sctlHooks `yaml:"hooks,omitempty"` +} + +type K0sctlSSH struct { + Address string `yaml:"address"` + User string `yaml:"user"` + Port int `yaml:"port"` + KeyPath string `yaml:"keyPath,omitempty"` + Bastion *K0sctlBastion `yaml:"bastion,omitempty"` +} + +type K0sctlBastion struct { + Address string `yaml:"address"` + User string `yaml:"user"` + Port int `yaml:"port"` + KeyPath string `yaml:"keyPath,omitempty"` +} + +type K0sctlK0s struct { + Version string `yaml:"version"` + Config map[string]interface{} `yaml:"config,omitempty"` +} + +type K0sctlHooks struct { + Apply *K0sctlApplyHooks `yaml:"apply,omitempty"` +} + +type K0sctlApplyHooks struct { + Before []string `yaml:"before,omitempty"` + After []string `yaml:"after,omitempty"` +} + +func createK0sctlHost(node files.K8sNode, role string, installFlags []string, sshKeyPath string, k0sBinaryPath string) K0sctlHost { + sshPort := node.SSHPort + if sshPort == 0 { + sshPort = 22 + } + + sshAddress := node.SSHAddress + if sshAddress == "" { + sshAddress = node.IPAddress + } + + host := K0sctlHost{ + Role: role, + SSH: K0sctlSSH{ + Address: sshAddress, + User: "root", + Port: sshPort, + KeyPath: sshKeyPath, + }, + InstallFlags: installFlags, + PrivateAddress: node.IPAddress, + Environment: map[string]string{ + "KUBELET_EXTRA_ARGS": fmt.Sprintf("--node-ip=%s", node.IPAddress), + }, + } + + if k0sBinaryPath != "" { + host.UploadBinary = true + host.K0sBinaryPath = k0sBinaryPath + } + + return host +} + +// GenerateK0sctlConfig generates a k0sctl configuration from a Codesphere install-config +func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, sshKeyPath string, k0sBinaryPath string) (*K0sctlConfig, error) { + if installConfig == nil { + return nil, fmt.Errorf("installConfig cannot be nil") + } + + if !installConfig.Kubernetes.ManagedByCodesphere { + return nil, fmt.Errorf("k0sctl is only supported for Codesphere-managed Kubernetes") + } + + // Generate k0s config that will be embedded in k0sctl config + k0sConfig, err := GenerateK0sConfig(installConfig) + if err != nil { + return nil, fmt.Errorf("failed to generate k0s config: %w", err) + } + + // Convert K0sConfig struct to map for k0sctl + k0sConfigYAML, err := k0sConfig.Marshal() + if err != nil { + return nil, fmt.Errorf("failed to marshal k0s config: %w", err) + } + + var k0sConfigMap map[string]interface{} + if err := yaml.Unmarshal(k0sConfigYAML, &k0sConfigMap); err != nil { + return nil, fmt.Errorf("failed to unmarshal k0s config to map: %w", err) + } + + k0sctlConfig := &K0sctlConfig{ + APIVersion: "k0sctl.k0sproject.io/v1beta1", + Kind: "Cluster", + Metadata: K0sctlMeta{ + Name: fmt.Sprintf("codesphere-%s", installConfig.Datacenter.Name), + }, + Spec: K0sctlSpec{ + Hosts: []K0sctlHost{}, + K0s: K0sctlK0s{ + Version: k0sVersion, + Config: k0sConfigMap, + }, + }, + } + + // Track added IPs to avoid duplicates + addedIPs := make(map[string]bool) + + // Add controller+worker nodes from control planes + controllerFlags := []string{"--enable-worker", "--no-taints"} + for _, cp := range installConfig.Kubernetes.ControlPlanes { + host := createK0sctlHost(cp, "controller+worker", controllerFlags, sshKeyPath, k0sBinaryPath) + k0sctlConfig.Spec.Hosts = append(k0sctlConfig.Spec.Hosts, host) + addedIPs[cp.IPAddress] = true + } + + // Add dedicated worker nodes if present + for _, worker := range installConfig.Kubernetes.Workers { + if addedIPs[worker.IPAddress] { + continue + } + host := createK0sctlHost(worker, "worker", nil, sshKeyPath, k0sBinaryPath) + k0sctlConfig.Spec.Hosts = append(k0sctlConfig.Spec.Hosts, host) + addedIPs[worker.IPAddress] = true + } + + return k0sctlConfig, nil +} + +// Marshal serializes the k0sctl config to YAML +func (c *K0sctlConfig) Marshal() ([]byte, error) { + return yaml.Marshal(c) +} + +// Unmarshal deserializes YAML to a k0sctl config +func (c *K0sctlConfig) Unmarshal(data []byte) error { + return yaml.Unmarshal(data, c) +} diff --git a/internal/installer/k0sctl_config_test.go b/internal/installer/k0sctl_config_test.go new file mode 100644 index 0000000..5b1a9c8 --- /dev/null +++ b/internal/installer/k0sctl_config_test.go @@ -0,0 +1,485 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" + + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" +) + +var _ = Describe("K0sctlConfig", func() { + Describe("GenerateK0sctlConfig", func() { + Context("with valid install-config", func() { + It("should generate k0sctl config with control plane nodes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + APIServerHost: "k8s.example.com", + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + {IPAddress: "10.0.1.11"}, + {IPAddress: "10.0.1.12"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "/path/to/k0s") + Expect(err).ToNot(HaveOccurred()) + Expect(k0sctlConfig).ToNot(BeNil()) + + // Check basic structure + Expect(k0sctlConfig.APIVersion).To(Equal("k0sctl.k0sproject.io/v1beta1")) + Expect(k0sctlConfig.Kind).To(Equal("Cluster")) + Expect(k0sctlConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + + // Check k0s version + Expect(k0sctlConfig.Spec.K0s.Version).To(Equal("v1.30.0+k0s.0")) + + // Check hosts count matches control planes + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(3)) + }) + + It("should assign controller+worker role to control plane nodes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) + Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller+worker")) + Expect(k0sctlConfig.Spec.Hosts[0].InstallFlags).To(ContainElements("--enable-worker", "--no-taints")) + }) + + It("should assign worker role to dedicated worker nodes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + Workers: []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + {IPAddress: "10.0.2.11"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(3)) + + // First host should be controller+worker + Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller+worker")) + + // Worker nodes should have worker role with no install flags + Expect(k0sctlConfig.Spec.Hosts[1].Role).To(Equal("worker")) + Expect(k0sctlConfig.Spec.Hosts[1].InstallFlags).To(BeNil()) + Expect(k0sctlConfig.Spec.Hosts[2].Role).To(Equal("worker")) + }) + + It("should use SSHAddress when specified", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + { + IPAddress: "10.0.1.10", + SSHAddress: "ssh.example.com", + }, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.Address).To(Equal("ssh.example.com")) + Expect(k0sctlConfig.Spec.Hosts[0].PrivateAddress).To(Equal("10.0.1.10")) + }) + + It("should default SSHAddress to IPAddress when not specified", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.Address).To(Equal("10.0.1.10")) + }) + + It("should use SSHPort when specified", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + { + IPAddress: "10.0.1.10", + SSHPort: 2222, + }, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.Port).To(Equal(2222)) + }) + + It("should default SSHPort to 22 when not specified", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.Port).To(Equal(22)) + }) + + It("should skip duplicate IPs between control planes and workers", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + Workers: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, // Duplicate + {IPAddress: "10.0.2.10"}, // Unique + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + // Should only have 2 hosts: 1 control plane + 1 unique worker + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(2)) + Expect(k0sctlConfig.Spec.Hosts[0].SSH.Address).To(Equal("10.0.1.10")) + Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller+worker")) + Expect(k0sctlConfig.Spec.Hosts[1].SSH.Address).To(Equal("10.0.2.10")) + Expect(k0sctlConfig.Spec.Hosts[1].Role).To(Equal("worker")) + }) + + It("should enable UploadBinary when k0sBinaryPath is provided", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "/path/to/k0s") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].UploadBinary).To(BeTrue()) + Expect(k0sctlConfig.Spec.Hosts[0].K0sBinaryPath).To(Equal("/path/to/k0s")) + }) + + It("should not enable UploadBinary when k0sBinaryPath is empty", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].UploadBinary).To(BeFalse()) + Expect(k0sctlConfig.Spec.Hosts[0].K0sBinaryPath).To(BeEmpty()) + }) + + It("should set SSH key path correctly", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/home/user/.ssh/id_rsa", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.KeyPath).To(Equal("/home/user/.ssh/id_rsa")) + }) + + It("should set SSH user to root", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.User).To(Equal("root")) + }) + + It("should set KUBELET_EXTRA_ARGS environment variable", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].Environment).To(HaveKeyWithValue("KUBELET_EXTRA_ARGS", "--node-ip=10.0.1.10")) + }) + + It("should generate valid YAML", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + PodCIDR: "10.244.0.0/16", + ServiceCIDR: "10.96.0.0/12", + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + yamlData, err := k0sctlConfig.Marshal() + Expect(err).ToNot(HaveOccurred()) + Expect(yamlData).ToNot(BeEmpty()) + + // Verify it can be unmarshalled back + var parsedConfig installer.K0sctlConfig + err = yaml.Unmarshal(yamlData, &parsedConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedConfig.Metadata.Name).To(Equal("codesphere-test-dc")) + }) + }) + + Context("with invalid input", func() { + It("should return error for nil install-config", func() { + k0sctlConfig, err := installer.GenerateK0sctlConfig(nil, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("installConfig cannot be nil")) + Expect(k0sctlConfig).To(BeNil()) + }) + + It("should return error for non-managed Kubernetes", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: false, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("k0sctl is only supported for Codesphere-managed Kubernetes")) + Expect(k0sctlConfig).To(BeNil()) + }) + }) + + Context("edge cases", func() { + It("should handle empty control plane list", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{}, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + Expect(k0sctlConfig.Spec.Hosts).To(BeEmpty()) + }) + + It("should handle nil control plane list", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: nil, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + Expect(k0sctlConfig.Spec.Hosts).To(BeEmpty()) + }) + + It("should handle only worker nodes (no control planes)", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{}, + Workers: []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + // Workers should still be added even without control planes + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) + Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("worker")) + }) + + It("should handle empty SSH key path", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "", "") + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].SSH.KeyPath).To(BeEmpty()) + }) + + It("should set PrivateAddress for all host types", func() { + installConfig := &files.RootConfig{ + Datacenter: files.DatacenterConfig{ + ID: 1, + Name: "test-dc", + }, + Kubernetes: files.KubernetesConfig{ + ManagedByCodesphere: true, + ControlPlanes: []files.K8sNode{ + {IPAddress: "10.0.1.10", SSHAddress: "public1.example.com"}, + }, + Workers: []files.K8sNode{ + {IPAddress: "10.0.2.10", SSHAddress: "public2.example.com"}, + }, + }, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + Expect(err).ToNot(HaveOccurred()) + + // Both hosts should have PrivateAddress set to the internal IP + Expect(k0sctlConfig.Spec.Hosts[0].PrivateAddress).To(Equal("10.0.1.10")) + Expect(k0sctlConfig.Spec.Hosts[1].PrivateAddress).To(Equal("10.0.2.10")) + }) + }) + }) +}) diff --git a/internal/installer/mocks.go b/internal/installer/mocks.go index 79d4d7a..0bf9fda 100644 --- a/internal/installer/mocks.go +++ b/internal/installer/mocks.go @@ -859,37 +859,64 @@ func (_c *MockK0sManager_GetLatestVersion_Call) RunAndReturn(run func() (string, return _c } -// Install provides a mock function for the type MockK0sManager -func (_mock *MockK0sManager) Install(configPath string, k0sPath string, force bool) error { - ret := _mock.Called(configPath, k0sPath, force) +// NewMockK0sctlManager creates a new instance of MockK0sctlManager. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockK0sctlManager(t interface { + mock.TestingT + Cleanup(func()) +}) *MockK0sctlManager { + mock := &MockK0sctlManager{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// MockK0sctlManager is an autogenerated mock type for the K0sctlManager type +type MockK0sctlManager struct { + mock.Mock +} + +type MockK0sctlManager_Expecter struct { + mock *mock.Mock +} + +func (_m *MockK0sctlManager) EXPECT() *MockK0sctlManager_Expecter { + return &MockK0sctlManager_Expecter{mock: &_m.Mock} +} + +// Apply provides a mock function for the type MockK0sctlManager +func (_mock *MockK0sctlManager) Apply(configPath string, k0sctlPath string, force bool) error { + ret := _mock.Called(configPath, k0sctlPath, force) if len(ret) == 0 { - panic("no return value specified for Install") + panic("no return value specified for Apply") } var r0 error if returnFunc, ok := ret.Get(0).(func(string, string, bool) error); ok { - r0 = returnFunc(configPath, k0sPath, force) + r0 = returnFunc(configPath, k0sctlPath, force) } else { r0 = ret.Error(0) } return r0 } -// MockK0sManager_Install_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Install' -type MockK0sManager_Install_Call struct { +// MockK0sctlManager_Apply_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Apply' +type MockK0sctlManager_Apply_Call struct { *mock.Call } -// Install is a helper method to define mock.On call +// Apply is a helper method to define mock.On call // - configPath string -// - k0sPath string +// - k0sctlPath string // - force bool -func (_e *MockK0sManager_Expecter) Install(configPath interface{}, k0sPath interface{}, force interface{}) *MockK0sManager_Install_Call { - return &MockK0sManager_Install_Call{Call: _e.mock.On("Install", configPath, k0sPath, force)} +func (_e *MockK0sctlManager_Expecter) Apply(configPath interface{}, k0sctlPath interface{}, force interface{}) *MockK0sctlManager_Apply_Call { + return &MockK0sctlManager_Apply_Call{Call: _e.mock.On("Apply", configPath, k0sctlPath, force)} } -func (_c *MockK0sManager_Install_Call) Run(run func(configPath string, k0sPath string, force bool)) *MockK0sManager_Install_Call { +func (_c *MockK0sctlManager_Apply_Call) Run(run func(configPath string, k0sctlPath string, force bool)) *MockK0sctlManager_Apply_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { @@ -912,12 +939,194 @@ func (_c *MockK0sManager_Install_Call) Run(run func(configPath string, k0sPath s return _c } -func (_c *MockK0sManager_Install_Call) Return(err error) *MockK0sManager_Install_Call { +func (_c *MockK0sctlManager_Apply_Call) Return(err error) *MockK0sctlManager_Apply_Call { + _c.Call.Return(err) + return _c +} + +func (_c *MockK0sctlManager_Apply_Call) RunAndReturn(run func(configPath string, k0sctlPath string, force bool) error) *MockK0sctlManager_Apply_Call { + _c.Call.Return(run) + return _c +} + +// Download provides a mock function for the type MockK0sctlManager +func (_mock *MockK0sctlManager) Download(version string, force bool, quiet bool) (string, error) { + ret := _mock.Called(version, force, quiet) + + if len(ret) == 0 { + panic("no return value specified for Download") + } + + var r0 string + var r1 error + if returnFunc, ok := ret.Get(0).(func(string, bool, bool) (string, error)); ok { + return returnFunc(version, force, quiet) + } + if returnFunc, ok := ret.Get(0).(func(string, bool, bool) string); ok { + r0 = returnFunc(version, force, quiet) + } else { + r0 = ret.Get(0).(string) + } + if returnFunc, ok := ret.Get(1).(func(string, bool, bool) error); ok { + r1 = returnFunc(version, force, quiet) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockK0sctlManager_Download_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Download' +type MockK0sctlManager_Download_Call struct { + *mock.Call +} + +// Download is a helper method to define mock.On call +// - version string +// - force bool +// - quiet bool +func (_e *MockK0sctlManager_Expecter) Download(version interface{}, force interface{}, quiet interface{}) *MockK0sctlManager_Download_Call { + return &MockK0sctlManager_Download_Call{Call: _e.mock.On("Download", version, force, quiet)} +} + +func (_c *MockK0sctlManager_Download_Call) Run(run func(version string, force bool, quiet bool)) *MockK0sctlManager_Download_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + var arg1 bool + if args[1] != nil { + arg1 = args[1].(bool) + } + var arg2 bool + if args[2] != nil { + arg2 = args[2].(bool) + } + run( + arg0, + arg1, + arg2, + ) + }) + return _c +} + +func (_c *MockK0sctlManager_Download_Call) Return(s string, err error) *MockK0sctlManager_Download_Call { + _c.Call.Return(s, err) + return _c +} + +func (_c *MockK0sctlManager_Download_Call) RunAndReturn(run func(version string, force bool, quiet bool) (string, error)) *MockK0sctlManager_Download_Call { + _c.Call.Return(run) + return _c +} + +// GetLatestVersion provides a mock function for the type MockK0sctlManager +func (_mock *MockK0sctlManager) GetLatestVersion() (string, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for GetLatestVersion") + } + + var r0 string + var r1 error + if returnFunc, ok := ret.Get(0).(func() (string, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() string); ok { + r0 = returnFunc() + } else { + r0 = ret.Get(0).(string) + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockK0sctlManager_GetLatestVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetLatestVersion' +type MockK0sctlManager_GetLatestVersion_Call struct { + *mock.Call +} + +// GetLatestVersion is a helper method to define mock.On call +func (_e *MockK0sctlManager_Expecter) GetLatestVersion() *MockK0sctlManager_GetLatestVersion_Call { + return &MockK0sctlManager_GetLatestVersion_Call{Call: _e.mock.On("GetLatestVersion")} +} + +func (_c *MockK0sctlManager_GetLatestVersion_Call) Run(run func()) *MockK0sctlManager_GetLatestVersion_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockK0sctlManager_GetLatestVersion_Call) Return(s string, err error) *MockK0sctlManager_GetLatestVersion_Call { + _c.Call.Return(s, err) + return _c +} + +func (_c *MockK0sctlManager_GetLatestVersion_Call) RunAndReturn(run func() (string, error)) *MockK0sctlManager_GetLatestVersion_Call { + _c.Call.Return(run) + return _c +} + +// Reset provides a mock function for the type MockK0sctlManager +func (_mock *MockK0sctlManager) Reset(configPath string, k0sctlPath string) error { + ret := _mock.Called(configPath, k0sctlPath) + + if len(ret) == 0 { + panic("no return value specified for Reset") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(string, string) error); ok { + r0 = returnFunc(configPath, k0sctlPath) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// MockK0sctlManager_Reset_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Reset' +type MockK0sctlManager_Reset_Call struct { + *mock.Call +} + +// Reset is a helper method to define mock.On call +// - configPath string +// - k0sctlPath string +func (_e *MockK0sctlManager_Expecter) Reset(configPath interface{}, k0sctlPath interface{}) *MockK0sctlManager_Reset_Call { + return &MockK0sctlManager_Reset_Call{Call: _e.mock.On("Reset", configPath, k0sctlPath)} +} + +func (_c *MockK0sctlManager_Reset_Call) Run(run func(configPath string, k0sctlPath string)) *MockK0sctlManager_Reset_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + var arg1 string + if args[1] != nil { + arg1 = args[1].(string) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockK0sctlManager_Reset_Call) Return(err error) *MockK0sctlManager_Reset_Call { _c.Call.Return(err) return _c } -func (_c *MockK0sManager_Install_Call) RunAndReturn(run func(configPath string, k0sPath string, force bool) error) *MockK0sManager_Install_Call { +func (_c *MockK0sctlManager_Reset_Call) RunAndReturn(run func(configPath string, k0sctlPath string) error) *MockK0sctlManager_Reset_Call { _c.Call.Return(run) return _c } diff --git a/internal/installer/node/mock_ssh_client.go b/internal/installer/node/mock_ssh_client.go new file mode 100644 index 0000000..6afa1aa --- /dev/null +++ b/internal/installer/node/mock_ssh_client.go @@ -0,0 +1,61 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package node + +import ( + "errors" + "io" + + "golang.org/x/crypto/ssh" +) + +// MockSSHClientFactory is a test implementation of SSHClientFactory. +type MockSSHClientFactory struct { + DialFunc func(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) +} + +func (m *MockSSHClientFactory) Dial(network, addr string, config *ssh.ClientConfig) (*ssh.Client, error) { + if m.DialFunc != nil { + return m.DialFunc(network, addr, config) + } + return nil, errors.New("mock SSH client factory not configured") +} + +// MockSSHSession is a mock SSH session for testing. +type MockSSHSession struct { + StartFunc func(cmd string) error + WaitFunc func() error + CloseFunc func() error + SetenvFunc func(name, value string) error + Stdout io.Writer + Stderr io.Writer +} + +func (m *MockSSHSession) Start(cmd string) error { + if m.StartFunc != nil { + return m.StartFunc(cmd) + } + return nil +} + +func (m *MockSSHSession) Wait() error { + if m.WaitFunc != nil { + return m.WaitFunc() + } + return nil +} + +func (m *MockSSHSession) Close() error { + if m.CloseFunc != nil { + return m.CloseFunc() + } + return nil +} + +func (m *MockSSHSession) Setenv(name, value string) error { + if m.SetenvFunc != nil { + return m.SetenvFunc(name, value) + } + return nil +} diff --git a/internal/installer/node/node.go b/internal/installer/node/node.go index 8223416..c889755 100644 --- a/internal/installer/node/node.go +++ b/internal/installer/node/node.go @@ -17,6 +17,7 @@ import ( "github.com/pkg/sftp" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/knownhosts" "golang.org/x/term" ) @@ -115,6 +116,85 @@ func (n *Node) GetName() string { return n.Name } +// getHostKeyCallback returns a host key callback that uses known_hosts file +// and auto-accepts new hosts for provisioning. +func (n *Node) getHostKeyCallback() (ssh.HostKeyCallback, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return nil, fmt.Errorf("failed to get user home directory: %w", err) + } + + knownHostsPath := filepath.Join(homeDir, ".ssh", "known_hosts") + + // Ensure known_hosts file exists + sshDir := filepath.Join(homeDir, ".ssh") + if err := os.MkdirAll(sshDir, 0700); err != nil { + return nil, fmt.Errorf("failed to create .ssh directory: %w", err) + } + + // Create file if it doesn't exist + if _, err := os.Stat(knownHostsPath); os.IsNotExist(err) { + f, err := os.OpenFile(knownHostsPath, os.O_CREATE|os.O_RDONLY, 0600) + if err != nil { + return nil, fmt.Errorf("failed to create known_hosts file: %w", err) + } + if err := f.Close(); err != nil { + return nil, fmt.Errorf("failed to close known_hosts file: %w", err) + } + } + + hostKeyCallback, err := knownhosts.New(knownHostsPath) + if err != nil { + return nil, fmt.Errorf("failed to load known_hosts: %w", err) + } + + // Wrap the callback to auto-accept new hosts for provisioning + return func(hostname string, remote net.Addr, key ssh.PublicKey) error { + err := hostKeyCallback(hostname, remote, key) + if err == nil { + // Host key is already known and valid + return nil + } + + // Check if this is a "host key not found" error (new host) + var keyErr *knownhosts.KeyError + if errors, ok := err.(*knownhosts.KeyError); ok { + keyErr = errors + } + + if keyErr != nil && len(keyErr.Want) == 0 { + // Host key not in known_hosts - auto-accept for provisioning + log.Printf("Warning: Adding new host %s to known_hosts (first connection)", hostname) + + // Append the new host key to known_hosts + f, err := os.OpenFile(knownHostsPath, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return fmt.Errorf("failed to open known_hosts for writing: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + log.Printf("Warning: failed to close known_hosts file: %v", err) + } + }() + + // Format: hostname ssh-keytype base64-encoded-key + normalizedHosts := []string{hostname} + if host, port, splitErr := net.SplitHostPort(hostname); splitErr == nil { + normalizedHosts = []string{net.JoinHostPort(host, port)} + } + line := knownhosts.Line(normalizedHosts, key) + if _, err := f.WriteString(line + "\n"); err != nil { + return fmt.Errorf("failed to write to known_hosts: %w", err) + } + + return nil + } + + // Host key mismatch (potential MITM attack) - reject + return fmt.Errorf("host key verification failed: %w", err) + }, nil +} + // WaitForSSH tries to connect to the node via SSH until timeout. // Once successful, the connection is cached for reuse. func (n *Node) WaitForSSH(timeout time.Duration) error { @@ -398,6 +478,11 @@ func (n *Node) createClient(jumpboxIp string, ip string, username string) (*ssh. return nil, fmt.Errorf("failed to get authentication methods: %w", err) } + hostKeyCallback, err := n.getHostKeyCallback() + if err != nil { + return nil, fmt.Errorf("failed to get host key callback: %w", err) + } + if jumpboxIp != "" { // Use the Jumpbox's cached client if available jbClient, err := n.Jumpbox.getOrCreateClient("", jumpboxIp, jumpboxUser) @@ -406,14 +491,10 @@ func (n *Node) createClient(jumpboxIp string, ip string, username string) (*ssh. } finalTargetConfig := &ssh.ClientConfig{ - User: username, - Auth: authMethods, - Timeout: 10 * time.Second, - // WARNING: This is INSECURE for production! - // It tells the client to accept any host key. - // For production, you should implement a proper HostKeyCallback - // to verify the remote server's identity. - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + User: username, + Auth: authMethods, + Timeout: 10 * time.Second, + HostKeyCallback: hostKeyCallback, } finalAddr := fmt.Sprintf("%s:22", ip) @@ -430,14 +511,10 @@ func (n *Node) createClient(jumpboxIp string, ip string, username string) (*ssh. } config := &ssh.ClientConfig{ - User: username, - Auth: authMethods, - Timeout: 10 * time.Second, - // WARNING: This is INSECURE for production! - // It tells the client to accept any host key. - // For production, you should implement a proper HostKeyCallback - // to verify the remote server's identity. - HostKeyCallback: ssh.InsecureIgnoreHostKey(), + User: username, + Auth: authMethods, + Timeout: 10 * time.Second, + HostKeyCallback: hostKeyCallback, } addr := fmt.Sprintf("%s:22", ip)