[sled-agent] Self assembling switch zone (#5593)

## Overview

This PR migrates the switch zone to a self assembling format. There are
a few bits of old code I'll be cleaning up, more logs I'll be adding and
some documentation about how the switch zone flow works, but I'll do
this in follow up PRs to keep this one as compact as possible.

## Caveats

I've tested this in a local single node deployment and in the a4x2
testbed. Unfortunately, this is not enough testing to make sure all of
the services play nice together on a real rack. We'll have to keep an
eye on dogfood when this is deployed.

The only services that depend on the [common
networking](https://github.com/oxidecomputer/omicron/blob/30eb1ee38987201ac71f2115fdd89da4b08710c7/zone-setup/src/bin/zone-setup.rs#L578-L690)
[service](https://github.com/oxidecomputer/omicron/blob/30eb1ee38987201ac71f2115fdd89da4b08710c7/smf/zone-network-setup/manifest.xml)
are dendrite and MGS. While this makes sense on the a4x2 testbed, I'd
like to verify that these dependencies make sense when running on a real
rack.

As several people have worked on different parts of this zone, I've
tagged a whole bunch of people for review, sorry if this is overkill!
Just want to make sure I've got the right eyes on each service of the
zone.

Related: #1898
Closes: #2884

TODO:

- [x] Update Dendrite hashes after merging
https://github.com/oxidecomputer/dendrite/pull/990

Author: karencfv

Cargo.lock

@@ -36,8 +36,8 @@ pub const PFEXEC: &str = "/usr/bin/pfexec";
 pub struct CommandFailureInfo {
     command: String,
     status: std::process::ExitStatus,
-    stdout: String,
-    stderr: String,
+    pub stdout: String,
+    pub stderr: String,
 }
 
 impl std::fmt::Display for CommandFailureInfo {

illumos-utils/src/lib.rs

@@ -107,4 +107,23 @@ impl Route {
         };
         Ok(())
     }
+
+    pub fn add_bootstrap_route(
+        bootstrap_prefix: u16,
+        gz_bootstrap_addr: Ipv6Addr,
+        zone_vnic_name: &str,
+    ) -> Result<(), ExecutionError> {
+        let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[
+            ROUTE,
+            "add",
+            "-inet6",
+            &format!("{bootstrap_prefix:x}::/16"),
+            &gz_bootstrap_addr.to_string(),
+            "-ifp",
+            zone_vnic_name,
+        ]);
+        execute(cmd)?;
+        Ok(())
+    }
 }

illumos-utils/src/route.rs

@@ -46,7 +46,7 @@ pub enum ServiceError {
 pub struct RunCommandError {
     zone: String,
     #[source]
-    err: crate::ExecutionError,
+    pub err: crate::ExecutionError,
 }
 
 /// Errors returned from [`RunningZone::boot`].
@@ -462,7 +462,7 @@ impl RunningZone {
     /// Note that the zone must already be configured to be booted.
     pub async fn boot(zone: InstalledZone) -> Result<Self, BootError> {
         // Boot the zone.
-        info!(zone.log, "Zone booting");
+        info!(zone.log, "Booting {} zone", zone.name);
 
         Zones::boot(&zone.name).await?;
 
@@ -480,6 +480,9 @@ impl RunningZone {
                 zone: zone.name.to_string(),
             })?;
 
+        // TODO https://github.com/oxidecomputer/omicron/issues/1898:
+        // Remove all non-self assembling code
+
         // If the zone is self-assembling, then SMF service(s) inside the zone
         // will be creating the listen address for the zone's service(s),
         // setting the appropriate ifprop MTU, and so on. The idea behind
@@ -575,7 +578,6 @@ impl RunningZone {
         &self,
         address: Ipv6Addr,
     ) -> Result<(), EnsureAddressError> {
-        info!(self.inner.log, "Adding bootstrap address");
         let vnic = self.inner.bootstrap_vnic.as_ref().ok_or_else(|| {
             EnsureAddressError::MissingBootstrapVnic {
                 address: address.to_string(),
@@ -735,15 +737,16 @@ impl RunningZone {
         gz_bootstrap_addr: Ipv6Addr,
         zone_vnic_name: &str,
     ) -> Result<(), RunCommandError> {
-        self.run_cmd([
+        let args = [
             "/usr/sbin/route",
             "add",
             "-inet6",
             &format!("{bootstrap_prefix:x}::/16"),
             &gz_bootstrap_addr.to_string(),
             "-ifp",
             zone_vnic_name,
-        ])?;
+        ];
+        self.run_cmd(args)?;
         Ok(())
     }
 
@@ -775,7 +778,7 @@ impl RunningZone {
 
     /// Return a reference to the links for this zone.
     pub fn links(&self) -> &Vec<Link> {
-        &self.inner.links
+        &self.inner.links()
     }
 
     /// Return a mutable reference to the links for this zone.
@@ -1010,6 +1013,11 @@ impl InstalledZone {
     pub fn root(&self) -> Utf8PathBuf {
         self.zonepath.path.join(Self::ROOT_FS_PATH)
     }
+
+    /// Return a reference to the links for this zone.
+    pub fn links(&self) -> &Vec<Link> {
+        &self.links
+    }
 }
 
 #[derive(Clone)]

illumos-utils/src/running_zone.rs

@@ -640,7 +640,7 @@ impl Zones {
     //
     // Does NOT check if the address already exists.
     #[allow(clippy::needless_lifetimes)]
-    fn create_address_internal<'a>(
+    pub fn create_address_internal<'a>(
         zone: Option<&'a str>,
         addrobj: &AddrObject,
         addrtype: AddressRequest,

illumos-utils/src/zone.rs

@@ -439,7 +439,6 @@ service_name = "switch_zone_setup"
 source.type = "local"
 source.paths = [
   { from = "smf/switch_zone_setup/manifest.xml", to = "/var/svc/manifest/site/switch_zone_setup/manifest.xml" },
-  { from = "smf/switch_zone_setup/switch_zone_setup", to = "/opt/oxide/bin/switch_zone_setup" },
   { from = "smf/switch_zone_setup/support_authorized_keys", to = "/opt/oxide/support/authorized_keys" },
   { from = "/opt/ooce/pgsql-13/lib/amd64", to = "/opt/ooce/pgsql-13/lib/amd64" },
 ]
@@ -645,8 +644,8 @@ only_for_targets.image = "standard"
 # the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "e83f4f164fd3dbb2100989a399a4fa087232ac36"
-source.sha256 = "b28247df4d301540b0a46e4d9fdf410ee6fbdb23d18c80acbd36c016a084e30e"
+source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297"
+source.sha256 = "c7971efca6500cee8edf2696ec6b38014af82bacfe88a0e583bb9bb3a591bc8d"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -672,8 +671,8 @@ only_for_targets.image = "standard"
 # the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "e83f4f164fd3dbb2100989a399a4fa087232ac36"
-source.sha256 = "caf988e39d800bdccb1b9423568a19ba10a79aa2b07f74bf7eb65589fd81f8b1"
+source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297"
+source.sha256 = "0a96670ce203bce7bed6a0e40842d319c2b4b8ee1a2e9210d3713423f8bd00b1"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -692,8 +691,8 @@ only_for_targets.image = "standard"
 # the other `source.*` keys.
 source.type = "prebuilt"
 source.repo = "dendrite"
-source.commit = "e83f4f164fd3dbb2100989a399a4fa087232ac36"
-source.sha256 = "378a2f32c1850a5a62fa9b320813e342a647647d2f014ab5eced7c2d1d4f9c95"
+source.commit = "fb571dc6512b24a777c5a9b2927a50501f6be297"
+source.sha256 = "a5bda6b899bff23fccd4dd74224fd1bc44703741054b50552921efa7470cb11a"
 output.type = "zone"
 output.intermediate_only = true
 
@@ -740,6 +739,8 @@ source.packages = [
   "switch_zone_setup.tar.gz",
   "xcvradm.tar.gz",
   "omicron-omdb.tar.gz",
+  "zone-setup.tar.gz",
+  "zone-network-install.tar.gz"
 ]
 output.type = "zone"
 
@@ -764,6 +765,8 @@ source.packages = [
   "switch_zone_setup.tar.gz",
   "sp-sim-stub.tar.gz",
   "omicron-omdb.tar.gz",
+  "zone-setup.tar.gz",
+  "zone-network-install.tar.gz"
 ]
 output.type = "zone"
 
@@ -788,6 +791,8 @@ source.packages = [
   "switch_zone_setup.tar.gz",
   "sp-sim-softnpu.tar.gz",
   "omicron-omdb.tar.gz",
+  "zone-setup.tar.gz",
+  "zone-network-install.tar.gz"
 ]
 output.type = "zone"
 

package-manifest.toml

@@ -597,9 +597,6 @@ impl crate::smf_helper::Service for OmicronZoneType {
     fn smf_name(&self) -> String {
         format!("svc:/oxide/{}", self.service_name())
     }
-    fn should_import(&self) -> bool {
-        true
-    }
 }
 
 impl From<OmicronZoneType> for sled_agent_client::types::OmicronZoneType {

sled-agent/src/params.rs

@@ -163,6 +163,7 @@ impl Display for ServiceInstanceBuilder {
     }
 }
 
+#[derive(Clone)]
 pub struct PropertyGroupBuilder {
     name: String,
     /// names of the properties that were added, in the order they were added
@@ -233,7 +234,7 @@ impl Display for PropertyGroupBuilder {
             if values.len() == 1 {
                 write!(
                     f,
-                    r#"        <propval type="{ty}" name="{name}" value="{value}"/>
+                    r#"        <propval type="{ty}" name="{name}" value='{value}'/>
 "#,
                     name = property_name,
                     value = &values[0],
@@ -302,7 +303,7 @@ mod tests {
 <service_bundle type="profile" name="myprofile">
   <service version="1" type="service" name="myservice">
       <property_group type="application" name="mypg">
-        <propval type="astring" name="myprop" value="myvalue"/>
+        <propval type="astring" name="myprop" value='myvalue'/>
       </property_group>
   </service>
 </service_bundle>"#,
@@ -384,7 +385,7 @@ mod tests {
   <service version="1" type="service" name="myservice">
     <instance enabled="true" name="default">
       <property_group type="application" name="mypg">
-        <propval type="type" name="prop" value="value"/>
+        <propval type="type" name="prop" value='value'/>
       </property_group>
     </instance>
   </service>
@@ -429,11 +430,11 @@ mod tests {
       </property_group>
     <instance enabled="true" name="default">
       <property_group type="application" name="mypg">
-        <propval type="type" name="prop" value="value"/>
-        <propval type="type" name="prop2" value="value2"/>
+        <propval type="type" name="prop" value='value'/>
+        <propval type="type" name="prop2" value='value2'/>
       </property_group>
       <property_group type="application" name="mypg2">
-        <propval type="type" name="prop3" value="value3"/>
+        <propval type="type" name="prop3" value='value3'/>
       </property_group>
     </instance>
   </service>