From ac7284dac71227c4a919acf9e500289e08e8ac35 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Thu, 25 Dec 2025 00:49:57 +0800 Subject: [PATCH 1/4] HDDS-14022. Move HddsUtils.getSCMAddressForDatanodes to HddsServerUtil --- .../org/apache/hadoop/hdds/HddsUtils.java | 92 ------------ .../org/apache/hadoop/hdds/TestHddsUtils.java | 133 ----------------- .../hadoop/ozone/HddsDatanodeService.java | 4 +- .../statemachine/DatanodeStateMachine.java | 3 +- .../states/datanode/InitDatanodeState.java | 2 +- .../hadoop/hdds/utils/HddsServerUtil.java | 98 +++++++++++++ .../hadoop/hdds/scm/TestHddsServerUtil.java | 137 ++++++++++++++++++ 7 files changed, 239 insertions(+), 230 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index e002de647a81..648f9428f636 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -27,13 +27,9 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_PORT_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEFAULT_SERVICE_ID; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY; @@ -58,12 +54,10 @@ import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; import java.util.TreeMap; import java.util.UUID; import javax.management.ObjectName; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@ -280,92 +274,6 @@ public static OptionalInt getPortNumberFromConfigKeys( return OptionalInt.empty(); } - /** - * Retrieve the socket addresses of all storage container managers. - * - * @return A collection of SCM addresses - * @throws IllegalArgumentException If the configuration is invalid - */ - public static Collection getSCMAddressForDatanodes( - ConfigurationSource conf) { - - // First check HA style config, if not defined fall back to OZONE_SCM_NAMES - - if (getScmServiceId(conf) != null) { - List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); - Collection scmAddressList = - new HashSet<>(scmNodeInfoList.size()); - for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { - scmAddressList.add( - NetUtils.createSocketAddr(scmNodeInfo.getScmDatanodeAddress())); - } - return scmAddressList; - } else { - // fall back to OZONE_SCM_NAMES. - Collection names = - conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES); - if (names.isEmpty()) { - throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES - + " need to be a set of valid DNS names or IP addresses." - + " Empty address list found."); - } - - Collection addresses = new HashSet<>(names.size()); - for (String address : names) { - Optional hostname = getHostName(address); - if (!hostname.isPresent()) { - throw new IllegalArgumentException("Invalid hostname for SCM: " - + address); - } - int port = getHostPort(address) - .orElse(conf.getInt(OZONE_SCM_DATANODE_PORT_KEY, - OZONE_SCM_DATANODE_PORT_DEFAULT)); - InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(), - port); - addresses.add(addr); - } - - if (addresses.size() > 1) { - LOG.warn("When SCM HA is configured, configure {} appended with " + - "serviceId and nodeId. {} is deprecated.", OZONE_SCM_ADDRESS_KEY, - OZONE_SCM_NAMES); - } - return addresses; - } - } - - /** - * Returns the SCM address for datanodes based on the service ID and the SCM addresses. - * @param conf Configuration - * @param scmServiceId SCM service ID - * @param scmNodeIds Requested SCM node IDs - * @return A collection with addresses of the request SCM node IDs. - * Null if there is any wrongly configured SCM address. Note that the returned collection - * might not be ordered the same way as the requested SCM node IDs - */ - public static Collection> getSCMAddressForDatanodes( - ConfigurationSource conf, String scmServiceId, Set scmNodeIds) { - Collection> scmNodeAddress = new HashSet<>(scmNodeIds.size()); - for (String scmNodeId : scmNodeIds) { - String addressKey = ConfUtils.addKeySuffixes( - OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId); - String scmAddress = conf.get(addressKey); - if (scmAddress == null) { - LOG.warn("The SCM address configuration {} is not defined, return nothing", addressKey); - return null; - } - - int scmDatanodePort = SCMNodeInfo.getPort(conf, scmServiceId, scmNodeId, - OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_DATANODE_PORT_KEY, - OZONE_SCM_DATANODE_PORT_DEFAULT); - - String scmDatanodeAddressStr = SCMNodeInfo.buildAddress(scmAddress, scmDatanodePort); - InetSocketAddress scmDatanodeAddress = NetUtils.createSocketAddr(scmDatanodeAddressStr); - scmNodeAddress.add(Pair.of(scmNodeId, scmDatanodeAddress)); - } - return scmNodeAddress; - } - /** * Retrieve the socket addresses of recon. * diff --git a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java index 5eb1b5e31fee..650db8c5439f 100644 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/TestHddsUtils.java @@ -17,25 +17,14 @@ package org.apache.hadoop.hdds; -import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.HddsUtils.processForLogging; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT; -import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import java.net.InetSocketAddress; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -103,128 +92,6 @@ void validatePathRejectsInvalidPath(String path, String ancestor) { () -> HddsUtils.validatePath(Paths.get(path), Paths.get(ancestor))); } - @Test - void testGetSCMAddresses() { - final OzoneConfiguration conf = new OzoneConfiguration(); - Collection addresses; - InetSocketAddress addr; - Iterator it; - - // Verify valid IP address setup - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4"); - addresses = getSCMAddressForDatanodes(conf); - assertEquals(1, addresses.size()); - addr = addresses.iterator().next(); - assertEquals("1.2.3.4", addr.getHostName()); - assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, addr.getPort()); - - // Verify valid hostname setup - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1"); - addresses = getSCMAddressForDatanodes(conf); - assertEquals(1, addresses.size()); - addr = addresses.iterator().next(); - assertEquals("scm1", addr.getHostName()); - assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, addr.getPort()); - - // Verify valid hostname and port - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234"); - addresses = getSCMAddressForDatanodes(conf); - assertEquals(1, addresses.size()); - addr = addresses.iterator().next(); - assertEquals("scm1", addr.getHostName()); - assertEquals(1234, addr.getPort()); - - final Map hostsAndPorts = new HashMap<>(); - hostsAndPorts.put("scm1", 1234); - hostsAndPorts.put("scm2", 2345); - hostsAndPorts.put("scm3", 3456); - - // Verify multiple hosts and port - conf.setStrings( - ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); - addresses = getSCMAddressForDatanodes(conf); - assertEquals(3, addresses.size()); - it = addresses.iterator(); - HashMap expected1 = new HashMap<>(hostsAndPorts); - while (it.hasNext()) { - InetSocketAddress current = it.next(); - assertTrue(expected1.remove(current.getHostName(), - current.getPort())); - } - assertThat(expected1).isEmpty(); - - // Verify names with spaces - conf.setStrings( - ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); - addresses = getSCMAddressForDatanodes(conf); - assertEquals(3, addresses.size()); - it = addresses.iterator(); - HashMap expected2 = new HashMap<>(hostsAndPorts); - while (it.hasNext()) { - InetSocketAddress current = it.next(); - assertTrue(expected2.remove(current.getHostName(), - current.getPort())); - } - assertThat(expected2).isEmpty(); - - // Verify empty value - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, ""); - assertThrows(IllegalArgumentException.class, - () -> getSCMAddressForDatanodes(conf), - "Empty value should cause an IllegalArgumentException"); - - // Verify invalid hostname - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234"); - assertThrows(IllegalArgumentException.class, - () -> getSCMAddressForDatanodes(conf), - "An invalid hostname should cause an IllegalArgumentException"); - - // Verify invalid port - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz"); - assertThrows(IllegalArgumentException.class, - () -> getSCMAddressForDatanodes(conf), - "An invalid port should cause an IllegalArgumentException"); - - // Verify a mixed case (valid and invalid value both appears) - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); - assertThrows(IllegalArgumentException.class, - () -> getSCMAddressForDatanodes(conf), - "An invalid value should cause an IllegalArgumentException"); - } - - @Test - void testGetSCMAddressesWithHAConfig() { - OzoneConfiguration conf = new OzoneConfiguration(); - String scmServiceId = "scmserviceId"; - String[] nodes = new String[]{"scm1", "scm2", "scm3"}; - conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId); - conf.set(ScmConfigKeys.OZONE_SCM_NODES_KEY + "." + scmServiceId, - "scm1,scm2,scm3"); - - int port = 9880; - List expected = new ArrayList<>(); - for (String nodeId : nodes) { - conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, - scmServiceId, nodeId), "scm"); - conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_PORT_KEY, - scmServiceId, nodeId), ++port); - expected.add("scm" + ":" + port); - } - - Collection scmAddressList = - HddsUtils.getSCMAddressForDatanodes(conf); - - assertNotNull(scmAddressList); - assertEquals(3, scmAddressList.size()); - - for (InetSocketAddress next : scmAddressList) { - expected.remove(next.getHostName() + ":" + next.getPort()); - } - - assertEquals(0, expected.size()); - - } - @Test void testGetNumberFromConfigKeys() { final String testnum1 = "8"; diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index 28d5d5a2cb12..9bd10a83b3ec 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@ -739,12 +739,12 @@ private String reconfigScmNodes(String value) { LOG.info("Reconfiguring SCM nodes for service ID {} with new SCM nodes {} and remove SCM nodes {}", scmServiceId, scmNodesIdsToAdd, scmNodesIdsToRemove); - Collection> scmToAdd = HddsUtils.getSCMAddressForDatanodes( + Collection> scmToAdd = HddsServerUtil.getSCMAddressForDatanodes( getConf(), scmServiceId, scmNodesIdsToAdd); if (scmToAdd == null) { throw new IllegalStateException("Reconfiguration failed to get SCM address to add due to wrong configuration"); } - Collection> scmToRemove = HddsUtils.getSCMAddressForDatanodes( + Collection> scmToRemove = HddsServerUtil.getSCMAddressForDatanodes( getConf(), scmServiceId, scmNodesIdsToRemove); if (scmToRemove == null) { throw new IllegalArgumentException( diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java index f05a14e21093..dfe89a17aa28 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java @@ -32,7 +32,6 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.conf.ReconfigurationHandler; @@ -314,7 +313,7 @@ private int getEndPointTaskThreadPoolSize() { int totalServerCount = 1; try { - totalServerCount += HddsUtils.getSCMAddressForDatanodes(conf).size(); + totalServerCount += HddsServerUtil.getSCMAddressForDatanodes(conf).size(); } catch (Exception e) { LOG.error("Fail to get scm addresses", e); } diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java index 6fe0e111bad2..db4eb59d0363 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java @@ -18,7 +18,7 @@ package org.apache.hadoop.ozone.container.common.states.datanode; import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; -import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import com.google.common.base.Strings; import java.io.File; diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index df4c31077aa6..5f64bf90e882 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -25,9 +25,15 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_RECON_HEARTBEAT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_RECON_INITIAL_HEARTBEAT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_RECON_INITIAL_HEARTBEAT_INTERVAL_DEFAULT; +import static org.apache.hadoop.hdds.HddsUtils.getHostName; import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys; +import static org.apache.hadoop.hdds.HddsUtils.getHostPort; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; +import static org.apache.hadoop.hdds.HddsUtils.getScmServiceId; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL_DEFAULT; @@ -40,6 +46,7 @@ import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_RETRY_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.server.ServerUtils.sanitizeUserArgs; @@ -58,6 +65,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.OptionalInt; @@ -69,6 +78,7 @@ import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.lang3.SystemUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; @@ -85,6 +95,7 @@ import org.apache.hadoop.hdds.ratis.ServerNotLeaderException; import org.apache.hadoop.hdds.recon.ReconConfigKeys; import org.apache.hadoop.hdds.scm.ScmConfigKeys; +import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; import org.apache.hadoop.hdds.scm.proxy.SCMClientConfig; import org.apache.hadoop.hdds.scm.proxy.SCMSecurityProtocolFailoverProxyProvider; @@ -104,6 +115,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ShutdownHookManager; import org.apache.ratis.util.Preconditions; @@ -757,4 +769,90 @@ public static void setPoolSize(ThreadPoolExecutor executor, int size, Logger log } } + /** + * Retrieve the socket addresses of all storage container managers. + * + * @return A collection of SCM addresses + * @throws IllegalArgumentException If the configuration is invalid + */ + public static Collection getSCMAddressForDatanodes( + ConfigurationSource conf) { + + // First check HA style config, if not defined fall back to OZONE_SCM_NAMES + + if (getScmServiceId(conf) != null) { + List scmNodeInfoList = SCMNodeInfo.buildNodeInfo(conf); + Collection scmAddressList = + new HashSet<>(scmNodeInfoList.size()); + for (SCMNodeInfo scmNodeInfo : scmNodeInfoList) { + scmAddressList.add( + NetUtils.createSocketAddr(scmNodeInfo.getScmDatanodeAddress())); + } + return scmAddressList; + } else { + // fall back to OZONE_SCM_NAMES. + Collection names = + conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES); + if (names.isEmpty()) { + throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES + + " need to be a set of valid DNS names or IP addresses." + + " Empty address list found."); + } + + Collection addresses = new HashSet<>(names.size()); + for (String address : names) { + Optional hostname = getHostName(address); + if (!hostname.isPresent()) { + throw new IllegalArgumentException("Invalid hostname for SCM: " + + address); + } + int port = getHostPort(address) + .orElse(conf.getInt(OZONE_SCM_DATANODE_PORT_KEY, + OZONE_SCM_DATANODE_PORT_DEFAULT)); + InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(), + port); + addresses.add(addr); + } + + if (addresses.size() > 1) { + LOG.warn("When SCM HA is configured, configure {} appended with " + + "serviceId and nodeId. {} is deprecated.", OZONE_SCM_ADDRESS_KEY, + OZONE_SCM_NAMES); + } + return addresses; + } + } + + /** + * Returns the SCM address for datanodes based on the service ID and the SCM addresses. + * @param conf Configuration + * @param scmServiceId SCM service ID + * @param scmNodeIds Requested SCM node IDs + * @return A collection with addresses of the request SCM node IDs. + * Null if there is any wrongly configured SCM address. Note that the returned collection + * might not be ordered the same way as the requested SCM node IDs + */ + public static Collection> getSCMAddressForDatanodes( + ConfigurationSource conf, String scmServiceId, Set scmNodeIds) { + Collection> scmNodeAddress = new HashSet<>(scmNodeIds.size()); + for (String scmNodeId : scmNodeIds) { + String addressKey = ConfUtils.addKeySuffixes( + OZONE_SCM_ADDRESS_KEY, scmServiceId, scmNodeId); + String scmAddress = conf.get(addressKey); + if (scmAddress == null) { + LOG.warn("The SCM address configuration {} is not defined, return nothing", addressKey); + return null; + } + + int scmDatanodePort = SCMNodeInfo.getPort(conf, scmServiceId, scmNodeId, + OZONE_SCM_DATANODE_ADDRESS_KEY, OZONE_SCM_DATANODE_PORT_KEY, + OZONE_SCM_DATANODE_PORT_DEFAULT); + + String scmDatanodeAddressStr = SCMNodeInfo.buildAddress(scmAddress, scmDatanodePort); + InetSocketAddress scmDatanodeAddress = NetUtils.createSocketAddr(scmDatanodeAddressStr); + scmNodeAddress.add(Pair.of(scmNodeId, scmDatanodeAddress)); + } + return scmNodeAddress; + } + } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java index 6feba1a48f31..2878304f3863 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestHddsServerUtil.java @@ -17,13 +17,28 @@ package org.apache.hadoop.hdds.scm; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.net.NetUtils; +import org.apache.hadoop.ozone.ha.ConfUtils; import org.junit.jupiter.api.Test; /** @@ -169,4 +184,126 @@ public void testScmDataNodeBindHostDefault() { assertEquals(200, addr.getPort()); } + @Test + void testGetSCMAddresses() { + final OzoneConfiguration conf = new OzoneConfiguration(); + Collection addresses; + InetSocketAddress addr; + Iterator it; + + // Verify valid IP address setup + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4"); + addresses = getSCMAddressForDatanodes(conf); + assertEquals(1, addresses.size()); + addr = addresses.iterator().next(); + assertEquals("1.2.3.4", addr.getHostName()); + assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, addr.getPort()); + + // Verify valid hostname setup + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1"); + addresses = getSCMAddressForDatanodes(conf); + assertEquals(1, addresses.size()); + addr = addresses.iterator().next(); + assertEquals("scm1", addr.getHostName()); + assertEquals(OZONE_SCM_DATANODE_PORT_DEFAULT, addr.getPort()); + + // Verify valid hostname and port + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234"); + addresses = getSCMAddressForDatanodes(conf); + assertEquals(1, addresses.size()); + addr = addresses.iterator().next(); + assertEquals("scm1", addr.getHostName()); + assertEquals(1234, addr.getPort()); + + final Map hostsAndPorts = new HashMap<>(); + hostsAndPorts.put("scm1", 1234); + hostsAndPorts.put("scm2", 2345); + hostsAndPorts.put("scm3", 3456); + + // Verify multiple hosts and port + conf.setStrings( + ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); + addresses = getSCMAddressForDatanodes(conf); + assertEquals(3, addresses.size()); + it = addresses.iterator(); + HashMap expected1 = new HashMap<>(hostsAndPorts); + while (it.hasNext()) { + InetSocketAddress current = it.next(); + assertTrue(expected1.remove(current.getHostName(), + current.getPort())); + } + assertThat(expected1).isEmpty(); + + // Verify names with spaces + conf.setStrings( + ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); + addresses = getSCMAddressForDatanodes(conf); + assertEquals(3, addresses.size()); + it = addresses.iterator(); + HashMap expected2 = new HashMap<>(hostsAndPorts); + while (it.hasNext()) { + InetSocketAddress current = it.next(); + assertTrue(expected2.remove(current.getHostName(), + current.getPort())); + } + assertThat(expected2).isEmpty(); + + // Verify empty value + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, ""); + assertThrows(IllegalArgumentException.class, + () -> getSCMAddressForDatanodes(conf), + "Empty value should cause an IllegalArgumentException"); + + // Verify invalid hostname + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234"); + assertThrows(IllegalArgumentException.class, + () -> getSCMAddressForDatanodes(conf), + "An invalid hostname should cause an IllegalArgumentException"); + + // Verify invalid port + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz"); + assertThrows(IllegalArgumentException.class, + () -> getSCMAddressForDatanodes(conf), + "An invalid port should cause an IllegalArgumentException"); + + // Verify a mixed case (valid and invalid value both appears) + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); + assertThrows(IllegalArgumentException.class, + () -> getSCMAddressForDatanodes(conf), + "An invalid value should cause an IllegalArgumentException"); + } + + @Test + void testGetSCMAddressesWithHAConfig() { + OzoneConfiguration conf = new OzoneConfiguration(); + String scmServiceId = "scmserviceId"; + String[] nodes = new String[]{"scm1", "scm2", "scm3"}; + conf.set(ScmConfigKeys.OZONE_SCM_SERVICE_IDS_KEY, scmServiceId); + conf.set(ScmConfigKeys.OZONE_SCM_NODES_KEY + "." + scmServiceId, + "scm1,scm2,scm3"); + + int port = 9880; + List expected = new ArrayList<>(); + for (String nodeId : nodes) { + conf.set(ConfUtils.addKeySuffixes(OZONE_SCM_ADDRESS_KEY, + scmServiceId, nodeId), "scm"); + conf.setInt(ConfUtils.addKeySuffixes(OZONE_SCM_DATANODE_PORT_KEY, + scmServiceId, nodeId), ++port); + expected.add("scm" + ":" + port); + } + + Collection scmAddressList = + getSCMAddressForDatanodes(conf); + + assertNotNull(scmAddressList); + assertEquals(3, scmAddressList.size()); + + for (InetSocketAddress next : scmAddressList) { + expected.remove(next.getHostName() + ":" + next.getPort()); + } + + assertEquals(0, expected.size()); + + } + } From 0ebda5cfaaf8c43603dff18a845a835560cf55cb Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Thu, 25 Dec 2025 01:10:29 +0800 Subject: [PATCH 2/4] Fix Build Error --- .../java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index 300a506305d3..1db6878e978a 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -20,7 +20,7 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; -import static org.apache.hadoop.hdds.HddsUtils.getSCMAddressForDatanodes; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmHeartbeatInterval; From f1d5531aa3d894c4d227529dff92f5af9703ef8f Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Thu, 25 Dec 2025 01:14:09 +0800 Subject: [PATCH 3/4] Fix CheckStyle Error --- .../java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index 1db6878e978a..810c296d9102 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -20,9 +20,9 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT; import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; -import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmHeartbeatInterval; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcRetryCount; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcRetryInterval; From 443dd2faf801a420fba5318d3e87d772f4146489 Mon Sep 17 00:00:00 2001 From: Russole <850905junior@gmail.com> Date: Sat, 27 Dec 2025 02:27:59 +0800 Subject: [PATCH 4/4] Move getReconAddresses to HddsServerUtil and rename to getReconAddressForDatanodes --- .../org/apache/hadoop/hdds/HddsUtils.java | 24 ------------------- .../states/datanode/InitDatanodeState.java | 4 ++-- .../hadoop/hdds/utils/HddsServerUtil.java | 23 ++++++++++++++++++ .../hadoop/ozone/freon/DatanodeSimulator.java | 4 ++-- .../hadoop/ozone/recon/ReconServer.java | 3 +-- 5 files changed, 28 insertions(+), 30 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java index 648f9428f636..d74cae839421 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsUtils.java @@ -25,8 +25,6 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_DNS_INTERFACE_KEY; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_DNS_NAMESERVER_KEY; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_HOST_NAME_KEY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; -import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY; @@ -57,7 +55,6 @@ import java.util.TreeMap; import java.util.UUID; import javax.management.ObjectName; -import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.hdds.annotation.InterfaceAudience; @@ -274,27 +271,6 @@ public static OptionalInt getPortNumberFromConfigKeys( return OptionalInt.empty(); } - /** - * Retrieve the socket addresses of recon. - * - * @return Recon address - * @throws IllegalArgumentException If the configuration is invalid - */ - public static InetSocketAddress getReconAddresses( - ConfigurationSource conf) { - String name = conf.get(OZONE_RECON_ADDRESS_KEY); - if (StringUtils.isEmpty(name)) { - return null; - } - Optional hostname = getHostName(name); - if (!hostname.isPresent()) { - throw new IllegalArgumentException("Invalid hostname for Recon: " - + name); - } - int port = getHostPort(name).orElse(OZONE_RECON_DATANODE_PORT_DEFAULT); - return NetUtils.createSocketAddr(hostname.get(), port); - } - /** * Returns the hostname for this datanode. If the hostname is not * explicitly configured in the given config, then it is determined diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java index db4eb59d0363..2787093b1bf4 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java @@ -17,7 +17,7 @@ package org.apache.hadoop.ozone.container.common.states.datanode; -import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getReconAddressForDatanodes; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import com.google.common.base.Strings; @@ -104,7 +104,7 @@ public DatanodeStateMachine.DatanodeStates call() throws Exception { connectionManager.addSCMServer(addr, context.getThreadNamePrefix()); this.context.addEndpoint(addr); } - InetSocketAddress reconAddress = getReconAddresses(conf); + InetSocketAddress reconAddress = getReconAddressForDatanodes(conf); if (reconAddress != null) { connectionManager.addReconServer(reconAddress, context.getThreadNamePrefix()); diff --git a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java index 5f64bf90e882..3d8b61780257 100644 --- a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java +++ b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/HddsServerUtil.java @@ -30,6 +30,8 @@ import static org.apache.hadoop.hdds.HddsUtils.getHostPort; import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys; import static org.apache.hadoop.hdds.HddsUtils.getScmServiceId; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_ADDRESS_KEY; +import static org.apache.hadoop.hdds.recon.ReconConfigKeys.OZONE_RECON_DATANODE_PORT_DEFAULT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_ADDRESS_KEY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY; @@ -77,6 +79,7 @@ import java.util.stream.Stream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.SystemUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; @@ -855,4 +858,24 @@ public static Collection> getSCMAddressForDatano return scmNodeAddress; } + /** + * Retrieve the socket addresses of recon. + * + * @return Recon address + * @throws IllegalArgumentException If the configuration is invalid + */ + public static InetSocketAddress getReconAddressForDatanodes( + ConfigurationSource conf) { + String name = conf.get(OZONE_RECON_ADDRESS_KEY); + if (StringUtils.isEmpty(name)) { + return null; + } + Optional hostname = getHostName(name); + if (!hostname.isPresent()) { + throw new IllegalArgumentException("Invalid hostname for Recon: " + + name); + } + int port = getHostPort(name).orElse(OZONE_RECON_DATANODE_PORT_DEFAULT); + return NetUtils.createSocketAddr(hostname.get(), port); + } } diff --git a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java index 810c296d9102..cb3769e7fd2d 100644 --- a/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java +++ b/hadoop-ozone/freon/src/main/java/org/apache/hadoop/ozone/freon/DatanodeSimulator.java @@ -19,9 +19,9 @@ import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL; import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL_DEFAULT; -import static org.apache.hadoop.hdds.HddsUtils.getReconAddresses; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT; +import static org.apache.hadoop.hdds.utils.HddsServerUtil.getReconAddressForDatanodes; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getSCMAddressForDatanodes; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmHeartbeatInterval; import static org.apache.hadoop.hdds.utils.HddsServerUtil.getScmRpcRetryCount; @@ -430,7 +430,7 @@ private void init() throws IOException { scmClients.put(address, createScmClient(address)); } - reconAddress = getReconAddresses(conf); + reconAddress = getReconAddressForDatanodes(conf); reconClient = createReconClient(reconAddress); heartbeatScheduler = Executors.newScheduledThreadPool(threadCount); diff --git a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java index 493e340bc5d4..c9fc0ab3470b 100644 --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java @@ -34,7 +34,6 @@ import java.util.concurrent.Callable; import java.util.concurrent.atomic.AtomicBoolean; import javax.sql.DataSource; -import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; @@ -381,7 +380,7 @@ private static void loginReconUser(OzoneConfiguration conf) reconConfig.getKerberosPrincipal(), reconConfig.getKerberosKeytab()); UserGroupInformation.setConfiguration(conf); - InetSocketAddress socAddr = HddsUtils.getReconAddresses(conf); + InetSocketAddress socAddr = HddsServerUtil.getReconAddressForDatanodes(conf); SecurityUtil.login(conf, OZONE_RECON_KERBEROS_KEYTAB_FILE_KEY, OZONE_RECON_KERBEROS_PRINCIPAL_KEY,