From d0a783faba503ff59413500fbf98b3009aa0b720 Mon Sep 17 00:00:00 2001 From: "d.garus" Date: Wed, 27 Jan 2021 15:11:51 +0300 Subject: [PATCH 01/19] IGNITE-14070 Protecting a snapshot from unauthorized changes. --- .../persistence/snapshot/SnapshotDigest.java | 52 +++++++++++++++++ .../snapshot/SnapshotDigestException.java | 58 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java new file mode 100644 index 0000000000000..b54f604015d9f --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.nio.file.Path; +import org.apache.ignite.internal.GridKernalContext; + +/** + * Interface to check snapshot files integrity before restoring them. + */ +public interface SnapshotDigest { + /** + * @return Instance of SnapshotDigest. + */ + public static SnapshotDigest getInstance(GridKernalContext ctx) { + SnapshotDigest res = ctx.plugins().createComponent(SnapshotDigest.class); + + if (res == null) { + res = new SnapshotDigest() { + @Override public void verify(Path snpDir) { + // No-op. + } + }; + } + + return res; + } + + /** + * Verify snapshot integrity. + * + * @param snpDir Path to a snapshot directory. + * @throws SnapshotDigestException Thrown if snapshot integrity is corrupted or impossible to check. + */ + public void verify(Path snpDir) throws SnapshotDigestException; + +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java new file mode 100644 index 0000000000000..a84f9026ea6ee --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import org.apache.ignite.IgniteCheckedException; +import org.jetbrains.annotations.Nullable; + +/** + * Thrown when snapshot integrity is corrupted or impossible to check. + */ +public class SnapshotDigestException extends IgniteCheckedException { + /** + * @param msg Message. + */ + public SnapshotDigestException(String msg) { + super(msg); + } + + /** + * @param cause Cause. + */ + public SnapshotDigestException(Throwable cause) { + super(cause); + } + + /** + * @param msg Message. + * @param cause Cause. + * @param writableStackTrace Writable stack trace. + */ + public SnapshotDigestException(String msg, @Nullable Throwable cause, + boolean writableStackTrace) { + super(msg, cause, writableStackTrace); + } + + /** + * @param msg Message. + * @param cause Cause. + */ + public SnapshotDigestException(String msg, @Nullable Throwable cause) { + super(msg, cause); + } +} From 2ab694361f8ac492161e846c2997a1903992a22e Mon Sep 17 00:00:00 2001 From: "d.garus" Date: Wed, 27 Jan 2021 16:14:39 +0300 Subject: [PATCH 02/19] fix comments --- ...pshotDigest.java => SnapshotVerifier.java} | 24 +++---------------- ...on.java => SnapshotVerifierException.java} | 10 ++++---- 2 files changed, 8 insertions(+), 26 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/{SnapshotDigest.java => SnapshotVerifier.java} (60%) rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/{SnapshotDigestException.java => SnapshotVerifierException.java} (81%) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java similarity index 60% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java index b54f604015d9f..f93e0dd318315 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java @@ -18,35 +18,17 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; import java.nio.file.Path; -import org.apache.ignite.internal.GridKernalContext; /** * Interface to check snapshot files integrity before restoring them. */ -public interface SnapshotDigest { - /** - * @return Instance of SnapshotDigest. - */ - public static SnapshotDigest getInstance(GridKernalContext ctx) { - SnapshotDigest res = ctx.plugins().createComponent(SnapshotDigest.class); - - if (res == null) { - res = new SnapshotDigest() { - @Override public void verify(Path snpDir) { - // No-op. - } - }; - } - - return res; - } - +public interface SnapshotVerifier { /** * Verify snapshot integrity. * * @param snpDir Path to a snapshot directory. - * @throws SnapshotDigestException Thrown if snapshot integrity is corrupted or impossible to check. + * @throws SnapshotVerifierException Thrown if snapshot integrity is corrupted or impossible to check. */ - public void verify(Path snpDir) throws SnapshotDigestException; + public void verify(Path snpDir) throws SnapshotVerifierException; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java similarity index 81% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java index a84f9026ea6ee..2a38c0fe3a02f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotDigestException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java @@ -23,18 +23,18 @@ /** * Thrown when snapshot integrity is corrupted or impossible to check. */ -public class SnapshotDigestException extends IgniteCheckedException { +public class SnapshotVerifierException extends IgniteCheckedException { /** * @param msg Message. */ - public SnapshotDigestException(String msg) { + public SnapshotVerifierException(String msg) { super(msg); } /** * @param cause Cause. */ - public SnapshotDigestException(Throwable cause) { + public SnapshotVerifierException(Throwable cause) { super(cause); } @@ -43,7 +43,7 @@ public SnapshotDigestException(Throwable cause) { * @param cause Cause. * @param writableStackTrace Writable stack trace. */ - public SnapshotDigestException(String msg, @Nullable Throwable cause, + public SnapshotVerifierException(String msg, @Nullable Throwable cause, boolean writableStackTrace) { super(msg, cause, writableStackTrace); } @@ -52,7 +52,7 @@ public SnapshotDigestException(String msg, @Nullable Throwable cause, * @param msg Message. * @param cause Cause. */ - public SnapshotDigestException(String msg, @Nullable Throwable cause) { + public SnapshotVerifierException(String msg, @Nullable Throwable cause) { super(msg, cause); } } From c7af289e3bf7e65c8f6378fa034a9a56173dfeb8 Mon Sep 17 00:00:00 2001 From: "d.garus" Date: Thu, 28 Jan 2021 09:55:49 +0300 Subject: [PATCH 03/19] fix error --- .../cache/persistence/snapshot/SnapshotVerifierException.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java index 2a38c0fe3a02f..c51fc2e514fc9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java @@ -24,6 +24,9 @@ * Thrown when snapshot integrity is corrupted or impossible to check. */ public class SnapshotVerifierException extends IgniteCheckedException { + /** */ + private static final long serialVersionUID = 0L; + /** * @param msg Message. */ From 9ff888f54c7eb37d403776aec622d0137a6c5bc3 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 9 Jun 2021 18:16:33 +0300 Subject: [PATCH 04/19] IGNITE-14070 Optional snapshot vrification. --- .../snapshot/IgniteSnapshotManager.java | 21 ++++++++ .../IgniteSnapshotVerifyException.java | 11 ++++ .../SnapshotMetadataCollectorTask.java | 14 ++++- .../snapshot/SnapshotVerifier.java | 1 - .../IgniteClusterSnapshotCheckTest.java | 54 +++++++++++++++++++ 5 files changed, 98 insertions(+), 3 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index f9ea6a5734d9a..0b5a1ab3331a8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -310,6 +310,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Last seen cluster snapshot operation. */ private volatile ClusterSnapshotFuture lastSeenSnpFut = new ClusterSnapshotFuture(); + /** Optional user snapshot verification. */ + private SnapshotVerifier optSnpCheck; + /** * @param ctx Kernal context. */ @@ -381,6 +384,8 @@ public static String partDeltaFileName(int partId) { U.ensureDirectory(locSnpDir, "snapshot work directory", log); U.ensureDirectory(tmpWorkDir, "temp directory for snapshot creation", log); + optSnpCheck = ctx.plugins().createComponent(SnapshotVerifier.class); + MetricRegistry mreg = cctx.kernalContext().metric().registry(SNAPSHOT_METRICS); mreg.register("LastSnapshotStartTime", () -> lastSeenSnpFut.startTime, @@ -1465,6 +1470,22 @@ public GridCloseableIterator partitionRowIterator(GridKernalContex }; } + /** + * @param metas List of snapshot metadata. + * @throws SnapshotVerifierException If optional verification has failed. + */ + void optionalSnapshotCheck(List metas) throws SnapshotVerifierException { + if (optSnpCheck == null) + return; + + SnapshotMetadata meta = F.first(metas); + + if (!meta.consistentId().equals(cctx.localNode().consistentId().toString())) + return; + + optSnpCheck.verify(snapshotLocalDir(meta.snapshotName()).toPath()); + } + /** * @param snpName Unique snapshot name. * @param srcNodeId Node id which cause snapshot operation. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java index bcaea4256d3bd..1b63df7960035 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java @@ -21,6 +21,7 @@ import java.util.Map; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.util.typedef.internal.SB; /** * Compound snapshot verification exception from the nodes where the verification process executed. @@ -45,4 +46,14 @@ public IgniteSnapshotVerifyException(Map map) public Map exceptions() { return exs; } + + /** {@inheritDoc} */ + @Override public String getMessage() { + SB buf = new SB(); + + for (Map.Entry entry : exs.entrySet()) + buf.a("Snapshot check failed [nodeId=").a(entry.getKey().id()).a(", reason=").a(entry.getValue()).a("]. "); + + return buf.toString(); + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java index 8ddd00a0e76e3..fc4030e37f3f0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java @@ -54,8 +54,18 @@ public class SnapshotMetadataCollectorTask private transient IgniteEx ignite; @Override public List execute() throws IgniteException { - return ignite.context().cache().context().snapshotMgr() - .readSnapshotMetadatas(snpName); + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + List metas = snpMgr.readSnapshotMetadatas(snpName); + + try { + snpMgr.optionalSnapshotCheck(metas); + } + catch (SnapshotVerifierException e) { + throw new IgniteException(e); + } + + return metas; } }, node); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java index f93e0dd318315..5111d02acde1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java @@ -30,5 +30,4 @@ public interface SnapshotVerifier { * @throws SnapshotVerifierException Thrown if snapshot integrity is corrupted or impossible to check. */ public void verify(Path snpDir) throws SnapshotVerifierException; - } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index c57ce1ae5f540..555195f7f9c9a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -42,6 +42,7 @@ import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridJobExecuteRequest; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteEx; @@ -75,6 +76,11 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginConfiguration; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; +import org.apache.ignite.testframework.GridTestUtils; import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -106,12 +112,25 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest { /** Optional cache name to be created on demand. */ private static final String OPTIONAL_CACHE_NAME = "CacheName"; + /** Optional snapshot check plugin. */ + private PluginProvider snpCheckPlugin; + /** Cleanup data of task execution results if need. */ @Before public void beforeCheck() { jobResults.clear(); } + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + if (snpCheckPlugin != null) + cfg.setPluginProviders(snpCheckPlugin); + + return cfg; + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCheck() throws Exception { @@ -132,6 +151,41 @@ public void testClusterSnapshotCheck() throws Exception { assertContains(log, b.toString(), "The check procedure has finished, no conflicts have been found"); } + /** @throws Exception If fails. */ + @Test + public void testClusterSnapshotOptionalCheck() throws Exception { + String exMsg = "Test verification exception message."; + + snpCheckPlugin = new AbstractTestPluginProvider() { + @Override public String name() { + return "SnapshotVerifier"; + } + + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (cls != SnapshotVerifier.class) + return null; + + return (T)(SnapshotVerifier)((path) -> { + throw new SnapshotVerifierException(exMsg); + }); + } + }; + + IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + startClientGrid(); + + ignite.snapshot().createSnapshot(SNAPSHOT_NAME) + .get(); + + GridTestUtils.assertThrowsAnyCause( + log, + () -> snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(), + IgniteSnapshotVerifyException.class, + exMsg + ); + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotCheckMissedPart() throws Exception { From f339bb125ce262619c48feaf8301cc7a20154d27 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 21 Jul 2021 13:10:42 +0300 Subject: [PATCH 05/19] IGNITE-14070 Improve snapshot check. --- .../snapshot/IgniteSnapshotManager.java | 11 +++------ .../IgniteSnapshotVerifyException.java | 2 +- .../SnapshotMetadataCollectorTask.java | 14 ++--------- .../SnapshotPartitionsVerifyTask.java | 8 +++++++ .../IgniteClusterSnapshotCheckTest.java | 24 ++++++++++++------- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 0b5a1ab3331a8..5f4581e3ac507 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -1471,19 +1471,14 @@ public GridCloseableIterator partitionRowIterator(GridKernalContex } /** - * @param metas List of snapshot metadata. + * @param snpName Snapshot name. * @throws SnapshotVerifierException If optional verification has failed. */ - void optionalSnapshotCheck(List metas) throws SnapshotVerifierException { + void optionalSnapshotCheck(String snpName) throws SnapshotVerifierException { if (optSnpCheck == null) return; - SnapshotMetadata meta = F.first(metas); - - if (!meta.consistentId().equals(cctx.localNode().consistentId().toString())) - return; - - optSnpCheck.verify(snapshotLocalDir(meta.snapshotName()).toPath()); + optSnpCheck.verify(snapshotLocalDir(snpName).toPath()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java index 1b63df7960035..9133501f2e70a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java @@ -52,7 +52,7 @@ public Map exceptions() { SB buf = new SB(); for (Map.Entry entry : exs.entrySet()) - buf.a("Snapshot check failed [nodeId=").a(entry.getKey().id()).a(", reason=").a(entry.getValue()).a("]. "); + buf.a(entry.getValue()).a(" [nodeId=").a(entry.getKey().id()).a("]. "); return buf.toString(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java index fc4030e37f3f0..8ddd00a0e76e3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotMetadataCollectorTask.java @@ -54,18 +54,8 @@ public class SnapshotMetadataCollectorTask private transient IgniteEx ignite; @Override public List execute() throws IgniteException { - IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); - - List metas = snpMgr.readSnapshotMetadatas(snpName); - - try { - snpMgr.optionalSnapshotCheck(metas); - } - catch (SnapshotVerifierException e) { - throw new IgniteException(e); - } - - return metas; + return ignite.context().cache().context().snapshotMgr() + .readSnapshotMetadatas(snpName); } }, node); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index 0e99e8cfc094c..5a375611517b2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -193,6 +193,7 @@ public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collectio this.rqGrps = rqGrps == null ? Collections.emptySet() : new HashSet<>(rqGrps); } + /** {@inheritDoc} */ @Override public Map execute() throws IgniteException { IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); @@ -201,6 +202,13 @@ public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collectio "[snpName=" + snpName + ", consId=" + consId + ']'); } + try { + snpMgr.optionalSnapshotCheck(snpName); + } + catch (SnapshotVerifierException e) { + throw new IgniteException(e); + } + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); Set grps = rqGrps.isEmpty() ? new HashSet<>(meta.partitions().keySet()) : rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index 555195f7f9c9a..adae479dfa931 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -37,6 +37,7 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; @@ -71,6 +72,7 @@ import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -80,7 +82,6 @@ import org.apache.ignite.plugin.PluginConfiguration; import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.plugin.PluginProvider; -import org.apache.ignite.testframework.GridTestUtils; import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -175,15 +176,20 @@ public void testClusterSnapshotOptionalCheck() throws Exception { startClientGrid(); - ignite.snapshot().createSnapshot(SNAPSHOT_NAME) - .get(); + ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(); + + IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); - GridTestUtils.assertThrowsAnyCause( - log, - () -> snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(), - IgniteSnapshotVerifyException.class, - exMsg - ); + assertTrue(!res.exceptions().isEmpty()); + + for (Ignite grid : G.allGrids()) { + if (grid.cluster().localNode().isClient()) + continue; + + Exception ex = res.exceptions().get(grid.cluster().localNode()); + + assertTrue(X.hasCause(ex, exMsg, SnapshotVerifierException.class)); + } } /** @throws Exception If fails. */ From e5a1f075d008e5ace659b3f364a19a3cc30117b6 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 26 Jul 2021 12:49:49 +0300 Subject: [PATCH 06/19] IGNITE-14070 Checks should be pluggable with default implementation (wip). --- .../persistence/snapshot/SnapshotVerify.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java new file mode 100644 index 0000000000000..87aee468f3831 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +public interface SnapshotVerify { +} From 2d0eac51e6cbfe5a69a777dea5608f3e4e2d5032 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Mon, 26 Jul 2021 22:48:11 +0300 Subject: [PATCH 07/19] IGNITE-14070 Rework optional snapshot verification. --- .../snapshot/IgniteSnapshotManager.java | 18 +-- .../IgniteSnapshotVerifyException.java | 11 -- .../SnapshotPartitionsVerifyTask.java | 8 -- .../snapshot/SnapshotRestoreProcess.java | 118 ++++++++++-------- .../snapshot/SnapshotVerifier.java | 17 ++- .../snapshot/SnapshotVerifierException.java | 61 --------- .../persistence/snapshot/SnapshotVerify.java | 21 ---- .../IgniteClusterSnapshotCheckTest.java | 60 --------- .../IgniteClusterSnapshotRestoreSelfTest.java | 54 +++++++- 9 files changed, 129 insertions(+), 239 deletions(-) delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java delete mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 5f4581e3ac507..8278e91a25cda 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -310,9 +310,6 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Last seen cluster snapshot operation. */ private volatile ClusterSnapshotFuture lastSeenSnpFut = new ClusterSnapshotFuture(); - /** Optional user snapshot verification. */ - private SnapshotVerifier optSnpCheck; - /** * @param ctx Kernal context. */ @@ -384,8 +381,6 @@ public static String partDeltaFileName(int partId) { U.ensureDirectory(locSnpDir, "snapshot work directory", log); U.ensureDirectory(tmpWorkDir, "temp directory for snapshot creation", log); - optSnpCheck = ctx.plugins().createComponent(SnapshotVerifier.class); - MetricRegistry mreg = cctx.kernalContext().metric().registry(SNAPSHOT_METRICS); mreg.register("LastSnapshotStartTime", () -> lastSeenSnpFut.startTime, @@ -403,6 +398,8 @@ public static String partDeltaFileName(int partId) { "The list of names of all snapshots currently saved on the local node with respect to " + "the configured via IgniteConfiguration snapshot working path."); + restoreCacheGrpProc.init(); + storeFactory = storeMgr::getPageStoreFactory; cctx.exchange().registerExchangeAwareComponent(this); @@ -1470,17 +1467,6 @@ public GridCloseableIterator partitionRowIterator(GridKernalContex }; } - /** - * @param snpName Snapshot name. - * @throws SnapshotVerifierException If optional verification has failed. - */ - void optionalSnapshotCheck(String snpName) throws SnapshotVerifierException { - if (optSnpCheck == null) - return; - - optSnpCheck.verify(snapshotLocalDir(snpName).toPath()); - } - /** * @param snpName Unique snapshot name. * @param srcNodeId Node id which cause snapshot operation. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java index 9133501f2e70a..bcaea4256d3bd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotVerifyException.java @@ -21,7 +21,6 @@ import java.util.Map; import org.apache.ignite.IgniteException; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.util.typedef.internal.SB; /** * Compound snapshot verification exception from the nodes where the verification process executed. @@ -46,14 +45,4 @@ public IgniteSnapshotVerifyException(Map map) public Map exceptions() { return exs; } - - /** {@inheritDoc} */ - @Override public String getMessage() { - SB buf = new SB(); - - for (Map.Entry entry : exs.entrySet()) - buf.a(entry.getValue()).a(" [nodeId=").a(entry.getKey().id()).a("]. "); - - return buf.toString(); - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index 5a375611517b2..0e99e8cfc094c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -193,7 +193,6 @@ public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collectio this.rqGrps = rqGrps == null ? Collections.emptySet() : new HashSet<>(rqGrps); } - /** {@inheritDoc} */ @Override public Map execute() throws IgniteException { IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); @@ -202,13 +201,6 @@ public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collectio "[snpName=" + snpName + ", consId=" + consId + ']'); } - try { - snpMgr.optionalSnapshotCheck(snpName); - } - catch (SnapshotVerifierException e) { - throw new IgniteException(e); - } - SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); Set grps = rqGrps.isEmpty() ? new HashSet<>(meta.partitions().keySet()) : rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 5061532d9e65d..553eddc16a9d5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -107,6 +107,9 @@ public class SnapshotRestoreProcess { /** Snapshot restore operation context. */ private volatile SnapshotRestoreContext opCtx; + /** Optional user-defined integrity check of the snapshot. */ + private SnapshotVerifier optSnpVerification; + /** * @param ctx Kernal context. */ @@ -143,6 +146,11 @@ protected void cleanup() throws IgniteCheckedException { } } + /** Initialization. */ + protected void init() { + optSnpVerification = ctx.plugins().createComponent(SnapshotVerifier.class); + } + /** * Start cache group restore operation. * @@ -185,88 +193,96 @@ public IgniteFuture start(String snpName, @Nullable Collection cac } ctx.cache().context().snapshotMgr().checkSnapshot(snpName, cacheGrpNames).listen(f -> { - if (f.error() != null) { - finishProcess(fut0.rqId, f.error()); + Throwable err = f.error() != null ? f.error() : fut0.interruptEx; - return; - } + if (err == null && !F.isEmpty(f.result().exceptions())) + err = F.first(f.result().exceptions().values()); - if (!F.isEmpty(f.result().exceptions())) { - finishProcess(fut0.rqId, F.first(f.result().exceptions().values())); + IdleVerifyResultV2 res = f.result().idleVerifyResult(); - return; + if (err == null && !F.isEmpty(res.exceptions()) || res.hasConflicts()) { + StringBuilder sb = new StringBuilder(); + + res.print(sb::append, true); + + err = new IgniteException(sb.toString()); } - if (fut0.interruptEx != null) { - finishProcess(fut0.rqId, fut0.interruptEx); + if (err != null) { + finishProcess(fut0.rqId, err); return; } - Set dataNodes = new HashSet<>(); - Set snpBltNodes = null; Map> metas = f.result().metas(); - Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : - cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); - for (Map.Entry> entry : metas.entrySet()) { - SnapshotMetadata meta = F.first(entry.getValue()); + IgniteInternalFuture optVerifyFut = + optSnpVerification != null ? optSnpVerification.verify(metas, cacheGrpNames) : new GridFinishedFuture<>(); - assert meta != null : entry.getKey().id(); + optVerifyFut.listen( + f0 -> { + Throwable err0 = f0.error() != null ? f0.error() : fut0.interruptEx; - if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) - continue; + if (err0 != null) { + finishProcess(fut0.rqId, err0); - if (snpBltNodes == null) - snpBltNodes = new HashSet<>(meta.baselineNodes()); + return; + } - dataNodes.add(entry.getKey().id()); + Set dataNodes = new HashSet<>(); + Set snpBltNodes = null; - reqGrpIds.keySet().removeAll(meta.partitions().keySet()); - } + Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : + cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); - if (snpBltNodes == null) { - finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + - "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); + for (Map.Entry> entry : metas.entrySet()) { + SnapshotMetadata meta = F.first(entry.getValue()); - return; - } + assert meta != null : entry.getKey().id(); + + if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) + continue; - if (!reqGrpIds.isEmpty()) { - finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + - "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); + if (snpBltNodes == null) + snpBltNodes = new HashSet<>(meta.baselineNodes()); - return; - } + dataNodes.add(entry.getKey().id()); - Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), - node -> node.consistentId().toString(), (node) -> CU.baselineNode(node, ctx.state().clusterState())); + reqGrpIds.keySet().removeAll(meta.partitions().keySet()); + } - snpBltNodes.removeAll(bltNodes); + if (snpBltNodes == null) { + finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + + "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - if (!snpBltNodes.isEmpty()) { - finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + - "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); + return; + } - return; - } + if (!reqGrpIds.isEmpty()) { + finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + + "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - IdleVerifyResultV2 res = f.result().idleVerifyResult(); + return; + } - if (!F.isEmpty(res.exceptions()) || res.hasConflicts()) { - StringBuilder sb = new StringBuilder(); + Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), + node -> node.consistentId().toString(), (node) -> CU.baselineNode(node, ctx.state().clusterState())); - res.print(sb::append, true); + snpBltNodes.removeAll(bltNodes); - finishProcess(fut0.rqId, new IgniteException(sb.toString())); + if (!snpBltNodes.isEmpty()) { + finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + + "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); - return; - } + return; + } - SnapshotOperationRequest req = new SnapshotOperationRequest( - fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); + SnapshotOperationRequest req = new SnapshotOperationRequest( + fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); - prepareRestoreProc.start(req.requestId(), req); + prepareRestoreProc.start(fut0.rqId, req); + } + ); }); return new IgniteFutureImpl<>(fut0); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java index 5111d02acde1e..7ea78b3a8dfad 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java @@ -17,17 +17,22 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.IgniteInternalFuture; /** - * Interface to check snapshot files integrity before restoring them. + * Optional user-defined integrity check of the snapshot. */ public interface SnapshotVerifier { /** - * Verify snapshot integrity. + * Performs an optional user-defined integrity check of the snapshot before restoring it. * - * @param snpDir Path to a snapshot directory. - * @throws SnapshotVerifierException Thrown if snapshot integrity is corrupted or impossible to check. + * @param metas The map of distribution of snapshot metadata pieces across the cluster. + * @param grps Cache groups to be restored or {@code null} to restore all cache groups from the snapshot. + * @return Future, which returns nothing if the check succeeds and throws an exception if the check fails. */ - public void verify(Path snpDir) throws SnapshotVerifierException; + public IgniteInternalFuture verify(Map> metas, Collection grps); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java deleted file mode 100644 index c51fc2e514fc9..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifierException.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.persistence.snapshot; - -import org.apache.ignite.IgniteCheckedException; -import org.jetbrains.annotations.Nullable; - -/** - * Thrown when snapshot integrity is corrupted or impossible to check. - */ -public class SnapshotVerifierException extends IgniteCheckedException { - /** */ - private static final long serialVersionUID = 0L; - - /** - * @param msg Message. - */ - public SnapshotVerifierException(String msg) { - super(msg); - } - - /** - * @param cause Cause. - */ - public SnapshotVerifierException(Throwable cause) { - super(cause); - } - - /** - * @param msg Message. - * @param cause Cause. - * @param writableStackTrace Writable stack trace. - */ - public SnapshotVerifierException(String msg, @Nullable Throwable cause, - boolean writableStackTrace) { - super(msg, cause, writableStackTrace); - } - - /** - * @param msg Message. - * @param cause Cause. - */ - public SnapshotVerifierException(String msg, @Nullable Throwable cause) { - super(msg, cause); - } -} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java deleted file mode 100644 index 87aee468f3831..0000000000000 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerify.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.internal.processors.cache.persistence.snapshot; - -public interface SnapshotVerify { -} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java index adae479dfa931..c57ce1ae5f540 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotCheckTest.java @@ -37,13 +37,11 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; -import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.configuration.CacheConfiguration; -import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.GridJobExecuteRequest; import org.apache.ignite.internal.GridTopic; import org.apache.ignite.internal.IgniteEx; @@ -72,16 +70,11 @@ import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.lang.GridIterator; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.X; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.internal.visor.verify.CacheFilterEnum; import org.apache.ignite.internal.visor.verify.VisorIdleVerifyTaskArg; -import org.apache.ignite.plugin.AbstractTestPluginProvider; -import org.apache.ignite.plugin.PluginConfiguration; -import org.apache.ignite.plugin.PluginContext; -import org.apache.ignite.plugin.PluginProvider; import org.jetbrains.annotations.Nullable; import org.junit.Before; import org.junit.Test; @@ -113,25 +106,12 @@ public class IgniteClusterSnapshotCheckTest extends AbstractSnapshotSelfTest { /** Optional cache name to be created on demand. */ private static final String OPTIONAL_CACHE_NAME = "CacheName"; - /** Optional snapshot check plugin. */ - private PluginProvider snpCheckPlugin; - /** Cleanup data of task execution results if need. */ @Before public void beforeCheck() { jobResults.clear(); } - /** {@inheritDoc} */ - @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { - IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - - if (snpCheckPlugin != null) - cfg.setPluginProviders(snpCheckPlugin); - - return cfg; - } - /** @throws Exception If fails. */ @Test public void testClusterSnapshotCheck() throws Exception { @@ -152,46 +132,6 @@ public void testClusterSnapshotCheck() throws Exception { assertContains(log, b.toString(), "The check procedure has finished, no conflicts have been found"); } - /** @throws Exception If fails. */ - @Test - public void testClusterSnapshotOptionalCheck() throws Exception { - String exMsg = "Test verification exception message."; - - snpCheckPlugin = new AbstractTestPluginProvider() { - @Override public String name() { - return "SnapshotVerifier"; - } - - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (cls != SnapshotVerifier.class) - return null; - - return (T)(SnapshotVerifier)((path) -> { - throw new SnapshotVerifierException(exMsg); - }); - } - }; - - IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); - - startClientGrid(); - - ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(); - - IdleVerifyResultV2 res = snp(ignite).checkSnapshot(SNAPSHOT_NAME).get(); - - assertTrue(!res.exceptions().isEmpty()); - - for (Ignite grid : G.allGrids()) { - if (grid.cluster().localNode().isClient()) - continue; - - Exception ex = res.exceptions().get(grid.cluster().localNode()); - - assertTrue(X.hasCause(ex, exMsg, SnapshotVerifierException.class)); - } - } - /** @throws Exception If fails. */ @Test public void testClusterSnapshotCheckMissedPart() throws Exception { diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index c4d6a7ee6f614..c7c2922c089a1 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -60,10 +60,15 @@ import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; +import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.PluginConfiguration; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.plugin.PluginProvider; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.testframework.GridTestUtils; import org.jetbrains.annotations.Nullable; @@ -93,25 +98,64 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR /** Default shared cache group name. */ private static final String SHARED_GRP = "shared"; + /** Optional snapshot check plugin. */ + private PluginProvider snpCheckPlugin; + /** Cache value builder. */ private Function valBuilder = String::valueOf; /** Reset consistent ID flag. */ private boolean resetConsistentId; + /** {@inheritDoc} */ + @Override protected Function valueBuilder() { + return valBuilder; + } + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - if (resetConsistentId) - cfg.setConsistentId(null); + if (snpCheckPlugin != null) + cfg.setPluginProviders(snpCheckPlugin); return cfg; } - /** {@inheritDoc} */ - @Override protected Function valueBuilder() { - return valBuilder; + /** @throws Exception If fails. */ + @Test + public void testClusterSnapshotOptionalVerification() throws Exception { + String exMsg = "Test verification exception message."; + + snpCheckPlugin = new AbstractTestPluginProvider() { + @Override public String name() { + return "SnapshotVerifier"; + } + + @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { + if (cls != SnapshotVerifier.class) + return null; + + return (T)(SnapshotVerifier)((metas, grps) -> new GridFinishedFuture<>(new IgniteCheckedException(exMsg))); + } + }; + + IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + + ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(); + + ignite.cache(DEFAULT_CACHE_NAME).destroy(); + + awaitPartitionMapExchange(); + + IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); + + GridTestUtils.assertThrowsAnyCause( + log, + () -> fut.get(TIMEOUT), + IgniteCheckedException.class, + exMsg + ); } /** @throws Exception If failed. */ From f343be1285a459dd2d94ccfc2ddf2bba48ec471c Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 29 Jul 2021 14:56:17 +0300 Subject: [PATCH 08/19] IGNITE-14070 (minor) Code cleanup. --- .../snapshot/SnapshotRestoreProcess.java | 34 +++++++++++-------- .../IgniteClusterSnapshotRestoreSelfTest.java | 24 ++++++------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 553eddc16a9d5..c53996922f3ee 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -193,25 +193,32 @@ public IgniteFuture start(String snpName, @Nullable Collection cac } ctx.cache().context().snapshotMgr().checkSnapshot(snpName, cacheGrpNames).listen(f -> { - Throwable err = f.error() != null ? f.error() : fut0.interruptEx; + if (f.error() != null) { + finishProcess(fut0.rqId, f.error()); - if (err == null && !F.isEmpty(f.result().exceptions())) - err = F.first(f.result().exceptions().values()); + return; + } - IdleVerifyResultV2 res = f.result().idleVerifyResult(); + if (!F.isEmpty(f.result().exceptions())) { + finishProcess(fut0.rqId, F.first(f.result().exceptions().values())); - if (err == null && !F.isEmpty(res.exceptions()) || res.hasConflicts()) { - StringBuilder sb = new StringBuilder(); + return; + } - res.print(sb::append, true); + if (fut0.interruptEx != null) { + finishProcess(fut0.rqId, fut0.interruptEx); - err = new IgniteException(sb.toString()); + return; } - if (err != null) { - finishProcess(fut0.rqId, err); + IdleVerifyResultV2 res = f.result().idleVerifyResult(); - return; + if (!F.isEmpty(res.exceptions()) || res.hasConflicts()) { + StringBuilder sb = new StringBuilder(); + + res.print(sb::append, true); + + finishProcess(fut0.rqId, new IgniteException(sb.toString())); } Map> metas = f.result().metas(); @@ -231,7 +238,6 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Set dataNodes = new HashSet<>(); Set snpBltNodes = null; - Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); @@ -240,8 +246,8 @@ public IgniteFuture start(String snpName, @Nullable Collection cac assert meta != null : entry.getKey().id(); - if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) - continue; + if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) + continue; if (snpBltNodes == null) snpBltNodes = new HashSet<>(meta.baselineNodes()); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index c7c2922c089a1..92fa714c482e3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -107,11 +107,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR /** Reset consistent ID flag. */ private boolean resetConsistentId; - /** {@inheritDoc} */ - @Override protected Function valueBuilder() { - return valBuilder; - } - /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); @@ -119,13 +114,21 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR if (snpCheckPlugin != null) cfg.setPluginProviders(snpCheckPlugin); + if (resetConsistentId) + cfg.setConsistentId(null); + return cfg; } + /** {@inheritDoc} */ + @Override protected Function valueBuilder() { + return valBuilder; + } + /** @throws Exception If fails. */ @Test public void testClusterSnapshotOptionalVerification() throws Exception { - String exMsg = "Test verification exception message."; + String expMsg = "Test verification exception message."; snpCheckPlugin = new AbstractTestPluginProvider() { @Override public String name() { @@ -136,7 +139,7 @@ public void testClusterSnapshotOptionalVerification() throws Exception { if (cls != SnapshotVerifier.class) return null; - return (T)(SnapshotVerifier)((metas, grps) -> new GridFinishedFuture<>(new IgniteCheckedException(exMsg))); + return (T)(SnapshotVerifier)((metas, grps) -> new GridFinishedFuture<>(new IgniteCheckedException(expMsg))); } }; @@ -150,12 +153,7 @@ public void testClusterSnapshotOptionalVerification() throws Exception { IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); - GridTestUtils.assertThrowsAnyCause( - log, - () -> fut.get(TIMEOUT), - IgniteCheckedException.class, - exMsg - ); + GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); } /** @throws Exception If failed. */ From 229c8c908c1789cbadf1cff27417bb6f885a1477 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 29 Jul 2021 15:51:55 +0300 Subject: [PATCH 09/19] IGNITE-14070 Sync call. --- .../snapshot/SnapshotRestoreProcess.java | 93 +++++++++---------- .../snapshot/SnapshotVerifier.java | 8 +- .../IgniteClusterSnapshotRestoreSelfTest.java | 5 +- 3 files changed, 52 insertions(+), 54 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index c53996922f3ee..131b8bfd7dd7e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -219,76 +219,73 @@ public IgniteFuture start(String snpName, @Nullable Collection cac res.print(sb::append, true); finishProcess(fut0.rqId, new IgniteException(sb.toString())); + + return; } Map> metas = f.result().metas(); - IgniteInternalFuture optVerifyFut = - optSnpVerification != null ? optSnpVerification.verify(metas, cacheGrpNames) : new GridFinishedFuture<>(); - - optVerifyFut.listen( - f0 -> { - Throwable err0 = f0.error() != null ? f0.error() : fut0.interruptEx; - - if (err0 != null) { - finishProcess(fut0.rqId, err0); + try { + if (optSnpVerification != null) + optSnpVerification.verify(metas, cacheGrpNames); + } + catch (IgniteCheckedException e) { + finishProcess(fut0.rqId, e); - return; - } + return; + } - Set dataNodes = new HashSet<>(); - Set snpBltNodes = null; - Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : - cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); + Set dataNodes = new HashSet<>(); + Set snpBltNodes = null; + Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : + cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); - for (Map.Entry> entry : metas.entrySet()) { - SnapshotMetadata meta = F.first(entry.getValue()); + for (Map.Entry> entry : metas.entrySet()) { + SnapshotMetadata meta = F.first(entry.getValue()); - assert meta != null : entry.getKey().id(); + assert meta != null : entry.getKey().id(); - if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) - continue; + if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) + continue; - if (snpBltNodes == null) - snpBltNodes = new HashSet<>(meta.baselineNodes()); + if (snpBltNodes == null) + snpBltNodes = new HashSet<>(meta.baselineNodes()); - dataNodes.add(entry.getKey().id()); + dataNodes.add(entry.getKey().id()); - reqGrpIds.keySet().removeAll(meta.partitions().keySet()); - } + reqGrpIds.keySet().removeAll(meta.partitions().keySet()); + } - if (snpBltNodes == null) { - finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + - "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); + if (snpBltNodes == null) { + finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + + "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - return; - } + return; + } - if (!reqGrpIds.isEmpty()) { - finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + - "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); + if (!reqGrpIds.isEmpty()) { + finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + + "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - return; - } + return; + } - Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), - node -> node.consistentId().toString(), (node) -> CU.baselineNode(node, ctx.state().clusterState())); + Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), + node -> node.consistentId().toString(), (node) -> CU.baselineNode(node, ctx.state().clusterState())); - snpBltNodes.removeAll(bltNodes); + snpBltNodes.removeAll(bltNodes); - if (!snpBltNodes.isEmpty()) { - finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + - "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); + if (!snpBltNodes.isEmpty()) { + finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + + "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); - return; - } + return; + } - SnapshotOperationRequest req = new SnapshotOperationRequest( - fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); + SnapshotOperationRequest req = new SnapshotOperationRequest( + fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); - prepareRestoreProc.start(fut0.rqId, req); - } - ); + prepareRestoreProc.start(req.requestId(), req); }); return new IgniteFutureImpl<>(fut0); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java index 7ea78b3a8dfad..37e996746fc0c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java @@ -20,8 +20,8 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; -import org.apache.ignite.internal.IgniteInternalFuture; /** * Optional user-defined integrity check of the snapshot. @@ -31,8 +31,8 @@ public interface SnapshotVerifier { * Performs an optional user-defined integrity check of the snapshot before restoring it. * * @param metas The map of distribution of snapshot metadata pieces across the cluster. - * @param grps Cache groups to be restored or {@code null} to restore all cache groups from the snapshot. - * @return Future, which returns nothing if the check succeeds and throws an exception if the check fails. + * @param grps Cache groups to be restored or {@code null} if all cache groups are being restored.. + * @throws IgniteCheckedException If the check fails. */ - public IgniteInternalFuture verify(Map> metas, Collection grps); + public void verify(Map> metas, Collection grps) throws IgniteCheckedException; } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 92fa714c482e3..6bc3ddc775acd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -60,7 +60,6 @@ import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory; import org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType; import org.apache.ignite.internal.util.distributed.SingleNodeMessage; -import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -139,7 +138,9 @@ public void testClusterSnapshotOptionalVerification() throws Exception { if (cls != SnapshotVerifier.class) return null; - return (T)(SnapshotVerifier)((metas, grps) -> new GridFinishedFuture<>(new IgniteCheckedException(expMsg))); + return (T)(SnapshotVerifier)((metas, grps) -> { + throw new IgniteCheckedException(expMsg); + }); } }; From 19cd1d2c7192203f7b453bca2443689151ab709d Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Fri, 30 Jul 2021 12:34:49 +0300 Subject: [PATCH 10/19] IGNITE-14070 Use plugin extensions to verify snapshot. --- .../snapshot/IgniteSnapshotManager.java | 2 - .../snapshot/SnapshotRestoreProcess.java | 28 ++++++---- .../snapshot/SnapshotVerifier.java | 3 +- .../IgniteClusterSnapshotRestoreSelfTest.java | 52 ++++++++++++++----- 4 files changed, 59 insertions(+), 26 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 8278e91a25cda..f9ea6a5734d9a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -398,8 +398,6 @@ public static String partDeltaFileName(int partId) { "The list of names of all snapshots currently saved on the local node with respect to " + "the configured via IgniteConfiguration snapshot working path."); - restoreCacheGrpProc.init(); - storeFactory = storeMgr::getPageStoreFactory; cctx.exchange().registerExchangeAwareComponent(this); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 131b8bfd7dd7e..ffd8eb7f02f7f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -22,6 +22,7 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -55,6 +56,7 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture; import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; +import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -107,9 +109,6 @@ public class SnapshotRestoreProcess { /** Snapshot restore operation context. */ private volatile SnapshotRestoreContext opCtx; - /** Optional user-defined integrity check of the snapshot. */ - private SnapshotVerifier optSnpVerification; - /** * @param ctx Kernal context. */ @@ -146,11 +145,6 @@ protected void cleanup() throws IgniteCheckedException { } } - /** Initialization. */ - protected void init() { - optSnpVerification = ctx.plugins().createComponent(SnapshotVerifier.class); - } - /** * Start cache group restore operation. * @@ -224,10 +218,24 @@ public IgniteFuture start(String snpName, @Nullable Collection cac } Map> metas = f.result().metas(); + SnapshotVerifier[] customVerifiocations = ctx.plugins().extensions(SnapshotVerifier.class); try { - if (optSnpVerification != null) - optSnpVerification.verify(metas, cacheGrpNames); + if (customVerifiocations != null) { + if (customVerifiocations.length == 1) + customVerifiocations[0].verify(metas, cacheGrpNames); + else { + IgniteUtils.doInParallel( + ctx.cache().context().snapshotMgr().snapshotExecutorService(), + Arrays.asList(customVerifiocations), + verifier -> { + verifier.verify(metas, cacheGrpNames); + + return null; + } + ); + } + } } catch (IgniteCheckedException e) { finishProcess(fut0.rqId, e); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java index 37e996746fc0c..e96106fdb112f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotVerifier.java @@ -22,11 +22,12 @@ import java.util.Map; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.plugin.Extension; /** * Optional user-defined integrity check of the snapshot. */ -public interface SnapshotVerifier { +public interface SnapshotVerifier extends Extension { /** * Performs an optional user-defined integrity check of the snapshot before restoring it. * diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 6bc3ddc775acd..4da22ec01869b 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -65,6 +65,7 @@ import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.ExtensionRegistry; import org.apache.ignite.plugin.PluginConfiguration; import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.plugin.PluginProvider; @@ -126,7 +127,7 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR /** @throws Exception If fails. */ @Test - public void testClusterSnapshotOptionalVerification() throws Exception { + public void testClusterSnapshotOptionalVerificationFailure() throws Exception { String expMsg = "Test verification exception message."; snpCheckPlugin = new AbstractTestPluginProvider() { @@ -134,27 +135,52 @@ public void testClusterSnapshotOptionalVerification() throws Exception { return "SnapshotVerifier"; } - @Override public @Nullable T createComponent(PluginContext ctx, Class cls) { - if (cls != SnapshotVerifier.class) - return null; - - return (T)(SnapshotVerifier)((metas, grps) -> { + /** {@inheritDoc} */ + @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { + registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { throw new IgniteCheckedException(expMsg); - }); + })); + + registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { + // No-op. + })); } }; - IgniteEx ignite = startGridsWithCache(3, dfltCacheCfg, CACHE_KEYS_RANGE); + IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); - ignite.snapshot().createSnapshot(SNAPSHOT_NAME).get(); + IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); - ignite.cache(DEFAULT_CACHE_NAME).destroy(); + GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); + } - awaitPartitionMapExchange(); + /** @throws Exception If fails. */ + @Test + public void testClusterSnapshotOptionalVerifications() throws Exception { + AtomicInteger checkCntr = new AtomicInteger(); - IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); + snpCheckPlugin = new AbstractTestPluginProvider() { + @Override public String name() { + return "SnapshotVerifier"; + } - GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); + /** {@inheritDoc} */ + @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { + registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { + checkCntr.incrementAndGet(); + })); + + registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { + checkCntr.incrementAndGet(); + })); + } + }; + + IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); + + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); + + assertEquals(2, checkCntr.get()); } /** @throws Exception If failed. */ From 707f094de695de045c21e04db394fed17a737fe7 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Fri, 30 Jul 2021 14:35:14 +0300 Subject: [PATCH 11/19] IGNITE-14070 Review note. --- .../snapshot/SnapshotRestoreProcess.java | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index ffd8eb7f02f7f..99a6709986073 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -22,7 +22,6 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -56,7 +55,6 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture; import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; -import org.apache.ignite.internal.util.IgniteUtils; import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.future.GridFinishedFuture; import org.apache.ignite.internal.util.future.GridFutureAdapter; @@ -222,19 +220,8 @@ public IgniteFuture start(String snpName, @Nullable Collection cac try { if (customVerifiocations != null) { - if (customVerifiocations.length == 1) - customVerifiocations[0].verify(metas, cacheGrpNames); - else { - IgniteUtils.doInParallel( - ctx.cache().context().snapshotMgr().snapshotExecutorService(), - Arrays.asList(customVerifiocations), - verifier -> { - verifier.verify(metas, cacheGrpNames); - - return null; - } - ); - } + for (SnapshotVerifier customVerification : customVerifiocations) + customVerification.verify(metas, cacheGrpNames); } } catch (IgniteCheckedException e) { From 241bca67579e2ff890b76925375f163a5bf293e7 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 3 Aug 2021 20:36:33 +0300 Subject: [PATCH 12/19] wip --- .../snapshot/IgniteSnapshotManager.java | 56 +++++ .../SnapshotLifeCycleListenerImpl.java | 219 ++++++++++++++++++ .../snapshot/SnapshotRestoreProcess.java | 157 +++++++++---- .../snapshot/lifecycle/RestoreHandleTask.java | 169 ++++++++++++++ .../lifecycle/SnapshotLifecycleListener.java | 71 ++++++ .../IgniteClusterSnapshotRestoreSelfTest.java | 25 ++ 6 files changed, 650 insertions(+), 47 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index f9ea6a5734d9a..d2c540c34f5e4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -100,6 +100,7 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadOnlyMetastorage; import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -301,6 +302,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Cluster snapshot operation requested by user. */ private ClusterSnapshotFuture clusterSnpFut; + // todo something like pipeline + private final List lifecycleListeners = new ArrayList<>(); + /** Current snapshot operation on local node. */ private volatile SnapshotOperationRequest clusterSnpReq; @@ -381,6 +385,13 @@ public static String partDeltaFileName(int partId) { U.ensureDirectory(locSnpDir, "snapshot work directory", log); U.ensureDirectory(tmpWorkDir, "temp directory for snapshot creation", log); + lifecycleListeners.add(new SnapshotLifeCycleListenerImpl(ctx)); + + SnapshotLifecycleListener[] lsnrs = cctx.kernalContext().plugins().extensions(SnapshotLifecycleListener.class); + + if (lsnrs != null) + Collections.addAll(lifecycleListeners, lsnrs); + MetricRegistry mreg = cctx.kernalContext().metric().registry(SNAPSHOT_METRICS); mreg.register("LastSnapshotStartTime", () -> lastSeenSnpFut.startTime, @@ -538,6 +549,10 @@ public void deleteSnapshot(File snpDir, String folderName) { } } + public Collection lifeCycleListeners() { + return lifecycleListeners; + } + /** * @param snpName Snapshot name. * @return Local snapshot directory for snapshot with given name. @@ -597,6 +612,13 @@ private IgniteInternalFuture initLocalSnapshotStartSt "prior to snapshot operation start: " + leftNodes)); } +// try { +// for (SnapshotLifecycleListener lsnr : lifecycleListeners) +// lsnr.beforeCreateSnapshot(cctx.localNode(), req); +// } catch (IgniteCheckedException e) { +// return new GridFinishedFuture<>(e); +// } + List grpIds = new ArrayList<>(F.viewReadOnly(req.groups(), CU::cacheId)); Set leftGrps = new HashSet<>(grpIds); @@ -669,6 +691,9 @@ private IgniteInternalFuture initLocalSnapshotStartSt log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); } + for (SnapshotLifecycleListener lsnr : lifecycleListeners) + lsnr.postCreate(req.snapshotName()); + return new SnapshotOperationResponse(); } catch (IOException | IgniteCheckedException e) { @@ -960,6 +985,37 @@ public IgniteInternalFuture checkSnapshot(String name) { }); } + /** + * @param name Snapshot name. + * @return Future with snapshot metadata obtained from nodes. + */ + IgniteInternalFuture>> collectSnapshotMetadata(String name) { + GridKernalContext kctx0 = cctx.kernalContext(); + + kctx0.security().authorize(ADMIN_SNAPSHOT); + + Collection bltNodes = F.view(cctx.discovery().serverNodes(AffinityTopologyVersion.NONE), + (node) -> CU.baselineNode(node, kctx0.state().clusterState())); + + kctx0.task().setThreadContext(TC_SKIP_AUTH, true); + kctx0.task().setThreadContext(TC_SUBGRID, bltNodes); + + return kctx0.task().execute(SnapshotMetadataCollectorTask.class, name); + } + +// /** +// * @param metas Nodes snapshot metadata. +// * @return Future with the verification results. +// */ +// IgniteInternalFuture runSnapshotVerification(Map> metas) { +// GridKernalContext kctx0 = cctx.kernalContext(); +// +// kctx0.task().setThreadContext(TC_SKIP_AUTH, true); +// kctx0.task().setThreadContext(TC_SUBGRID, new ArrayList<>(metas.keySet())); +// +// return kctx0.task().execute(SnapshotPartitionsVerifyTask.class, metas); +// } + /** * The check snapshot procedure performs compute operation over the whole cluster to verify the snapshot * entirety and partitions consistency. The result future will be completed with an exception if this diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java new file mode 100644 index 0000000000000..722d2cc1f8b83 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java @@ -0,0 +1,219 @@ +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.internal.GridComponent; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; +import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2; +import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2; +import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; + +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId; +import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId; +import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; +import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; + +public class SnapshotLifeCycleListenerImpl implements SnapshotLifecycleListener { + /** */ + private final GridKernalContext ctx; + + /** */ + private final IgniteLogger log; + + public SnapshotLifeCycleListenerImpl(GridKernalContext ctx) { + this.ctx = ctx; + + log = ctx.log(getClass()); + } + + @Override public String name() { + return "system:consistency-check"; + } + + /** {@inheritDoc} */ + @Override public T handleRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { + IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr(); + + if (log.isInfoEnabled()) { + log.info("Verify snapshot partitions procedure has been initiated " + + "[snpName=" + snpName + ", consId=" + consId + ']'); + } + + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); + Set grps = F.isEmpty(rqGrps) ? new HashSet<>(meta.partitions().keySet()) : + rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); + Set partFiles = new HashSet<>(); + + for (File dir : snpMgr.snapshotCacheDirectories(snpName, meta.folderName())) { + int grpId = CU.cacheId(cacheGroupName(dir)); + + if (!grps.remove(grpId)) + continue; + + Set parts = new HashSet<>(meta.partitions().get(grpId)); + + for (File part : cachePartitionFiles(dir)) { + int partId = partId(part.getName()); + + if (!parts.remove(partId)) + continue; + + partFiles.add(part); + } + + if (!parts.isEmpty()) { + throw new IgniteException("Snapshot data doesn't contain required cache group partition " + + "[grpId=" + grpId + ", snpName=" + snpName + ", consId=" + consId + + ", missed=" + parts + ", meta=" + meta + ']'); + } + } + + if (!grps.isEmpty()) { + throw new IgniteException("Snapshot data doesn't contain required cache groups " + + "[grps=" + grps + ", snpName=" + snpName + ", consId=" + consId + + ", meta=" + meta + ']'); + } + + Map res = new ConcurrentHashMap<>(); + ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) + .order(ByteOrder.nativeOrder())); + + try { + GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(snpName, meta.folderName()); + + for (GridComponent comp : snpCtx) + comp.start(); + + try { + U.doInParallel( + snpMgr.snapshotExecutorService(), + partFiles, + part -> { + String grpName = cacheGroupName(part.getParentFile()); + int grpId = CU.cacheId(grpName); + int partId = partId(part.getName()); + + FilePageStoreManager storeMgr = (FilePageStoreManager)ctx.cache().context().pageStore(); + + try (FilePageStore pageStore = (FilePageStore)storeMgr.getPageStoreFactory(grpId, false) + .createPageStore(getTypeByPartId(partId), + part::toPath, + val -> { + }) + ) { + if (partId == INDEX_PARTITION) { + checkPartitionsPageCrcSum(() -> pageStore, INDEX_PARTITION, FLAG_IDX); + + return null; + } + + if (grpId == MetaStorage.METASTORAGE_CACHE_ID) { + checkPartitionsPageCrcSum(() -> pageStore, partId, FLAG_DATA); + + return null; + } + + ByteBuffer pageBuff = buff.get(); + pageBuff.clear(); + pageStore.read(0, pageBuff, true); + + long pageAddr = GridUnsafe.bufferAddress(pageBuff); + + PagePartitionMetaIO io = PageIO.getPageIO(pageBuff); + GridDhtPartitionState partState = fromOrdinal(io.getPartitionState(pageAddr)); + + if (partState != OWNING) { + throw new IgniteCheckedException("Snapshot partitions must be in the OWNING " + + "state only: " + partState); + } + + long updateCntr = io.getUpdateCounter(pageAddr); + long size = io.getSize(pageAddr); + + if (log.isDebugEnabled()) { + log.debug("Partition [grpId=" + grpId + + ", id=" + partId + + ", counter=" + updateCntr + + ", size=" + size + "]"); + } + + // Snapshot partitions must always be in OWNING state. + // There is no `primary` partitions for snapshot. + PartitionKeyV2 key = new PartitionKeyV2(grpId, partId, grpName); + + PartitionHashRecordV2 hash = calculatePartitionHash(key, + updateCntr, + consId, + GridDhtPartitionState.OWNING, + false, + size, + snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore)); + + assert hash != null : "OWNING must have hash: " + key; + + res.put(key, hash); + } + catch (IOException e) { + throw new IgniteCheckedException(e); + } + + return null; + } + ); + } + finally { + for (GridComponent comp : snpCtx) + comp.stop(true); + } + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + + return (T)res; + } + + @Override public void reduce(List res) throws IgniteCheckedException { + IdleVerifyResultV2 verifyResult = VerifyBackupPartitionsTaskV2.reduce0(res); + + if (!F.isEmpty(verifyResult.exceptions()) || verifyResult.hasConflicts()) { + StringBuilder sb = new StringBuilder(); + + verifyResult.print(sb::append, true); + + throw new IgniteCheckedException(sb.toString()); + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 99a6709986073..b5b0c0a860b1e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -53,6 +53,8 @@ import org.apache.ignite.internal.processors.cache.StoredCacheData; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.RestoreHandleTask; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.distributed.DistributedProcess; @@ -72,9 +74,12 @@ import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_GRP_DIR_PREFIX; import static org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage.METASTORAGE_CACHE_NAME; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.databaseRelativePath; +import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SKIP_AUTH; +import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_ROLLBACK; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_SNAPSHOT; /** * Distributed process to restore cache group from the snapshot. @@ -179,57 +184,27 @@ public IgniteFuture start(String snpName, @Nullable Collection cac fut0 = fut; } - } - catch (IgniteException e) { - return new IgniteFinishedFutureImpl<>(e); - } - - ctx.cache().context().snapshotMgr().checkSnapshot(snpName, cacheGrpNames).listen(f -> { - if (f.error() != null) { - finishProcess(fut0.rqId, f.error()); - - return; - } - - if (!F.isEmpty(f.result().exceptions())) { - finishProcess(fut0.rqId, F.first(f.result().exceptions().values())); - - return; - } - - if (fut0.interruptEx != null) { - finishProcess(fut0.rqId, fut0.interruptEx); - - return; - } - - IdleVerifyResultV2 res = f.result().idleVerifyResult(); - if (!F.isEmpty(res.exceptions()) || res.hasConflicts()) { - StringBuilder sb = new StringBuilder(); + // todo - async execution + Map> metas = + ctx.cache().context().snapshotMgr().collectSnapshotMetadata(snpName).get(); - res.print(sb::append, true); + Collection lsnrs = ctx.cache().context().snapshotMgr().lifeCycleListeners(); - finishProcess(fut0.rqId, new IgniteException(sb.toString())); + if (!lsnrs.isEmpty()) { + ctx.security().authorize(ADMIN_SNAPSHOT); - return; - } + Collection bltNodes = F.view(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), + (node) -> CU.baselineNode(node, ctx.state().clusterState())); - Map> metas = f.result().metas(); - SnapshotVerifier[] customVerifiocations = ctx.plugins().extensions(SnapshotVerifier.class); + ctx.task().setThreadContext(TC_SKIP_AUTH, true); + ctx.task().setThreadContext(TC_SUBGRID, bltNodes); - try { - if (customVerifiocations != null) { - for (SnapshotVerifier customVerification : customVerifiocations) - customVerification.verify(metas, cacheGrpNames); - } - } - catch (IgniteCheckedException e) { - finishProcess(fut0.rqId, e); - - return; + ctx.task().execute(RestoreHandleTask.class, new SnapshotPartitionsVerifyTaskArg(cacheGrpNames, metas)).get(); } + // +// Map> metas = f.result().metas(); Set dataNodes = new HashSet<>(); Set snpBltNodes = null; Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : @@ -255,14 +230,14 @@ public IgniteFuture start(String snpName, @Nullable Collection cac finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - return; + return new IgniteFutureImpl<>(fut0); } if (!reqGrpIds.isEmpty()) { finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); - return; + return new IgniteFutureImpl<>(fut0); } Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), @@ -274,14 +249,102 @@ public IgniteFuture start(String snpName, @Nullable Collection cac finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); - return; + return new IgniteFutureImpl<>(fut0); } SnapshotOperationRequest req = new SnapshotOperationRequest( fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); prepareRestoreProc.start(req.requestId(), req); - }); + } + catch (IgniteException | IgniteCheckedException e) { + return new IgniteFinishedFutureImpl<>(e); + } + +// ctx.cache().context().snapshotMgr().checkSnapshot(snpName, cacheGrpNames).listen(f -> { +// if (f.error() != null) { +// finishProcess(fut0.rqId, f.error()); +// +// return; +// } +// +// if (!F.isEmpty(f.result().exceptions())) { +// finishProcess(fut0.rqId, F.first(f.result().exceptions().values())); +// +// return; +// } +// +// if (fut0.interruptEx != null) { +// finishProcess(fut0.rqId, fut0.interruptEx); +// +// return; +// } +// +// IdleVerifyResultV2 res = f.result().idleVerifyResult(); +// +// if (!F.isEmpty(res.exceptions()) || res.hasConflicts()) { +// StringBuilder sb = new StringBuilder(); +// +// res.print(sb::append, true); +// +// finishProcess(fut0.rqId, new IgniteException(sb.toString())); +// +// return; +// } +// +// Map> metas = f.result().metas(); +// Set dataNodes = new HashSet<>(); +// Set snpBltNodes = null; +// Map reqGrpIds = cacheGrpNames == null ? Collections.emptyMap() : +// cacheGrpNames.stream().collect(Collectors.toMap(CU::cacheId, v -> v)); +// +// for (Map.Entry> entry : metas.entrySet()) { +// SnapshotMetadata meta = F.first(entry.getValue()); +// +// assert meta != null : entry.getKey().id(); +// +// if (!entry.getKey().consistentId().toString().equals(meta.consistentId())) +// continue; +// +// if (snpBltNodes == null) +// snpBltNodes = new HashSet<>(meta.baselineNodes()); +// +// dataNodes.add(entry.getKey().id()); +// +// reqGrpIds.keySet().removeAll(meta.partitions().keySet()); +// } +// +// if (snpBltNodes == null) { +// finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "No snapshot data " + +// "has been found [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); +// +// return; +// } +// +// if (!reqGrpIds.isEmpty()) { +// finishProcess(fut0.rqId, new IllegalArgumentException(OP_REJECT_MSG + "Cache group(s) was not " + +// "found in the snapshot [groups=" + reqGrpIds.values() + ", snapshot=" + snpName + ']')); +// +// return; +// } +// +// Collection bltNodes = F.viewReadOnly(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), +// node -> node.consistentId().toString(), (node) -> CU.baselineNode(node, ctx.state().clusterState())); +// +// snpBltNodes.removeAll(bltNodes); +// +// if (!snpBltNodes.isEmpty()) { +// finishProcess(fut0.rqId, new IgniteIllegalStateException(OP_REJECT_MSG + "Some nodes required to " + +// "restore a cache group are missing [nodeId(s)=" + snpBltNodes + ", snapshot=" + snpName + ']')); +// +// return; +// } +// +// SnapshotOperationRequest req = new SnapshotOperationRequest( +// fut0.rqId, F.first(dataNodes), snpName, cacheGrpNames, dataNodes); +// +// prepareRestoreProc.start(req.requestId(), req); +// }); return new IgniteFutureImpl<>(fut0); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java new file mode 100644 index 0000000000000..c425e364d72fc --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -0,0 +1,169 @@ +package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.compute.ComputeJob; +import org.apache.ignite.compute.ComputeJobAdapter; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.compute.ComputeTaskAdapter; +import org.apache.ignite.internal.GridJobResultImpl; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotVerifyException; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyTaskArg; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.resources.LoggerResource; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +public class RestoreHandleTask extends ComputeTaskAdapter> { + /** Ignite instance. */ + @IgniteInstanceResource + private IgniteEx ignite; + + /** Task argument. */ + private final Map> metas = new HashMap<>(); + + /** {@inheritDoc} */ + @Override public @NotNull Map map(List subgrid, + @Nullable SnapshotPartitionsVerifyTaskArg arg) throws IgniteException { + + Map> clusterMetas = arg.clusterMetadata(); + + if (!subgrid.containsAll(clusterMetas.keySet())) { + throw new IgniteSnapshotVerifyException(F.asMap(ignite.localNode(), + new IgniteException("Some of Ignite nodes left the cluster during the snapshot verification " + + "[curr=" + F.viewReadOnly(subgrid, F.node2id()) + + ", init=" + F.viewReadOnly(clusterMetas.keySet(), F.node2id()) + ']'))); + } + + Map jobs = new HashMap<>(); + Set allMetas = new HashSet<>(); + clusterMetas.values().forEach(allMetas::addAll); + + Set missed = null; + + for (SnapshotMetadata meta : allMetas) { + if (missed == null) + missed = new HashSet<>(meta.baselineNodes()); + + missed.remove(meta.consistentId()); + + if (missed.isEmpty()) + break; + } + + if (!missed.isEmpty()) { + throw new IgniteSnapshotVerifyException(F.asMap(ignite.localNode(), + new IgniteException("Some metadata is missing from the snapshot: " + missed))); + } + + metas.putAll(clusterMetas); + + while (!allMetas.isEmpty()) { + for (Map.Entry> e : clusterMetas.entrySet()) { + SnapshotMetadata meta = F.find(e.getValue(), null, allMetas::remove); + + if (meta == null) + continue; + + jobs.put(new RestoreHandleJob(meta.snapshotName(), meta.consistentId(), arg.cacheGroupNames()), + e.getKey()); + + if (allMetas.isEmpty()) + break; + } + } + + return jobs; + +// return null; + } + + /** {@inheritDoc} */ + @Nullable @Override public Map reduce(List results) throws IgniteException { + // re-map + Map> resMap = new HashMap<>(); + + for (ComputeJobResult res : results) { + Map nodeDataMap = res.getData(); + + for (Map.Entry entry : nodeDataMap.entrySet()) { + // Copy job result. + GridJobResultImpl jobRes = + new GridJobResultImpl(res.getJob(), res.getJobContext().getJobId(), res.getNode(), ((GridJobResultImpl)res).getSibling()); + + jobRes.onResponse(entry.getValue(), res.getException(), (Map)res.getJobContext().getAttributes(), res.isCancelled()); + + resMap.computeIfAbsent(entry.getKey(), v -> new ArrayList<>()).add(jobRes); + } + } + + for (SnapshotLifecycleListener lsnr : ignite.context().cache().context().snapshotMgr().lifeCycleListeners()) { + List nodeResults = resMap.get(lsnr.name()); + + try { + lsnr.reduce(nodeResults); + } catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + } + + return null; + } + + private static class RestoreHandleJob extends ComputeJobAdapter { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Ignite instance. */ + @IgniteInstanceResource + private IgniteEx ignite; + + /** Injected logger. */ + @LoggerResource + private IgniteLogger log; + + String snapshotName; + + String consistentId; + + Collection groupNames; + + public RestoreHandleJob(String snapshotName, String consistentId, Collection groupNames) { + this.snapshotName = snapshotName; + this.consistentId = consistentId; + this.groupNames = groupNames; + } + + /** {@inheritDoc} */ + @Override public Map execute() throws IgniteException { + // todo expand restore hadnler separately + Collection lsnrs = ignite.context().cache().context().snapshotMgr().lifeCycleListeners(); + + Map resMap = new HashMap<>(); + + try { + for (SnapshotLifecycleListener lsnr : lsnrs) { + Object res = lsnr.handleRestore(snapshotName, consistentId, groupNames); + + resMap.put(lsnr.name(), res); + } + } catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + + return resMap; + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java new file mode 100644 index 0000000000000..759e05e1fe0e7 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java @@ -0,0 +1,71 @@ +package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; + +import java.util.Collection; +import java.util.List; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.plugin.Extension; + +public interface SnapshotLifecycleListener extends Extension { + public String name(); + + public default void postCreate(String snpName) throws IgniteCheckedException { + // No-op. + } + + public default T handleRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { + return null; + } + + public default void reduce(List res) throws IgniteCheckedException { + // No-op. + } + + // global +// public default void beforeCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException +// // local +// public default void beforeCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; +// public default void afterCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req, File snapshotMetadata) throws IgniteCheckedException; +// +// // global +// public default void beforeRestoreSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException; +// public default void afterRestoreSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException; +// +// // local +// public default void beforeRestoreSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; +// public default void afterRestoreSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; + + + + +// // global +// public default void beforeCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException { +// +// } +// +// // local +// public default void beforeCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException { +// // No-op. +// }; +// +// // global +// public default void afterCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException { +// // No-op. +// }; +// +// // local +// public default void afterCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req, File snapshotMetadata) throws IgniteCheckedException { +// // No-op. +// }; +// +// // global +// public default void beforeRestoreSnapshot(Map> metas, Collection grps) throws IgniteCheckedException { +// +// } +// +// // local +// public void beforeRestoreSnapshot(ClusterNode locNode, SnapshotMetadata metadata, Collection grps); +// +// +// public void afterRestoreSnapshot(); +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 4da22ec01869b..a1ff8e720c2d7 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -20,6 +20,7 @@ import java.io.File; import java.io.IOException; import java.nio.file.OpenOption; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.Collections; @@ -77,9 +78,11 @@ import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.CACHE_DIR_PREFIX; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.FILE_SUFFIX; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_PREFIX; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName; import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreProcess.TMP_CACHE_DIR_PREFIX; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_PREPARE; import static org.apache.ignite.internal.util.distributed.DistributedProcess.DistributedProcessType.RESTORE_CACHE_GROUP_SNAPSHOT_START; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; import static org.apache.ignite.testframework.GridTestUtils.runAsync; /** @@ -183,6 +186,28 @@ public void testClusterSnapshotOptionalVerifications() throws Exception { assertEquals(2, checkCntr.get()); } + /** @throws Exception If fails. */ + @Test + public void testRestoreWithMissedPart() throws Exception { + IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); + + Path part0 = U.searchFileRecursively(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME).toPath(), + getPartitionFileName(0)); + + assertNotNull(part0); + assertTrue(part0.toString(), part0.toFile().exists()); + assertTrue(part0.toFile().delete()); + + assertThrowsAnyCause( + log, + () -> ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(), + IgniteException.class, + "Snapshot data doesn't contain required cache group partition" + ); + + ensureCacheAbsent(dfltCacheCfg); + } + /** @throws Exception If failed. */ @Test public void testRestoreAllGroups() throws Exception { From caaffd58e7c64629f5171434305e5a03771f0fa2 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Tue, 3 Aug 2021 23:58:46 +0300 Subject: [PATCH 13/19] wipwip --- .../snapshot/IgniteClusterSnapshotRestoreBaseTest.java | 10 ++++++++++ .../snapshot/IgniteClusterSnapshotRestoreSelfTest.java | 9 ++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java index 18d02f5242b15..b649e0fcd56d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java @@ -20,7 +20,10 @@ import java.util.function.Function; import org.apache.ignite.IgniteCache; import org.apache.ignite.binary.BinaryObjectBuilder; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.testframework.GridTestUtils; /** * Snapshot restore test base. @@ -29,6 +32,13 @@ public abstract class IgniteClusterSnapshotRestoreBaseTest extends AbstractSnaps /** Cache value builder. */ protected abstract Function valueBuilder(); + protected final int PARTS_NUMBER = GridTestUtils.SF.apply(512); + + /** {@inheritDoc} */ + @Override protected CacheConfiguration txCacheConfig(CacheConfiguration ccfg) { + return super.txCacheConfig(ccfg).setAffinity(new RendezvousAffinityFunction(false, 8)); + } + /** * @param nodesCnt Nodes count. * @param keysCnt Number of keys to create. diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index a1ff8e720c2d7..fe7aface6f1d6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -205,7 +205,9 @@ public void testRestoreWithMissedPart() throws Exception { "Snapshot data doesn't contain required cache group partition" ); + System.out.println(">xxx> ensure cache absent"); ensureCacheAbsent(dfltCacheCfg); + System.out.println(">xxx> ensure cache absent :: end"); } /** @throws Exception If failed. */ @@ -823,10 +825,15 @@ private void ensureCacheAbsent(CacheConfiguration ccfg) throws IgniteCheck assertNull("nodeId=" + kctx.localNodeId() + ", cache=" + cacheName, desc); - GridTestUtils.waitForCondition( + System.out.println(">xxx> await restore..."); + boolean cond = GridTestUtils.waitForCondition( () -> !kctx.cache().context().snapshotMgr().isRestoring(), TIMEOUT); + assertTrue(cond); + + System.out.println(">xxx> await restore... DONE"); + File dir = ((FilePageStoreManager)kctx.cache().context().pageStore()).cacheWorkDir(ccfg); String errMsg = String.format("%s, dir=%s, exists=%b, files=%s", From 9a43edc99bd6ed62fd26b0720d871feb9d12210d Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Wed, 4 Aug 2021 19:55:48 +0300 Subject: [PATCH 14/19] IGNITE-14070 Snapshot listeners management. --- .../snapshot/IgniteSnapshotManager.java | 17 +- ...a => SnapshotRestoreConsistencyCheck.java} | 13 +- .../snapshot/SnapshotRestoreProcess.java | 11 +- .../snapshot/lifecycle/RestoreHandleTask.java | 25 ++- .../lifecycle/SnapshotLifecycleListener.java | 23 +++ .../snapshot/lifecycle/SnapshotListeners.java | 106 ++++++++++++ ...eClusterSnapshotLifecycleListenerTest.java | 83 ++++++++++ .../IgniteClusterSnapshotRestoreBaseTest.java | 43 ++++- .../IgniteClusterSnapshotRestoreSelfTest.java | 155 +++++++----------- 9 files changed, 360 insertions(+), 116 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/{SnapshotLifeCycleListenerImpl.java => SnapshotRestoreConsistencyCheck.java} (96%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index d2c540c34f5e4..36d9d0e6140eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -101,6 +101,7 @@ import org.apache.ignite.internal.processors.cache.persistence.metastorage.ReadWriteMetastorage; import org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId; import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotListeners; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; @@ -303,7 +304,7 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter private ClusterSnapshotFuture clusterSnpFut; // todo something like pipeline - private final List lifecycleListeners = new ArrayList<>(); + private SnapshotListeners snpLsnrs; /** Current snapshot operation on local node. */ private volatile SnapshotOperationRequest clusterSnpReq; @@ -385,12 +386,10 @@ public static String partDeltaFileName(int partId) { U.ensureDirectory(locSnpDir, "snapshot work directory", log); U.ensureDirectory(tmpWorkDir, "temp directory for snapshot creation", log); - lifecycleListeners.add(new SnapshotLifeCycleListenerImpl(ctx)); + snpLsnrs = new SnapshotListeners(ctx.plugins()); - SnapshotLifecycleListener[] lsnrs = cctx.kernalContext().plugins().extensions(SnapshotLifecycleListener.class); - - if (lsnrs != null) - Collections.addAll(lifecycleListeners, lsnrs); + // Register default. + snpLsnrs.register(new SnapshotRestoreConsistencyCheck(ctx)); MetricRegistry mreg = cctx.kernalContext().metric().registry(SNAPSHOT_METRICS); @@ -549,8 +548,8 @@ public void deleteSnapshot(File snpDir, String folderName) { } } - public Collection lifeCycleListeners() { - return lifecycleListeners; + public SnapshotListeners listeners() { + return snpLsnrs; } /** @@ -691,7 +690,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); } - for (SnapshotLifecycleListener lsnr : lifecycleListeners) + for (SnapshotLifecycleListener lsnr : listeners().list()) lsnr.postCreate(req.snapshotName()); return new SnapshotOperationResponse(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java similarity index 96% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java index 722d2cc1f8b83..8af4253b0c392 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotLifeCycleListenerImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java @@ -45,21 +45,28 @@ import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; -public class SnapshotLifeCycleListenerImpl implements SnapshotLifecycleListener { +public class SnapshotRestoreConsistencyCheck implements SnapshotLifecycleListener { + /** */ + public static final String LSNR_NAME = "system:consistency-check"; + /** */ private final GridKernalContext ctx; /** */ private final IgniteLogger log; - public SnapshotLifeCycleListenerImpl(GridKernalContext ctx) { + @Override public int priority() { + return Integer.MIN_VALUE; + } + + public SnapshotRestoreConsistencyCheck(GridKernalContext ctx) { this.ctx = ctx; log = ctx.log(getClass()); } @Override public String name() { - return "system:consistency-check"; + return LSNR_NAME; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index b5b0c0a860b1e..25c94a5e4f3c2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -55,7 +55,6 @@ import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager.ClusterSnapshotFuture; import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.RestoreHandleTask; import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; -import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.processors.cluster.DiscoveryDataClusterState; import org.apache.ignite.internal.util.distributed.DistributedProcess; import org.apache.ignite.internal.util.future.GridFinishedFuture; @@ -156,7 +155,7 @@ protected void cleanup() throws IgniteCheckedException { * @return Future that will be completed when the restore operation is complete and the cache groups are started. */ public IgniteFuture start(String snpName, @Nullable Collection cacheGrpNames) { - ClusterSnapshotFuture fut0; + ClusterSnapshotFuture fut0 = null; try { if (ctx.clientNode()) @@ -189,9 +188,9 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Map> metas = ctx.cache().context().snapshotMgr().collectSnapshotMetadata(snpName).get(); - Collection lsnrs = ctx.cache().context().snapshotMgr().lifeCycleListeners(); + Iterable lsnrs = ctx.cache().context().snapshotMgr().listeners().list(); - if (!lsnrs.isEmpty()) { + if (!F.isEmpty(lsnrs)) { ctx.security().authorize(ADMIN_SNAPSHOT); Collection bltNodes = F.view(ctx.discovery().serverNodes(AffinityTopologyVersion.NONE), @@ -202,7 +201,6 @@ public IgniteFuture start(String snpName, @Nullable Collection cac ctx.task().execute(RestoreHandleTask.class, new SnapshotPartitionsVerifyTaskArg(cacheGrpNames, metas)).get(); } - // // Map> metas = f.result().metas(); Set dataNodes = new HashSet<>(); @@ -258,6 +256,9 @@ public IgniteFuture start(String snpName, @Nullable Collection cac prepareRestoreProc.start(req.requestId(), req); } catch (IgniteException | IgniteCheckedException e) { + if (fut0 != null) + finishProcess(fut0.rqId, e); + return new IgniteFinishedFutureImpl<>(e); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java index c425e364d72fc..b7c215aa140b0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; import java.util.ArrayList; @@ -109,7 +126,7 @@ public class RestoreHandleTask extends ComputeTaskAdapter nodeResults = resMap.get(lsnr.name()); try { @@ -149,12 +166,12 @@ public RestoreHandleJob(String snapshotName, String consistentId, Collection execute() throws IgniteException { // todo expand restore hadnler separately - Collection lsnrs = ignite.context().cache().context().snapshotMgr().lifeCycleListeners(); - Map resMap = new HashMap<>(); + SnapshotListeners lsnrs = ignite.context().cache().context().snapshotMgr().listeners(); + try { - for (SnapshotLifecycleListener lsnr : lsnrs) { + for (SnapshotLifecycleListener lsnr : lsnrs.list()) { Object res = lsnr.handleRestore(snapshotName, consistentId, groupNames); resMap.put(lsnr.name(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java index 759e05e1fe0e7..bc042ec443eb9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; import java.util.Collection; @@ -7,8 +24,14 @@ import org.apache.ignite.plugin.Extension; public interface SnapshotLifecycleListener extends Extension { + public static final int DEFAULT_PRIORITY = 0; + public String name(); + public default int priority() { + return DEFAULT_PRIORITY; + } + public default void postCreate(String snpName) throws IgniteCheckedException { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java new file mode 100644 index 0000000000000..9544830c78cfa --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; +import org.apache.ignite.internal.util.typedef.F; + +public class SnapshotListeners { + private final IgnitePluginProcessor plugins; + private final Map lsnrsState = new HashMap<>(); + private final List lsnrsByPriority = new ArrayList<>(); + + public SnapshotListeners(IgnitePluginProcessor plugins) { + this.plugins = plugins; + + SnapshotLifecycleListener[] lsnrs = plugins.extensions(SnapshotLifecycleListener.class); + + if (lsnrs == null) + return; + + for (SnapshotLifecycleListener lsnr : lsnrs) + register(lsnr); + } + + public Iterable list() { + return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.name())); + } + +// public @Nullable SnapshotLifecycleListener find(String name) { +// T2 lsnrState = lsnrsByName.get(name); +// +// if (lsnrState == null) +// return null; +// +// return lsnrState.get1() ? lsnrState.get2() : null; +// } + + public void register(SnapshotLifecycleListener lsnr) { + if (lsnrsState.putIfAbsent(lsnr.name(), true) != null) + throw new IllegalArgumentException("Listener named " + lsnr.name() + " is already registered."); + + int idx = Collections.binarySearch(lsnrsByPriority, lsnr, Comparator.comparingInt(SnapshotLifecycleListener::priority)); + + if (idx < 0) + lsnrsByPriority.add(-idx - 1, lsnr); + else + lsnrsByPriority.add(idx, lsnr); + } + + public boolean disable(String name) { + Boolean enabled = lsnrsState.get(name); + + if (!Boolean.TRUE.equals(enabled)) + return false; + + return lsnrsState.put(name, false); + } + + public boolean enable(String name) { + Boolean enabled = lsnrsState.get(name); + + if (!Boolean.FALSE.equals(enabled)) + return false; + + return !lsnrsState.put(name, true); +// +// SnapshotLifecycleListener lsnr = lsnrsState.get(name); +// +// if (lsnr == null) +// throw new IllegalArgumentException("No listener with name " + lsnr.name() + " has been registered."); +// +// for (String ) +// +// for (SnapshotLifecycleListener lsnr : lsnrsState.values()) { +// if (name.equals(lsnr.name())) { +// register(lsnr); +// +// return true; +// } +// } +// +// return false; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java new file mode 100644 index 0000000000000..c76d2234fbbf0 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.nio.file.Path; +import java.util.function.Function; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.junit.Test; + +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreConsistencyCheck.LSNR_NAME; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; + +public class IgniteClusterSnapshotLifecycleListenerTest extends IgniteClusterSnapshotRestoreBaseTest { + /** @throws Exception If fails. */ + @Test + public void testRestoreWithMissedPart() throws Exception { + IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); + + Path part0 = U.searchFileRecursively(snp(ignite).snapshotLocalDir(SNAPSHOT_NAME).toPath(), + getPartitionFileName(0)); + + assertNotNull(part0); + assertTrue(part0.toString(), part0.toFile().exists()); + assertTrue(part0.toFile().delete()); + +// assertThrowsAnyCause( +// log, +// () -> ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(), +// IgniteException.class, +// "Snapshot data doesn't contain required cache group partition" +// ); +// +// ensureCacheAbsent(dfltCacheCfg); + + for (Ignite g : G.allGrids()) { + boolean disabled = ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().disable(LSNR_NAME); + assertTrue(disabled); + } + + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); + + ignite.cache(DEFAULT_CACHE_NAME).destroy(); + + awaitPartitionMapExchange(); + + ensureCacheAbsent(dfltCacheCfg); + + for (Ignite g : G.allGrids()) + ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().enable(LSNR_NAME); + + assertThrowsAnyCause( + log, + () -> ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(), + IgniteException.class, + "Snapshot data doesn't contain required cache group partition" + ); + } + + /** {@inheritDoc} */ + @Override protected Function valueBuilder() { + return Integer::new; + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java index b649e0fcd56d6..95970554e6e78 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreBaseTest.java @@ -17,12 +17,21 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; +import java.util.Arrays; import java.util.function.Function; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.binary.BinaryObjectBuilder; import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.testframework.GridTestUtils; /** @@ -36,7 +45,7 @@ public abstract class IgniteClusterSnapshotRestoreBaseTest extends AbstractSnaps /** {@inheritDoc} */ @Override protected CacheConfiguration txCacheConfig(CacheConfiguration ccfg) { - return super.txCacheConfig(ccfg).setAffinity(new RendezvousAffinityFunction(false, 8)); + return super.txCacheConfig(ccfg).setAffinity(new RendezvousAffinityFunction(false, PARTS_NUMBER)); } /** @@ -82,6 +91,38 @@ protected void assertCacheKeys(IgniteCache cache, int keysCnt) { assertEquals(valueBuilder().apply(i), cache.get(i)); } + /** + * @param ccfg Cache configuration. + * @throws IgniteCheckedException if failed. + */ + protected void ensureCacheAbsent(CacheConfiguration ccfg) throws IgniteCheckedException { + String cacheName = ccfg.getName(); + + for (Ignite ignite : G.allGrids()) { + GridKernalContext kctx = ((IgniteEx)ignite).context(); + + if (kctx.clientNode()) + continue; + + CacheGroupDescriptor desc = kctx.cache().cacheGroupDescriptors().get(CU.cacheId(cacheName)); + + assertNull("nodeId=" + kctx.localNodeId() + ", cache=" + cacheName, desc); + + boolean success = GridTestUtils.waitForCondition( + () -> !kctx.cache().context().snapshotMgr().isRestoring(), + TIMEOUT); + + assertTrue("The process has not finished on the node " + kctx.localNodeId(), success); + + File dir = ((FilePageStoreManager)kctx.cache().context().pageStore()).cacheWorkDir(ccfg); + + String errMsg = String.format("%s, dir=%s, exists=%b, files=%s", + ignite.name(), dir, dir.exists(), Arrays.toString(dir.list())); + + assertTrue(errMsg, !dir.exists() || dir.list().length == 0); + } + } + /** */ protected class BinaryValueBuilder implements Function { /** Binary type name. */ diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index fe7aface6f1d6..75c37e868e7e4 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -128,63 +128,63 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR return valBuilder; } - /** @throws Exception If fails. */ - @Test - public void testClusterSnapshotOptionalVerificationFailure() throws Exception { - String expMsg = "Test verification exception message."; - - snpCheckPlugin = new AbstractTestPluginProvider() { - @Override public String name() { - return "SnapshotVerifier"; - } - - /** {@inheritDoc} */ - @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { - registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { - throw new IgniteCheckedException(expMsg); - })); - - registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { - // No-op. - })); - } - }; - - IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); - - IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); - - GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); - } - - /** @throws Exception If fails. */ - @Test - public void testClusterSnapshotOptionalVerifications() throws Exception { - AtomicInteger checkCntr = new AtomicInteger(); - - snpCheckPlugin = new AbstractTestPluginProvider() { - @Override public String name() { - return "SnapshotVerifier"; - } - - /** {@inheritDoc} */ - @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { - registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { - checkCntr.incrementAndGet(); - })); - - registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { - checkCntr.incrementAndGet(); - })); - } - }; - - IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); - - ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); - - assertEquals(2, checkCntr.get()); - } +// /** @throws Exception If fails. */ +// @Test +// public void testClusterSnapshotOptionalVerificationFailure() throws Exception { +// String expMsg = "Test verification exception message."; +// +// snpCheckPlugin = new AbstractTestPluginProvider() { +// @Override public String name() { +// return "SnapshotVerifier"; +// } +// +// /** {@inheritDoc} */ +// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// throw new IgniteCheckedException(expMsg); +// })); +// +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// // No-op. +// })); +// } +// }; +// +// IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); +// +// IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); +// +// GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); +// } + +// /** @throws Exception If fails. */ +// @Test +// public void testClusterSnapshotOptionalVerifications() throws Exception { +// AtomicInteger checkCntr = new AtomicInteger(); +// +// snpCheckPlugin = new AbstractTestPluginProvider() { +// @Override public String name() { +// return "SnapshotVerifier"; +// } +// +// /** {@inheritDoc} */ +// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// checkCntr.incrementAndGet(); +// })); +// +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// checkCntr.incrementAndGet(); +// })); +// } +// }; +// +// IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); +// +// ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); +// +// assertEquals(2, checkCntr.get()); +// } /** @throws Exception If fails. */ @Test @@ -205,9 +205,11 @@ public void testRestoreWithMissedPart() throws Exception { "Snapshot data doesn't contain required cache group partition" ); - System.out.println(">xxx> ensure cache absent"); ensureCacheAbsent(dfltCacheCfg); - System.out.println(">xxx> ensure cache absent :: end"); + + for (Ignite g : G.allGrids()) { + + } } /** @throws Exception If failed. */ @@ -808,41 +810,6 @@ private void checkClusterStateChange( assertCacheKeys(ignite.cache(cacheName), CACHE_KEYS_RANGE); } - /** - * @param ccfg Cache configuration. - * @throws IgniteCheckedException if failed. - */ - private void ensureCacheAbsent(CacheConfiguration ccfg) throws IgniteCheckedException { - String cacheName = ccfg.getName(); - - for (Ignite ignite : G.allGrids()) { - GridKernalContext kctx = ((IgniteEx)ignite).context(); - - if (kctx.clientNode()) - continue; - - CacheGroupDescriptor desc = kctx.cache().cacheGroupDescriptors().get(CU.cacheId(cacheName)); - - assertNull("nodeId=" + kctx.localNodeId() + ", cache=" + cacheName, desc); - - System.out.println(">xxx> await restore..."); - boolean cond = GridTestUtils.waitForCondition( - () -> !kctx.cache().context().snapshotMgr().isRestoring(), - TIMEOUT); - - assertTrue(cond); - - System.out.println(">xxx> await restore... DONE"); - - File dir = ((FilePageStoreManager)kctx.cache().context().pageStore()).cacheWorkDir(ccfg); - - String errMsg = String.format("%s, dir=%s, exists=%b, files=%s", - ignite.name(), dir, dir.exists(), Arrays.toString(dir.list())); - - assertTrue(errMsg, !dir.exists() || dir.list().length == 0); - } - } - /** * @param spi Test communication spi. * @param restorePhase The type of distributed process on which communication is blocked. From 5b37b52ddbfd94a98b3129e4830a40fab4d11f41 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 5 Aug 2021 14:09:50 +0300 Subject: [PATCH 15/19] wip (extract) job --- .../snapshot/IgniteSnapshotManager.java | 18 +- ...=> SnapshotConsistencyCheckOnRestore.java} | 8 +- .../SnapshotPartitionsVerifyTask.java | 231 ----------------- .../VisorVerifySnapshotPartitionsJob.java | 241 ++++++++++++++++++ .../snapshot/lifecycle/RestoreHandleTask.java | 4 +- .../lifecycle/SnapshotLifecycleListener.java | 87 +++---- .../snapshot/lifecycle/SnapshotListeners.java | 18 +- ...eClusterSnapshotLifecycleListenerTest.java | 11 +- 8 files changed, 299 insertions(+), 319 deletions(-) rename modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/{SnapshotRestoreConsistencyCheck.java => SnapshotConsistencyCheckOnRestore.java} (96%) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/VisorVerifySnapshotPartitionsJob.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 36d9d0e6140eb..c3520f2b9890d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -270,6 +270,9 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Distributed process to restore cache group from the snapshot. */ private final SnapshotRestoreProcess restoreCacheGrpProc; + // todo something like pipeline + private final SnapshotListeners snpLsnrs = new SnapshotListeners(); + /** Resolved persistent data storage settings. */ private volatile PdsFolderSettings pdsSettings; @@ -303,9 +306,6 @@ public class IgniteSnapshotManager extends GridCacheSharedManagerAdapter /** Cluster snapshot operation requested by user. */ private ClusterSnapshotFuture clusterSnpFut; - // todo something like pipeline - private SnapshotListeners snpLsnrs; - /** Current snapshot operation on local node. */ private volatile SnapshotOperationRequest clusterSnpReq; @@ -386,10 +386,14 @@ public static String partDeltaFileName(int partId) { U.ensureDirectory(locSnpDir, "snapshot work directory", log); U.ensureDirectory(tmpWorkDir, "temp directory for snapshot creation", log); - snpLsnrs = new SnapshotListeners(ctx.plugins()); + snpLsnrs.register(new SnapshotConsistencyCheckOnRestore(ctx)); + + SnapshotLifecycleListener[] lsnrs = ctx.plugins().extensions(SnapshotLifecycleListener.class); - // Register default. - snpLsnrs.register(new SnapshotRestoreConsistencyCheck(ctx)); + if (lsnrs != null) { + for (SnapshotLifecycleListener lsnr : lsnrs) + snpLsnrs.register(lsnr); + } MetricRegistry mreg = cctx.kernalContext().metric().registry(SNAPSHOT_METRICS); @@ -691,7 +695,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt } for (SnapshotLifecycleListener lsnr : listeners().list()) - lsnr.postCreate(req.snapshotName()); + lsnr.afterCreate(req.snapshotName()); return new SnapshotOperationResponse(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java similarity index 96% rename from modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java rename to modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java index 8af4253b0c392..9e77c6d8378f8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreConsistencyCheck.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java @@ -45,7 +45,7 @@ import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; -public class SnapshotRestoreConsistencyCheck implements SnapshotLifecycleListener { +public class SnapshotConsistencyCheckOnRestore implements SnapshotLifecycleListener { /** */ public static final String LSNR_NAME = "system:consistency-check"; @@ -59,7 +59,7 @@ public class SnapshotRestoreConsistencyCheck implements SnapshotLifecycleListene return Integer.MIN_VALUE; } - public SnapshotRestoreConsistencyCheck(GridKernalContext ctx) { + public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { this.ctx = ctx; log = ctx.log(getClass()); @@ -70,7 +70,7 @@ public SnapshotRestoreConsistencyCheck(GridKernalContext ctx) { } /** {@inheritDoc} */ - @Override public T handleRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { + @Override public T beforeRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr(); if (log.isInfoEnabled()) { @@ -212,7 +212,7 @@ public SnapshotRestoreConsistencyCheck(GridKernalContext ctx) { return (T)res; } - @Override public void reduce(List res) throws IgniteCheckedException { + @Override public void handlePreRestoreResults(List res) throws IgniteCheckedException { IdleVerifyResultV2 verifyResult = VerifyBackupPartitionsTaskV2.reduce0(res); if (!F.isEmpty(verifyResult.exceptions()) || verifyResult.hasConflicts()) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java index 0e99e8cfc094c..61e1998087229 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotPartitionsVerifyTask.java @@ -17,62 +17,25 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; -import java.io.File; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.compute.ComputeJob; -import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeJobResultPolicy; import org.apache.ignite.compute.ComputeTaskAdapter; -import org.apache.ignite.internal.GridComponent; -import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; -import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; -import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; -import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; -import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; -import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2; -import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2; import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2; import org.apache.ignite.internal.processors.task.GridInternal; -import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.F; -import org.apache.ignite.internal.util.typedef.internal.CU; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.resources.IgniteInstanceResource; -import org.apache.ignite.resources.LoggerResource; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; -import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; -import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION; -import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; -import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal; -import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName; -import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles; -import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId; -import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId; -import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; -import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; import static org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2.reduce0; /** @@ -159,198 +122,4 @@ public class SnapshotPartitionsVerifyTask return ComputeJobResultPolicy.WAIT; } - /** Job that collects update counters of snapshot partitions on the node it executes. */ - private static class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter { - /** Serial version uid. */ - private static final long serialVersionUID = 0L; - - /** Ignite instance. */ - @IgniteInstanceResource - private IgniteEx ignite; - - /** Injected logger. */ - @LoggerResource - private IgniteLogger log; - - /** Snapshot name to validate. */ - private final String snpName; - - /** Consistent snapshot metadata file name. */ - private final String consId; - - /** Set of cache groups to be checked in the snapshot or {@code empty} to check everything. */ - private final Set rqGrps; - - /** - * @param snpName Snapshot name to validate. - * @param consId Consistent snapshot metadata file name. - * @param rqGrps Set of cache groups to be checked in the snapshot or {@code empty} to check everything. - */ - public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collection rqGrps) { - this.snpName = snpName; - this.consId = consId; - - this.rqGrps = rqGrps == null ? Collections.emptySet() : new HashSet<>(rqGrps); - } - - @Override public Map execute() throws IgniteException { - IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); - - if (log.isInfoEnabled()) { - log.info("Verify snapshot partitions procedure has been initiated " + - "[snpName=" + snpName + ", consId=" + consId + ']'); - } - - SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); - Set grps = rqGrps.isEmpty() ? new HashSet<>(meta.partitions().keySet()) : - rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); - Set partFiles = new HashSet<>(); - - for (File dir : snpMgr.snapshotCacheDirectories(snpName, meta.folderName())) { - int grpId = CU.cacheId(cacheGroupName(dir)); - - if (!grps.remove(grpId)) - continue; - - Set parts = new HashSet<>(meta.partitions().get(grpId)); - - for (File part : cachePartitionFiles(dir)) { - int partId = partId(part.getName()); - - if (!parts.remove(partId)) - continue; - - partFiles.add(part); - } - - if (!parts.isEmpty()) { - throw new IgniteException("Snapshot data doesn't contain required cache group partition " + - "[grpId=" + grpId + ", snpName=" + snpName + ", consId=" + consId + - ", missed=" + parts + ", meta=" + meta + ']'); - } - } - - if (!grps.isEmpty()) { - throw new IgniteException("Snapshot data doesn't contain required cache groups " + - "[grps=" + grps + ", snpName=" + snpName + ", consId=" + consId + - ", meta=" + meta + ']'); - } - - Map res = new ConcurrentHashMap<>(); - ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) - .order(ByteOrder.nativeOrder())); - - try { - GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(snpName, meta.folderName()); - - for (GridComponent comp : snpCtx) - comp.start(); - - try { - U.doInParallel( - snpMgr.snapshotExecutorService(), - partFiles, - part -> { - String grpName = cacheGroupName(part.getParentFile()); - int grpId = CU.cacheId(grpName); - int partId = partId(part.getName()); - - FilePageStoreManager storeMgr = (FilePageStoreManager)ignite.context().cache().context().pageStore(); - - try (FilePageStore pageStore = (FilePageStore)storeMgr.getPageStoreFactory(grpId, false) - .createPageStore(getTypeByPartId(partId), - part::toPath, - val -> { - }) - ) { - if (partId == INDEX_PARTITION) { - checkPartitionsPageCrcSum(() -> pageStore, INDEX_PARTITION, FLAG_IDX); - - return null; - } - - if (grpId == MetaStorage.METASTORAGE_CACHE_ID) { - checkPartitionsPageCrcSum(() -> pageStore, partId, FLAG_DATA); - - return null; - } - - ByteBuffer pageBuff = buff.get(); - pageBuff.clear(); - pageStore.read(0, pageBuff, true); - - long pageAddr = GridUnsafe.bufferAddress(pageBuff); - - PagePartitionMetaIO io = PageIO.getPageIO(pageBuff); - GridDhtPartitionState partState = fromOrdinal(io.getPartitionState(pageAddr)); - - if (partState != OWNING) { - throw new IgniteCheckedException("Snapshot partitions must be in the OWNING " + - "state only: " + partState); - } - - long updateCntr = io.getUpdateCounter(pageAddr); - long size = io.getSize(pageAddr); - - if (log.isDebugEnabled()) { - log.debug("Partition [grpId=" + grpId - + ", id=" + partId - + ", counter=" + updateCntr - + ", size=" + size + "]"); - } - - // Snapshot partitions must always be in OWNING state. - // There is no `primary` partitions for snapshot. - PartitionKeyV2 key = new PartitionKeyV2(grpId, partId, grpName); - - PartitionHashRecordV2 hash = calculatePartitionHash(key, - updateCntr, - consId, - GridDhtPartitionState.OWNING, - false, - size, - snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore)); - - assert hash != null : "OWNING must have hash: " + key; - - res.put(key, hash); - } - catch (IOException e) { - throw new IgniteCheckedException(e); - } - - return null; - } - ); - } - finally { - for (GridComponent comp : snpCtx) - comp.stop(true); - } - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } - - return res; - } - - /** {@inheritDoc} */ - @Override public boolean equals(Object o) { - if (this == o) - return true; - - if (o == null || getClass() != o.getClass()) - return false; - - VisorVerifySnapshotPartitionsJob job = (VisorVerifySnapshotPartitionsJob)o; - - return snpName.equals(job.snpName) && consId.equals(job.consId); - } - - /** {@inheritDoc} */ - @Override public int hashCode() { - return Objects.hash(snpName, consId); - } - } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/VisorVerifySnapshotPartitionsJob.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/VisorVerifySnapshotPartitionsJob.java new file mode 100644 index 0000000000000..9f08da9ea1cd6 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/VisorVerifySnapshotPartitionsJob.java @@ -0,0 +1,241 @@ +package org.apache.ignite.internal.processors.cache.persistence.snapshot; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.compute.ComputeJobAdapter; +import org.apache.ignite.internal.GridComponent; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore; +import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; +import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO; +import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2; +import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2; +import org.apache.ignite.internal.util.GridUnsafe; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.resources.LoggerResource; + +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; +import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING; +import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.fromOrdinal; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cacheGroupName; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.cachePartitionFiles; +import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.partId; +import static org.apache.ignite.internal.processors.cache.persistence.partstate.GroupPartitionId.getTypeByPartId; +import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; +import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; + +/** Job that collects update counters of snapshot partitions on the node it executes. */ +class VisorVerifySnapshotPartitionsJob extends ComputeJobAdapter { + /** Serial version uid. */ + private static final long serialVersionUID = 0L; + + /** Ignite instance. */ + @IgniteInstanceResource + private IgniteEx ignite; + + /** Injected logger. */ + @LoggerResource + private IgniteLogger log; + + /** Snapshot name to validate. */ + private final String snpName; + + /** Consistent snapshot metadata file name. */ + private final String consId; + + /** Set of cache groups to be checked in the snapshot or {@code empty} to check everything. */ + private final Set rqGrps; + + /** + * @param snpName Snapshot name to validate. + * @param consId Consistent snapshot metadata file name. + * @param rqGrps Set of cache groups to be checked in the snapshot or {@code empty} to check everything. + */ + public VisorVerifySnapshotPartitionsJob(String snpName, String consId, Collection rqGrps) { + this.snpName = snpName; + this.consId = consId; + + this.rqGrps = rqGrps == null ? Collections.emptySet() : new HashSet<>(rqGrps); + } + + @Override public Map execute() throws IgniteException { + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + + if (log.isInfoEnabled()) { + log.info("Verify snapshot partitions procedure has been initiated " + + "[snpName=" + snpName + ", consId=" + consId + ']'); + } + + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); + Set grps = rqGrps.isEmpty() ? new HashSet<>(meta.partitions().keySet()) : + rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); + Set partFiles = new HashSet<>(); + + for (File dir : snpMgr.snapshotCacheDirectories(snpName, meta.folderName())) { + int grpId = CU.cacheId(cacheGroupName(dir)); + + if (!grps.remove(grpId)) + continue; + + Set parts = new HashSet<>(meta.partitions().get(grpId)); + + for (File part : cachePartitionFiles(dir)) { + int partId = partId(part.getName()); + + if (!parts.remove(partId)) + continue; + + partFiles.add(part); + } + + if (!parts.isEmpty()) { + throw new IgniteException("Snapshot data doesn't contain required cache group partition " + + "[grpId=" + grpId + ", snpName=" + snpName + ", consId=" + consId + + ", missed=" + parts + ", meta=" + meta + ']'); + } + } + + if (!grps.isEmpty()) { + throw new IgniteException("Snapshot data doesn't contain required cache groups " + + "[grps=" + grps + ", snpName=" + snpName + ", consId=" + consId + + ", meta=" + meta + ']'); + } + + Map res = new ConcurrentHashMap<>(); + ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) + .order(ByteOrder.nativeOrder())); + + try { + GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(snpName, meta.folderName()); + + for (GridComponent comp : snpCtx) + comp.start(); + + try { + U.doInParallel( + snpMgr.snapshotExecutorService(), + partFiles, + part -> { + String grpName = cacheGroupName(part.getParentFile()); + int grpId = CU.cacheId(grpName); + int partId = partId(part.getName()); + + FilePageStoreManager storeMgr = (FilePageStoreManager)ignite.context().cache().context().pageStore(); + + try (FilePageStore pageStore = (FilePageStore)storeMgr.getPageStoreFactory(grpId, false) + .createPageStore(getTypeByPartId(partId), + part::toPath, + val -> { + }) + ) { + if (partId == INDEX_PARTITION) { + checkPartitionsPageCrcSum(() -> pageStore, INDEX_PARTITION, FLAG_IDX); + + return null; + } + + if (grpId == MetaStorage.METASTORAGE_CACHE_ID) { + checkPartitionsPageCrcSum(() -> pageStore, partId, FLAG_DATA); + + return null; + } + + ByteBuffer pageBuff = buff.get(); + pageBuff.clear(); + pageStore.read(0, pageBuff, true); + + long pageAddr = GridUnsafe.bufferAddress(pageBuff); + + PagePartitionMetaIO io = PageIO.getPageIO(pageBuff); + GridDhtPartitionState partState = fromOrdinal(io.getPartitionState(pageAddr)); + + if (partState != OWNING) { + throw new IgniteCheckedException("Snapshot partitions must be in the OWNING " + + "state only: " + partState); + } + + long updateCntr = io.getUpdateCounter(pageAddr); + long size = io.getSize(pageAddr); + + if (log.isDebugEnabled()) { + log.debug("Partition [grpId=" + grpId + + ", id=" + partId + + ", counter=" + updateCntr + + ", size=" + size + "]"); + } + + // Snapshot partitions must always be in OWNING state. + // There is no `primary` partitions for snapshot. + PartitionKeyV2 key = new PartitionKeyV2(grpId, partId, grpName); + + PartitionHashRecordV2 hash = calculatePartitionHash(key, + updateCntr, + consId, + GridDhtPartitionState.OWNING, + false, + size, + snpMgr.partitionRowIterator(snpCtx, grpName, partId, pageStore)); + + assert hash != null : "OWNING must have hash: " + key; + + res.put(key, hash); + } + catch (IOException e) { + throw new IgniteCheckedException(e); + } + + return null; + } + ); + } + finally { + for (GridComponent comp : snpCtx) + comp.stop(true); + } + } + catch (IgniteCheckedException e) { + throw new IgniteException(e); + } + + return res; + } + + /** {@inheritDoc} */ + @Override public boolean equals(Object o) { + if (this == o) + return true; + + if (o == null || getClass() != o.getClass()) + return false; + + VisorVerifySnapshotPartitionsJob job = (VisorVerifySnapshotPartitionsJob)o; + + return snpName.equals(job.snpName) && consId.equals(job.consId); + } + + /** {@inheritDoc} */ + @Override public int hashCode() { + return Objects.hash(snpName, consId); + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java index b7c215aa140b0..a6e15ca2b1e69 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -130,7 +130,7 @@ public class RestoreHandleTask extends ComputeTaskAdapter nodeResults = resMap.get(lsnr.name()); try { - lsnr.reduce(nodeResults); + lsnr.handlePreRestoreResults(nodeResults); } catch (IgniteCheckedException e) { throw new IgniteException(e); } @@ -172,7 +172,7 @@ public RestoreHandleJob(String snapshotName, String consistentId, Collection T handleRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { + /** + * Called locally before restore snapshot files. + * + * @param snpName Snapshot name. + * @param consId Consistent snapshot metadata file name. + * @param grps Cache groups to be restored ({@code null} if all cache groups are restored from the snapshot). + * @param Type of the result. + * @return Local node result, or {@code null} if cluster-wide aggregation is not required. + * @throws IgniteCheckedException If failed. + */ + @Nullable public default T beforeRestore( + String snpName, + String consId, + @Nullable Collection grps + ) throws IgniteCheckedException { return null; } - public default void reduce(List res) throws IgniteCheckedException { + /** + * Process the results of a pre-restore operation across the cluster. + * + * @param res Results from all nodes. + * @throws IgniteCheckedException If failed. + */ + public default void handlePreRestoreResults(List res) throws IgniteCheckedException { // No-op. } - - // global -// public default void beforeCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException -// // local -// public default void beforeCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; -// public default void afterCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req, File snapshotMetadata) throws IgniteCheckedException; -// -// // global -// public default void beforeRestoreSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException; -// public default void afterRestoreSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException; -// -// // local -// public default void beforeRestoreSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; -// public default void afterRestoreSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException; - - - - -// // global -// public default void beforeCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException { -// -// } -// -// // local -// public default void beforeCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req) throws IgniteCheckedException { -// // No-op. -// }; -// -// // global -// public default void afterCreateSnapshot(SnapshotOperationRequest req) throws IgniteCheckedException { -// // No-op. -// }; -// -// // local -// public default void afterCreateSnapshot(ClusterNode locNode, SnapshotOperationRequest req, File snapshotMetadata) throws IgniteCheckedException { -// // No-op. -// }; -// -// // global -// public default void beforeRestoreSnapshot(Map> metas, Collection grps) throws IgniteCheckedException { -// -// } -// -// // local -// public void beforeRestoreSnapshot(ClusterNode locNode, SnapshotMetadata metadata, Collection grps); -// -// -// public void afterRestoreSnapshot(); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java index 9544830c78cfa..9ceb724b66df6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java @@ -18,31 +18,21 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.ignite.internal.processors.plugin.IgnitePluginProcessor; import org.apache.ignite.internal.util.typedef.F; public class SnapshotListeners { - private final IgnitePluginProcessor plugins; +// private final IgnitePluginProcessor plugins; private final Map lsnrsState = new HashMap<>(); private final List lsnrsByPriority = new ArrayList<>(); - public SnapshotListeners(IgnitePluginProcessor plugins) { - this.plugins = plugins; - - SnapshotLifecycleListener[] lsnrs = plugins.extensions(SnapshotLifecycleListener.class); - - if (lsnrs == null) - return; - - for (SnapshotLifecycleListener lsnr : lsnrs) - register(lsnr); - } +// public SnapshotListeners(IgnitePluginProcessor plugins) { +//// this.plugins = plugins; +// } public Iterable list() { return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.name())); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java index c76d2234fbbf0..0bc3ca6c17920 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java @@ -27,7 +27,7 @@ import org.junit.Test; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName; -import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotRestoreConsistencyCheck.LSNR_NAME; +import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotConsistencyCheckOnRestore.LSNR_NAME; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; public class IgniteClusterSnapshotLifecycleListenerTest extends IgniteClusterSnapshotRestoreBaseTest { @@ -43,15 +43,6 @@ public void testRestoreWithMissedPart() throws Exception { assertTrue(part0.toString(), part0.toFile().exists()); assertTrue(part0.toFile().delete()); -// assertThrowsAnyCause( -// log, -// () -> ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(), -// IgniteException.class, -// "Snapshot data doesn't contain required cache group partition" -// ); -// -// ensureCacheAbsent(dfltCacheCfg); - for (Ignite g : G.allGrids()) { boolean disabled = ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().disable(LSNR_NAME); assertTrue(disabled); From 624fd7d1fa4bc65f9f2d185d95a546e920f21a73 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 5 Aug 2021 15:46:13 +0300 Subject: [PATCH 16/19] remove name() --- .../SnapshotConsistencyCheckOnRestore.java | 11 +++++------ .../snapshot/lifecycle/RestoreHandleTask.java | 4 ++-- .../lifecycle/SnapshotLifecycleListener.java | 4 ++-- .../snapshot/lifecycle/SnapshotListeners.java | 18 +++++++++--------- ...teClusterSnapshotLifecycleListenerTest.java | 8 +++++--- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java index 9e77c6d8378f8..17767e26b9999 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java @@ -46,9 +46,8 @@ import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; public class SnapshotConsistencyCheckOnRestore implements SnapshotLifecycleListener { - /** */ - public static final String LSNR_NAME = "system:consistency-check"; - +// /** */ +// public static final String LSNR_NAME = "system:consistency-check"; /** */ private final GridKernalContext ctx; @@ -65,9 +64,9 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { log = ctx.log(getClass()); } - @Override public String name() { - return LSNR_NAME; - } +// @Override public String name() { +// return LSNR_NAME; +// } /** {@inheritDoc} */ @Override public T beforeRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java index a6e15ca2b1e69..de0195407b174 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -127,7 +127,7 @@ public class RestoreHandleTask extends ComputeTaskAdapter nodeResults = resMap.get(lsnr.name()); + List nodeResults = resMap.get(lsnr.getClass().getSimpleName()); try { lsnr.handlePreRestoreResults(nodeResults); @@ -174,7 +174,7 @@ public RestoreHandleJob(String snapshotName, String consistentId, Collection list() { - return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.name())); + return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.getClass().getSimpleName())); } // public @Nullable SnapshotLifecycleListener find(String name) { @@ -48,8 +48,8 @@ public Iterable list() { // } public void register(SnapshotLifecycleListener lsnr) { - if (lsnrsState.putIfAbsent(lsnr.name(), true) != null) - throw new IllegalArgumentException("Listener named " + lsnr.name() + " is already registered."); + if (lsnrsState.putIfAbsent(lsnr.getClass().getSimpleName(), true) != null) + throw new IllegalArgumentException("Listener " + lsnr.getClass().getSimpleName() + " is already registered."); int idx = Collections.binarySearch(lsnrsByPriority, lsnr, Comparator.comparingInt(SnapshotLifecycleListener::priority)); @@ -59,22 +59,22 @@ public void register(SnapshotLifecycleListener lsnr) { lsnrsByPriority.add(idx, lsnr); } - public boolean disable(String name) { - Boolean enabled = lsnrsState.get(name); + public boolean disable(String clsName) { + Boolean enabled = lsnrsState.get(clsName); if (!Boolean.TRUE.equals(enabled)) return false; - return lsnrsState.put(name, false); + return lsnrsState.put(clsName, false); } - public boolean enable(String name) { - Boolean enabled = lsnrsState.get(name); + public boolean enable(String clsName) { + Boolean enabled = lsnrsState.get(clsName); if (!Boolean.FALSE.equals(enabled)) return false; - return !lsnrsState.put(name, true); + return !lsnrsState.put(clsName, true); // // SnapshotLifecycleListener lsnr = lsnrsState.get(name); // diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java index 0bc3ca6c17920..842b470f12518 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java @@ -22,12 +22,12 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteException; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotListeners; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; import org.junit.Test; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName; -import static org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotConsistencyCheckOnRestore.LSNR_NAME; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; public class IgniteClusterSnapshotLifecycleListenerTest extends IgniteClusterSnapshotRestoreBaseTest { @@ -44,7 +44,9 @@ public void testRestoreWithMissedPart() throws Exception { assertTrue(part0.toFile().delete()); for (Ignite g : G.allGrids()) { - boolean disabled = ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().disable(LSNR_NAME); + SnapshotListeners lsnrs = ((IgniteEx)g).context().cache().context().snapshotMgr().listeners(); + + boolean disabled = lsnrs.disable(SnapshotConsistencyCheckOnRestore.class.getSimpleName()); assertTrue(disabled); } @@ -57,7 +59,7 @@ public void testRestoreWithMissedPart() throws Exception { ensureCacheAbsent(dfltCacheCfg); for (Ignite g : G.allGrids()) - ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().enable(LSNR_NAME); + ((IgniteEx)g).context().cache().context().snapshotMgr().listeners().enable(SnapshotConsistencyCheckOnRestore.class.getSimpleName()); assertThrowsAnyCause( log, From 64f7f13d664d6eff47c5ac6074370ff1021ba383 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 5 Aug 2021 18:00:20 +0300 Subject: [PATCH 17/19] handleResults arguments --- .../snapshot/IgniteSnapshotManager.java | 12 ++- .../SnapshotConsistencyCheckOnRestore.java | 71 ++++++++------ .../snapshot/lifecycle/RestoreHandleTask.java | 95 +++++++++++++------ .../lifecycle/SnapshotLifecycleListener.java | 70 ++++++++++---- 4 files changed, 170 insertions(+), 78 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index c3520f2b9890d..630b4ce853516 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -694,8 +694,14 @@ private IgniteInternalFuture initLocalSnapshotStartSt log.info("Snapshot metafile has been created: " + smf.getAbsolutePath()); } + File snpDir = snapshotLocalDir(req.snapshotName()); + + Path binaryWorkDir = binaryWorkDir(snpDir.getAbsolutePath(), pdsSettings.folderName()).toPath(); + Path marshallerDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath()).toPath(); + Path cacheDir = Paths.get(snpDir.toString(), DB_DEFAULT_FOLDER, pdsSettings.folderName()); + for (SnapshotLifecycleListener lsnr : listeners().list()) - lsnr.afterCreate(req.snapshotName()); + lsnr.afterCreate(req.snapshotName(), smf, binaryWorkDir, marshallerDir, cacheDir); return new SnapshotOperationResponse(); } @@ -1128,7 +1134,7 @@ public SnapshotMetadata readSnapshotMetadata(String snpName, String consId) { * @param smf File denoting to snapshot metafile. * @return Snapshot metadata instance. */ - private SnapshotMetadata readSnapshotMetadata(File smf) { + public SnapshotMetadata readSnapshotMetadata(File smf) { if (!smf.exists()) throw new IgniteException("Snapshot metafile cannot be read due to it doesn't exist: " + smf); @@ -1417,7 +1423,7 @@ public void onCacheGroupsStopped(List grps) { * @param consId Consistent node id. * @return Snapshot metadata file name. */ - private static String snapshotMetaFileName(String consId) { + public static String snapshotMetaFileName(String consId) { return U.maskForFileName(consId) + SNAPSHOT_METAFILE_EXT; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java index 17767e26b9999..22fd4698345ca 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotConsistencyCheckOnRestore.java @@ -4,7 +4,10 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -14,7 +17,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; -import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.GridComponent; import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState; @@ -27,11 +30,11 @@ import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2; import org.apache.ignite.internal.processors.cache.verify.PartitionHashRecordV2; import org.apache.ignite.internal.processors.cache.verify.PartitionKeyV2; -import org.apache.ignite.internal.processors.cache.verify.VerifyBackupPartitionsTaskV2; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; +import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA; import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX; @@ -45,9 +48,7 @@ import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.calculatePartitionHash; import static org.apache.ignite.internal.processors.cache.verify.IdleVerifyUtility.checkPartitionsPageCrcSum; -public class SnapshotConsistencyCheckOnRestore implements SnapshotLifecycleListener { -// /** */ -// public static final String LSNR_NAME = "system:consistency-check"; +public class SnapshotConsistencyCheckOnRestore implements SnapshotLifecycleListener> { /** */ private final GridKernalContext ctx; @@ -64,25 +65,28 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { log = ctx.log(getClass()); } -// @Override public String name() { -// return LSNR_NAME; -// } - /** {@inheritDoc} */ - @Override public T beforeRestore(String snpName, String consId, Collection rqGrps) throws IgniteCheckedException { + @Override public HashMap beforeRestore( + String name, + @Nullable Collection rqGrps, + File metadata, + Path binaryDir, + Path marshallerDir, + Path cacheDir + ) throws IgniteCheckedException { IgniteSnapshotManager snpMgr = ctx.cache().context().snapshotMgr(); + SnapshotMetadata meta = snpMgr.readSnapshotMetadata(metadata); + if (log.isInfoEnabled()) { log.info("Verify snapshot partitions procedure has been initiated " + - "[snpName=" + snpName + ", consId=" + consId + ']'); + "[snpName=" + name + ", consId=" + meta.consistentId() + ']'); } - - SnapshotMetadata meta = snpMgr.readSnapshotMetadata(snpName, consId); Set grps = F.isEmpty(rqGrps) ? new HashSet<>(meta.partitions().keySet()) : rqGrps.stream().map(CU::cacheId).collect(Collectors.toSet()); Set partFiles = new HashSet<>(); - for (File dir : snpMgr.snapshotCacheDirectories(snpName, meta.folderName())) { + for (File dir : snpMgr.snapshotCacheDirectories(name, meta.folderName())) { int grpId = CU.cacheId(cacheGroupName(dir)); if (!grps.remove(grpId)) @@ -101,14 +105,14 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { if (!parts.isEmpty()) { throw new IgniteException("Snapshot data doesn't contain required cache group partition " + - "[grpId=" + grpId + ", snpName=" + snpName + ", consId=" + consId + + "[grpId=" + grpId + ", snpName=" + name + ", consId=" + meta.consistentId() + ", missed=" + parts + ", meta=" + meta + ']'); } } if (!grps.isEmpty()) { throw new IgniteException("Snapshot data doesn't contain required cache groups " + - "[grps=" + grps + ", snpName=" + snpName + ", consId=" + consId + + "[grps=" + grps + ", snpName=" + name + ", consId=" + meta.consistentId() + ", meta=" + meta + ']'); } @@ -116,8 +120,8 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { ThreadLocal buff = ThreadLocal.withInitial(() -> ByteBuffer.allocateDirect(meta.pageSize()) .order(ByteOrder.nativeOrder())); - try { - GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(snpName, meta.folderName()); +// try { + GridKernalContext snpCtx = snpMgr.createStandaloneKernalContext(name, meta.folderName()); for (GridComponent comp : snpCtx) comp.start(); @@ -181,7 +185,7 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { PartitionHashRecordV2 hash = calculatePartitionHash(key, updateCntr, - consId, + meta.consistentId(), GridDhtPartitionState.OWNING, false, size, @@ -203,18 +207,31 @@ public SnapshotConsistencyCheckOnRestore(GridKernalContext ctx) { for (GridComponent comp : snpCtx) comp.stop(true); } - } - catch (IgniteCheckedException e) { - throw new IgniteException(e); - } +// } +// catch (IgniteCheckedException e) { +// throw new IgniteException(e); +// } - return (T)res; + return new HashMap<>(res); } - @Override public void handlePreRestoreResults(List res) throws IgniteCheckedException { - IdleVerifyResultV2 verifyResult = VerifyBackupPartitionsTaskV2.reduce0(res); + /** {@inheritDoc} */ + @Override public void handlePreRestoreResults(String name, @Nullable Collection grps, + Map> res, + Map errs) throws IgniteCheckedException { + Map> clusterHashes = new HashMap<>(); + + for (Map nodeHashes : res.values()) { + for (Map.Entry e : nodeHashes.entrySet()) { + List records = clusterHashes.computeIfAbsent(e.getKey(), k -> new ArrayList<>()); + + records.add(e.getValue()); + } + } + + IdleVerifyResultV2 verifyResult = new IdleVerifyResultV2(clusterHashes, errs); - if (!F.isEmpty(verifyResult.exceptions()) || verifyResult.hasConflicts()) { + if (!F.isEmpty(errs) || verifyResult.hasConflicts()) { StringBuilder sb = new StringBuilder(); verifyResult.print(sb::append, true); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java index de0195407b174..167b00708d23b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -17,7 +17,10 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; -import java.util.ArrayList; +import java.io.File; +import java.io.Serializable; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -32,18 +35,23 @@ import org.apache.ignite.compute.ComputeJobAdapter; import org.apache.ignite.compute.ComputeJobResult; import org.apache.ignite.compute.ComputeTaskAdapter; -import org.apache.ignite.internal.GridJobResultImpl; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotManager; import org.apache.ignite.internal.processors.cache.persistence.snapshot.IgniteSnapshotVerifyException; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotMetadata; import org.apache.ignite.internal.processors.cache.persistence.snapshot.SnapshotPartitionsVerifyTaskArg; import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.T2; import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.resources.LoggerResource; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -public class RestoreHandleTask extends ComputeTaskAdapter> { +import static org.apache.ignite.internal.MarshallerContextImpl.mappingFileStoreWorkDir; +import static org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl.binaryWorkDir; +import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderResolver.DB_DEFAULT_FOLDER; + +public class RestoreHandleTask extends ComputeTaskAdapter> { /** Ignite instance. */ @IgniteInstanceResource private IgniteEx ignite; @@ -51,9 +59,14 @@ public class RestoreHandleTask extends ComputeTaskAdapter> metas = new HashMap<>(); + /** Cache groups to be restored ({@code null} if all cache groups are restored from the snapshot). */ + private Collection grps; + /** {@inheritDoc} */ @Override public @NotNull Map map(List subgrid, - @Nullable SnapshotPartitionsVerifyTaskArg arg) throws IgniteException { + SnapshotPartitionsVerifyTaskArg arg) throws IgniteException { + + grps = arg.cacheGroupNames(); Map> clusterMetas = arg.clusterMetadata(); @@ -103,34 +116,47 @@ public class RestoreHandleTask extends ComputeTaskAdapter reduce(List results) throws IgniteException { + @Nullable @Override public Map reduce(List results) throws IgniteException { // re-map - Map> resMap = new HashMap<>(); + Map, Map>> resMap = new HashMap<>(); for (ComputeJobResult res : results) { - Map nodeDataMap = res.getData(); + // Unahndled exception. + if (res.getException() != null) + throw res.getException(); + + Map nodeDataMap = res.getData(); + + for (Map.Entry entry : nodeDataMap.entrySet()) { + String lsnrName = entry.getKey(); - for (Map.Entry entry : nodeDataMap.entrySet()) { - // Copy job result. - GridJobResultImpl jobRes = - new GridJobResultImpl(res.getJob(), res.getJobContext().getJobId(), res.getNode(), ((GridJobResultImpl)res).getSibling()); + T2, Map> mapPair = + resMap.computeIfAbsent(lsnrName, v -> new T2<>(new HashMap<>(), new HashMap<>())); - jobRes.onResponse(entry.getValue(), res.getException(), (Map)res.getJobContext().getAttributes(), res.isCancelled()); + Serializable val = entry.getValue(); - resMap.computeIfAbsent(entry.getKey(), v -> new ArrayList<>()).add(jobRes); + if (val instanceof Exception) + mapPair.get2().put(res.getNode(), (Exception)val); + else + mapPair.get1().put(res.getNode(), val); } } + // todo validate count of listeners on nodes + String snpName = F.first(F.first(metas.values())).snapshotName(); + for (SnapshotLifecycleListener lsnr : ignite.context().cache().context().snapshotMgr().listeners().list()) { - List nodeResults = resMap.get(lsnr.getClass().getSimpleName()); + T2, Map> mapPair = resMap.get(lsnr.getClass().getSimpleName()); + + // The listener might have been enabled at runtime. + if (mapPair == null) + continue; try { - lsnr.handlePreRestoreResults(nodeResults); + lsnr.handlePreRestoreResults(snpName, grps, mapPair.get1(), mapPair.get2()); } catch (IgniteCheckedException e) { throw new IgniteException(e); } @@ -151,28 +177,43 @@ private static class RestoreHandleJob extends ComputeJobAdapter { @LoggerResource private IgniteLogger log; - String snapshotName; + String snpName; String consistentId; - Collection groupNames; + Collection grps; - public RestoreHandleJob(String snapshotName, String consistentId, Collection groupNames) { - this.snapshotName = snapshotName; + public RestoreHandleJob(String snpName, String consistentId, Collection grps) { + this.snpName = snpName; this.consistentId = consistentId; - this.groupNames = groupNames; + this.grps = grps; } /** {@inheritDoc} */ - @Override public Map execute() throws IgniteException { + @Override public Map execute() throws IgniteException { // todo expand restore hadnler separately - Map resMap = new HashMap<>(); - - SnapshotListeners lsnrs = ignite.context().cache().context().snapshotMgr().listeners(); + Map resMap = new HashMap<>(); + IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); + SnapshotListeners lsnrs = snpMgr.listeners(); + File snpDir = snpMgr.snapshotLocalDir(snpName); + File smf = new File(snpDir, snpMgr.snapshotMetaFileName(consistentId)); try { + String pdsFolderName = ignite.context().pdsFolderResolver().resolveFolders().folderName(); + + Path binaryWorkDir = binaryWorkDir(snpDir.getAbsolutePath(), pdsFolderName).toPath(); + Path marshallerDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath()).toPath(); + Path cacheDir = Paths.get(snpDir.toString(), DB_DEFAULT_FOLDER, pdsFolderName); + for (SnapshotLifecycleListener lsnr : lsnrs.list()) { - Object res = lsnr.beforeRestore(snapshotName, consistentId, groupNames); + Serializable res; + + try { + res = lsnr.beforeRestore(snpName, grps, smf, binaryWorkDir, marshallerDir, cacheDir); + } + catch (Exception e) { + res = e; + } resMap.put(lsnr.getClass().getSimpleName(), res); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java index 519e24d9ab096..cea8468d4acb1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java @@ -17,52 +17,66 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle; +import java.io.File; +import java.io.Serializable; +import java.nio.file.Path; import java.util.Collection; -import java.util.List; +import java.util.Map; import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.internal.util.typedef.F; import org.apache.ignite.plugin.Extension; import org.jetbrains.annotations.Nullable; /** * Snapshot lifecycle listener. */ -public interface SnapshotLifecycleListener extends Extension { - /** Default listener priority. */ - public static final int DEFAULT_PRIORITY = 0; - -// /** Listener name (must be unique). */ -// public String name(); - +public interface SnapshotLifecycleListener extends Extension { /** Listener invocation priority (ascending order is used). */ public default int priority() { - return DEFAULT_PRIORITY; + return 0; } /** * Called locally after the snapshot files have been created on the node. * - * @param snpName Snapshot name. + * @param name Snapshot name. + * @param metadata Snapshot metadata file. + * @param binaryDir Snapshot binary metadata directory. + * @param marshallerDir Snapshot marshaller data directory. + * @param cacheDir Snapshot cache data directory. * @throws IgniteCheckedException If failed. */ - public default void afterCreate(String snpName) throws IgniteCheckedException { + public default void afterCreate( + String name, + File metadata, + Path binaryDir, + Path marshallerDir, + Path cacheDir + ) throws IgniteCheckedException { // No-op. } /** * Called locally before restore snapshot files. * - * @param snpName Snapshot name. - * @param consId Consistent snapshot metadata file name. + * @param name Snapshot name. * @param grps Cache groups to be restored ({@code null} if all cache groups are restored from the snapshot). - * @param Type of the result. + * @param metadata Snapshot metadata file. + * @param binaryDir Snapshot binary metadata directory. + * @param marshallerDir Snapshot marshaller data directory. + * @param cacheDir Snapshot cache data directory. + * * @return Local node result, or {@code null} if cluster-wide aggregation is not required. * @throws IgniteCheckedException If failed. */ - @Nullable public default T beforeRestore( - String snpName, - String consId, - @Nullable Collection grps + @Nullable public default T beforeRestore( + String name, + @Nullable Collection grps, + File metadata, + Path binaryDir, + Path marshallerDir, + Path cacheDir ) throws IgniteCheckedException { return null; } @@ -70,10 +84,24 @@ public default void afterCreate(String snpName) throws IgniteCheckedException { /** * Process the results of a pre-restore operation across the cluster. * + * @param name Snapshot name. + * @param grps Cache groups to be restored ({@code null} if all cache groups are restored from the snapshot). * @param res Results from all nodes. + * @param errs Errors from all nodes. * @throws IgniteCheckedException If failed. */ - public default void handlePreRestoreResults(List res) throws IgniteCheckedException { - // No-op. + public default void handlePreRestoreResults( + String name, + @Nullable Collection grps, + Map res, + Map errs + ) throws IgniteCheckedException { + Map.Entry errEntry = F.first(errs.entrySet()); + + if (errEntry == null) + return; + + throw new IgniteCheckedException("Snapshot restore handler " + getClass().getSimpleName() + + " has failed on node " + errEntry.getKey().id() + '.', errEntry.getValue()); } } From 5619505ed32a76ec6e2a7c8b38387285076d8140 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 5 Aug 2021 21:21:13 +0300 Subject: [PATCH 18/19] added name() --- .../lifecycle/SnapshotLifecycleListener.java | 7 +- .../snapshot/lifecycle/SnapshotListeners.java | 18 +-- ...eClusterSnapshotLifecycleListenerTest.java | 105 ++++++++++++++++++ .../IgniteClusterSnapshotRestoreSelfTest.java | 64 ----------- 4 files changed, 120 insertions(+), 74 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java index cea8468d4acb1..9ff2dbccda578 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotLifecycleListener.java @@ -32,6 +32,11 @@ * Snapshot lifecycle listener. */ public interface SnapshotLifecycleListener extends Extension { + /** Listener name. */ + public default String name() { + return getClass().getSimpleName(); + } + /** Listener invocation priority (ascending order is used). */ public default int priority() { return 0; @@ -101,7 +106,7 @@ public default void handlePreRestoreResults( if (errEntry == null) return; - throw new IgniteCheckedException("Snapshot restore handler " + getClass().getSimpleName() + + throw new IgniteCheckedException("Snapshot restore handler " + name() + " has failed on node " + errEntry.getKey().id() + '.', errEntry.getValue()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java index 248ddb8e38881..eec78490f5afa 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java @@ -35,7 +35,7 @@ public class SnapshotListeners { // } public Iterable list() { - return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.getClass().getSimpleName())); + return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.name())); } // public @Nullable SnapshotLifecycleListener find(String name) { @@ -48,8 +48,8 @@ public Iterable list() { // } public void register(SnapshotLifecycleListener lsnr) { - if (lsnrsState.putIfAbsent(lsnr.getClass().getSimpleName(), true) != null) - throw new IllegalArgumentException("Listener " + lsnr.getClass().getSimpleName() + " is already registered."); + if (lsnrsState.putIfAbsent(lsnr.name(), true) != null) + throw new IllegalArgumentException("Listener " + lsnr.name() + " is already registered."); int idx = Collections.binarySearch(lsnrsByPriority, lsnr, Comparator.comparingInt(SnapshotLifecycleListener::priority)); @@ -59,22 +59,22 @@ public void register(SnapshotLifecycleListener lsnr) { lsnrsByPriority.add(idx, lsnr); } - public boolean disable(String clsName) { - Boolean enabled = lsnrsState.get(clsName); + public boolean disable(String name) { + Boolean enabled = lsnrsState.get(name); if (!Boolean.TRUE.equals(enabled)) return false; - return lsnrsState.put(clsName, false); + return lsnrsState.put(name, false); } - public boolean enable(String clsName) { - Boolean enabled = lsnrsState.get(clsName); + public boolean enable(String name) { + Boolean enabled = lsnrsState.get(name); if (!Boolean.FALSE.equals(enabled)) return false; - return !lsnrsState.put(clsName, true); + return !lsnrsState.put(name, true); // // SnapshotLifecycleListener lsnr = lsnrsState.get(name); // diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java index 842b470f12518..a10b317cb81b3 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java @@ -17,20 +17,44 @@ package org.apache.ignite.internal.processors.cache.persistence.snapshot; +import java.io.File; +import java.io.Serializable; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.function.Function; import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; +import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotLifecycleListener; import org.apache.ignite.internal.processors.cache.persistence.snapshot.lifecycle.SnapshotListeners; import org.apache.ignite.internal.util.typedef.G; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteFuture; +import org.apache.ignite.plugin.AbstractTestPluginProvider; +import org.apache.ignite.plugin.ExtensionRegistry; +import org.apache.ignite.plugin.PluginContext; +import org.apache.ignite.testframework.GridTestUtils; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.getPartitionFileName; import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; public class IgniteClusterSnapshotLifecycleListenerTest extends IgniteClusterSnapshotRestoreBaseTest { + /** */ + private List extensions = new ArrayList<>(); + + private SnapshotLifecyclePluginProvider testPluginProvider = new SnapshotLifecyclePluginProvider(extensions); + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setPluginProviders(testPluginProvider); + } + /** @throws Exception If fails. */ @Test public void testRestoreWithMissedPart() throws Exception { @@ -69,6 +93,87 @@ public void testRestoreWithMissedPart() throws Exception { ); } + /** @throws Exception If fails. */ + @Test + public void testClusterSnapshotOptionalVerificationFailure() throws Exception { + String expMsg = "Test verification exception message."; + + extensions.add(new SnapshotLifecycleListener() { + @Override public String name() { + return "listener-1"; + } + + @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, + Path binaryDir, Path marshallerDir, Path cacheDir) throws IgniteCheckedException { + throw new IgniteCheckedException(expMsg); + } + }); + + extensions.add(new SnapshotLifecycleListener() { + @Override public String name() { + return "listener-2"; + } + + @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, + Path binaryDir, Path marshallerDir, Path cacheDir) throws IgniteCheckedException { + return null; + } + }); + + IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); + + IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); + + GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); + } +// +// /** @throws Exception If fails. */ +// @Test +// public void testClusterSnapshotOptionalVerifications() throws Exception { +// AtomicInteger checkCntr = new AtomicInteger(); +// +// snpCheckPlugin = new AbstractTestPluginProvider() { +// @Override public String name() { +// return "SnapshotVerifier"; +// } +// +// /** {@inheritDoc} */ +// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// checkCntr.incrementAndGet(); +// })); +// +// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { +// checkCntr.incrementAndGet(); +// })); +// } +// }; +// +// IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); +// +// ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); +// +// assertEquals(2, checkCntr.get()); +// } + + private static class SnapshotLifecyclePluginProvider extends AbstractTestPluginProvider { + private final List extensions; + + public SnapshotLifecyclePluginProvider(List extensions) { + this.extensions = extensions; + } + + @Override public String name() { + return "SnapshotVerifier"; + } + + /** {@inheritDoc} */ + @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { + for (SnapshotLifecycleListener lsnr : extensions) + registry.registerExtension(SnapshotLifecycleListener.class, lsnr); + } + }; + /** {@inheritDoc} */ @Override protected Function valueBuilder() { return Integer::new; diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java index 75c37e868e7e4..78d4b04b24f8c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotRestoreSelfTest.java @@ -101,9 +101,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR /** Default shared cache group name. */ private static final String SHARED_GRP = "shared"; - /** Optional snapshot check plugin. */ - private PluginProvider snpCheckPlugin; - /** Cache value builder. */ private Function valBuilder = String::valueOf; @@ -114,9 +111,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); - if (snpCheckPlugin != null) - cfg.setPluginProviders(snpCheckPlugin); - if (resetConsistentId) cfg.setConsistentId(null); @@ -128,64 +122,6 @@ public class IgniteClusterSnapshotRestoreSelfTest extends IgniteClusterSnapshotR return valBuilder; } -// /** @throws Exception If fails. */ -// @Test -// public void testClusterSnapshotOptionalVerificationFailure() throws Exception { -// String expMsg = "Test verification exception message."; -// -// snpCheckPlugin = new AbstractTestPluginProvider() { -// @Override public String name() { -// return "SnapshotVerifier"; -// } -// -// /** {@inheritDoc} */ -// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// throw new IgniteCheckedException(expMsg); -// })); -// -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// // No-op. -// })); -// } -// }; -// -// IgniteEx ignite = startGridsWithSnapshot(2, CACHE_KEYS_RANGE); -// -// IgniteFuture fut = ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null); -// -// GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); -// } - -// /** @throws Exception If fails. */ -// @Test -// public void testClusterSnapshotOptionalVerifications() throws Exception { -// AtomicInteger checkCntr = new AtomicInteger(); -// -// snpCheckPlugin = new AbstractTestPluginProvider() { -// @Override public String name() { -// return "SnapshotVerifier"; -// } -// -// /** {@inheritDoc} */ -// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// checkCntr.incrementAndGet(); -// })); -// -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// checkCntr.incrementAndGet(); -// })); -// } -// }; -// -// IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); -// -// ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); -// -// assertEquals(2, checkCntr.get()); -// } - /** @throws Exception If fails. */ @Test public void testRestoreWithMissedPart() throws Exception { From b5bfa9e17e7c0c524f6d9370c56d6d6d81635840 Mon Sep 17 00:00:00 2001 From: Pavel Pereslegin Date: Thu, 5 Aug 2021 23:06:23 +0300 Subject: [PATCH 19/19] pririty test --- .../snapshot/IgniteSnapshotManager.java | 2 +- .../snapshot/SnapshotRestoreProcess.java | 2 +- .../snapshot/lifecycle/RestoreHandleTask.java | 6 +- .../snapshot/lifecycle/SnapshotListeners.java | 6 +- ...eClusterSnapshotLifecycleListenerTest.java | 80 +++++++++++-------- 5 files changed, 55 insertions(+), 41 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java index 630b4ce853516..550f61ded5f2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteSnapshotManager.java @@ -700,7 +700,7 @@ private IgniteInternalFuture initLocalSnapshotStartSt Path marshallerDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath()).toPath(); Path cacheDir = Paths.get(snpDir.toString(), DB_DEFAULT_FOLDER, pdsSettings.folderName()); - for (SnapshotLifecycleListener lsnr : listeners().list()) + for (SnapshotLifecycleListener lsnr : listeners().list()) lsnr.afterCreate(req.snapshotName(), smf, binaryWorkDir, marshallerDir, cacheDir); return new SnapshotOperationResponse(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java index 25c94a5e4f3c2..fd40063ac28e4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/SnapshotRestoreProcess.java @@ -188,7 +188,7 @@ public IgniteFuture start(String snpName, @Nullable Collection cac Map> metas = ctx.cache().context().snapshotMgr().collectSnapshotMetadata(snpName).get(); - Iterable lsnrs = ctx.cache().context().snapshotMgr().listeners().list(); + Iterable> lsnrs = ctx.cache().context().snapshotMgr().listeners().list(); if (!F.isEmpty(lsnrs)) { ctx.security().authorize(ADMIN_SNAPSHOT); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java index 167b00708d23b..8166cdb463cb6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/RestoreHandleTask.java @@ -148,7 +148,7 @@ public class RestoreHandleTask extends ComputeTaskAdapter lsnr : ignite.context().cache().context().snapshotMgr().listeners().list()) { T2, Map> mapPair = resMap.get(lsnr.getClass().getSimpleName()); // The listener might have been enabled at runtime. @@ -196,7 +196,7 @@ public RestoreHandleJob(String snpName, String consistentId, Collection IgniteSnapshotManager snpMgr = ignite.context().cache().context().snapshotMgr(); SnapshotListeners lsnrs = snpMgr.listeners(); File snpDir = snpMgr.snapshotLocalDir(snpName); - File smf = new File(snpDir, snpMgr.snapshotMetaFileName(consistentId)); + File smf = new File(snpDir, IgniteSnapshotManager.snapshotMetaFileName(consistentId)); try { String pdsFolderName = ignite.context().pdsFolderResolver().resolveFolders().folderName(); @@ -205,7 +205,7 @@ public RestoreHandleJob(String snpName, String consistentId, Collection Path marshallerDir = mappingFileStoreWorkDir(snpDir.getAbsolutePath()).toPath(); Path cacheDir = Paths.get(snpDir.toString(), DB_DEFAULT_FOLDER, pdsFolderName); - for (SnapshotLifecycleListener lsnr : lsnrs.list()) { + for (SnapshotLifecycleListener lsnr : lsnrs.list()) { Serializable res; try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java index eec78490f5afa..787791e229f08 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/lifecycle/SnapshotListeners.java @@ -28,13 +28,13 @@ public class SnapshotListeners { // private final IgnitePluginProcessor plugins; private final Map lsnrsState = new HashMap<>(); - private final List lsnrsByPriority = new ArrayList<>(); + private final List> lsnrsByPriority = new ArrayList<>(); // public SnapshotListeners(IgnitePluginProcessor plugins) { //// this.plugins = plugins; // } - public Iterable list() { + public Iterable> list() { return F.viewReadOnly(lsnrsByPriority, lsnr -> lsnr, lsnr -> lsnrsState.get(lsnr.name())); } @@ -47,7 +47,7 @@ public Iterable list() { // return lsnrState.get1() ? lsnrState.get2() : null; // } - public void register(SnapshotLifecycleListener lsnr) { + public void register(SnapshotLifecycleListener lsnr) { if (lsnrsState.putIfAbsent(lsnr.name(), true) != null) throw new IllegalArgumentException("Listener " + lsnr.name() + " is already registered."); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java index a10b317cb81b3..4b2f9683db4fd 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/snapshot/IgniteClusterSnapshotLifecycleListenerTest.java @@ -22,8 +22,16 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; @@ -98,23 +106,23 @@ public void testRestoreWithMissedPart() throws Exception { public void testClusterSnapshotOptionalVerificationFailure() throws Exception { String expMsg = "Test verification exception message."; - extensions.add(new SnapshotLifecycleListener() { + extensions.add(new SnapshotLifecycleListener() { @Override public String name() { return "listener-1"; } - @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, + @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, Path binaryDir, Path marshallerDir, Path cacheDir) throws IgniteCheckedException { throw new IgniteCheckedException(expMsg); } }); - extensions.add(new SnapshotLifecycleListener() { + extensions.add(new SnapshotLifecycleListener() { @Override public String name() { return "listener-2"; } - @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, + @Nullable @Override public Serializable beforeRestore(String name, @Nullable Collection grps, File metadata, Path binaryDir, Path marshallerDir, Path cacheDir) throws IgniteCheckedException { return null; } @@ -126,35 +134,41 @@ public void testClusterSnapshotOptionalVerificationFailure() throws Exception { GridTestUtils.assertThrowsAnyCause(log, () -> fut.get(TIMEOUT), IgniteCheckedException.class, expMsg); } -// -// /** @throws Exception If fails. */ -// @Test -// public void testClusterSnapshotOptionalVerifications() throws Exception { -// AtomicInteger checkCntr = new AtomicInteger(); -// -// snpCheckPlugin = new AbstractTestPluginProvider() { -// @Override public String name() { -// return "SnapshotVerifier"; -// } -// -// /** {@inheritDoc} */ -// @Override public void initExtensions(PluginContext ctx, ExtensionRegistry registry) { -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// checkCntr.incrementAndGet(); -// })); -// -// registry.registerExtension(SnapshotVerifier.class, ((metas, grps) -> { -// checkCntr.incrementAndGet(); -// })); -// } -// }; -// -// IgniteEx ignite = startGridsWithSnapshot(3, CACHE_KEYS_RANGE); -// -// ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); -// -// assertEquals(2, checkCntr.get()); -// } + + /** @throws Exception If fails. */ + @Test + public void testClusterSnapshotListenerPriority() throws Exception { + AtomicReference checkCntr = new AtomicReference<>(0); + + List ints = IntStream.range(0, 100).boxed().collect(Collectors.toList()); + + Collections.shuffle(ints); + + for (int num : ints) { + extensions.add(new SnapshotLifecycleListener() { + @Override public int priority() { + return num; + } + + @Override public String name() { + return "listener-" + num; + } + + @Nullable @Override public Integer beforeRestore(String name, @Nullable Collection grps, File metadata, + Path binaryDir, Path marshallerDir, Path cacheDir) throws IgniteCheckedException { + assertEquals(checkCntr.get().intValue(), num); + + checkCntr.set(num + 1); + + return null; + } + }); + } + + IgniteEx ignite = startGridsWithSnapshot(1, CACHE_KEYS_RANGE); + + ignite.snapshot().restoreSnapshot(SNAPSHOT_NAME, null).get(); + } private static class SnapshotLifecyclePluginProvider extends AbstractTestPluginProvider { private final List extensions;