From 819f72a3be691f9b4cbe18d713fb83ae3efdeb67 Mon Sep 17 00:00:00 2001 From: Suresh Kumar Anaparti Date: Tue, 17 Dec 2024 17:15:22 +0530 Subject: [PATCH] Support to enable/disable VM High Availability manager and related alerts - Adds new config 'vm.ha.enabled' with Zone scope, to enable/disable VM High Availability manager. This is enable by default (for backward compatibilty). When enabled, the VM HA WorkItems (for VM Stop, Restart, Migration, Destroy) can be created and the scheduled items are executed. When disabled, new VM HA WorkItems are not allowed and the scheduled items are retried until max retries configured at 'vm.ha.migration.max.retries' (executed in case HA is re-enabled during retry attempts), and then purged after 'time.between.failures' by the cleanup thread that runs regularly at 'time.between.cleanup'. - Adds new config 'vm.ha.alerts.enabled' with Zone scope, to enable/disable alerts for the VM HA operations. This is enabled by default. --- .../com/cloud/ha/HighAvailabilityManager.java | 6 +- .../cloud/ha/HighAvailabilityManagerImpl.java | 129 +++++++++++++-- .../cloud/resource/ResourceManagerImpl.java | 10 +- .../ha/HighAvailabilityManagerImplTest.java | 156 ++++++++++++++++++ 4 files changed, 281 insertions(+), 20 deletions(-) diff --git a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java index ae47b1d76ed8..728f5a2b1808 100644 --- a/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java +++ b/engine/components-api/src/main/java/com/cloud/ha/HighAvailabilityManager.java @@ -32,7 +32,7 @@ */ public interface HighAvailabilityManager extends Manager { - public ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", + ConfigKey ForceHA = new ConfigKey<>("Advanced", Boolean.class, "force.ha", "false", "Force High-Availability to happen even if the VM says no.", true, Cluster); ConfigKey HAWorkers = new ConfigKey<>("Advanced", Integer.class, "ha.workers", "5", @@ -112,7 +112,7 @@ enum Step { void cancelDestroy(VMInstanceVO vm, Long hostId); - void scheduleDestroy(VMInstanceVO vm, long hostId); + boolean scheduleDestroy(VMInstanceVO vm, long hostId); /** * Schedule restarts for all vms running on the host. @@ -143,7 +143,7 @@ enum Step { * @param host host the virtual machine is on. * @param type which type of stop is requested. */ - void scheduleStop(VMInstanceVO vm, long hostId, WorkType type); + boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type); void cancelScheduledMigrations(HostVO host); diff --git a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java index d8fc99a0934e..20435f48b526 100644 --- a/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java +++ b/server/src/main/java/com/cloud/ha/HighAvailabilityManagerImpl.java @@ -16,6 +16,8 @@ // under the License. package com.cloud.ha; +import static org.apache.cloudstack.framework.config.ConfigKey.Scope.Zone; + import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -121,6 +123,16 @@ public class HighAvailabilityManagerImpl extends ManagerBase implements Configur "Total number of attempts for trying migration of a VM.", true, ConfigKey.Scope.Global); + public static ConfigKey VmHaEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.ha.enabled", "true", + "Enable/Disable VM High Availability manager, it is enabled by default." + + " When enabled, the VM HA WorkItems (for VM Stop, Restart, Migration, Destroy) can be created and the scheduled items are executed; and" + + " When disabled, new VM HA WorkItems are not allowed and the scheduled items are retried until max retries configured at 'vm.ha.migration.max.retries'" + + " (executed in case HA is re-enabled during retry attempts), and then purged after 'time.between.failures' by the cleanup thread that runs" + + " regularly at 'time.between.cleanup'", true, Zone); + + protected static ConfigKey VmHaAlertsEnabled = new ConfigKey<>("Advanced", Boolean.class, "vm.ha.alerts.enabled", "true", + "Enable/Disable alerts for the VM HA operations, it is enabled by default.", true, Zone); + WorkerThread[] _workers; boolean _stopped; long _timeToSleep; @@ -185,7 +197,6 @@ public void setHaPlanners(List haPlanners) { _haPlanners = haPlanners; } - @Inject AgentManager _agentMgr; @Inject @@ -231,6 +242,15 @@ public Status investigate(final long hostId) { return Status.Alert; } + if (!VmHaEnabled.valueIn(host.getDataCenterId())) { + String message = String.format("Unable to investigate the host %s (%d), VM high availability manager is disabled.", host.getName(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendHostAlert(host, message); + return Status.Alert; + } + Status hostState = null; for (Investigator investigator : investigators) { hostState = investigator.isAgentAlive(host); @@ -260,6 +280,15 @@ public void scheduleRestartForVmsOnHost(final HostVO host, boolean investigate) return; } + if (!VmHaEnabled.valueIn(host.getDataCenterId())) { + String message = String.format("Unable to schedule restart for VMs on host %s (%d), VM high availability manager is disabled.", host.getName(), host.getId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendHostAlert(host, message); + return; + } + logger.warn("Scheduling restart for VMs on host " + host.getId() + "-" + host.getName()); final List vms = _instanceDao.listByHostId(host.getId()); @@ -314,12 +343,21 @@ public void scheduleRestartForVmsOnHost(final HostVO host, boolean investigate) } @Override - public void scheduleStop(VMInstanceVO vm, long hostId, WorkType type) { + public boolean scheduleStop(VMInstanceVO vm, long hostId, WorkType type) { assert (type == WorkType.CheckStop || type == WorkType.ForceStop || type == WorkType.Stop); if (_haDao.hasBeenScheduled(vm.getId(), type)) { logger.info("There's already a job scheduled to stop " + vm); - return; + return false; + } + + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule stop for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; } HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), type, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated()); @@ -328,6 +366,7 @@ public void scheduleStop(VMInstanceVO vm, long hostId, WorkType type) { logger.debug("Scheduled " + work); } wakeupWorkers(); + return true; } protected void wakeupWorkers() { @@ -339,17 +378,37 @@ protected void wakeupWorkers() { @Override public boolean scheduleMigration(final VMInstanceVO vm) { - if (vm.getHostId() != null) { - final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated()); - _haDao.persist(work); - logger.info("Scheduled migration work of VM " + vm.getUuid() + " from host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work); - wakeupWorkers(); + if (vm.getHostId() == null) { + return false; + } + + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule migration for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), vm.getHostId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; } + + final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Migration, Step.Scheduled, vm.getHostId(), vm.getState(), 0, vm.getUpdated()); + _haDao.persist(work); + logger.info("Scheduled migration work of VM " + vm.getUuid() + " from host " + _hostDao.findById(vm.getHostId()) + " with HAWork " + work); + wakeupWorkers(); return true; } @Override public void scheduleRestart(VMInstanceVO vm, boolean investigate) { + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule restart for the VM %s (%d), VM high availability manager is disabled.", vm.getName(), vm.getId()); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return; + } + logger.debug("HA schedule restart"); Long hostId = vm.getHostId(); if (hostId == null) { @@ -440,7 +499,6 @@ public void scheduleRestart(VMInstanceVO vm, boolean investigate) { } wakeupWorkers(); - } private void startVm(VirtualMachine vm, Map params, @@ -737,13 +795,23 @@ public Long migrate(final HaWorkVO work) { } @Override - public void scheduleDestroy(VMInstanceVO vm, long hostId) { + public boolean scheduleDestroy(VMInstanceVO vm, long hostId) { + if (!VmHaEnabled.valueIn(vm.getDataCenterId())) { + String message = String.format("Unable to schedule destroy for the VM %s (%d) on host %d, VM high availability manager is disabled.", vm.getName(), vm.getId(), hostId); + if (logger.isDebugEnabled()) { + logger.debug(message); + } + sendVMAlert(vm, message); + return false; + } + final HaWorkVO work = new HaWorkVO(vm.getId(), vm.getType(), WorkType.Destroy, Step.Scheduled, hostId, vm.getState(), 0, vm.getUpdated()); _haDao.persist(work); if (logger.isDebugEnabled()) { logger.debug("Scheduled " + work.toString()); } wakeupWorkers(); + return true; } @Override @@ -892,7 +960,17 @@ private long getRescheduleTime(WorkType workType) { private void processWork(final HaWorkVO work) { final WorkType wt = work.getWorkType(); + final VMInstanceVO vm = _instanceDao.findById(work.getInstanceId()); try { + if (vm != null && !VmHaEnabled.valueIn(vm.getDataCenterId())) { + if (logger.isDebugEnabled()) { + logger.debug(String.format("VM high availability manager is disabled, rescheduling the HA work %s, for the VM %s (id) to retry later in case VM high availability manager is enabled on retry attempt", work, vm.getName(), vm.getId())); + } + long nextTime = getRescheduleTime(wt); + rescheduleWork(work, nextTime); + return; + } + Long nextTime = null; if (wt == WorkType.Migration) { nextTime = migrate(work); @@ -921,9 +999,10 @@ private void processWork(final HaWorkVO work) { // if restart failed in the middle due to exception, VM state may has been changed // recapture into the HA worker so that it can really continue in it next turn - VMInstanceVO vm = _instanceDao.findById(work.getInstanceId()); - work.setUpdateTime(vm.getUpdated()); - work.setPreviousState(vm.getState()); + if (vm != null) { + work.setUpdateTime(vm.getUpdated()); + work.setPreviousState(vm.getState()); + } } finally { if (!Step.Done.equals(work.getStep())) { if (work.getTimesTried() >= _maxRetries) { @@ -1128,11 +1207,33 @@ public String getConfigComponentName() { public ConfigKey[] getConfigKeys() { return new ConfigKey[] {TimeBetweenCleanup, MigrationMaxRetries, TimeToSleep, TimeBetweenFailures, StopRetryInterval, RestartRetryInterval, MigrateRetryInterval, InvestigateRetryInterval, - HAWorkers, ForceHA, KvmHAFenceHostIfHeartbeatFailsOnStorage}; + HAWorkers, ForceHA, VmHaEnabled, VmHaAlertsEnabled, KvmHAFenceHostIfHeartbeatFailsOnStorage}; } @Override public int expungeWorkItemsByVmList(List vmIds, Long batchSize) { return _haDao.expungeByVmList(vmIds, batchSize); } + + private void sendVMAlert(VMInstanceVO vm, String message) { + if (vm == null || !VmHaAlertsEnabled.valueIn(vm.getDataCenterId())) { + return; + } + AlertManager.AlertType alertType = AlertManager.AlertType.ALERT_TYPE_USERVM; + if (VirtualMachine.Type.DomainRouter.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_DOMAIN_ROUTER; + } else if (VirtualMachine.Type.ConsoleProxy.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_CONSOLE_PROXY; + } else if (VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) { + alertType = AlertManager.AlertType.ALERT_TYPE_SSVM; + } + _alertMgr.sendAlert(alertType, vm.getDataCenterId(), vm.getPodIdToDeployIn(), message, message); + } + + private void sendHostAlert(HostVO host, String message) { + if (host == null || !VmHaAlertsEnabled.valueIn(host.getDataCenterId())) { + return; + } + _alertMgr.sendAlert(AlertManager.AlertType.ALERT_TYPE_HOST, host.getDataCenterId(), host.getPodId(), message, message); + } } diff --git a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java index 228373896204..94dc91698232 100755 --- a/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java +++ b/server/src/main/java/com/cloud/resource/ResourceManagerImpl.java @@ -41,6 +41,7 @@ import com.cloud.cpu.CPU; import com.cloud.exception.StorageConflictException; import com.cloud.exception.StorageUnavailableException; +import com.cloud.ha.HighAvailabilityManagerImpl; import com.cloud.host.HostTagVO; import com.cloud.storage.Volume; import com.cloud.storage.VolumeVO; @@ -1363,6 +1364,11 @@ private boolean doMaintain(final long hostId) { throw new CloudRuntimeException("Cannot perform maintain when resource state is " + hostState + ", hostId = " + hostId); } + final List vms = _vmDao.listByHostId(hostId); + if (CollectionUtils.isNotEmpty(vms) && !HighAvailabilityManagerImpl.VmHaEnabled.valueIn(host.getDataCenterId())) { + throw new CloudRuntimeException(String.format("Cannot perform maintain for the host %s (%d) as there are running VMs on it and VM high availability manager is disabled", host.getName(), hostId)); + } + final MaintainAnswer answer = (MaintainAnswer)_agentMgr.easySend(hostId, new MaintainCommand()); if (answer == null || !answer.getResult()) { logger.warn("Unable to send MaintainCommand to host: " + hostId); @@ -1382,8 +1388,6 @@ private boolean doMaintain(final long hostId) { /* TODO: move below to listener */ if (host.getType() == Host.Type.Routing) { - - final List vms = _vmDao.listByHostId(hostId); if (vms.size() == 0) { return true; } @@ -2841,7 +2845,7 @@ public void deleteRoutingHost(final HostVO host, final boolean isForced, final b logger.debug("Cannot transmit host " + host.getId() + " to Disabled state", e); } for (final VMInstanceVO vm : vms) { - if ((! HighAvailabilityManager.ForceHA.value() && !vm.isHaEnabled()) || vm.getState() == State.Stopping) { + if ((!HighAvailabilityManager.ForceHA.value() && !vm.isHaEnabled()) || vm.getState() == State.Stopping) { logger.debug(String.format("Stopping %s as a part of hostDelete for %s",vm, host)); try { _haMgr.scheduleStop(vm, host.getId(), WorkType.Stop); diff --git a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java index 74897967a2f9..27b6e1b1b4cc 100644 --- a/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java +++ b/server/src/test/java/com/cloud/ha/HighAvailabilityManagerImplTest.java @@ -16,6 +16,7 @@ // under the License. package com.cloud.ha; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; @@ -32,6 +33,7 @@ import org.apache.cloudstack.engine.orchestration.service.VolumeOrchestrationService; import org.apache.cloudstack.engine.subsystem.api.storage.DataStoreProviderManager; +import org.apache.cloudstack.framework.config.ConfigKey; import org.apache.cloudstack.framework.config.dao.ConfigurationDao; import org.apache.cloudstack.managed.context.ManagedContext; import org.apache.logging.log4j.LogManager; @@ -174,10 +176,15 @@ public void setup() throws IllegalArgumentException, public void scheduleRestartForVmsOnHost() { Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); Mockito.lenient().when(_instanceDao.listByHostId(42l)).thenReturn(Arrays.asList(Mockito.mock(VMInstanceVO.class))); Mockito.when(_podDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(HostPodVO.class)); Mockito.when(_dcDao.findById(Mockito.anyLong())).thenReturn(Mockito.mock(DataCenterVO.class)); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } @@ -189,11 +196,25 @@ public void scheduleRestartForVmsOnHostNotSupported() { highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } + @Test + public void scheduleRestartForVmsOnHostHADisabled() { + Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); + Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.KVM); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); + } + @Test public void scheduleRestartForVmsOnHostNonEmptyVMList() { Mockito.when(hostVO.getId()).thenReturn(1l); Mockito.when(hostVO.getType()).thenReturn(Host.Type.Routing); Mockito.when(hostVO.getHypervisorType()).thenReturn(HypervisorType.XenServer); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); List vms = new ArrayList(); VMInstanceVO vm1 = Mockito.mock(VMInstanceVO.class); Mockito.lenient().when(vm1.getHostId()).thenReturn(1l); @@ -206,6 +227,7 @@ public void scheduleRestartForVmsOnHostNonEmptyVMList() { //Mockito.when(vm2.getInstanceName()).thenReturn("r-2-VM"); Mockito.when(vm2.getType()).thenReturn(VirtualMachine.Type.DomainRouter); Mockito.when(vm2.isHaEnabled()).thenReturn(true); + Mockito.when(vm2.getDataCenterId()).thenReturn(1L); vms.add(vm2); Mockito.when(_instanceDao.listByHostId(Mockito.anyLong())).thenReturn(vms); Mockito.when(_instanceDao.findByUuid(vm1.getUuid())).thenReturn(vm1); @@ -216,12 +238,125 @@ public void scheduleRestartForVmsOnHostNonEmptyVMList() { Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); Mockito.when(_serviceOfferingDao.findById(vm1.getServiceOfferingId())).thenReturn(Mockito.mock(ServiceOfferingVO.class)); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + highAvailabilityManager.scheduleRestartForVmsOnHost(hostVO, true); } + @Test + public void scheduleRestartHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + highAvailabilityManager.scheduleRestart(vm, true); + } + + @Test + public void scheduleRestartHostNotSupported() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.VMware); + + highAvailabilityManager.scheduleRestart(vm, true); + } + + @Test + public void scheduleStop() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + + assertTrue(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop)); + } + + @Test + public void scheduleStopHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(_haDao.hasBeenScheduled(vm.getId(), WorkType.Stop)).thenReturn(false); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleStop(vm, 1L, WorkType.Stop)); + } + + @Test + public void scheduleMigration() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + + assertTrue(highAvailabilityManager.scheduleMigration(vm)); + } + + @Test + public void scheduleMigrationHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getHostId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleMigration(vm)); + } + + @Test + public void scheduleDestroy() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getId()).thenReturn(1L); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + Mockito.when(vm.getType()).thenReturn(VirtualMachine.Type.User); + Mockito.when(vm.getState()).thenReturn(VirtualMachine.State.Running); + Mockito.when(_haDao.persist((HaWorkVO)Mockito.any())).thenReturn(Mockito.mock(HaWorkVO.class)); + + assertTrue(highAvailabilityManager.scheduleDestroy(vm, 1L)); + } + + @Test + public void scheduleDestroyHADisabled() { + VMInstanceVO vm = Mockito.mock(VMInstanceVO.class); + Mockito.when(vm.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertFalse(highAvailabilityManager.scheduleDestroy(vm, 1L)); + } + @Test public void investigateHostStatusSuccess() { Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); // Set the list of investigators, CheckOnAgentInvestigator suffices for now Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class); List investigators = new ArrayList(); @@ -230,12 +365,17 @@ public void investigateHostStatusSuccess() { // Mock isAgentAlive to return host status as Down Mockito.when(investigator.isAgentAlive(hostVO)).thenReturn(Status.Down); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + assertTrue(highAvailabilityManager.investigate(1l) == Status.Down); } @Test public void investigateHostStatusFailure() { Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); // Set the list of investigators, CheckOnAgentInvestigator suffices for now // Also no need to mock isAgentAlive() as actual implementation returns null Investigator investigator = Mockito.mock(CheckOnAgentInvestigator.class); @@ -243,9 +383,25 @@ public void investigateHostStatusFailure() { investigators.add(investigator); highAvailabilityManager.setInvestigators(investigators); + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(true); + assertNull(highAvailabilityManager.investigate(1l)); } + @Test + public void investigateHostStatusHADisabled() { + Mockito.when(_hostDao.findById(Mockito.anyLong())).thenReturn(hostVO); + Mockito.when(hostVO.getDataCenterId()).thenReturn(1L); + + ConfigKey haEnabled = Mockito.mock(ConfigKey.class); + highAvailabilityManager.VmHaEnabled = haEnabled; + Mockito.when(highAvailabilityManager.VmHaEnabled.valueIn(1L)).thenReturn(false); + + assertTrue(highAvailabilityManager.investigate(1L) == Status.Alert); + } + private void processWorkWithRetryCount(int count, Step expectedStep) { assertNotNull(processWorkMethod); HaWorkVO work = new HaWorkVO(1l, VirtualMachine.Type.User, WorkType.Migration, Step.Scheduled, 1l, VirtualMachine.State.Running, count, 12345678l);