From 4d36b93df0695e4512dfa8a950d3a102a9a8c2da Mon Sep 17 00:00:00 2001 From: masokol <97948057+masokol@users.noreply.github.com> Date: Thu, 29 Jun 2023 08:55:16 +0200 Subject: [PATCH] Fix priority for repair jobs (#516) Closes #515 --- CHANGES.md | 1 + .../ecchronos/core/HostStatesImpl.java | 11 ++- .../ecchronos/core/repair/TableRepairJob.java | 37 ++++++-- .../core/repair/state/RepairedAt.java | 9 +- .../core/repair/state/ReplicaRepairGroup.java | 15 ++- .../repair/state/VnodeRepairGroupFactory.java | 9 +- .../core/scheduling/ScheduledJob.java | 9 +- .../ecchronos/core/TestHostStatesImpl.java | 6 +- ...taCenterAndVnodeRepairResourceFactory.java | 2 +- .../TestDataCenterRepairResourceFactory.java | 2 +- .../core/repair/TestRepairGroup.java | 8 +- .../core/repair/TestRepairGroupTasks.java | 5 +- .../core/repair/TestTableRepairJob.java | 93 ++++++++++++++++++- .../TestVnodeRepairResourceFactory.java | 2 +- .../repair/state/TestReplicaRepairGroup.java | 4 +- 15 files changed, 180 insertions(+), 33 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7d8da63a9..a21de57e0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,7 @@ ## Version 1.0.8 +* Fix repair job priority - Issue #515 * Fix malformed IPv6 for JMX - Issue #306 * Step karaf to 4.2.8 * Improve Alarm logging - Issue #191 diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java index 7ba78929c..9aa8ce9b4 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java @@ -20,6 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,14 +76,20 @@ private void refreshNodeStatus() { synchronized (myRefreshLock) { - if (shouldRefreshNodeStatus()) + if (shouldRefreshNodeStatus() && !tryRefreshHostStates()) { - tryRefreshHostStates(); + myHostStates.clear(); } } } } + @VisibleForTesting + void resetLastRefresh() + { + myLastRefresh = -1; + } + private boolean shouldRefreshNodeStatus() { return myLastRefresh == -1 || myLastRefresh < (System.currentTimeMillis() - myRefreshIntervalInMs); diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java index dbb1053af..58c563408 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java @@ -181,15 +181,12 @@ else if (msSinceLastRepair >= myRepairConfiguration.getRepairWarningTimeInMs()) @Override public boolean runnable() { - if (super.runnable()) + try { - try - { - myRepairState.update(); - } catch (Exception e) - { - LOG.warn("Unable to check repair history, {}", this, e); - } + myRepairState.update(); + } catch (Exception e) + { + LOG.warn("Unable to check repair history, {}", this, e); } RepairStateSnapshot repairStateSnapshot = myRepairState.getSnapshot(); @@ -204,6 +201,30 @@ public boolean runnable() return repairStateSnapshot.canRepair() && super.runnable(); } + /** + * Calculate real priority based on available tasks. + */ + @Override + public final int getRealPriority() + { + RepairStateSnapshot repairStateSnapshot = myRepairState.getSnapshot(); + int priority = -1; + if (repairStateSnapshot.canRepair()) + { + long minRepairedAt = System.currentTimeMillis(); + for (ReplicaRepairGroup replicaRepairGroup : repairStateSnapshot.getRepairGroups()) + { + long replicaGroupCompletedAt = replicaRepairGroup.getLastCompletedAt(); + if (replicaGroupCompletedAt < minRepairedAt) + { + minRepairedAt = replicaGroupCompletedAt; + } + } + priority = getRealPriority(minRepairedAt); + } + return priority; + } + @Override public String toString() { diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java index c3991ef3c..c7450d7f2 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java @@ -14,6 +14,8 @@ */ package com.ericsson.bss.cassandra.ecchronos.core.repair.state; +import java.util.Collection; + /** * Utility class to determine collective repaired at information for {@link VnodeRepairStates}. * @@ -78,11 +80,16 @@ public String toString() } public static RepairedAt generate(VnodeRepairStates vnodeRepairStates) + { + return RepairedAt.generate(vnodeRepairStates.getVnodeRepairStates()); + } + + public static RepairedAt generate(final Collection vnodeRepairStates) { long minRepairedAt = Long.MAX_VALUE; long maxRepairedAt = Long.MIN_VALUE; - for (VnodeRepairState vnodeRepairState : vnodeRepairStates.getVnodeRepairStates()) + for (VnodeRepairState vnodeRepairState : vnodeRepairStates) { long repairedAt = vnodeRepairState.lastRepairedAt(); diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java index aa3a515d8..2d4622e09 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java @@ -30,11 +30,14 @@ public class ReplicaRepairGroup implements Iterable { private final ImmutableSet myReplicas; private final ImmutableList myVnodes; + private final long myLastCompletedAt; - public ReplicaRepairGroup(ImmutableSet replicas, ImmutableList vnodes) + public ReplicaRepairGroup(ImmutableSet replicas, ImmutableList vnodes, + final long lastCompletedAt) { myReplicas = replicas; myVnodes = vnodes; + myLastCompletedAt = lastCompletedAt; } public Set getReplicas() @@ -47,6 +50,16 @@ public Set getDataCenters() return myReplicas.stream().map(Host::getDatacenter).collect(Collectors.toSet()); } + /** + * Get last completed at. + * + * @return Last completed at for this repair group. + */ + public long getLastCompletedAt() + { + return myLastCompletedAt; + } + @Override public Iterator iterator() { diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java index 7318c17b7..1f2092ef6 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java @@ -57,12 +57,15 @@ public List generateReplicaRepairGroups(List commonVnodes = availableVnodeRepairStates.stream() - .filter(v -> v.getReplicas().equals(replicas)) + List vnodesForReplicas = availableVnodeRepairStates.stream() + .filter(v -> v.getReplicas().equals(replicas)).collect(Collectors.toList()); + RepairedAt repairedAt = RepairedAt.generate(vnodesForReplicas); + List commonVnodes = vnodesForReplicas.stream() .map(VnodeRepairState::getTokenRange) .collect(Collectors.toList()); - sortedRepairGroups.add(new ReplicaRepairGroup(replicas, ImmutableList.copyOf(commonVnodes))); + sortedRepairGroups.add(new ReplicaRepairGroup(replicas, ImmutableList.copyOf(commonVnodes), + repairedAt.getMinRepairedAt())); } } diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java index 48c48e793..63972bee9 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java @@ -108,11 +108,16 @@ public Priority getPriority() * @return The current priority or -1 if the job shouldn't run now. * @see #getPriority() */ - public final int getRealPriority() + public int getRealPriority() + { + return getRealPriority(getLastSuccessfulRun()); + } + + public final int getRealPriority(final long lastSuccessfulRun) { long now = System.currentTimeMillis(); - long diff = now - (getLastSuccessfulRun() + myRunIntervalInMs); + long diff = now - (lastSuccessfulRun + myRunIntervalInMs); if (diff < 0) { diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java index a8f5ebf18..344a13748 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java @@ -89,7 +89,7 @@ public void testIsInetAddressUp() throws UnknownHostException } @Test - public void testIsHostUp() throws UnknownHostException + public void testIsHostUp() throws IOException { InetAddress expectedAddress = InetAddress.getLocalHost(); Host expectedHost = mock(Host.class); @@ -103,6 +103,10 @@ public void testIsHostUp() throws UnknownHostException when(expectedHost.getBroadcastAddress()).thenReturn(expectedAddress); assertThat(myHostStates.isUp(expectedHost)).isTrue(); + + myHostStates.resetLastRefresh(); + when(myJmxProxyFactory.connect()).thenThrow(new IOException("Unittest")); + assertThat(myHostStates.isUp(expectedHost)).isFalse(); } @Test diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java index 15d04b9b5..a3fad89fa 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java @@ -66,7 +66,7 @@ public void testMultipleDataCenterHosts() private ReplicaRepairGroup generateReplicaRepairGroup(Host... hosts) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range), System.currentTimeMillis()); } private Host mockHost(String dataCenter, UUID hostId) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java index b51edff39..88860c312 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java @@ -56,7 +56,7 @@ public void testMultipleDataCenters() private ReplicaRepairGroup generateReplicaRepairGroup(Host... hosts) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range), System.currentTimeMillis()); } private Host mockHost(String dataCenter) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java index d7f35c9d6..a0f7f8c6b 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java @@ -102,7 +102,7 @@ public void testGetLock() throws LockException Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of()); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of(), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); doReturn(repairResources).when(myRepairResourceFactory).getRepairResources(eq(replicaRepairGroup)); @@ -124,7 +124,7 @@ public void testGetLockWithThrowingLockingStrategy() throws LockException Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of()); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of(), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); doReturn(repairResources).when(myRepairResourceFactory).getRepairResources(eq(replicaRepairGroup)); @@ -152,7 +152,7 @@ public void testGetRepairTask() Set ranges = new HashSet<>(); ranges.add(range); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(hosts, ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(hosts, ImmutableList.of(range), System.currentTimeMillis()); RepairGroup repairGroup = new RepairGroup(priority, tableReference, repairConfiguration, replicaRepairGroup, myJmxProxyFactory, myTableRepairMetrics, @@ -181,7 +181,7 @@ public void testGetPartialRepairTasks() new LongTokenRange(2, 3), new LongTokenRange(4, 5)); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host, host2), vnodes); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host, host2), vnodes, System.currentTimeMillis()); RepairGroup repairGroup = new RepairGroup(priority, tableReference, repairConfiguration, replicaRepairGroup, myJmxProxyFactory, myTableRepairMetrics, diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java index ae49389f7..82e9989ad 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java @@ -91,7 +91,8 @@ public void testExecute() throws Exception Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withHost("127.0.0.1")), ImmutableList.of(range(1, 2))); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withHost("127.0.0.1")), ImmutableList.of(range(1, 2)), + System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); when(mockJmxProxyFactory.connect()).thenReturn(new CustomJmxProxy((notificationListener, i) -> progressAndComplete(notificationListener, range(1, 2)))); @@ -112,7 +113,7 @@ public void testExecuteWithPolicyStoppingSecondTask() throws Exception Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withHost("127.0.0.1")), ImmutableList.of(range(1, 2), range(2, 3))); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withHost("127.0.0.1")), ImmutableList.of(range(1, 2), range(2, 3)), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); final AtomicBoolean shouldRun = new AtomicBoolean(true); diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java index 7f8352417..2110b79f3 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java @@ -22,21 +22,30 @@ import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.ignoreStubs; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; +import static org.mockito.Mockito.when; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import com.datastax.driver.core.Host; import com.ericsson.bss.cassandra.ecchronos.core.JmxProxyFactory; import com.ericsson.bss.cassandra.ecchronos.core.MockedClock; import com.ericsson.bss.cassandra.ecchronos.core.repair.state.RepairState; import com.ericsson.bss.cassandra.ecchronos.core.repair.state.RepairStateSnapshot; +import com.ericsson.bss.cassandra.ecchronos.core.repair.state.ReplicaRepairGroup; +import com.ericsson.bss.cassandra.ecchronos.core.utils.LongTokenRange; import com.ericsson.bss.cassandra.ecchronos.core.utils.TableReference; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -165,11 +174,11 @@ public void testPrevalidateNeedRepair() { // mock doReturn(true).when(myRepairStateSnapshot).canRepair(); - + mockRepairGroup(0L); assertThat(myRepairJob.runnable()).isTrue(); verify(myRepairState, times(1)).update(); - verify(myRepairStateSnapshot, times(1)).canRepair(); + verify(myRepairStateSnapshot, times(2)).canRepair(); } @Test @@ -177,12 +186,12 @@ public void testPrevalidateNotRepairableThenRepairable() { // mock doReturn(false).doReturn(true).when(myRepairStateSnapshot).canRepair(); - + mockRepairGroup(0L); assertThat(myRepairJob.runnable()).isFalse(); assertThat(myRepairJob.runnable()).isTrue(); verify(myRepairState, times(2)).update(); - verify(myRepairStateSnapshot, times(2)).canRepair(); + verify(myRepairStateSnapshot, times(3)).canRepair(); } @Test @@ -293,6 +302,7 @@ public void testThatWarningAlarmIsSentAndCeased() // mock - not repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); assertThat(myRepairJob.runnable()).isTrue(); @@ -307,6 +317,7 @@ public void testThatWarningAlarmIsSentAndCeased() // mock - repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); myRepairJob.runnable(); @@ -336,6 +347,7 @@ public void testThatWarningAlarmIsSentAndCeasedExternalRepair() // mock - not repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); assertThat(myRepairJob.runnable()).isTrue(); @@ -350,6 +362,7 @@ public void testThatWarningAlarmIsSentAndCeasedExternalRepair() // mock - repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); assertThat(myRepairJob.runnable()).isFalse(); @@ -373,6 +386,7 @@ public void testThatErrorAlarmIsSentAndCeased() // mock - not repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); assertThat(myRepairJob.runnable()).isTrue(); @@ -387,6 +401,7 @@ public void testThatErrorAlarmIsSentAndCeased() // mock - repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); myRepairJob.postExecute(true); @@ -410,6 +425,7 @@ public void testThatErrorAlarmIsSentAndCeasedExternalRepair() // mock - not repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); assertThat(myRepairJob.runnable()).isTrue(); @@ -424,6 +440,7 @@ public void testThatErrorAlarmIsSentAndCeasedExternalRepair() // mock - repaired doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); myClock.setTime(start); myRepairJob.runnable(); @@ -466,6 +483,7 @@ public void testLastSuccessfulRunIsBasedOnRepairHistory() // We have waited 2 days to repair, send alarm and run repair doReturn(lastRepairedAtWarning).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepairedAtWarning); assertThat(myRepairJob.runnable()).isTrue(); @@ -477,6 +495,7 @@ public void testLastSuccessfulRunIsBasedOnRepairHistory() // Repair has been completed doReturn(lastRepairedAtAfterRepair).when(myRepairStateSnapshot).lastRepairedAt(); doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepairedAtAfterRepair); myRepairJob.postExecute(true); @@ -492,4 +511,70 @@ public void testLastSuccessfulRunIsBasedOnRepairHistory() assertThat(myRepairJob.getLastSuccessfulRun()).isEqualTo(lastRepairedAtAfterRepair); verify(myFaultReporter, times(0)).raise(any(FaultCode.class), anyMapOf(String.class, Object.class)); } + + @Test + public void testGetRealPriority() + { + long lastRepaired = System.currentTimeMillis(); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); + doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(-1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) - TimeUnit.HOURS.toMillis(1)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(-1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) + TimeUnit.HOURS.toMillis(1)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastRepairedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(2); + } + + @Test + public void testGetRealPrioritySnapshotLastRepairedAtLowerThanRepairGroups() + { + long lastRepairedAtSnapshot = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(14); + doReturn(lastRepairedAtSnapshot).when(myRepairStateSnapshot).lastRepairedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + long firstRepairGroupLastRepairedAt = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS)); + ReplicaRepairGroup firstReplicaRepairGroup = getRepairGroup(new LongTokenRange(1, 2), firstRepairGroupLastRepairedAt); + mockRepairGroup(firstReplicaRepairGroup); + assertThat(myRepairJob.getRealPriority()).isEqualTo(1); + + long secondRepairGroupLastRepairedAt = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) + TimeUnit.HOURS.toMillis(1)); + ReplicaRepairGroup secondReplicaRepairGroup = getRepairGroup(new LongTokenRange(2, 3), secondRepairGroupLastRepairedAt); + mockRepairGroup(secondReplicaRepairGroup, firstReplicaRepairGroup); + assertThat(myRepairJob.getRealPriority()).isEqualTo(2); + } + + private void mockRepairGroup(long lastRepairedAt) + { + mockRepairGroup(getRepairGroup(new LongTokenRange(1, 2), lastRepairedAt)); + } + + private void mockRepairGroup(ReplicaRepairGroup ...replicaRepairGroups) + { + List repairGroups = new ArrayList<>(); + for (ReplicaRepairGroup replicaRepairGroup : replicaRepairGroups) + { + repairGroups.add(replicaRepairGroup); + } + when(myRepairStateSnapshot.getRepairGroups()).thenReturn(repairGroups); + } + + private ReplicaRepairGroup getRepairGroup(LongTokenRange range, long lastRepairedAt) + { + ImmutableSet replicas = ImmutableSet.of(mock(Host.class), mock(Host.class)); + return new ReplicaRepairGroup(replicas, ImmutableList.of(range), lastRepairedAt); + } } diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java index 3fb7bd230..889aef598 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java @@ -62,7 +62,7 @@ public void testMultipleDataCenterHosts() private ReplicaRepairGroup generateReplicaRepairGroup(Host... hosts) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(hosts), ImmutableList.of(range), System.currentTimeMillis()); } private Host mockHost(String dataCenter, UUID hostId) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java index 42726cc93..a32cba468 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java @@ -35,7 +35,7 @@ public void testMultipleDataCenters() Host host4 = mockHost("DC1"); LongTokenRange range = new LongTokenRange(1, 2); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host1, host2, host3, host4), ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host1, host2, host3, host4), ImmutableList.of(range), System.currentTimeMillis()); assertThat(replicaRepairGroup.getDataCenters()).containsExactlyInAnyOrder("DC1", "DC2", "DC3"); assertThat(replicaRepairGroup.getReplicas()).containsExactlyInAnyOrder(host1, host2, host3, host4); @@ -52,7 +52,7 @@ public void testMultipleRanges() LongTokenRange range2 = new LongTokenRange(3, 4); LongTokenRange range3 = new LongTokenRange(5, 6); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host1, host2, host3), ImmutableList.of(range, range2, range3)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(host1, host2, host3), ImmutableList.of(range, range2, range3), System.currentTimeMillis()); assertThat(replicaRepairGroup.getDataCenters()).containsExactlyInAnyOrder("DC1"); assertThat(replicaRepairGroup.getReplicas()).containsExactlyInAnyOrder(host1, host2, host3);