diff --git a/CHANGES.md b/CHANGES.md index 12a44fa2d..5878534ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -12,6 +12,9 @@ * Support configuring backoff for failed jobs - Issue #475 * Dropping keyspaces does not clean up schedules - Issue #469 +### Merged from 1.0 +* Fix repair job priority - Issue #515 + ## Version 4.0.3 * Bump jackson-databind from 2.13.4.1 to 2.13.4.2 diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java index d37388a2b..2d1f9c69e 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/HostStatesImpl.java @@ -23,6 +23,7 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.ericsson.bss.cassandra.ecchronos.core.utils.DriverNode; import com.ericsson.bss.cassandra.ecchronos.core.utils.logging.ThrottlingLogger; +import com.google.common.annotations.VisibleForTesting; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,14 +85,20 @@ private void refreshNodeStatus() { synchronized (myRefreshLock) { - if (shouldRefreshNodeStatus()) + if (shouldRefreshNodeStatus() && !tryRefreshHostStates()) { - tryRefreshHostStates(); + myHostStates.clear(); } } } } + @VisibleForTesting + void resetLastRefresh() + { + myLastRefresh = -1; + } + private boolean shouldRefreshNodeStatus() { return myLastRefresh == -1 || myLastRefresh < (System.currentTimeMillis() - myRefreshIntervalInMs); diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java index 6ec362fad..830f3b386 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TableRepairJob.java @@ -266,21 +266,43 @@ public long getRunOffset() @Override public boolean runnable() { - if (super.runnable()) + try { - try - { - myRepairState.update(); - } - catch (Exception e) - { - LOG.warn("Unable to check repair history, {}", this, e); - } + myRepairState.update(); + } + catch (Exception e) + { + LOG.warn("Unable to check repair history, {}", this, e); } return myRepairState.getSnapshot().canRepair() && super.runnable(); } + /** + * Calculate real priority based on available tasks. + * @return priority + */ + @Override + public final int getRealPriority() + { + RepairStateSnapshot repairStateSnapshot = myRepairState.getSnapshot(); + int priority = -1; + if (repairStateSnapshot.canRepair()) + { + long minRepairedAt = System.currentTimeMillis(); + for (ReplicaRepairGroup replicaRepairGroup : repairStateSnapshot.getRepairGroups()) + { + long replicaGroupCompletedAt = replicaRepairGroup.getLastCompletedAt(); + if (replicaGroupCompletedAt < minRepairedAt) + { + minRepairedAt = replicaGroupCompletedAt; + } + } + priority = getRealPriority(minRepairedAt); + } + return priority; + } + /** * String representation. * diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java index bc1b57ad1..2c081537b 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/RepairedAt.java @@ -14,6 +14,8 @@ */ package com.ericsson.bss.cassandra.ecchronos.core.repair.state; +import java.util.Collection; + /** * Utility class to determine collective repaired at information for {@link VnodeRepairStates}. * @@ -85,11 +87,16 @@ public String toString() * @return RepairedAt */ public static RepairedAt generate(final VnodeRepairStates vnodeRepairStates) + { + return RepairedAt.generate(vnodeRepairStates.getVnodeRepairStates()); + } + + public static RepairedAt generate(final Collection vnodeRepairStates) { long minRepairedAt = Long.MAX_VALUE; long maxRepairedAt = Long.MIN_VALUE; - for (VnodeRepairState vnodeRepairState : vnodeRepairStates.getVnodeRepairStates()) + for (VnodeRepairState vnodeRepairState : vnodeRepairStates) { long repairedAt = vnodeRepairState.lastRepairedAt(); diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java index 358e6ec5b..66d8f3c9e 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/ReplicaRepairGroup.java @@ -30,17 +30,21 @@ public class ReplicaRepairGroup implements Iterable { private final ImmutableSet myReplicas; private final ImmutableList myVnodes; + private final long myLastCompletedAt; /** * Constructor. * * @param replicas The nodes. * @param vnodes The token ranges. + * @param lastCompletedAt last repair completed */ - public ReplicaRepairGroup(final ImmutableSet replicas, final ImmutableList vnodes) + public ReplicaRepairGroup(final ImmutableSet replicas, final ImmutableList vnodes, + final long lastCompletedAt) { myReplicas = replicas; myVnodes = vnodes; + myLastCompletedAt = lastCompletedAt; } /** @@ -63,6 +67,16 @@ public Set getDataCenters() return myReplicas.stream().map(DriverNode::getDatacenter).collect(Collectors.toSet()); } + /** + * Get last completed at. + * + * @return Last completed at for this repair group. + */ + public long getLastCompletedAt() + { + return myLastCompletedAt; + } + /** * Iterate. * diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java index 1438c081a..cd7a2f324 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/VnodeRepairGroupFactory.java @@ -57,12 +57,15 @@ public List generateReplicaRepairGroups(final List commonVnodes = availableVnodeRepairStates.stream() - .filter(v -> v.getReplicas().equals(replicas)) + List vnodesForReplicas = availableVnodeRepairStates.stream() + .filter(v -> v.getReplicas().equals(replicas)).collect(Collectors.toList()); + RepairedAt repairedAt = RepairedAt.generate(vnodesForReplicas); + List commonVnodes = vnodesForReplicas.stream() .map(VnodeRepairState::getTokenRange) .collect(Collectors.toList()); - sortedRepairGroups.add(new ReplicaRepairGroup(replicas, ImmutableList.copyOf(commonVnodes))); + sortedRepairGroups.add(new ReplicaRepairGroup(replicas, ImmutableList.copyOf(commonVnodes), + repairedAt.getMinRepairedAt())); } } diff --git a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java index 347483360..f8bec9360 100644 --- a/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java +++ b/core/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/scheduling/ScheduledJob.java @@ -145,10 +145,16 @@ public Priority getPriority() * @return The current priority or -1 if the job shouldn't run now. * @see #getPriority() */ - public final int getRealPriority() + public int getRealPriority() + { + return getRealPriority(getLastSuccessfulRun()); + } + + public final int getRealPriority(final long lastSuccessfulRun) { long now = System.currentTimeMillis(); - long diff = now - (getLastSuccessfulRun() + myRunIntervalInMs - getRunOffset()); + + long diff = now - (lastSuccessfulRun + myRunIntervalInMs - getRunOffset()); if (diff < 0) { diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java index 290f0fe5b..3c3364c68 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/TestHostStatesImpl.java @@ -90,7 +90,7 @@ public void testIsInetAddressUp() throws UnknownHostException } @Test - public void testIsHostUp() throws UnknownHostException + public void testIsHostUp() throws IOException { InetSocketAddress expectedAddress = new InetSocketAddress(InetAddress.getLocalHost(), 9042); Node expectedHost = mock(Node.class); @@ -104,6 +104,10 @@ public void testIsHostUp() throws UnknownHostException when(expectedHost.getBroadcastAddress()).thenReturn(Optional.of(expectedAddress)); assertThat(myHostStates.isUp(expectedHost)).isTrue(); + + myHostStates.resetLastRefresh(); + when(myJmxProxyFactory.connect()).thenThrow(new IOException("Unittest")); + assertThat(myHostStates.isUp(expectedHost)).isFalse(); } @Test diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java index 1651a1625..c2f121024 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterAndVnodeRepairResourceFactory.java @@ -65,7 +65,7 @@ public void testMultipleDataCenterHosts() private ReplicaRepairGroup generateReplicaRepairGroup(DriverNode... nodes) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range), System.currentTimeMillis()); } private DriverNode mockNode(String dataCenter, UUID nodeId) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java index 0db79de21..bab23899c 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestDataCenterRepairResourceFactory.java @@ -57,7 +57,7 @@ public void testMultipleDataCenters() private ReplicaRepairGroup generateReplicaRepairGroup(DriverNode... nodes) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range), System.currentTimeMillis()); } private DriverNode mockNode(String dataCenter) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java index 014f8e8be..3b7706c3a 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroup.java @@ -128,7 +128,7 @@ public void testGetLock() throws LockException Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of()); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of(), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); doReturn(repairResources).when(myRepairResourceFactory).getRepairResources(eq(replicaRepairGroup)); @@ -148,7 +148,7 @@ public void testGetLockWithThrowingLockingStrategy() throws LockException Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of()); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(), ImmutableList.of(), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); doReturn(repairResources).when(myRepairResourceFactory).getRepairResources(eq(replicaRepairGroup)); @@ -171,7 +171,10 @@ public void testGetRepairTask() ImmutableSet nodes = ImmutableSet.of(node); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range)); + Set ranges = new HashSet<>(); + ranges.add(range); + + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range), System.currentTimeMillis()); RepairGroup repairGroup = builderFor(replicaRepairGroup).build(priority); @@ -205,7 +208,7 @@ public void testGetRepairTaskWithSubRange() ImmutableSet nodes = ImmutableSet.of(node); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(vnode)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(vnode), System.currentTimeMillis()); RepairGroup repairGroup = builderFor(replicaRepairGroup) .withTokensPerRepair(tokensPerRange) @@ -240,7 +243,7 @@ public void testGetPartialRepairTasks() new LongTokenRange(2, 3), new LongTokenRange(4, 5)); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node, node2), vnodes); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node, node2), vnodes, System.currentTimeMillis()); RepairGroup repairGroup = builderFor(replicaRepairGroup).build(priority); @@ -271,7 +274,7 @@ public void testExecuteAllTasksSuccessful() throws ScheduledJobException DriverNode node = mockNode("DC1"); LongTokenRange range = new LongTokenRange(1, 2); ImmutableSet nodes = ImmutableSet.of(node); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range), System.currentTimeMillis()); RepairGroup repairGroup = spy(builderFor(replicaRepairGroup).build(priority)); RepairTask repairTask1 = mock(RepairTask.class); @@ -296,7 +299,7 @@ public void testExecuteAllTasksFailed() throws ScheduledJobException DriverNode node = mockNode("DC1"); LongTokenRange range = new LongTokenRange(1, 2); ImmutableSet nodes = ImmutableSet.of(node); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range), System.currentTimeMillis()); RepairGroup repairGroup = spy(builderFor(replicaRepairGroup).build(priority)); RepairTask repairTask1 = mock(RepairTask.class); @@ -321,7 +324,7 @@ public void testExecuteSomeTasksFailed() throws ScheduledJobException DriverNode node = mockNode("DC1"); LongTokenRange range = new LongTokenRange(1, 2); ImmutableSet nodes = ImmutableSet.of(node); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(nodes, ImmutableList.of(range), System.currentTimeMillis()); RepairGroup repairGroup = spy(builderFor(replicaRepairGroup).build(priority)); RepairTask repairTask1 = mock(RepairTask.class); diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java index 23b5e0bb4..582d1860a 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestRepairGroupTasks.java @@ -121,8 +121,8 @@ public void testExecute() throws Exception Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withNode("127.0.0.1")), - ImmutableList.of(range(1, 2))); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withNode("127.0.0.1")), ImmutableList.of(range(1, 2)), + System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); when(mockJmxProxyFactory.connect()).thenReturn(new CustomJmxProxy((notificationListener, i) -> progressAndComplete(notificationListener, range(1, 2)))); @@ -144,8 +144,7 @@ public void testExecuteWithPolicyStoppingSecondTask() throws Exception Map metadata = new HashMap<>(); metadata.put("keyspace", keyspaceName); metadata.put("table", tableName); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withNode("127.0.0.1")), - ImmutableList.of(range(1, 2), range(2, 3))); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(withNode("127.0.0.1")), ImmutableList.of(range(1, 2), range(2, 3)), System.currentTimeMillis()); Set repairResources = Sets.newHashSet(new RepairResource("DC1", "my-resource")); final AtomicBoolean shouldRun = new AtomicBoolean(true); diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java index e5a09986f..91ce853e7 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestTableRepairJob.java @@ -42,6 +42,7 @@ import org.mockito.junit.MockitoJUnitRunner; import java.util.Arrays; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Iterator; @@ -169,11 +170,11 @@ public void testPrevalidateNeedRepair() { // mock doReturn(true).when(myRepairStateSnapshot).canRepair(); - + mockRepairGroup(0L); assertThat(myRepairJob.runnable()).isTrue(); verify(myRepairState, times(1)).update(); - verify(myRepairStateSnapshot, times(1)).canRepair(); + verify(myRepairStateSnapshot, times(2)).canRepair(); } @Test @@ -181,12 +182,12 @@ public void testPrevalidateNotRepairableThenRepairable() { // mock doReturn(false).doReturn(true).when(myRepairStateSnapshot).canRepair(); - + mockRepairGroup(0L); assertThat(myRepairJob.runnable()).isFalse(); assertThat(myRepairJob.runnable()).isTrue(); verify(myRepairState, times(2)).update(); - verify(myRepairStateSnapshot, times(2)).canRepair(); + verify(myRepairStateSnapshot, times(3)).canRepair(); } @Test @@ -309,7 +310,7 @@ public void testIterator() VnodeRepairStates vnodeRepairStates = VnodeRepairStatesImpl .newBuilder(ImmutableList.of(new VnodeRepairState(tokenRange, replicas, 1234L))) .build(); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(replicas, vnodes); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(replicas, vnodes, System.currentTimeMillis()); RepairStateSnapshot repairStateSnapshot = RepairStateSnapshot.newBuilder() .withReplicaRepairGroups(Collections.singletonList(replicaRepairGroup)) @@ -354,7 +355,7 @@ public void testIteratorWithTargetSize() VnodeRepairStates vnodeRepairStates = VnodeRepairStatesImpl.newBuilder( ImmutableList.of(new VnodeRepairState(tokenRange, replicas, 1234L))).build(); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(replicas, vnodes); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(replicas, vnodes, System.currentTimeMillis()); RepairStateSnapshot repairStateSnapshot = RepairStateSnapshot.newBuilder() .withReplicaRepairGroups(Collections.singletonList(replicaRepairGroup)) @@ -445,7 +446,9 @@ public void testStatusBlocked() VnodeRepairState vnodeRepairState = TestUtils.createVnodeRepairState(1, 2, ImmutableSet.of(), repairedAt); VnodeRepairStatesImpl vnodeRepairStates = VnodeRepairStatesImpl.newBuilder(Arrays.asList(vnodeRepairState)) .build(); + mockRepairGroup(repairedAt); when(myRepairStateSnapshot.getVnodeRepairStates()).thenReturn(vnodeRepairStates); + when(myRepairStateSnapshot.canRepair()).thenReturn(true); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); myRepairJob.setRunnableIn(TimeUnit.HOURS.toMillis(1)); @@ -510,46 +513,119 @@ public void testRunnable() doReturn(true).when(myRepairStateSnapshot).canRepair(); //Runinterval is 1 day long repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(25); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isTrue(); repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(24); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isTrue(); // Make sure we don't repair too early repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(23); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isFalse(); repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(22); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isFalse(); } @Test - public void testRunnableWithOffset() - { + public void testRunnableWithOffset() { doReturn(true).when(myRepairStateSnapshot).canRepair(); //Runinterval is 1 day long offset = TimeUnit.HOURS.toMillis(1); doReturn(offset).when(myRepairStateSnapshot).getEstimatedRepairTime(); long repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(25); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isTrue(); repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(24); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isTrue(); repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(23); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isTrue(); // Make sure we don't repair too early repairedAt = System.currentTimeMillis() - TimeUnit.HOURS.toMillis(22); + mockRepairGroup(repairedAt); doReturn(repairedAt).when(myRepairStateSnapshot).lastCompletedAt(); assertThat(myRepairJob.runnable()).isFalse(); } + + @Test + public void testGetRealPriority() + { + long lastRepaired = System.currentTimeMillis(); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastCompletedAt(); + doReturn(false).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(-1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) - TimeUnit.HOURS.toMillis(1)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastCompletedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(-1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastCompletedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(1); + + lastRepaired = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) + TimeUnit.HOURS.toMillis(1)); + doReturn(lastRepaired).when(myRepairStateSnapshot).lastCompletedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + mockRepairGroup(lastRepaired); + assertThat(myRepairJob.getRealPriority()).isEqualTo(2); + } + + @Test + public void testGetRealPrioritySnapshotLastRepairedAtLowerThanRepairGroups() + { + long lastRepairedAtSnapshot = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(14); + doReturn(lastRepairedAtSnapshot).when(myRepairStateSnapshot).lastCompletedAt(); + doReturn(true).when(myRepairStateSnapshot).canRepair(); + long firstRepairGroupLastRepairedAt = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS)); + ReplicaRepairGroup firstReplicaRepairGroup = getRepairGroup(new LongTokenRange(1, 2), firstRepairGroupLastRepairedAt); + mockRepairGroup(firstReplicaRepairGroup); + assertThat(myRepairJob.getRealPriority()).isEqualTo(1); + + long secondRepairGroupLastRepairedAt = System.currentTimeMillis() - (TimeUnit.DAYS.toMillis(RUN_INTERVAL_IN_DAYS) + TimeUnit.HOURS.toMillis(1)); + ReplicaRepairGroup secondReplicaRepairGroup = getRepairGroup(new LongTokenRange(2, 3), secondRepairGroupLastRepairedAt); + mockRepairGroup(secondReplicaRepairGroup, firstReplicaRepairGroup); + assertThat(myRepairJob.getRealPriority()).isEqualTo(2); + } + + private void mockRepairGroup(long lastRepairedAt) + { + mockRepairGroup(getRepairGroup(new LongTokenRange(1, 2), lastRepairedAt)); + } + + private void mockRepairGroup(ReplicaRepairGroup ...replicaRepairGroups) + { + List repairGroups = new ArrayList<>(); + for (ReplicaRepairGroup replicaRepairGroup : replicaRepairGroups) + { + repairGroups.add(replicaRepairGroup); + } + when(myRepairStateSnapshot.getRepairGroups()).thenReturn(repairGroups); + } + + private ReplicaRepairGroup getRepairGroup(LongTokenRange range, long lastRepairedAt) + { + ImmutableSet replicas = ImmutableSet.of(mock(DriverNode.class), mock(DriverNode.class)); + return new ReplicaRepairGroup(replicas, ImmutableList.of(range), lastRepairedAt); + } } diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java index bfd2d0a18..43dcd76a6 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeRepairResourceFactory.java @@ -62,7 +62,7 @@ public void testMultipleDataCenterHosts() private ReplicaRepairGroup generateReplicaRepairGroup(DriverNode... nodes) { LongTokenRange range = new LongTokenRange(1, 2); - return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range)); + return new ReplicaRepairGroup(ImmutableSet.copyOf(nodes), ImmutableList.of(range), System.currentTimeMillis()); } private DriverNode mockNode(String dataCenter, UUID id) diff --git a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java index cf83be577..3d3670ea1 100644 --- a/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java +++ b/core/src/test/java/com/ericsson/bss/cassandra/ecchronos/core/repair/state/TestReplicaRepairGroup.java @@ -34,7 +34,7 @@ public void testMultipleDataCenters() DriverNode node4 = mockNode("DC1"); LongTokenRange range = new LongTokenRange(1, 2); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node1, node2, node3, node4), ImmutableList.of(range)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node1, node2, node3, node4), ImmutableList.of(range), System.currentTimeMillis()); assertThat(replicaRepairGroup.getDataCenters()).containsExactlyInAnyOrder("DC1", "DC2", "DC3"); assertThat(replicaRepairGroup.getReplicas()).containsExactlyInAnyOrder(node1, node2, node3, node4); @@ -51,7 +51,7 @@ public void testMultipleRanges() LongTokenRange range2 = new LongTokenRange(3, 4); LongTokenRange range3 = new LongTokenRange(5, 6); - ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node1, node2, node3), ImmutableList.of(range, range2, range3)); + ReplicaRepairGroup replicaRepairGroup = new ReplicaRepairGroup(ImmutableSet.of(node1, node2, node3), ImmutableList.of(range, range2, range3), System.currentTimeMillis()); assertThat(replicaRepairGroup.getDataCenters()).containsExactlyInAnyOrder("DC1"); assertThat(replicaRepairGroup.getReplicas()).containsExactlyInAnyOrder(node1, node2, node3);