Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Progress for on demand repair jobs is either 0% or 100% #593 #618

Merged
merged 2 commits into from
Nov 28, 2023
Merged

Progress for on demand repair jobs is either 0% or 100% #593 #618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7299f5a
Fix progress tracking for On demand Repair Jobs #593
Nov 27, 2023
666669f
updated changes.md
Nov 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Version 5.0.0 (Not yet released)

* Progress for on demand repair jobs is either 0% or 100% - Issue #593
* Make priority granularity configurable - Issue #599
* Bump springboot from 2.7.12 to 2.7.17 - Issue #604
* Bump io.micrometer from 1.9.2 to 1.9.16 - Issue #604
Expand Down
19 changes: 14 additions & 5 deletions ...rc/main/java/com/ericsson/bss/cassandra/ecchronos/core/repair/VnodeOnDemandRepairJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

package com.ericsson.bss.cassandra.ecchronos.core.repair;

import static com.ericsson.bss.cassandra.ecchronos.core.repair.OngoingJob.Status;

import com.ericsson.bss.cassandra.ecchronos.core.JmxProxyFactory;
import com.ericsson.bss.cassandra.ecchronos.core.metrics.TableRepairMetrics;
import com.ericsson.bss.cassandra.ecchronos.core.repair.state.RepairHistory;
Expand Down Expand Up @@ -52,17 +54,16 @@ public final class VnodeOnDemandRepairJob extends OnDemandRepairJob
private static final Logger LOG = LoggerFactory.getLogger(VnodeOnDemandRepairJob.class);
private final RepairHistory myRepairHistory;
private final Map<ScheduledTask, Set<LongTokenRange>> myTasks;
private final int myTotalTasks;
private final int myTotalTokens;

private VnodeOnDemandRepairJob(final Builder builder)
{
super(builder.configuration, builder.jmxProxyFactory, builder.repairConfiguration,
builder.repairLockType, builder.onFinishedHook, builder.tableRepairMetrics, builder.ongoingJob);
myRepairHistory = Preconditions.checkNotNull(builder.repairHistory,
"Repair history must be set");

myTotalTokens = getOngoingJob().getTokens().size();
myTasks = createRepairTasks(getOngoingJob().getTokens(), getOngoingJob().getRepairedTokens());
myTotalTasks = myTasks.size();
}

private Map<ScheduledTask, Set<LongTokenRange>> createRepairTasks(
Expand Down Expand Up @@ -187,8 +188,16 @@ public ScheduledJob.State getState()

public double getProgress()
{
int finishedTasks = myTotalTasks - myTasks.size();
return myTotalTasks == 0 ? 1 : (double) finishedTasks / myTotalTasks;
if (myTotalTokens == 0)
{
LOG.debug("Total tokens for this job are 0");
return 0;
}

OngoingJob ongoingJob = getOngoingJob();
Status state = ongoingJob.getStatus();
int repairedTokens = ongoingJob.getRepairedTokens().size();
return state == Status.finished ? 1 : (double) repairedTokens / myTotalTokens;
}

@Override
Expand Down
42 changes: 29 additions & 13 deletions ...est/java/com/ericsson/bss/cassandra/ecchronos/core/repair/TestVnodeOnDemandRepairJob.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public void finalVerification()
@Test
public void testJobCorrectlyReturned()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob(0);
OnDemandRepairJobView expectedView = new OnDemandRepairJobView(repairJob.getId(), myHostId, myTableReference,
OnDemandRepairJobView.Status.IN_QUEUE, 0, System.currentTimeMillis(), RepairOptions.RepairType.VNODE);
assertThat(repairJob.getId()).isEqualTo(repairJob.getId());
Expand All @@ -106,7 +106,7 @@ public void testJobCorrectlyReturned()
@Test
public void testFailedJobCorrectlyReturned()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob(0);
Iterator<ScheduledTask> it = repairJob.iterator();
repairJob.postExecute(false, it.next());
OnDemandRepairJobView expectedView = new OnDemandRepairJobView(repairJob.getId(), myHostId, myTableReference,
Expand All @@ -121,7 +121,7 @@ public void testFailedJobCorrectlyReturned()
@Test
public void testJobFinishedAfterExecution()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob(0);
Iterator<ScheduledTask> it = repairJob.iterator();
assertThat(repairJob.getState()).isEqualTo(ScheduledJob.State.RUNNABLE);
repairJob.postExecute(true, it.next());
Expand All @@ -143,15 +143,15 @@ public void testJobFinishedAfterRestart()
@Test
public void testJobFailedWhenTopologyChange()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob(0);
when(myOngoingJob.hasTopologyChanged()).thenReturn(true);
assertThat(repairJob.getState()).isEqualTo(ScheduledJob.State.FAILED);
}

@Test
public void testJobUnsuccessful()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob(0);
Iterator<ScheduledTask> it = repairJob.iterator();
repairJob.postExecute(true, it.next());
assertThat(repairJob.getState()).isEqualTo(ScheduledJob.State.RUNNABLE);
Expand All @@ -162,16 +162,20 @@ public void testJobUnsuccessful()
@Test
public void testGetProgress()
{
VnodeOnDemandRepairJob repairJob = createVnodeOnDemandRepairJob();
assertThat(repairJob.getProgress()).isEqualTo(0);
Iterator<ScheduledTask> it = repairJob.iterator();
repairJob.postExecute(true, it.next());
assertThat(repairJob.getProgress()).isEqualTo(0.5);
repairJob.postExecute(true, it.next());
assertThat(repairJob.getProgress()).isEqualTo(1);
VnodeOnDemandRepairJob repairJobZeroProgress = createVnodeOnDemandRepairJob(0);
assertThat(repairJobZeroProgress.getProgress()).isEqualTo(0);

VnodeOnDemandRepairJob repairJobHalfProgress = createVnodeOnDemandRepairJob(50);
assertThat(repairJobHalfProgress.getProgress()).isEqualTo(0.5);

VnodeOnDemandRepairJob repairJobFullProgress = createVnodeOnDemandRepairJob(100);
assertThat(repairJobFullProgress.getProgress()).isEqualTo(1.0);

when(repairJobHalfProgress.getOngoingJob().getStatus()).thenReturn(OngoingJob.Status.finished);
assertThat(repairJobHalfProgress.getProgress()).isEqualTo(1.0);
}

private VnodeOnDemandRepairJob createVnodeOnDemandRepairJob()
private VnodeOnDemandRepairJob createVnodeOnDemandRepairJob(int repairedTokenPercentage)
{
LongTokenRange range1 = new LongTokenRange(1, 2);
LongTokenRange range2 = new LongTokenRange(1, 3);
Expand All @@ -180,7 +184,19 @@ private VnodeOnDemandRepairJob createVnodeOnDemandRepairJob()
ImmutableSet.of(mockReplica1, mockReplica2, mockReplica3));
tokenRangeToReplicas.put(range2,
ImmutableSet.of(mockReplica1, mockReplica2));

Set<LongTokenRange> repairedTokens = new HashSet<>();
if (repairedTokenPercentage >= 50)
{
repairedTokens.add(range1);
}
if (repairedTokenPercentage == 100)
{
repairedTokens.add(range2);
}

when(myOngoingJob.getTokens()).thenReturn(tokenRangeToReplicas);
when(myOngoingJob.getRepairedTokens()).thenReturn(repairedTokens);

return new VnodeOnDemandRepairJob.Builder()
.withJmxProxyFactory(myJmxProxyFactory)
Expand Down