Skip to content

Commit

Permalink
pool: update sweeper purge command to accept option storage class
Browse files Browse the repository at this point in the history
Motivation:
Sometimes admin want to purge only one particular storage class, but
`sweep purge` command doesn't accept a filter.

Modification:
Update `sweeper purge` command to accept optional `-storageClass` option.

Result:
More flexible file purging in pool

Acked-by: Dmitry Litvintsev
Target: master
Require-book: no
Require-notes: yes
  • Loading branch information
kofemann committed Sep 25, 2024
1 parent 7b2f1a3 commit be9d1c7
Showing 1 changed file with 26 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.function.Predicate;

import org.dcache.namespace.FileAttribute;
import org.dcache.pool.PoolDataBeanProvider;
import org.dcache.pool.classic.json.SweeperData;
Expand Down Expand Up @@ -224,19 +226,25 @@ public String call() {
"even if it has been marked for removal.")
public class SweeperPurgeCommand implements Callable<String> {


@Option(name="storageClass", usage = "Purge only files of the specified storage class.", metaVar = "storage-class")
String storageClass;

@Override
public String call() {
new Thread("sweeper-purge") {
@Override
public void run() {
try {
long bytes = reclaim(Long.MAX_VALUE, "'sweeper purge' command");
Predicate<CacheEntry> filter = storageClass == null ? e -> true
: e -> e.getFileAttributes().getStorageClass().equals(storageClass);
long bytes = reclaim(Long.MAX_VALUE, "'sweeper purge' command", filter);
LOGGER.info("'sweeper purge' reclaimed {} bytes.", bytes);
} catch (InterruptedException e) {
}
}
}.start();
return "Purging all removable files from pool.";
return "Purging " + (storageClass == null ? "all" : storageClass) + " removable files from pool.";
}
}

Expand All @@ -254,7 +262,7 @@ public String call() {
@Override
public void run() {
try {
long bytes = reclaim(bytesToFree, "'sweeper free' command");
long bytes = reclaim(bytesToFree, "'sweeper free' command", e -> true);
LOGGER.info("'sweeper free {}' reclaimed {} bytes.", bytesToFree, bytes);
} catch (InterruptedException e) {
}
Expand Down Expand Up @@ -402,7 +410,15 @@ public String call() {
}
}

private long reclaim(long amount, String why)
/**
* Reclaims space by removing removable files from the pool.
* @param amount the amount of space to reclaim.
* @param why the reason for reclaiming space.
* @param filter remove only files that match the filter
* @return the amount of space reclaimed.
* @throws InterruptedException
*/
private long reclaim(long amount, String why, Predicate<CacheEntry> filter)
throws InterruptedException {
LOGGER.debug("Sweeper tries to reclaim {} bytes.", amount);

Expand All @@ -429,6 +445,11 @@ private long reclaim(long amount, String why)
continue;
}

if (!filter.test(entry)) {
LOGGER.debug("File skipped by sweeper (filter): {}", entry);
continue;
}

long size = entry.getReplicaSize();
LOGGER.debug("Sweeper removes {}.", id);
_repository.setState(id, ReplicaState.REMOVED, why);
Expand Down Expand Up @@ -475,7 +496,7 @@ public long waitForRequests()
public void run() {
try {
while (true) {
if (reclaim(waitForRequests(), "sweeper making space for new data") == 0) {
if (reclaim(waitForRequests(), "sweeper making space for new data", e -> true) == 0) {
/* The list maintained by the sweeper is imperfect
* in the sense that it can contain locked entries
* or entries in use. Thus we could be caught in a
Expand Down

0 comments on commit be9d1c7

Please sign in to comment.