Merge pull request #32157 from vespa-engine/hmusum/configurable-backo…

…ff-time Make initial backoff time for file distribution configurable
vespa-engine · Aug 16, 2024 · 6c76772 · 6c76772
2 parents b198dce + 1ff58cd
commit 6c76772
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 12 deletions.
diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileDownloader.java
@@ -27,9 +27,11 @@
 public class FileDownloader implements AutoCloseable {
 
     private static final Logger log = Logger.getLogger(FileDownloader.class.getName());
-    private static final Duration defaultSleepBetweenRetries = Duration.ofSeconds(5);
-    public static final File defaultDownloadDirectory = new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"));
-    private static final boolean forceDownload = Boolean.parseBoolean(System.getenv("VESPA_CONFIG_PROXY_FORCE_DOWNLOAD_OF_FILE_REFERENCES"));
+    private static final Duration backoffInitialTime;
+    public static final File defaultDownloadDirectory =
+            new File(Defaults.getDefaults().underVespaHome("var/db/vespa/filedistribution"));
+    // Undocumented on purpose, might change or be removed at any time
+    private static final boolean forceDownload = Boolean.parseBoolean(System.getenv("VESPA_FORCE_DOWNLOAD_OF_FILE_REFERENCES"));
 
     private final ConnectionPool connectionPool;
     private final Supervisor supervisor;
@@ -38,19 +40,25 @@ public class FileDownloader implements AutoCloseable {
     private final FileReferenceDownloader fileReferenceDownloader;
     private final Downloads downloads = new Downloads();
 
+    static {
+        // Undocumented on purpose, might change or be removed at any time
+        var backOff = System.getenv("VESPA_FILE_DOWNLOAD_BACKOFF_INITIAL_TIME_MS");
+        backoffInitialTime = Duration.ofMillis(backOff == null ? 1000 : Long.parseLong(backOff));
+    }
+
     public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, Duration timeout) {
-        this(connectionPool, supervisor, defaultDownloadDirectory, timeout, defaultSleepBetweenRetries);
+        this(connectionPool, supervisor, defaultDownloadDirectory, timeout, backoffInitialTime);
     }
 
     public FileDownloader(ConnectionPool connectionPool, Supervisor supervisor, File downloadDirectory, Duration timeout) {
-        this(connectionPool, supervisor, downloadDirectory, timeout, defaultSleepBetweenRetries);
+        this(connectionPool, supervisor, downloadDirectory, timeout, backoffInitialTime);
     }
 
     public FileDownloader(ConnectionPool connectionPool,
                           Supervisor supervisor,
                           File downloadDirectory,
                           Duration timeout,
-                          Duration sleepBetweenRetries) {
+                          Duration backoffInitialTime) {
         this.connectionPool = connectionPool;
         this.supervisor = supervisor;
         this.downloadDirectory = downloadDirectory;
@@ -60,7 +68,7 @@ public FileDownloader(ConnectionPool connectionPool,
         this.fileReferenceDownloader = new FileReferenceDownloader(connectionPool,
                                                                    downloads,
                                                                    timeout,
-                                                                   sleepBetweenRetries,
+                                                                   backoffInitialTime,
                                                                    downloadDirectory);
         if (forceDownload)
             log.log(Level.INFO, "Force download of file references (download even if file reference exists on disk)");

diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceDownloader.java
@@ -46,22 +46,23 @@ public class FileReferenceDownloader {
     private final ConnectionPool connectionPool;
     private final Downloads downloads;
     private final Duration downloadTimeout;
-    private final Duration sleepBetweenRetries;
+    private final Duration backoffInitialTime;
     private final Duration rpcTimeout;
     private final File downloadDirectory;
     private final AtomicBoolean shutDown = new AtomicBoolean(false);
 
     FileReferenceDownloader(ConnectionPool connectionPool,
                             Downloads downloads,
                             Duration timeout,
-                            Duration sleepBetweenRetries,
+                            Duration backoffInitialTime,
                             File downloadDirectory) {
         this.connectionPool = connectionPool;
         this.downloads = downloads;
         this.downloadTimeout = timeout;
-        this.sleepBetweenRetries = sleepBetweenRetries;
+        this.backoffInitialTime = backoffInitialTime;
         this.downloadDirectory = downloadDirectory;
-        String timeoutString = System.getenv("VESPA_CONFIGPROXY_FILEDOWNLOAD_RPC_TIMEOUT");
+        // Undocumented on purpose, might change or be removed at any time
+        String timeoutString = System.getenv("VESPA_FILE_DOWNLOAD_RPC_TIMEOUT");
         this.rpcTimeout = Duration.ofSeconds(timeoutString == null ? 30 : Integer.parseInt(timeoutString));
     }
 
@@ -95,7 +96,7 @@ private void waitUntilDownloadStarted(FileReferenceDownload fileReferenceDownloa
     private void backoff(int retryCount, Instant end) {
         try {
             long sleepTime = Math.min(120_000,
-                                      Math.min((long) (Math.pow(2, retryCount)) * sleepBetweenRetries.toMillis(),
+                                      Math.min((long) (Math.pow(2, retryCount)) * backoffInitialTime.toMillis(),
                                                Duration.between(Instant.now(), end).toMillis()));
             if (sleepTime <= 0) return;