allow to crawl bitstream download endpoint

DSpace · Oct 18, 2024 · e0267ef · e0267ef
1 parent 862d0d9
commit e0267ef
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/robots.txt.ejs b/src/robots.txt.ejs
@@ -18,7 +18,8 @@ Disallow: /profile
 Disallow: /workflowitems
 # Crawlers should be able to access entity pages, but not the facet search links present on entity pages
 Disallow: /entities/*?f
-# do not crawl REST API endpoints and HAL browser
+# do not crawl REST API endpoints and HAL browser (with the exception of the bitstream download endpoint)
+Allow: /server/api/core/bitstreams/*
 Disallow: /server/*
 # do not crawl health status
 Disallow: /app/health