tests: Speed up test_rm_with_1k_objects_or_more by x10 by asyncio par…

…allelism (#294)

pyproject.toml

@@ -34,7 +34,10 @@ classifiers = [
     "Typing :: Typed",
 ]
 
-dependencies = ["fsspec>=2023.12.0", "lakefs>=0.2.0"]
+dependencies = [
+    "fsspec>=2023.12.0",
+    "lakefs>=0.2.0",
+]
 
 dynamic = ["version"]
 
@@ -56,6 +59,7 @@ dev = [
     "pandas[parquet]",
     "polars",
     "duckdb",
+    "pytest-asyncio>=0.24.0",
 ]
 docs = [
     "mkdocs",
@@ -133,6 +137,7 @@ exclude_dirs = ["tests", "docs/tutorials"]
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "WARNING"
+asyncio_default_fixture_loop_scope = "function"
 
 [tool.pydoclint]
 style = 'numpy'

tests/test_rm.py

@@ -1,3 +1,6 @@
+import asyncio
+
+import pytest
 from lakefs.branch import Branch
 from lakefs.repository import Repository
 
@@ -69,7 +72,8 @@ def test_rm_recursive_with_maxdepth(
     assert fs.exists(f"{prefix}/dir1/dir2/c.txt")
 
 
-def test_rm_with_1k_objects_or_more(
+@pytest.mark.asyncio
+async def test_rm_with_1k_objects_or_more(
     fs: LakeFSFileSystem,
     repository: Repository,
     temp_branch: Branch,
@@ -80,12 +84,17 @@ def test_rm_with_1k_objects_or_more(
     """
     testdir = f"{repository.id}/{temp_branch.id}/subfolder"
 
-    # create and put 1001 objects into the above lakeFS directory.
+    # Create and put 1001 objects into the above lakeFS directory (to exceed the 1k API batch limit)
+    # Doing this async since we are I/O bound and get a significant speedup.
+    # Unfortunately, we cannot use a TaskGroup, since it was only introduced in Python 3.11.
+    tasks = []
     for i in range(1002):
         f = random_file_factory.make()
         lpath = str(f)
         rpath = testdir + f"/test_{i}.txt"
-        fs.put_file(lpath, rpath)
+        task = asyncio.create_task(asyncio.to_thread(fs.put_file, lpath, rpath))
+        tasks.append(task)
+    await asyncio.gather(*tasks)
 
     assert len(fs.ls(testdir, detail=False)) > 1000