Fix for HTTP storage_options without 'headers'

huggingface · Jul 24, 2024 · 11f56e4 · 11f56e4
1 parent 85aa4d0
commit 11f56e4
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py
@@ -1179,7 +1179,8 @@ def _prepare_single_hop_path_and_storage_options(
             urlpath += "&confirm=t"
         if urlpath.startswith("https://raw.githubusercontent.com/"):
             # Workaround for served data with gzip content-encoding: https://github.com/fsspec/filesystem_spec/issues/389
-            storage_options[protocol]["headers"]["Accept-Encoding"] = "identity"
+            headers = storage_options[protocol].setdefault("headers", {})
+            headers["Accept-Encoding"] = "identity"
     elif protocol == "hf":
         storage_options[protocol] = {
             "token": token,