Can a range request cause a connection reset by peer, and can I disable it?

[itcarroll]

I'm getting ClientOSError: [Errno 54] Connection reset by peer while trying to read from a binary file opened with fsspec.HTTPFileSystem, despite having no trouble downloading the file with get.

Unfortunately to reproduce you will need Earthdata login credentials. And although it's not imported, you'll need h5netcdf.

import fsspec
import aiohttp
import xarray

auth = aiohttp.BasicAuth(<user>, <pwd>)
url = "https://oceandata.sci.gsfc.nasa.gov/ob/getfile/AQUA_MODIS.20200613T003001.L2.OC.nc"
fs = fsspec.filesystem("https", client_kwargs={"auth": auth})

I can fs.get the file at the url just fine, but the following occurs with fs.open

with fs.open(url) as fl:
    ds = xarray.open_dataset(fl, engine="h5netcdf")

---------------------------------------------------------------------------
ClientOSError                             Traceback (most recent call last)
Cell In[3], line 2
      1 with fs.open(url) as fl:
----> 2     ds = xarray.open_dataset(fl, engine="h5netcdf")

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/xarray/backends/api.py:570, in open_dataset(filename_or_obj, engine, chunks, cache, decode_cf, mask_and_scale, decode_times, decode_timedelta, use_cftime, concat_characters, decode_coords, drop_variables, inline_array, chunked_array_type, from_array_kwargs, backend_kwargs, **kwargs)
    558 decoders = _resolve_decoders_kwargs(
    559     decode_cf,
    560     open_backend_dataset_parameters=backend.open_dataset_parameters,
   (...)
    566     decode_coords=decode_coords,
    567 )
    569 overwrite_encoded_chunks = kwargs.pop("overwrite_encoded_chunks", None)
--> 570 backend_ds = backend.open_dataset(
    571     filename_or_obj,
    572     drop_variables=drop_variables,
    573     **decoders,
    574     **kwargs,
    575 )
    576 ds = _dataset_from_backend_dataset(
    577     backend_ds,
    578     filename_or_obj,
   (...)
    588     **kwargs,
    589 )
    590 return ds

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/xarray/backends/h5netcdf_.py:400, in H5netcdfBackendEntrypoint.open_dataset(self, filename_or_obj, mask_and_scale, decode_times, concat_characters, decode_coords, drop_variables, use_cftime, decode_timedelta, format, group, lock, invalid_netcdf, phony_dims, decode_vlen_strings)
    381 def open_dataset(  # type: ignore[override]  # allow LSP violation, not supporting **kwargs
    382     self,
    383     filename_or_obj: str | os.PathLike[Any] | BufferedIOBase | AbstractDataStore,
   (...)
    397     decode_vlen_strings=True,
    398 ) -> Dataset:
    399     filename_or_obj = _normalize_path(filename_or_obj)
--> 400     store = H5NetCDFStore.open(
    401         filename_or_obj,
    402         format=format,
    403         group=group,
    404         lock=lock,
    405         invalid_netcdf=invalid_netcdf,
    406         phony_dims=phony_dims,
    407         decode_vlen_strings=decode_vlen_strings,
    408     )
    410     store_entrypoint = StoreBackendEntrypoint()
    412     ds = store_entrypoint.open_dataset(
    413         store,
    414         mask_and_scale=mask_and_scale,
   (...)
    420         decode_timedelta=decode_timedelta,
    421     )

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/xarray/backends/h5netcdf_.py:152, in H5NetCDFStore.open(cls, filename, mode, format, group, lock, autoclose, invalid_netcdf, phony_dims, decode_vlen_strings)
    147     raise ValueError(
    148         "can't open netCDF4/HDF5 as bytes "
    149         "try passing a path or file-like object"
    150     )
    151 elif isinstance(filename, io.IOBase):
--> 152     magic_number = read_magic_number_from_file(filename)
    153     if not magic_number.startswith(b"\211HDF\r\n\032\n"):
    154         raise ValueError(
    155             f"{magic_number} is not the signature of a valid netCDF4 file"
    156         )

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/xarray/core/utils.py:661, in read_magic_number_from_file(filename_or_obj, count)
    659     if filename_or_obj.tell() != 0:
    660         filename_or_obj.seek(0)
--> 661     magic_number = filename_or_obj.read(count)
    662     filename_or_obj.seek(0)
    663 else:

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/implementations/http.py:600, in HTTPFile.read(self, length)
    598 else:
    599     length = min(self.size - self.loc, length)
--> 600 return super().read(length)

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/spec.py:1790, in AbstractBufferedFile.read(self, length)
   1787 if length == 0:
   1788     # don't even bother calling fetch
   1789     return b""
-> 1790 out = self.cache._fetch(self.loc, self.loc + length)
   1791 self.loc += len(out)
   1792 return out

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/caching.py:379, in BytesCache._fetch(self, start, end)
    373     return b""
    375 if (self.start is None or start < self.start) and (
    376     self.end is None or end > self.end
    377 ):
    378     # First read, or extending both before and after
--> 379     self.cache = self.fetcher(start, bend)
    380     self.start = start
    381 elif start < self.start:

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/asyn.py:121, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
    118 @functools.wraps(func)
    119 def wrapper(*args, **kwargs):
    120     self = obj or args[0]
--> 121     return sync(self.loop, func, *args, **kwargs)

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/asyn.py:106, in sync(loop, func, timeout, *args, **kwargs)
    104     raise FSTimeoutError from return_result
    105 elif isinstance(return_result, BaseException):
--> 106     raise return_result
    107 else:
    108     return return_result

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/asyn.py:61, in _runner(event, coro, result, timeout)
     59     coro = asyncio.wait_for(coro, timeout=timeout)
     60 try:
---> 61     result[0] = await coro
     62 except Exception as ex:
     63     result[0] = ex

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/fsspec/implementations/http.py:648, in HTTPFile.async_fetch_range(self, start, end)
    646 headers["Range"] = "bytes=%i-%i" % (start, end - 1)
    647 logger.debug(str(self.url) + " : " + headers["Range"])
--> 648 r = await self.session.get(
    649     self.fs.encode_url(self.url), headers=headers, **kwargs
    650 )
    651 async with r:
    652     if r.status == 416:
    653         # range request outside file

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/aiohttp/client.py:560, in ClientSession._request(self, method, str_or_url, params, data, json, cookies, headers, skip_auto_headers, auth, allow_redirects, max_redirects, compress, chunked, expect100, raise_for_status, read_until_eof, proxy, proxy_auth, timeout, verify_ssl, fingerprint, ssl_context, ssl, proxy_headers, trace_request_ctx, read_bufsize)
    558 resp = await req.send(conn)
    559 try:
--> 560     await resp.start(conn)
    561 except BaseException:
    562     resp.close()

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/aiohttp/client_reqrep.py:899, in ClientResponse.start(self, connection)
    897 try:
    898     protocol = self._protocol
--> 899     message, payload = await protocol.read()  # type: ignore[union-attr]
    900 except http.HttpProcessingError as exc:
    901     raise ClientResponseError(
    902         self.request_info,
    903         self.history,
   (...)
    906         headers=exc.headers,
    907     ) from exc

File ~/Library/Caches/pypoetry/virtualenvs/dotfiles-S-yQfRXO-py3.10/lib/python3.10/site-packages/aiohttp/streams.py:616, in DataQueue.read(self)
    614 self._waiter = self._loop.create_future()
    615 try:
--> 616     await self._waiter
    617 except (asyncio.CancelledError, asyncio.TimeoutError):
    618     self._waiter = None

ClientOSError: [Errno 54] Connection reset by peer

I believe it's the range request in particular that is causing trouble, but I'm not sure why a range request is issued when the server does not provide an Accept-Ranges option. I also don't know how to prevent the range request to test that theory. Mostly I don't know what to do!

[martindurant]

No, a Range should not be able to cause this. A server may refuse connection due to unexpected headers (but should return a normal exception) or just ignore the header (which is annoying!). A poorly configures server, however...

h5netcdf needs range requests to read metadata and chunks in a random-access manner. To get around it, you could use local file caching (e.g., prefix "simplecache::" to the URL) or in-memory caching (cache_type="all" in open()), but you will of course download all bytes doing this.

[itcarroll]

Thanks for the help! Using a "filecache" wrapper on the https filesystem works, but downloads the entire file of course. Which is too bad. I guess the following indicates it really is a poorly configured server:

% curl -I -Ln -b ~/.urs_cookies -c ~/.urs_cookies https://oceandata.sci.gsfc.nasa.gov/ob/getfile/AQUA_MODIS.20200613T003001.L2.OC.nc
HTTP/2 200 
server: nginx
date: Fri, 01 Sep 2023 15:55:36 GMT
content-type: application/octet-stream
content-length: 28512070
set-cookie: app-obdaac=6ca80ad5db507080bc26777b208385904fc0dad8; path=/; secure
last-modified: Fri, 09 Sep 2022 06:02:26 GMT
content-disposition: attachment; filename=AQUA_MODIS.20200613T003001.L2.OC.nc
content-security-policy: upgrade-insecure-requests; default-src 'self' 'unsafe-inline' 'unsafe-eval' data: oceancolor.gsfc.nasa.gov *.earthdata.nasa.gov *.google.com *.gstatic.com *.googleapis.com www.googletagmanager.com www.google-analytics.com www.youtube.com s.ytimg.com stats.g.doubleclick.net dap.digitalgov.gov cdn.earthdata.nasa.gov api.ipify.org *.cloudflare.com maxcdn.bootstrapcdn.com platform.twitter.com;
strict-transport-security: max-age=31536000; includesubdomains; preload
x-frame-options: SAMEORIGIN

^ Says nothing about accepting ranges.

curl -r 0-512 -Ln -b ~/.urs_cookies -c ~/.urs_cookies -O https://oceandata.sci.gsfc.nasa.gov/ob/getfile/AQUA_MODIS.20200613T003001.L2.OC.nc                                      01T16:00
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
  0     0    0     0    0     0      0      0 --:--:-- --:--:-- --:--:--     0
curl: (56) HTTP/2 stream 1 was reset

^ Does not exit gracefully nor ignore range request.

[martindurant]

I guess the following indicates it really is a poorly configured server

I believe so. You could let them know maybe.

[itcarroll]

I let them know, and they fixed it. Go NASA!

One question on your initial reply though: what kwargs in fsspec.open("simplecache::" + url, **kwargs) does the equivalent of:

fsspec.filesystem("filecache", target_protocol="https", target_options={"client_kwargs": {"auth": auth}}).open(url)

Neither the nested target_options or client_kwargs works for me.

[martindurant]

when using chained urls, you generally use the protocol string as the key for kwargs to pass in:

fsspec.open("simplecache::" + url, https={"client_kwargs": {"auth": auth}})

and simplecache= for any arguments to the cache instance. Any kwargs not matching a protocol should I think go to the outermost instance, in this cache simplecache.

[itcarroll]

Doh, now that I know it's called "URL chaining" I see it in the docs. Thank you!