diff --git a/parfive/tests/test_downloader.py b/parfive/tests/test_downloader.py index 16ea9d1..b57635a 100644 --- a/parfive/tests/test_downloader.py +++ b/parfive/tests/test_downloader.py @@ -869,3 +869,16 @@ def test_invalid_server_checksum(httpserver, tmpdir, caplog, checksum): assert len(f.errors) == 0 assert "Got invalid checksum:" in caplog.messages[0] + + +@pytest.mark.parametrize( + "url", ["https://gong2.nso.edu/oQR/zqs/201912/mrzqs191231/mrzqs191231t2304c2225_011.fits.gz"] +) +@pytest.mark.allow_hosts(True) +def test_problematic_http_urls(url, tmpdir): + """ + This test checks that certain URLs which have caused trouble continue to work. + """ + res = Downloader.simple_download([url], path=tmpdir) + assert len(res) == 1, res.errors + assert not res.errors diff --git a/parfive/utils.py b/parfive/utils.py index 4a455bf..0fcc644 100644 --- a/parfive/utils.py +++ b/parfive/utils.py @@ -333,10 +333,14 @@ async def session_head_or_get(session: aiohttp.ClientSession, url: str, **kwargs Try and make a HEAD request to the resource and fallback to a get request if that fails. """ - async with session.head(url, **kwargs) as resp: - if resp.status == 200: - yield resp - return + try: + async with session.head(url, **kwargs) as resp: + if resp.status == 200: + yield resp + return + # Catch the situation where the server just ignores the HEAD request + except aiohttp.client_exceptions.ServerDisconnectedError: + pass async with session.get(url, **kwargs) as resp: yield resp