Fix failed downloads being marked as downloaded, add rip repair

streamrip/exceptions.py

@@ -68,3 +68,11 @@ def print_msg(self, item) -> str:
 
 class ConversionError(Exception):
     """ConversionError."""
+
+
+class TrackDownloadFailedError(Exception):
+    """Raised when a track fails to download after retrying.
+
+    Signals to Media.rip() that postprocess (tagging, marking downloaded)
+    must not run for this track.
+    """

streamrip/media/track.py

@@ -7,7 +7,7 @@
 from ..client import Client, Downloadable
 from ..config import Config
 from ..db import Database
-from ..exceptions import NonStreamableError
+from ..exceptions import NonStreamableError, TrackDownloadFailedError
 from ..filepath_utils import clean_filename
 from ..metadata import AlbumMetadata, Covers, TrackMetadata, tag_file
 from ..progress import add_title, get_progress_callback, remove_title
@@ -71,6 +71,16 @@ async def download(self):
                     self.db.set_failed(
                         self.downloadable.source, "track", self.meta.info.id
                     )
+                    if os.path.isfile(self.download_path):
+                        os.remove(self.download_path)
+                    # postprocess() normally does this, but raising below
+                    # skips it, which would leave a phantom title in the
+                    # progress display for the rest of the run.
+                    if self.is_single:
+                        remove_title(self.meta.title)
+                    raise TrackDownloadFailedError(
+                        f"{self.meta.title} ({self.meta.info.id})"
+                    ) from e
 
     async def postprocess(self):
         if self.is_single:
@@ -129,16 +139,22 @@ async def resolve(self) -> Track | None:
             return None
 
         source = self.client.source
+        # Every failure below has to be recorded, not just logged. An unlogged
+        # failure leaves no trace anywhere: no file, no downloads.db row, and
+        # nothing in the failed db for `rip repair` to retry -- the track just
+        # silently goes missing from the album.
         try:
             resp = await self.client.get_metadata(self.id, "track")
         except NonStreamableError as e:
             logger.error(f"Track {self.id} not available for stream on {source}: {e}")
+            self.db.set_failed(source, "track", self.id)
             return None
 
         try:
             meta = TrackMetadata.from_resp(self.album, source, resp)
         except Exception as e:
             logger.error(f"Error building track metadata for {self.id}: {e}")
+            self.db.set_failed(source, "track", self.id)
             return None
 
         if meta is None:
@@ -153,6 +169,7 @@ async def resolve(self) -> Track | None:
             logger.error(
                 f"Error getting downloadable data for track {meta.tracknumber} [{self.id}]: {e}"
             )
+            self.db.set_failed(source, "track", self.id)
             return None
 
         downloads_config = self.config.session.downloads
@@ -191,16 +208,20 @@ async def resolve(self) -> Track | None:
             )
             return None
 
+        # As in PendingTrack.resolve: record every failure, so a track that
+        # dies here is retryable by `rip repair` instead of vanishing.
         try:
             resp = await self.client.get_metadata(self.id, "track")
         except NonStreamableError as e:
             logger.error(f"Error fetching track {self.id}: {e}")
+            self.db.set_failed(self.client.source, "track", self.id)
             return None
         # Patch for soundcloud
         try:
             album = AlbumMetadata.from_track_resp(resp, self.client.source)
         except Exception as e:
             logger.error(f"Error building album metadata for track {id=}: {e}")
+            self.db.set_failed(self.client.source, "track", self.id)
             return None
 
         if album is None:
@@ -214,6 +235,7 @@ async def resolve(self) -> Track | None:
             meta = TrackMetadata.from_resp(album, self.client.source, resp)
         except Exception as e:
             logger.error(f"Error building track metadata for track {id=}: {e}")
+            self.db.set_failed(self.client.source, "track", self.id)
             return None
 
         if meta is None:

streamrip/rip/cli.py

@@ -361,6 +361,137 @@ def database_browse(ctx, table):
         )
 
 
+async def _albums_for(main, failed_items):
+    """Map failed tracks onto the albums that contain them.
+
+    Returns (album targets, items to retry as they are). Anything whose album
+    cannot be determined is passed through untouched rather than dropped.
+    """
+    targets: list[tuple[str, str, str]] = []
+    unresolved: list[tuple[str, str, str]] = []
+    seen: set[tuple[str, str]] = set()
+
+    for source, media_type, item_id in failed_items:
+        if media_type != "track":
+            targets.append((source, media_type, item_id))
+            continue
+        try:
+            client = await main.get_logged_in_client(source)
+            resp = await client.get_metadata(item_id, "track")
+            album_id = str((resp.get("album") or {}).get("id") or "")
+        except Exception as e:
+            logger.debug("Could not find the album for %s: %s", item_id, e)
+            album_id = ""
+
+        if not album_id:
+            unresolved.append((source, media_type, item_id))
+            continue
+        if (source, album_id) not in seen:
+            seen.add((source, album_id))
+            targets.append((source, "album", album_id))
+
+    return targets, unresolved
+
+
+@rip.command()
+@click.option("-y", "--yes", help="Don't ask for confirmation.", is_flag=True)
+@click.option(
+    "--flat",
+    help="Put repaired tracks straight in the download folder instead of "
+    "their album folder.",
+    is_flag=True,
+)
+@click.pass_context
+@coro
+async def repair(ctx, yes, flat):
+    """Retry downloads that previously failed.
+
+    Reads the failed downloads database, retries each item, and clears it
+    from the failed database on success. Items that fail again stay logged
+    so they can be retried later.
+
+    Failed tracks are retried individually, but are placed in their album's
+    folder so they rejoin the album they were originally missing from. Pass
+    --flat to put them in the download folder instead.
+    """
+    if ctx.obj["config"] is None:
+        return
+
+    with ctx.obj["config"] as cfg:
+        cfg: Config
+        # A repaired track is nearly always a track missing from an album that
+        # was otherwise downloaded, so it needs to land in that album's folder
+        # rather than loose in the download root. This only touches the
+        # in-memory session copy, so config.toml is left alone.
+        if not flat:
+            cfg.session.filepaths.add_singles_to_folder = True
+        failed_db = db.Failed(cfg.session.database.failed_downloads_path)
+        downloads_db = db.Downloads(cfg.session.database.downloads_path)
+        failed_items = failed_db.all()
+
+        if not failed_items:
+            console.print("[green]No failed downloads to repair!")
+            return
+
+        console.print(
+            f"Found [yellow]{len(failed_items)}[/yellow] failed download(s)."
+        )
+        if not yes and not Confirm.ask("Retry them now?"):
+            console.print("[green]Repair aborted")
+            return
+
+        # A failed item should never also be logged as downloaded, but older
+        # versions of streamrip could mark one downloaded even after it
+        # failed. Clear that stale state so the retry below isn't skipped.
+        for _source, _media_type, item_id in failed_items:
+            downloads_db.remove(id=item_id)
+
+        async with Main(cfg) as main:
+            # Retry through the album rather than track by track. Resolving a
+            # single track builds its album metadata from the track response,
+            # which on Tidal carries only an id, title and cover -- no track
+            # count, and the track's artists in place of the album artist. The
+            # folder that produces differs from the album's own in both, so
+            # repaired tracks land in a separate folder instead of rejoining
+            # the album.
+            #
+            # Going through the album gets the real metadata, the right
+            # folder, disc subfolders and cover art, and costs nothing extra:
+            # tracks already in the downloads db are skipped, so only what is
+            # missing gets fetched.
+            targets, unresolved = await _albums_for(main, failed_items)
+            if unresolved:
+                console.print(
+                    f"[yellow]{len(unresolved)} item(s) had no album to retry "
+                    "through; fetching them individually."
+                )
+            await main.add_all_by_id(targets + unresolved)
+            await main.resolve()
+            await main.rip()
+
+        # Nothing in the download pipeline removes rows from the failed db, so
+        # success can't be detected by diffing it. Instead rely on the
+        # invariant this patch establishes: set_downloaded() is only reached
+        # via postprocess(), which a failed download never gets to. So an item
+        # present in the downloads db now is one that just succeeded.
+        repaired = [
+            item_id
+            for _, _, item_id in failed_items
+            if downloads_db.contains(id=item_id)
+        ]
+        for item_id in repaired:
+            failed_db.remove(id=item_id)
+
+        console.print(
+            f"[green]Repaired {len(repaired)}/{len(failed_items)} item(s).[/green]"
+        )
+        if len(repaired) < len(failed_items):
+            console.print(
+                f"[yellow]{len(failed_items) - len(repaired)} item(s) failed again "
+                "and are still logged. Run [bold]rip repair[/bold] to try again."
+            )
+
+
 @rip.command()
 @click.option(
     "-f",