From 373dafb767bd6c5d551227404988034001ee56e3 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Sat, 8 Jun 2024 14:46:08 -0400 Subject: [PATCH 1/3] Read error from xklb-metadata.db --- cps/tasks/download.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/cps/tasks/download.py b/cps/tasks/download.py index a6ac3dda82..3569081a94 100644 --- a/cps/tasks/download.py +++ b/cps/tasks/download.py @@ -120,8 +120,7 @@ def run(self, worker_thread): except Exception as e: log.error("An error occurred during the subprocess execution: %s", e) - self.message = f"{self.media_url_link} failed to download: {e}" - self.record_error_in_database(str(e)) + self.message = f"{self.media_url_link} failed to download: {self.read_error_from_database()}" finally: self.end_time = datetime.now() @@ -134,18 +133,12 @@ def run(self, worker_thread): else: log.info("No media URL provided - skipping download task") - def record_error_in_database(self, error_message): - """Record the error in the database""" + def read_error_from_database(self): + """Read the error from the database""" with sqlite3.connect(XKLB_DB_FILE) as conn: - # Check if the error column exists, if not, create it at the rightmost position - cursor = conn.cursor() - cursor.execute("PRAGMA table_info(media)") - columns = [column[1] for column in cursor.fetchall()] - if "error" not in columns: - conn.execute("ALTER TABLE media ADD COLUMN error TEXT") - conn.execute("UPDATE media SET error = ? WHERE webpath = ?", (error_message, self.media_url)) - conn.commit() + error = conn.execute("SELECT error FROM media WHERE webpath = ?", (self.media_url,)).fetchone()[0] conn.close() + return error @property def name(self): From 6609ef15751e7f34e31601d2af57e572e17717df Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Sat, 8 Jun 2024 14:58:57 -0400 Subject: [PATCH 2/3] Timestamp both path and webpath of the failed video This prevents metada fetching hanging on retry. Metadata fetching identifies requested videos by reading xklb-metadata for videos which path and webpath are identical and error column is null. --- cps/tasks/download.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cps/tasks/download.py b/cps/tasks/download.py index 3569081a94..cd028697cc 100644 --- a/cps/tasks/download.py +++ b/cps/tasks/download.py @@ -138,6 +138,7 @@ def read_error_from_database(self): with sqlite3.connect(XKLB_DB_FILE) as conn: error = conn.execute("SELECT error FROM media WHERE webpath = ?", (self.media_url,)).fetchone()[0] conn.close() + conn.execute("UPDATE media SET webpath = ? WHERE webpath = ?", (f"{self.media_url}×tamp={int(datetime.now().timestamp())}", self.media_url)) return error @property From e16a5cdd855571fa0a537c13f5bb4a1d23f1b659 Mon Sep 17 00:00:00 2001 From: Blondel MONDESIR Date: Mon, 10 Jun 2024 06:44:54 -0400 Subject: [PATCH 3/3] Keep path and webpath of failed video intact --- cps/tasks/download.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cps/tasks/download.py b/cps/tasks/download.py index cd028697cc..3569081a94 100644 --- a/cps/tasks/download.py +++ b/cps/tasks/download.py @@ -138,7 +138,6 @@ def read_error_from_database(self): with sqlite3.connect(XKLB_DB_FILE) as conn: error = conn.execute("SELECT error FROM media WHERE webpath = ?", (self.media_url,)).fetchone()[0] conn.close() - conn.execute("UPDATE media SET webpath = ? WHERE webpath = ?", (f"{self.media_url}×tamp={int(datetime.now().timestamp())}", self.media_url)) return error @property