Save file transfer error histories for further processing

CITATION.cff

@@ -17,5 +17,5 @@ license: "CECILL-1.0"
 message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/Prodiguer/synda"
 title: Synda
-version: "3.10"
+version: "3.11"
 ...0

synda/sdt/sdapp.py

@@ -22,5 +22,5 @@
 os.umask(0002)
 
 name = 'transfer'
-version = '3.10'
+version = '3.11'
 sdapputils.set_exception_handler()

synda/sdt/sddbversion.py

@@ -63,6 +63,13 @@ def upgrade_db(conn,current_db_version,new_db_version):
 
 # -- upgrade procs -- #
 
+def upgrade_311(conn):
+
+    conn.execute("ALTER TABLE file ADD COLUMN error_history")
+    conn.commit()
+
+    sddbversionutils.update_db_version(conn,'3.11')
+
 def upgrade_310(conn):
 
     conn.execute("CREATE TABLE failed_url ( url_id INTEGER PRIMARY KEY, url TEXT, file_id INTEGER)")
@@ -145,6 +152,7 @@ def upgrade_30(conn):
 # init.
 
 upgrade_procs={
+    '3.11': upgrade_311,
     '3.10': upgrade_310,
     '3.9': upgrade_39,
     '3.8': upgrade_38,

synda/sdt/sddmdefault.py

@@ -120,6 +120,7 @@ def start_transfer_script(cls,tr):
                     tr.error_msg = ""
                 else:
                     # checksum is not ok
+                    update_error_history( tr, "bad checksum" )
 
                     if incorrect_checksum_action == "remove":
                         tr.status = TRANSFER["status"]['error']
@@ -225,9 +226,12 @@ def start_transfer_script(cls,tr):
                         tr.local_path,
                     ),
                 )
-
+                update_error_history( tr, 'killed' )
+            elif tr.sdget_error_msg.find('ERROR 404')>0:
+                # wget reported 'ERROR 404', i.e. the url was not found.
+                update_error_history( tr, 'ERROR 404' )
             else:
-                pass
+                update_error_history( tr, 'other' )
 
             next_url_on_error = preferences.is_download_http_fallback
 
@@ -259,6 +263,16 @@ def start_transfer_script(cls,tr):
                 tr.priority -= 1
                 tr.error_msg = 'Error occurs during download.'
 
+def update_error_history( tr, error ):
+    """Update the error_history field of a transfer, to add the specified error, a string.
+     The error should be short, e.g. 'ERROR 404'.  And it should be specific to the file
+     represented by tr. Thus an error 503 or "Connection refused" should not be provided."""
+    if tr.error_history is None:
+        tr.error_history = str([])
+    error_history = eval(tr.error_history)
+    error_history.append( ( sdtime.now(), error ) )
+    tr.error_history = str(error_history)
+    sdlog.info( "SDDMDEFA-080","error history for %s is %s" % (tr.filename,tr.error_history) )
 
 def end_of_transfer(tr):
 

synda/sdt/sdfiledao.py

@@ -310,6 +310,7 @@ def update_file(_file, commit=True, conn=sddb.conn):
     keys = [
         'status',
         'error_msg',
+        'error_history',
         'sdget_status',
         'sdget_error_msg',
         'start_date',