From 48b47e0e7fdcd6ba31b19d95d3572044eacfd6a3 Mon Sep 17 00:00:00 2001 From: Jeffrey Farrar Painter Date: Tue, 9 Mar 2021 14:57:56 -0800 Subject: [PATCH] There was a one-second "sleep" before every call (but the first) of start_transfer_thread(). I moved the "sleep" into the thread itself. The data node doesn't see the difference, but this can have a very noticeable effect on performance if there are active transfers from many data nodes in parallel. --- synda/sdt/sddmdefault.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/synda/sdt/sddmdefault.py b/synda/sdt/sddmdefault.py index 19d3aeaf..5333ee69 100755 --- a/synda/sdt/sddmdefault.py +++ b/synda/sdt/sddmdefault.py @@ -60,6 +60,10 @@ def run(cls, tr): @classmethod def start_transfer_script(cls,tr): + if hasattr( tr, 'delay' ): + time.sleep( tr.delay ) + else: + sdlog.warning("SDDMDEFA-001","delay attribute not set for transfer %s"%(tr,)) sdlog.info("JFPDMDEF-001", "Will download url={}".format(tr.url,)) if sdconfig.fake_download: tr.status = TRANSFER["status"]['done'] @@ -346,10 +350,20 @@ def transfers_begin(transfers): raise + datanode_delays = {} + # An item in datanode_delays will be datanode:delay, e.g. 'esg-dn1.nsc.liu.se':3. + # The delay, in seconds, will precede the attempt to connect to the data node. + # This feature will replace the 1-second 'sleep' which previously separated every call of + # start_transfer_thread. The reason given for it was "not to be too agressive with datanodes". + for tr in transfers: + if tr.data_node not in datanode_delays: + datanode_delays[tr.data_node] = 0 + else: + datanode_delays[tr.data_node] += 1 + tr.delay = datanode_delays[tr.data_node] + start_transfer_thread(tr) - # this sleep is not to be too agressive with datanodes - time.sleep(1) def can_leave():