Skip to content

Minor improvements in sdnexturl.py #175

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
painter1 wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Minor improvements in sdnexturl.py #175

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
72291ff
- don't crash if tr.url doesn't exist
painter1 Mar 19, 2021
7f6b451
a change I had forgotten to include: correctly comptue newurl in
painter1 Mar 22, 2021
03d0163
Merge in recent master branch changes for sdnexturl.py.
painter1 Mar 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 17 additions & 16 deletions synda/sdt/sdnexturl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
# multiple changes by JfP to make fallback more flexible.

import argparse
import sdlog
import sdquicksearch
import sdexception
from synda.sdt import sdlog
from synda.sdt import sdquicksearch
from synda.sdt import sdexception
import sqlite3

from synda.source.config.file.user.preferences.models import Config as Preferences
Expand All @@ -43,8 +43,10 @@ def run(tr):
(tr.url, tr.file_functional_id) )
conn.commit()
except sqlite3.IntegrityError as e:
# url is already in the failed_url table
sdlog.info("SDNEXTUR-001","During database operations, IntegrityError %s"%(e,))
# url,file_id is already in the failed_url table
sdlog.info("SDNEXTUR-001","During database operations, IntegrityError %s on %s with new file_id=%s"
%(e,url,tr.file_id))
conn.commit()
except Exception as e:
sdlog.info("SDNEXTUR-002","During database operations, unknown exception %s"%(e,))
return False
Expand Down Expand Up @@ -79,14 +81,11 @@ def next_url(tr,conn):
sdlog.info("SDNEXTUR-007","failed_urls= %s"%(failed_urls,))
urlps = [urlp for urlp in all_urlps if urlp[0] not in failed_urls]
# ... Note that list comprehensions preserve order.

# At this point urls is just a list of urls. We no longer have to keep track of the
# protocol because only http and gsiftp are possible (OpenDAP is a different protocol
# but it also uses a http url).
urls = [urlp[0] for urlp in urlps]

if len(urlps)>0:
if len(urls)>0:
old_url=tr.url
new_url=urlps[0]
new_url=urls[0]
tr.url=new_url
sdlog.info("SDNEXTUR-008","Url successfully switched (file_functional_id=%s,old_url=%s,new_url=%s)"%(tr.file_functional_id,old_url,new_url))
else:
Expand All @@ -103,11 +102,11 @@ def get_urls(file_functional_id):
file_functional_id],
post_pipeline_mode=None )
except Exception as e:
sdlog.debug("SDNEXTUR-015", "exception %s. instance_id=%s"%(e,file_functional_id))
sdlog.debug("SDNEXTUR-010", "exception %s. instance_id=%s"%(e,file_functional_id))
raise e

li=result.get_files()
sdlog.info("SDNEXTUR-016","sdquicksearch returned %s sets of file urls: %s"%(len(li),li))
sdlog.info("SDNEXTUR-011","sdquicksearch returned %s sets of file urls: %s"%(len(li),li))
if li==[]:
# No urls found. Try again, but wildcard the file id. (That leads to a string search on all
# fields for the wildcarded file id, rather than a match of the instance_id field only.)
Expand All @@ -116,7 +115,7 @@ def get_urls(file_functional_id):
file_functional_id+'*'],
post_pipeline_mode=None )
li=result.get_files()
sdlog.info("SDNEXTUR-017","sdquicksearch 2nd call %s sets of file urls: %s"%(len(li),li))
sdlog.info("SDNEXTUR-012","sdquicksearch 2nd call %s sets of file urls: %s"%(len(li),li))
# result looks like
# [ {protocol11:url11, protocol12:url12, attached_parameters:dict, score:number, type:'File',
# size:number} }, {[another dict of the same format}, {another dict},... ]
Expand Down Expand Up @@ -160,6 +159,8 @@ def priurl(url):
if __name__ == '__main__':
parser = argparse.ArgumentParser()

file_functional_id='cmip5.output1.LASG-CESS.FGOALS-g2.decadal1985.day.atmos.day.r1i1p1.v1.va_day_FGOALS-g2_decadal1985_r1i1p1_19880101-19881231.nc'
#file_functional_id='cmip5.output1.LASG-CESS.FGOALS-g2.decadal1985.day.atmos.day.r1i1p1.v1.va_day_FGOALS-g2_decadal1985_r1i1p1_19880101-19881231.nc'
#file_functional_id='cmip5.output1.CMCC.CMCC-CMS.piControl.mon.atmos.Amon.r1i1p1.v20120717.pr_Amon_CMCC-CMS_piControl_r1i1p1_416401-417312.nc'
file_functional_id='cmip5.output1.NOAA-GFDL.GFDL-CM2p1.historical.mon.atmos.Amon.r3i1p1.v20110601.cl_Amon_GFDL-CM2p1_historical_r3i1p1_190601-191012.nc'
from pprint import pprint
pprint( get_urls(file_functional_id) )
pprint( get_urls(file_functional_id,'esgf-node.llnl.gov') )