From c10debecc71dad619079614c53f61dd6ea8e8b9d Mon Sep 17 00:00:00 2001 From: dengzq1234 Date: Wed, 19 Feb 2020 09:35:56 +0100 Subject: [PATCH 1/5] fix path option for external apps & subcommand --- ete3/tools/ete.py | 17 ++- ete3/tools/ete_build.py | 285 ++++++++++++++++++++++++---------------- 2 files changed, 179 insertions(+), 123 deletions(-) diff --git a/ete3/tools/ete.py b/ete3/tools/ete.py index d407bc339..1dce60589 100755 --- a/ete3/tools/ete.py +++ b/ete3/tools/ete.py @@ -138,15 +138,14 @@ def _main(arguments): elif subcommand == "build": from . import ete_build del arguments[1] - - builtin_apps_path = None - ete3_path = which("ete3") - - if ete3_path: - builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") #Ziqi - #builtin_apps_path = os.path.split(ete3_path)[0] - ete_build._main(arguments, builtin_apps_path) - + #builtin_apps_path = None + #ete3_path = which("ete3") + + # is it a good place to make judgement? + #if ete3_path: + # builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") #Ziqi + # builtin_apps_path = os.path.split(ete3_path)[0] + ete_build._main(arguments) return # CREATE REUSABLE PARSER OPTIONS diff --git a/ete3/tools/ete_build.py b/ete3/tools/ete_build.py index e50dfec9a..46f3f496c 100755 --- a/ete3/tools/ete_build.py +++ b/ete3/tools/ete_build.py @@ -93,7 +93,6 @@ block_detail, list_apps) from .ete_build_lib import seqio - try: from ..version import __version__ except ImportError: @@ -259,8 +258,11 @@ def parse_workflows(names, target_wtype, parse_filters=False): parsed_workflows.extend(temp_workflows) return parsed_workflows - genetree_workflows = parse_workflows(args.workflow, "genetree") - supermatrix_workflows = parse_workflows(args.supermatrix_workflow, "supermatrix") + if args.workflow: + genetree_workflows = parse_workflows(args.workflow, "genetree") + supermatrix_workflows = parse_workflows(args.supermatrix_workflow, "supermatrix") + else: + raise ConfigError("the following arguments are required: -w") # Stop if mixing types of meta-workflows if supermatrix_workflows and len(genetree_workflows) > 1: @@ -649,115 +651,40 @@ def parse_workflows(names, target_wtype, parse_filters=False): raise DataError("Errors found in some tasks") -def _main(arguments, builtin_apps_path=None): +# def _main(arguments, builtin_apps_path=None): +def _main(arguments): global BASEPATH, APPSPATH, args - #print('this is buildin_apps_path', builtin_apps_path) #builtin_apps_path call by ete.py - + + #builtin_apps_path call by ete.py # read standard path or binary # 1)binary 2)internal toolchain 3)customize apps path etet_toolchain with argvs flags - if builtin_apps_path: - APPSPATH = builtin_apps_path - print("the apps path", APPSPATH) + # to determine if "ete3_apps/" exsist + + #if builtin_apps_path: + # if '-l' in arguments: + # APPSPATH = builtin_apps_path + # elif '--local-binary' in arguments: + # APPSPATH = builtin_apps_path + # else: + # builtin_apps_path = os.path.join(builtin_apps_path, "ete3_apps/bin") + # APPSPATH = builtin_apps_path ETEHOMEDIR = os.path.expanduser("~/.etetoolkit/") - if len(arguments) == 1: - if not pexist(APPSPATH): - print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) - print(colorify('Install using conda (recomended):', "lgreen"), file=sys.stderr) - print(colorify(' conda install -c etetoolkit ete_toolchain', "white"), file=sys.stderr) - print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) - print(colorify(' https://github.com/etetoolkit/ete_toolchain', "white"), file=sys.stderr) - print() - - - if len(arguments) > 1: - _config_path = pjoin(BASEPATH, 'ete_build.cfg') - if arguments[1] == "check": - if not pexist(APPSPATH): - print('this is apps_path', APPSPATH) - print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) - print(colorify('Install using conda (recomended):', "lgreen"), file=sys.stderr) - print(colorify(' conda install -c etetoolkit ete_toolchain', "white"), file=sys.stderr) - print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) - print(colorify(' https://github.com/etetoolkit/ete_toolchain', "white"), file=sys.stderr) - print() - sys.exit(0) - - try: - toolchain_version = open(pjoin(APPSPATH, "__version__")).readline() - except IOError: - toolchain_version = "unknown" - - print("Current Toolchain path: %s " %APPSPATH) - print("Current Toolchain version: %s" %toolchain_version) - - # setup portable apps - config = {} - for k in apps.builtin_apps: - # concatnate the path with "bin" - cmd = apps.get_call(k, APPSPATH, "/tmp", "1") - config[k] = cmd - apps.test_apps(config) - sys.exit(0) - - elif arguments[1] in ("workflows", "wl"): - if arguments[1] == "wl": - print(colorify("WARNING: 'wl' is obsolete and will be removed in the future, use 'workflows' instead", "orange"), file=sys.stderr) - - base_config = check_config(_config_path) - try: - wf_type = arguments[2] - except IndexError: - wf_type = None - list_workflows(base_config, wf_type) - sys.exit(0) - - elif arguments[1] == "apps": - base_config = check_config(_config_path) - list_apps(base_config, set(arguments[2:])) - sys.exit(0) - - elif arguments[1] == "show": - base_config = check_config(_config_path) - try: - block = arguments[2] - except IndexError: - print("Expected a block name, found none") - sys.exit(1) - - block_detail(block, base_config) - sys.exit(0) - - elif arguments[1] == "dump": - if len(arguments) > 2: - base_config = check_config(_config_path) - block_detail(arguments[2], base_config, color=False) - else: - print(open(_config_path).read()) - sys.exit(0) - - elif arguments[1] == "validate": - print('Validating configuration file ', arguments[2]) - if pexist(arguments[2]): - base_config = check_config(arguments[2]) - print('Everything ok') - else: - print('File does not exist') - sys.exit(-1) - sys.exit(0) - - elif arguments[1] == "version": - print(__version__) - sys.exit(0) parser = argparse.ArgumentParser(description=__DESCRIPTION__ + __EXAMPLES__, formatter_class=argparse.RawDescriptionHelpFormatter) - # Input data related flags - input_group = parser.add_argument_group('==== Input Options ====') - - input_group.add_argument('[check | workflows | apps | show | dump | validate]', + #subcommand group [check | workflows | apps | show | dump | validate] + + subcommand_group = parser.add_argument_group('==== Subcommands ====') + + subcommand_group.add_argument("check", nargs='?') + subcommand_group.add_argument("workflows", nargs='?', default=['genetree', 'supermatrix']) + subcommand_group.add_argument("apps", type=str, nargs='*', default=list(apps.APPTYPES.keys())) + subcommand_group.add_argument("show", type=str, nargs='*') + subcommand_group.add_argument("validate", type=is_file, nargs='*') + subcommand_group.add_argument('[check | workflows | apps | show | dump | validate]', nargs='?', help=("Utility commands:\n" "check: check that external applications are executable.\n" @@ -768,6 +695,9 @@ def _main(arguments, builtin_apps_path=None): "version: Show current version.\n" )) + # Input data related flags + input_group = parser.add_argument_group('==== Input Options ====') + input_group.add_argument("-c", "--custom-config", dest="custom_config", type=is_file, help="Custom configuration file.") @@ -776,12 +706,14 @@ def _main(arguments, builtin_apps_path=None): type=is_file, default=BASEPATH+'/ete_build.cfg', help="Base configuration file.") - input_group.add_argument("--tools-dir", dest="tools_dir", + # Ziqi check this + input_group.add_argument("-L", "--tools-dir", dest="tools_dir", type=str, - help="Custom path where external software is avaiable.") + #required=True, #Ziqi + help="How to find binaries to external software: [custom path], TOOLCHAIN, LOCAL") input_group.add_argument("-w", dest="workflow", - required=True, + #required=True, #ziqi nargs='+', help="One or more gene-tree workflow names. All the specified workflows will be executed using the same input data.") @@ -912,7 +844,8 @@ def _main(arguments, builtin_apps_path=None): # Output data related flags output_group = parser.add_argument_group('==== Output Options ====') output_group.add_argument("-o", "--outdir", dest="outdir", - type=str, required=True, + type=str, + #required=True, #ziqi help="""Output directory for results.""") output_group.add_argument("--scratch-dir", dest="scratch_dir", @@ -1026,8 +959,130 @@ def _main(arguments, builtin_apps_path=None): " debugging will start from such task on.") args = parser.parse_args(arguments) - if args.tools_dir: - APPSPATH = args.tools_dir + + ete3_path = shutil.which("ete3") + if ete3_path: + if args.tools_dir: + if args.tools_dir == 'LOCAL': + builtin_apps_path = os.path.split(ete3_path)[0] + APPSPATH = builtin_apps_path + + elif args.tools_dir == 'TOOLCHAIN': + builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") + APPSPATH = builtin_apps_path + + else: # here for the customize path + if os.path.isdir(args.tools_dir): + APPSPATH = os.path.realpath(args.tools_dir) + #builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") + #APPSPATH = builtin_apps_path + else: + raise ValueError("The provided custom path for external tools is not a valid directory") + else: + print(colorify('\nWARNING: path of external tools is not provided, now loading TOOLCHAIN', "yellow"), file=sys.stderr) + builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") + if not os.path.exists(builtin_apps_path): + raise ValueError("you path please") + else: + APPSPATH = builtin_apps_path + + else: + raise ValueError("ETE's toolchain directory is missing") + + ####################################################### + if len(arguments) == 1: + #print(arguments) + if not pexist(APPSPATH): + print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) + print(colorify('Install using conda (recomended):', "lgreen"), file=sys.stderr) + print(colorify(' conda install -c etetoolkit ete_toolchain', "white"), file=sys.stderr) + print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) + print(colorify(' https://github.com/etetoolkit/ete_toolchain', "white"), file=sys.stderr) + print() + + + if len(arguments) > 1: + _config_path = pjoin(BASEPATH, 'ete_build.cfg') + #print(args) + + if arguments[1] == 'check': + if not pexist(APPSPATH): + #print('this is apps_path', APPSPATH) + print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) + print(colorify('Install using conda (recomended):', "lgreen"), file=sys.stderr) + print(colorify(' conda install -c etetoolkit ete_toolchain', "white"), file=sys.stderr) + print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) + print(colorify(' https://github.com/etetoolkit/ete_toolchain', "white"), file=sys.stderr) + print() + sys.exit(0) + + try: + toolchain_version = open(pjoin(APPSPATH, "__version__")).readline() + except IOError: + toolchain_version = "unknown" + + print("Current Toolchain path: %s " %APPSPATH) + print("Current Toolchain version: %s" %toolchain_version) + + # setup portable apps + config = {} + for k in apps.builtin_apps: + # concatnate the path with "bin" + cmd = apps.get_call(k, APPSPATH, "/tmp", "1") + config[k] = cmd + apps.test_apps(config) + sys.exit(0) + + elif arguments[1] == 'workflows': + #if arguments[1] == "wl": + # print(colorify("WARNING: 'wl' is obsolete and will be removed in the future, use 'workflows' instead", "orange"), file=sys.stderr) + base_config = check_config(_config_path) + try: + wf_type = arguments[2] + except IndexError: + wf_type = None + list_workflows(base_config, wf_type) + sys.exit(0) + + elif arguments[1] == 'apps': + #print(arguments[2:]) + base_config = check_config(_config_path) + list_apps(base_config, set(arguments[2:])) + sys.exit(0) + + elif arguments[1] == 'show': + base_config = check_config(_config_path) + try: + block = arguments[2] + except IndexError: + print("Expected a block name, found none") + sys.exit(1) + + block_detail(block, base_config) + sys.exit(0) + + elif arguments[1] == "dump": + if len(arguments) > 2: + base_config = check_config(_config_path) + block_detail(arguments[2], base_config, color=False) + else: + print(open(_config_path).read()) + sys.exit(0) + + elif arguments[1] == "validate": + print('Validating configuration file ', arguments[2]) + if pexist(arguments[2]): + base_config = check_config(arguments[2]) + print('Everything ok') + else: + print('File does not exist') + sys.exit(-1) + sys.exit(0) + + elif arguments[1] == "version": + print(__version__) + sys.exit(0) +################################################################ try: toolchain_version = open(pjoin(APPSPATH, "__version__")).readline() @@ -1053,12 +1108,14 @@ def _main(arguments, builtin_apps_path=None): if not args.aa_seed_file and not args.nt_seed_file: parser.error('At least one input file argument (-a, -n) is required') - outdir = os.path.abspath(args.outdir) - final_dir, runpath = os.path.split(outdir) - if not runpath: - raise ValueError("Invalid outdir") - - GLOBALS["output_dir"] = os.path.abspath(args.outdir) + if args.outdir: + outdir = os.path.abspath(args.outdir) + final_dir, runpath = os.path.split(outdir) + if not runpath: + raise ValueError("Invalid outdir") + GLOBALS["output_dir"] = os.path.abspath(args.outdir) + else: + parser.error('The following arguments are required: -o/--outdir') if args.scratch_dir: # set paths for scratch folder for sqlite files From d667382061d5b2c1fee666fd9a944ee1630dbdf4 Mon Sep 17 00:00:00 2001 From: dengzq1234 Date: Wed, 19 Feb 2020 10:57:05 +0100 Subject: [PATCH 2/5] add error feedback --- ete3/tools/ete_build.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ete3/tools/ete_build.py b/ete3/tools/ete_build.py index 46f3f496c..25c33a463 100755 --- a/ete3/tools/ete_build.py +++ b/ete3/tools/ete_build.py @@ -979,15 +979,28 @@ def _main(arguments): else: raise ValueError("The provided custom path for external tools is not a valid directory") else: - print(colorify('\nWARNING: path of external tools is not provided, now loading TOOLCHAIN', "yellow"), file=sys.stderr) + print(colorify('\nWARNING: path of external tools is not provided, now loading TOOLCHAIN\n', "yellow"), file=sys.stderr) builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") + + if not os.path.exists(builtin_apps_path): - raise ValueError("you path please") + print(colorify('\nWARNING: external applications directory are not found at %s' %builtin_apps_path, "yellow"), file=sys.stderr) + print(colorify('Use "ete build install_tools" to install or upgrade tools', "orange"), file=sys.stderr) + print(colorify('\nWARNING: external applications not found', "yellow"), file=sys.stderr) + print(colorify('Install using conda (recomended):', "lgreen"), file=sys.stderr) + print(colorify(' conda install -c etetoolkit ete_toolchain', "white"), file=sys.stderr) + print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) + print(colorify(' https://github.com/etetoolkit/ete_toolchain', "white"), file=sys.stderr) + print(colorify('or manually compile from:', "lgreen"), file=sys.stderr) + print(colorify(' https://anaconda.org/bioconda', "white"), file=sys.stderr) + raise ValueError(colorify("Please select binary path of your external apps by adding -L [custom path]| LOCAL| TOOLCHAIN.", "yellow")) else: APPSPATH = builtin_apps_path else: - raise ValueError("ETE's toolchain directory is missing") + #print(colorify('\nWARNING: external applications directory are not found at %s' % ete3_path, "yellow"), file=sys.stderr) + #print(colorify('Use "ete build install_tools" to install or upgrade tools', "orange"), file=sys.stderr) + raise ValueError("ETE's toolchain directory is missing!\n Please select binary path of your external apps by -L [custom path]| LOCAL| TOOLCHAIN.") ####################################################### if len(arguments) == 1: From e493454fbfa86da1e5c09bf2d72d138ba90bc262 Mon Sep 17 00:00:00 2001 From: dengzq1234 Date: Sun, 19 Apr 2020 10:47:35 +0200 Subject: [PATCH 3/5] add comments --- ete3/tools/ete.py | 7 ++++--- ete3/tools/ete_build.py | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ete3/tools/ete.py b/ete3/tools/ete.py index 1dce60589..5ab65d22a 100755 --- a/ete3/tools/ete.py +++ b/ete3/tools/ete.py @@ -141,11 +141,12 @@ def _main(arguments): #builtin_apps_path = None #ete3_path = which("ete3") - # is it a good place to make judgement? + #if ete3_path: - # builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") #Ziqi + # builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") # builtin_apps_path = os.path.split(ete3_path)[0] - ete_build._main(arguments) + + ete_build._main(arguments) # pass arguments to ete_build including flag of apps_bin, so as to determine apps_path there return # CREATE REUSABLE PARSER OPTIONS diff --git a/ete3/tools/ete_build.py b/ete3/tools/ete_build.py index 25c33a463..9152e98a4 100755 --- a/ete3/tools/ete_build.py +++ b/ete3/tools/ete_build.py @@ -960,6 +960,7 @@ def _main(arguments): args = parser.parse_args(arguments) + # start to load apps_path depending on flags, therefore need to load args before load the path ete3_path = shutil.which("ete3") if ete3_path: if args.tools_dir: From 5e71df2b0c54ff40225489f06ba35b0af79af07e Mon Sep 17 00:00:00 2001 From: dengzq1234 Date: Sun, 19 Apr 2020 10:58:50 +0200 Subject: [PATCH 4/5] add commend --- ete3/tools/ete_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ete3/tools/ete_build.py b/ete3/tools/ete_build.py index 9152e98a4..6281b3baf 100755 --- a/ete3/tools/ete_build.py +++ b/ete3/tools/ete_build.py @@ -706,7 +706,6 @@ def _main(arguments): type=is_file, default=BASEPATH+'/ete_build.cfg', help="Base configuration file.") - # Ziqi check this input_group.add_argument("-L", "--tools-dir", dest="tools_dir", type=str, #required=True, #Ziqi @@ -980,6 +979,7 @@ def _main(arguments): else: raise ValueError("The provided custom path for external tools is not a valid directory") else: + # default path is ete3_apps/bin print(colorify('\nWARNING: path of external tools is not provided, now loading TOOLCHAIN\n', "yellow"), file=sys.stderr) builtin_apps_path = os.path.join(os.path.split(ete3_path)[0], "ete3_apps/bin") From b665433482ffc224955e4eb25b6ca6a0fcd1167a Mon Sep 17 00:00:00 2001 From: dengzq1234 Date: Sun, 3 May 2020 22:19:03 +0200 Subject: [PATCH 5/5] reverse version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7daab976b..50e47c89c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.1.18.1 \ No newline at end of file +3.1.1 \ No newline at end of file