From b9a97a5c7fbee3a50550c187307d9a75c39a9ab3 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 27 Oct 2021 12:33:13 -0400 Subject: [PATCH 01/18] Revise all download links to use HTTPS --- doc/ot-synthesis-v10.2.md | 6 +++--- doc/ot-synthesis-v10.3.md | 4 ++-- doc/ot-synthesis-v10.4.md | 6 +++--- doc/ot-synthesis-v11.4.md | 8 ++++---- doc/ot-synthesis-v12.3.md | 6 +++--- doc/ot-synthesis-v13.4.md | 6 +++--- doc/ot-synthesis-v5.0.md | 6 +++--- doc/ot-synthesis-v5.md | 6 +++--- doc/ot-synthesis-v6.1.md | 6 +++--- doc/ot-synthesis-v7.0.md | 6 +++--- doc/ot-synthesis-v8.0.md | 6 +++--- doc/ot-synthesis-v9.1.md | 6 +++--- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/doc/ot-synthesis-v10.2.md b/doc/ot-synthesis-v10.2.md index 047f57f..fd484ff 100644 --- a/doc/ot-synthesis-v10.2.md +++ b/doc/ot-synthesis-v10.2.md @@ -3,10 +3,10 @@ Version 10.2 of the synthetic tree was generated on 9 January 2018 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.2/opentree10.2_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.2/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v10.3.md b/doc/ot-synthesis-v10.3.md index 48ed88b..cd9cc0e 100644 --- a/doc/ot-synthesis-v10.3.md +++ b/doc/ot-synthesis-v10.3.md @@ -5,8 +5,8 @@ Version 10.3 of the synthetic tree was generated on 10 July 2018 using the [prop ## Downloads There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.3/opentree10.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.3/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v10.4.md b/doc/ot-synthesis-v10.4.md index 2f8ec0f..dc2bfd0 100644 --- a/doc/ot-synthesis-v10.4.md +++ b/doc/ot-synthesis-v10.4.md @@ -3,10 +3,10 @@ Version 10.4 of the synthetic tree was generated on 24 September 2018 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree10.4/opentree10.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree10.4/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v11.4.md b/doc/ot-synthesis-v11.4.md index 1a08a46..de053f6 100644 --- a/doc/ot-synthesis-v11.4.md +++ b/doc/ot-synthesis-v11.4.md @@ -3,10 +3,10 @@ Version 11.4 of the synthetic tree was generated on 09 October 2019 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree11.4/opentree11.4_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree11.4/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes) ## Release notes @@ -60,4 +60,4 @@ There are two downloads. The first (smaller download) contain only tree and anno 9860 2090 - \ No newline at end of file + diff --git a/doc/ot-synthesis-v12.3.md b/doc/ot-synthesis-v12.3.md index f3f9a84..cd34328 100644 --- a/doc/ot-synthesis-v12.3.md +++ b/doc/ot-synthesis-v12.3.md @@ -3,10 +3,10 @@ Version 12.3 of the synthetic tree was generated on 23 December 2019 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree12.3/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree12.3/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree12.3/opentree12.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree12.3/opentree12.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree12.3/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree12.3/opentree12.3_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 40 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree12.3/opentree12.3_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree12.3/output/index.html) rather than downloading. (compressed tar archive; 301 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v13.4.md b/doc/ot-synthesis-v13.4.md index 01448c6..a50c41f 100644 --- a/doc/ot-synthesis-v13.4.md +++ b/doc/ot-synthesis-v13.4.md @@ -3,10 +3,10 @@ Version 13.4 of the synthetic tree was generated on 18 June 2021 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree13.4/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree13.4/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree13.4/opentree13.4tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 36 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree13.4/opentree13.4.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree13.4/output/index.html) rather than downloading. (compressed tar archive; 561 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree13.4/opentree13.4tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 36 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree13.4/opentree13.4.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree13.4/output/index.html) rather than downloading. (compressed tar archive; 561 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v5.0.md b/doc/ot-synthesis-v5.0.md index 0afeaa7..bda66df 100644 --- a/doc/ot-synthesis-v5.0.md +++ b/doc/ot-synthesis-v5.0.md @@ -4,11 +4,11 @@ Version 5.0 was generated on 28 March 2016. ## Downloads -* [Synthetic tree](http://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_tree.tgz) : includes the full tree, annotations file, and a phylo-only tree (tips only from taxonomy pruned off). See the enclosed README for details. (compressed tar archive; 9 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. You can also [browse the contents](http://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) rather than downloading. (compressed tar archive; 68 Mbyes) +* [Synthetic tree](https://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_tree.tgz) : includes the full tree, annotations file, and a phylo-only tree (tips only from taxonomy pruned off). See the enclosed README for details. (compressed tar archive; 9 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. You can also [browse the contents](https://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) rather than downloading. (compressed tar archive; 68 Mbyes) ## Release notes This version of the tree was generated with a completely new synthesis method. There is a [make-based pipeline](https://github.com/OpenTreeOfLife/propinquity) that uses functions from the [otcetera library](https://github.com/OpenTreeOfLife/otcetera). -The method is currently unpublished, but the [documentation pages](http://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) have detailed descriptions of all steps, with links to inputs and outputs. There is also a [draft manuscript](https://github.com/OpenTreeOfLife/otcetera/blob/master/doc/summarizing-taxonomy-plus-trees.tex) (tex format). +The method is currently unpublished, but the [documentation pages](https://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) have detailed descriptions of all steps, with links to inputs and outputs. There is also a [draft manuscript](https://github.com/OpenTreeOfLife/otcetera/blob/master/doc/summarizing-taxonomy-plus-trees.tex) (tex format). diff --git a/doc/ot-synthesis-v5.md b/doc/ot-synthesis-v5.md index 0afeaa7..bda66df 100644 --- a/doc/ot-synthesis-v5.md +++ b/doc/ot-synthesis-v5.md @@ -4,11 +4,11 @@ Version 5.0 was generated on 28 March 2016. ## Downloads -* [Synthetic tree](http://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_tree.tgz) : includes the full tree, annotations file, and a phylo-only tree (tips only from taxonomy pruned off). See the enclosed README for details. (compressed tar archive; 9 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. You can also [browse the contents](http://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) rather than downloading. (compressed tar archive; 68 Mbyes) +* [Synthetic tree](https://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_tree.tgz) : includes the full tree, annotations file, and a phylo-only tree (tips only from taxonomy pruned off). See the enclosed README for details. (compressed tar archive; 9 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree5.0/opentree5.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. You can also [browse the contents](https://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) rather than downloading. (compressed tar archive; 68 Mbyes) ## Release notes This version of the tree was generated with a completely new synthesis method. There is a [make-based pipeline](https://github.com/OpenTreeOfLife/propinquity) that uses functions from the [otcetera library](https://github.com/OpenTreeOfLife/otcetera). -The method is currently unpublished, but the [documentation pages](http://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) have detailed descriptions of all steps, with links to inputs and outputs. There is also a [draft manuscript](https://github.com/OpenTreeOfLife/otcetera/blob/master/doc/summarizing-taxonomy-plus-trees.tex) (tex format). +The method is currently unpublished, but the [documentation pages](https://files.opentreeoflife.org/synthesis/opentree5.0/output/index.html) have detailed descriptions of all steps, with links to inputs and outputs. There is also a [draft manuscript](https://github.com/OpenTreeOfLife/otcetera/blob/master/doc/summarizing-taxonomy-plus-trees.tex) (tex format). diff --git a/doc/ot-synthesis-v6.1.md b/doc/ot-synthesis-v6.1.md index 1302aba..b815a75 100644 --- a/doc/ot-synthesis-v6.1.md +++ b/doc/ot-synthesis-v6.1.md @@ -3,10 +3,10 @@ Version 6.1 of the synthetic tree was generated on 2 August 2016 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree6.1/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree6.1/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree6.1/opentree6.1_tree.tar.gz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 32 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree6.1/opentree6.1_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree6.1/output/index.html) rather than downloading. (compressed tar archive; 138 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree6.1/opentree6.1_tree.tar.gz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 32 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree6.1/opentree6.1_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree6.1/output/index.html) rather than downloading. (compressed tar archive; 138 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v7.0.md b/doc/ot-synthesis-v7.0.md index 3fdd60d..d39058f 100644 --- a/doc/ot-synthesis-v7.0.md +++ b/doc/ot-synthesis-v7.0.md @@ -3,10 +3,10 @@ Version 7.0 of the synthetic tree was generated on 11 September 2016 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree7.0/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree7.0/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree7.0/opentree7.0_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 49 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree7.0/opentree7.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree7.0/output/index.html) rather than downloading. (compressed tar archive; 128 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree7.0/opentree7.0_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 49 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree7.0/opentree7.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree7.0/output/index.html) rather than downloading. (compressed tar archive; 128 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v8.0.md b/doc/ot-synthesis-v8.0.md index a6ec940..d6615c4 100644 --- a/doc/ot-synthesis-v8.0.md +++ b/doc/ot-synthesis-v8.0.md @@ -3,10 +3,10 @@ Version 8.0 of the synthetic tree was generated on 29 November 2016 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree8.0/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree8.0/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree8.0/opentree8.0_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 29 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree8.0/opentree8.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree8.0/output/index.html) rather than downloading. (compressed tar archive; 151 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree8.0/opentree8.0_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 29 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree8.0/opentree8.0_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree8.0/output/index.html) rather than downloading. (compressed tar archive; 151 Mbytes) ## Release notes diff --git a/doc/ot-synthesis-v9.1.md b/doc/ot-synthesis-v9.1.md index 6760b42..d821791 100644 --- a/doc/ot-synthesis-v9.1.md +++ b/doc/ot-synthesis-v9.1.md @@ -3,10 +3,10 @@ Version 9.1 of the synthetic tree was generated on 22 March 2017 using the [propinquity pipeline](https://github.com/OpenTreeOfLife/propinquity). ## Downloads -There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](http://files.opentreeoflife.org/synthesis/opentree9.1/output/index.html). +There are two downloads. The first (smaller download) contain only tree and annotations files. The second (larger download) is the full output from the synthesis procedure, including documentation. You can also [browse the full output](https://files.opentreeoflife.org/synthesis/opentree9.1/output/index.html). -* [Tree and annotations](http://files.opentreeoflife.org/synthesis/opentree9.1/opentree9.1_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) -* [All pipeline outputs](http://files.opentreeoflife.org/synthesis/opentree9.1/opentree9.1_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](http://files.opentreeoflife.org/synthesis/opentree9.1/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) +* [Tree and annotations](https://files.opentreeoflife.org/synthesis/opentree9.1/opentree9.1_tree.tgz) : Several versions of the synthetic tree, along with the annotations file. See the enclosed README for details. (compressed tar archive; 35 Mbytes) +* [All pipeline outputs](https://files.opentreeoflife.org/synthesis/opentree9.1/opentree9.1_output.tgz) : Outputs and documentation from all stages of the synthesis pipeline. Or, you can [browse the output](https://files.opentreeoflife.org/synthesis/opentree9.1/output/index.html) rather than downloading. (compressed tar archive; 189 Mbytes) ## Release notes From 673702e1bdf4429d6047e2945a383af57cf2724f Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 31 May 2022 10:31:44 -0400 Subject: [PATCH 02/18] Add console chatter with configuration specs --- ws-tests/run_tests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ws-tests/run_tests.sh b/ws-tests/run_tests.sh index a350e6a..f722468 100755 --- a/ws-tests/run_tests.sh +++ b/ws-tests/run_tests.sh @@ -58,6 +58,9 @@ fi first_config_spec=$1 shift +echo '+ first_config_spec:' +echo $first_config_spec +echo '+' if [[ ! "$first_config_spec" =~ "=" ]]; then if [[ "$first_config_spec" =~ ^http: ]]; then @@ -71,6 +74,11 @@ config_specs="$first_config_spec $*" if [[ ! "$config_specs" =~ "host:allowwrite=" ]]; then config_specs="$first_config_spec $* host:allowwrite=false" fi +echo '+' +echo '+ config_specs:' +echo $config_specs +echo '+' +echo '+' # The python test scripts all use the opentreetesting.py library, From 24a4595553cf41a7cc2f0aa41084fb5368ed19f4 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 31 May 2022 10:32:17 -0400 Subject: [PATCH 03/18] Update to our latest admin/unprivileged users --- deploy/push.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/push.sh b/deploy/push.sh index 9382750..81bb87c 100755 --- a/deploy/push.sh +++ b/deploy/push.sh @@ -80,10 +80,10 @@ done [ "x$OPENTREE_HOST" != x ] || err "OPENTREE_HOST not specified" # On ubuntu, the admin user is called 'ubuntu'; on debian it's 'admin' -[ "x$OPENTREE_ADMIN" != x ] || OPENTREE_ADMIN=admin +[ "x$OPENTREE_ADMIN" != x ] || OPENTREE_ADMIN=ubuntu # Unprivileged user that runs all the services -[ "x$OPENTREE_USER" != x ] || OPENTREE_USER=opentree +[ "x$OPENTREE_USER" != x ] || OPENTREE_USER=deploy # OPENTREE_SECRETS is the *local* directory where .pem and other # private files are kept From 71086bfa8228d84a8000256715a17b1d61af5528 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Tue, 31 May 2022 11:24:28 -0400 Subject: [PATCH 04/18] Fix lingering python2 syntax (for python3 testing) --- deploy/setup/web2py-patches/oauth20_account.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/setup/web2py-patches/oauth20_account.py b/deploy/setup/web2py-patches/oauth20_account.py index 1cf1c7e..869da86 100644 --- a/deploy/setup/web2py-patches/oauth20_account.py +++ b/deploy/setup/web2py-patches/oauth20_account.py @@ -83,7 +83,7 @@ def get_user(self): user = None try: user = self.graph.get_object("me") - except GraphAPIError, e: + except GraphAPIError as e: self.session.token = None self.graph = None @@ -226,7 +226,7 @@ def accessToken(self): opener.close() try: return current.session.token['access_token'] - except Exception, e: + except Exception as e: raise Exception("No access_token found in data: %s %s" % (current.session.token, e)) return None From bdd043100936df53921238b039ec5855350b105c Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:05:00 -0400 Subject: [PATCH 05/18] Add helpful diagnostic chatter around as-admin.sh --- deploy/push.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/push.sh b/deploy/push.sh index 81bb87c..9fbcc35 100755 --- a/deploy/push.sh +++ b/deploy/push.sh @@ -280,9 +280,21 @@ function sync_system { if [ $DRYRUN = "yes" ]; then echo "[sync]"; return; fi # Do privileged stuff # Don't use rsync - might not be installed yet + + ##echo ">>>>>>>>>>>>" + ##echo "BEFORE as-admin.sh..." + ##echo " scp -p -i '${ADMIN_IDENTITY}' as-admin.sh '$OPENTREE_ADMIN@$OPENTREE_HOST':" + ##echo " ${ASSH} '$ADMIN@$OPENTREE_HOST' ./as-admin.sh '$OPENTREE_HOST' '$OPENTREE_USER' '$CERTIFICATE_FILE' '$CERTIFICATE_KEY_FILE'" + ## echo ">>>>>>>>>>>>" + scp -p -i "${ADMIN_IDENTITY}" as-admin.sh "$OPENTREE_ADMIN@$OPENTREE_HOST": ${ASSH} "$ADMIN@$OPENTREE_HOST" ./as-admin.sh "$OPENTREE_HOST" "$OPENTREE_USER" \ "$CERTIFICATE_FILE" "$CERTIFICATE_KEY_FILE" + + ##echo "<<<<<<<<<<<<" + ##echo "AFTER as-admin.sh!" + ##echo "<<<<<<<<<<<<" + # Copy files over rsync -pr -e "${SSH}" "--exclude=*~" "--exclude=#*" setup "$OT_USER@$OPENTREE_HOST": # Bleh From 3808b6233a89a096b180228d4097534bddba6dee Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:09:59 -0400 Subject: [PATCH 06/18] Added as-admin log file for troubleshooting This script has an annoying tendency to fail silently, with no user feedback. I've adapted the logging feature used by other scripts to write a log to ~/log/messages (currently /home/ubuntu/log/messages). --- deploy/as-admin.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index fc485b2..5e8d9b3 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -14,6 +14,13 @@ OPENTREE_USER=$2 CERTIFICATE_FILE=$3 CERTIFICATE_KEY_FILE=$4 +# copied from setup/functions.sh (not yet available), so we can log admin stuff +ADMIN_HOME=$(bash <<< "echo ~") +function log() { + mkdir -p "$ADMIN_HOME/log" || exit 1 + (echo `date` as-admin.sh: " $*") >>"$ADMIN_HOME/log/messages" || exit +} + if [ x$OPENTREE_USER = x ]; then OPENTREE_USER=opentree fi @@ -287,6 +294,8 @@ else sudo rm -f /etc/apache2/sites-enabled/001-opentree-ssl.conf fi +log "before UNPRIVILEGED USER" + # Apache 2.4 is finicky about protection of the key file @@ -304,6 +313,8 @@ if [ ! -e $OTHOME ]; then sudo chsh -s /bin/bash $OPENTREE_USER fi +log "OPENTREE_USER added" + if [ ! -e $OTHOME/.ssh ]; then sudo mkdir $OTHOME/.ssh sudo cp -p .ssh/authorized_keys $OTHOME/.ssh/ @@ -311,6 +322,7 @@ if [ ! -e $OTHOME/.ssh ]; then sudo chown -R $OPENTREE_USER:$OPENTREE_USER $OTHOME fi +log ".ssh/authorized_keys created" # Ideally stowing the hostname one would be done every time, but we # want to avoid unsatisfiable sudo prompt demands, so let's assume it # stays the same. @@ -322,4 +334,8 @@ if [ x$OPENTREE_HOST != x -a ! -r $OTHOME/hostname ]; then chmod go+r $HOSTFILE chown $OPENTREE_USER $HOSTFILE EOF + + log "hostname updated to $OPENTREE_HOST" fi + +log "did we SKIP the hostname?" From e586745df6200cc24f25228625e64cf71d818bea Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:13:21 -0400 Subject: [PATCH 07/18] Automate changing canonical hostname Apparently this was formerly a manual step? Now we make it official, setting the `hostname` and `/etc/hostname` to survive reboots of the deployed server. This is necessary for apache to trust some configuration. Any chance this goes wrong if we have deployed multiple systems to the same server? e.g. `devtree.opentreeoflife.org` and `devapi.opentreeoflife.org`. Comments welcome! --- deploy/as-admin.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 5e8d9b3..71638a2 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -323,6 +323,10 @@ if [ ! -e $OTHOME/.ssh ]; then fi log ".ssh/authorized_keys created" + +# Update the canonical hostname on this server +sudo hostnamectl set-hostname "$OPENTREE_HOST" + # Ideally stowing the hostname one would be done every time, but we # want to avoid unsatisfiable sudo prompt demands, so let's assume it # stays the same. From 07e1fce879138c681560ee27f7c86d74c34b4d4a Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:18:08 -0400 Subject: [PATCH 08/18] Restore support for python2 packages Many of the packages we need for web2py apps (still on python2.*) are no longer provided in modern Ubuntu. Add the necessary repositories to fetch these anyway, alongside their python3 counterparts for our newer code. --- deploy/as-admin.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 71638a2..28dde4d 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -49,6 +49,16 @@ function apt_get_install_one fi } +# Add Bionic repository for these python2.* packages (in newer Ubuntu) +# python-venv (vs python3-venv) +# python-pip (vs python3-pip) +# python-requests (vs python3-requests) +# These are required by our current version of web2py +apt_get_install_one software-properties-common # needed for add-apt-repository +sudo apt-add-repository "deb http://us.archive.ubuntu.com/ubuntu/ bionic main universe" +$APTGET update + + # ---------- UPDATE ---------- if [ `which dialog`x = x ]; then @@ -97,7 +107,8 @@ apt_get_install_one libboost-system-dev apt_get_install_one libtool apt_get_install_one pkg-config apt_get_install_one psmisc # for killall -apt_get_install_one python3-venv +apt_get_install_one elpa-pyvenv # for web2py +apt_get_install_one python3-venv # for everyone else #apt_get_install_one g++-8 apt_get_install_one ninja-build @@ -174,6 +185,8 @@ fi # ---------- PIP ---------- # Get pip if [ `which pip`x = x ]; then + apt_get_install --allow-downgrades python-pip-whl=9.0.1-2 + # this python-pip dependency is very fussy about version apt_get_install python-pip fi @@ -206,10 +219,10 @@ fi # ---------- PYTHON VIRTUALENV ---------- # Get virtualenv if [ `which virtualenv`x = x ]; then - apt_get_install python-virtualenv virtualenv + # install for both python2 and python3 + apt_get_install python-virtualenv python3-virtualenv virtualenv fi - # ---------- MAVEN 3 ---------- if [ `which mvn`x = x ]; then apt_get_install maven @@ -226,7 +239,6 @@ if [ ! -r /etc/ntp.conf ]; then apt_get_install ntp fi - # ---------- BACKUP EXISTING CONFIG FILES --------- # Copy all possible Open Tree config files. A given target machine will have From e51dcf2ebbfd0d3b4cd3fea1a8c8443b05a2a523 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:20:08 -0400 Subject: [PATCH 09/18] Some commands now require `sudo` on Ubuntu --- deploy/as-admin.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 28dde4d..7eb71d3 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -244,8 +244,8 @@ fi # Copy all possible Open Tree config files. A given target machine will have # some but not all of these, so suppress "file not found" messages. CONFIG_BACKUP_DIR="/home/admin/otol-backups.$(date +'%b-%d-%Y-%H%M%S')" -mkdir -p $CONFIG_BACKUP_DIR -cp --parents --preserve=all \ +sudo mkdir -p $CONFIG_BACKUP_DIR +sudo cp --parents --preserve=all \ /etc/apache2/sites-available/opentree.conf \ /etc/apache2/sites-available/opentree-ssl.conf \ /etc/apache2/opentree-shared.conf \ From dfe14e4fbc0f00b4e5d139931fd5c6e588aeca3a Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:21:07 -0400 Subject: [PATCH 10/18] Tweak backups to fail without stopping as-admin.sh Later steps in as-admin.sh would be skipped (w/o warning!) if some of the files we want to backup are missing, which is entirely likely if only some systems are installed. --- deploy/as-admin.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 7eb71d3..86ee34a 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -252,8 +252,10 @@ sudo cp --parents --preserve=all \ /home/opentree/repo/opentree/curator/private/config \ /home/opentree/repo/opentree/webapp/private/config \ /home/phylesystemapi/repo/phylesystem-api/private/config \ - $CONFIG_BACKUP_DIR 2>/dev/null - + $CONFIG_BACKUP_DIR 2>/dev/null || true +# NB the final '|| true' (alternately, '|| :') makes sure we keep on running even if +# any of the above source files are missing. Else this script fails silently before +# doing the steps below! # ---------- APACHE VHOST ---------- From 89f7c70e45dfcc7d690819893f69673069f7222b Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 1 Jun 2022 13:34:30 -0400 Subject: [PATCH 11/18] Recognize new user `deploy` vs. old `opentree` Apache (rightly) complained about the missing user, but would also have failed with the old paths to /home/opentree/... --- deploy/setup/opentree-shared.conf | 2 +- deploy/setup/opentree-ssl.conf | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/deploy/setup/opentree-shared.conf b/deploy/setup/opentree-shared.conf index 42bfe05..6d32b64 100644 --- a/deploy/setup/opentree-shared.conf +++ b/deploy/setup/opentree-shared.conf @@ -26,7 +26,7 @@ Alias "/taxonomy/browse" "/home/opentree/repo/opentree/taxonomy/cgi-bin/browse.py" - + Require all granted Options +ExecCGI AddHandler cgi-script .py diff --git a/deploy/setup/opentree-ssl.conf b/deploy/setup/opentree-ssl.conf index 3093040..8f92bd2 100644 --- a/deploy/setup/opentree-ssl.conf +++ b/deploy/setup/opentree-ssl.conf @@ -43,14 +43,14 @@ # Moving all WSGI configuration here, so we only run one instance of web2py on HTTPS. # This should prevent our broken session problem as documented here: # https://github.com/OpenTreeOfLife/opentree/issues/804 - WSGIDaemonProcess web2py user=opentree group=opentree display-name=%{GROUP} + WSGIDaemonProcess web2py user=deploy group=deploy display-name=%{GROUP} WSGIProcessGroup web2py - WSGIScriptAlias / /home/opentree/web2py/wsgihandler.py + WSGIScriptAlias / /home/deploy/web2py/wsgihandler.py # We don't want the scriptalias to override mappings for # treemachine and taxomachine... seems not to. - + AllowOverride None Require all denied @@ -58,8 +58,8 @@ - AliasMatch ^/([^/]+)/static/(.*) /home/opentree/web2py/applications/$1/static/$2 - + AliasMatch ^/([^/]+)/static/(.*) /home/deploy/web2py/applications/$1/static/$2 + Require all granted From 3062c408f00a7d27ab2db2001fa9fa424406ec9f Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:21:30 -0400 Subject: [PATCH 12/18] Ensure python2.7 venv on new Ubuntu --- deploy/setup/functions.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/setup/functions.sh b/deploy/setup/functions.sh index 4e4d4d3..d4f1860 100644 --- a/deploy/setup/functions.sh +++ b/deploy/setup/functions.sh @@ -22,7 +22,8 @@ fi # ---------- Setup TWO VIRTUALENVs ---------- # Set up python env if [ ! -d venv ]; then - virtualenv venv || exit 1 + # specify python2.* or you'll now get python3 + virtualenv --python=`which python2` venv || exit 1 fi source venv/bin/activate || exit 1 From 8f5a82c248f0e4b7c9649dd8ffeeb6dc2c558da9 Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:21:52 -0400 Subject: [PATCH 13/18] Revised log chatter (start and end) --- deploy/as-admin.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 86ee34a..3e476e7 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -20,6 +20,7 @@ function log() { mkdir -p "$ADMIN_HOME/log" || exit 1 (echo `date` as-admin.sh: " $*") >>"$ADMIN_HOME/log/messages" || exit } +log ">>> as-admin.sh starting" if [ x$OPENTREE_USER = x ]; then OPENTREE_USER=opentree @@ -356,4 +357,4 @@ EOF log "hostname updated to $OPENTREE_HOST" fi -log "did we SKIP the hostname?" +log "<<< as-admin.sh finished" From 7f0d9495301d76c752a02d674a3e2ee742e795df Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:23:00 -0400 Subject: [PATCH 14/18] Global apache ServerName (stop warnings) --- deploy/as-admin.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index 3e476e7..fe0097e 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -309,7 +309,17 @@ else sudo rm -f /etc/apache2/sites-enabled/001-opentree-ssl.conf fi -log "before UNPRIVILEGED USER" +# Add global ServerName to main apache config, to silence warnings on restart +# NB we use 'localhost' here as a reminder that we might have multiple virtual +# hosts defined for this server. +APACHE_GLOBAL_CONFIG_FILE=/etc/apache2/apache2.conf +if grep -q "^ServerName" $APACHE_GLOBAL_CONFIG_FILE; then + log "global ServerName found in main apache config" +else + log "adding global ServerName to main apache config" + echo "ServerName localhost" | sudo tee -a $APACHE_GLOBAL_CONFIG_FILE > /dev/null + log "DONE!" +fi # Apache 2.4 is finicky about protection of the key file From 009061e2138fa1efd8fb4cc5852597e2ac13fd7a Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:25:58 -0400 Subject: [PATCH 15/18] Briefly use a newer apt repo? Stashing this for reference, but it seems to "leak" newer packages and cause trouble. Backing it out next... --- deploy/as-admin.sh | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index fe0097e..c392704 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -221,7 +221,33 @@ fi # Get virtualenv if [ `which virtualenv`x = x ]; then # install for both python2 and python3 - apt_get_install python-virtualenv python3-virtualenv virtualenv + #apt_get_install python-virtualenv python3-virtualenv + # No, it seems both of these install a buggy virtualenv (20.0.17) + + # Now for something really special. We need a bug fix for python2.7 venvs + # that only appears in a newer version of virtualenv (20.0.19+) than the + # current Ubuntu repos. The newer version we need is in the 'jammy' + # repository, but we should ONLY enable this as needed, or it may try to + # update other packages too aggressively. + # + # For background and bug details, see + # + + # Briefly add (or enable) the jammy repo + # WORKS if added manually: "deb http://archive.ubuntu.com/ubuntu/ jammy main universe" + FUTURE_REPO="deb http://archive.ubuntu.com/ubuntu/ jammy main universe" + sudo apt-add-repository "$FUTURE_REPO" + $APTGET update + + #apt_get_install virtualenv # FAILS with missing/broken dependencies! + # Let's try again, pulling all dependencies from the newer repo; see + # for examples + apt-get -t jammy install virtualenv + + + # Remove (or disable) the jammy repo to avoid surprises + sudo apt-add-repository --remove "$FUTURE_REPO" + $APTGET update fi # ---------- MAVEN 3 ---------- From 08fe8e3ecac82b7463b15db35125ed806476c8cf Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:26:53 -0400 Subject: [PATCH 16/18] Revert "Briefly use a newer apt repo?" This reverts commit 009061e2138fa1efd8fb4cc5852597e2ac13fd7a. We'll use a single-file hot fix to venv `activate_this.py` instead --- deploy/as-admin.sh | 28 +--------------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/deploy/as-admin.sh b/deploy/as-admin.sh index c392704..fe0097e 100755 --- a/deploy/as-admin.sh +++ b/deploy/as-admin.sh @@ -221,33 +221,7 @@ fi # Get virtualenv if [ `which virtualenv`x = x ]; then # install for both python2 and python3 - #apt_get_install python-virtualenv python3-virtualenv - # No, it seems both of these install a buggy virtualenv (20.0.17) - - # Now for something really special. We need a bug fix for python2.7 venvs - # that only appears in a newer version of virtualenv (20.0.19+) than the - # current Ubuntu repos. The newer version we need is in the 'jammy' - # repository, but we should ONLY enable this as needed, or it may try to - # update other packages too aggressively. - # - # For background and bug details, see - # - - # Briefly add (or enable) the jammy repo - # WORKS if added manually: "deb http://archive.ubuntu.com/ubuntu/ jammy main universe" - FUTURE_REPO="deb http://archive.ubuntu.com/ubuntu/ jammy main universe" - sudo apt-add-repository "$FUTURE_REPO" - $APTGET update - - #apt_get_install virtualenv # FAILS with missing/broken dependencies! - # Let's try again, pulling all dependencies from the newer repo; see - # for examples - apt-get -t jammy install virtualenv - - - # Remove (or disable) the jammy repo to avoid surprises - sudo apt-add-repository --remove "$FUTURE_REPO" - $APTGET update + apt_get_install python-virtualenv python3-virtualenv virtualenv fi # ---------- MAVEN 3 ---------- From 952ad2d4832704977b9940e390ac040da1115b6d Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 13:54:53 -0400 Subject: [PATCH 17/18] Add patch for broken virtualenv `activate_this` This is just adding the file and README, not yet patching... --- deploy/setup/virtualenv-patches/README.md | 3 ++ .../setup/virtualenv-patches/activate_this.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 deploy/setup/virtualenv-patches/README.md create mode 100644 deploy/setup/virtualenv-patches/activate_this.py diff --git a/deploy/setup/virtualenv-patches/README.md b/deploy/setup/virtualenv-patches/README.md new file mode 100644 index 0000000..bcf5fa0 --- /dev/null +++ b/deploy/setup/virtualenv-patches/README.md @@ -0,0 +1,3 @@ +Once again, we want some fixes that aren't easily available in newer Ubuntu +(mid-2022). In this case we want to fix bugs in `activate_this.py`, a helper +that lets us use the python packages in a venv from Apache's mod_wsgi. diff --git a/deploy/setup/virtualenv-patches/activate_this.py b/deploy/setup/virtualenv-patches/activate_this.py new file mode 100644 index 0000000..bc9df9b --- /dev/null +++ b/deploy/setup/virtualenv-patches/activate_this.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +"""Activate virtualenv for current interpreter: + +Use exec(open(this_file).read(), {'__file__': this_file}). + +This can be used when you must use an existing Python interpreter, not the virtualenv bin/python. +""" +import os +import site +import sys + +try: + abs_file = os.path.abspath(__file__) +except NameError: + raise AssertionError("You must use exec(open(this_file).read(), {'__file__': this_file}))") + +bin_dir = os.path.dirname(abs_file) +base = bin_dir[: -len("bin") - 1] # strip away the bin part from the __file__, plus the path separator + +# prepend bin to PATH (this file is inside the bin directory) +os.environ["PATH"] = os.pathsep.join([bin_dir] + os.environ.get("PATH", "").split(os.pathsep)) +os.environ["VIRTUAL_ENV"] = base # virtual env is right above bin directory + +# add the virtual environments libraries to the host python import mechanism +prev_length = len(sys.path) +for lib in "../lib/python2.7/site-packages".split(os.pathsep): + path = os.path.realpath(os.path.join(bin_dir, lib)) + site.addsitedir(path.decode("utf-8") if "" else path) +sys.path[:] = sys.path[prev_length:] + sys.path[0:prev_length] + +sys.real_prefix = sys.prefix +sys.prefix = base From 3ba04d531ee1d2b463d1d75a968b0b67cabeb24e Mon Sep 17 00:00:00 2001 From: Jim Allman Date: Wed, 8 Jun 2022 14:04:27 -0400 Subject: [PATCH 18/18] Use the virtualenv patch file added earlier. --- deploy/setup/functions.sh | 1 + deploy/setup/install-web2py.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/deploy/setup/functions.sh b/deploy/setup/functions.sh index d4f1860..8ec3e12 100644 --- a/deploy/setup/functions.sh +++ b/deploy/setup/functions.sh @@ -24,6 +24,7 @@ fi if [ ! -d venv ]; then # specify python2.* or you'll now get python3 virtualenv --python=`which python2` venv || exit 1 + # NB - later we'll probably apply our single-file patch to `activate_this.py` fi source venv/bin/activate || exit 1 diff --git a/deploy/setup/install-web2py.sh b/deploy/setup/install-web2py.sh index 58ec705..1090242 100755 --- a/deploy/setup/install-web2py.sh +++ b/deploy/setup/install-web2py.sh @@ -38,6 +38,10 @@ cp -p setup/web2py-patches/oauth20_account.py web2py/gluon/contrib/login_methods # ---------- VIRTUALENV + WEB2PY + WSGI ---------- +# Patch broken virtualenv activation helper (mid-June 2022), used below +echo "Applying patch (hot fix) to buggy virtualenv..." || exit +cp -p setup/virtualenv-patches/activate_this.py "$PWD/venv/bin/" || exit 1 + # Patch web2py's wsgihandler so that it does the equivalent of 'venv/activate' # when started by Apache.